diff --git a/CHANGES b/CHANGES index 14f0ec041d..7cbbc74e4f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,169 @@ +2.1-824 | 2013-07-22 14:25:14 -0400 + + * Fixed a scriptland state issue that manifested especially badly on proxies. (Seth Hall) + + * Another test fix. (Robin Sommer) + + * Canonyfying the output of core.print-bpf-filters. (Robin Sommer) + +2.1-820 | 2013-07-18 12:30:04 -0700 + + * Extending external canonifier to remove fractional values from + capture_loss.log. (Robin Sommer) + + * Canonifying internal order for plugins and their components to + make it deterministic. (Robin Sommer) + + * Small raw reader tweaks that got left our earlier. (Robin Sommer) + +2.1-814 | 2013-07-15 18:18:20 -0700 + + * Fixing raw reader crash when accessing nonexistant file, and + memory leak when reading from file. Addresses #1038. (Bernhard + Amann) + +2.1-811 | 2013-07-14 08:01:54 -0700 + + * Bump sqlite to 3.7.17. (Bernhard Amann) + + * Small test fixes. (Seth Hall) + + * Fix a bug where the same analyzer tag was reused for two different + analyzers. (Seth Hall) + + * Moved DPD signatures into script specific directories. Left out + the BitTorrent signatures pending further updates to that + analyzer. (Seth Hall) + +2.1-802 | 2013-07-10 10:55:14 -0700 + + * Const adjustment for methods. (Jon Siwek) + +2.1-798 | 2013-07-08 13:05:37 -0700 + + * Rewrite of the packet filter framework. (Seth Hall) + + This includes: + + - Plugin interface for adding filtering mechanisms. + + - Integrated the packet filter framework with the analyzer + framework to retrieve well-known ports from there. + + - Support for BPF-based load balancing (IPv4 and IPv6). This will + tie in with upcoming BroControl support for configuring this. + + - Support for BPF-based connection sampling. + + - Support for "shunting" traffic with BPF filters. + + - Replaced PacketFilter::all_packets with + PacketFilter::enable_auto_protocol_capture_filters. + +2.1-784 | 2013-07-04 22:28:48 -0400 + + * Add a call to lookup_connection in SSH scripts to update connval. (Seth Hall) + + * Updating submodule(s). (Robin Sommer) + +2.1-782 | 2013-07-03 17:00:39 -0700 + + * Remove the SSL log queueing mechanism that was included with the + log delay mechanism. (Seth Hall) + +2.1-780 | 2013-07-03 16:46:26 -0700 + + * Rewrite of the RAW input reader for improved robustness and new + features. (Bernhard Amann) This includes: + + - Send "end_of_data" event for all kind of streams. + - Send "process_finished" event with exit code of child + process at process termination. + - Expose name of input stream to readers. + - Better error handling. + - New "force_kill" option which SIGKILLs processes on reader termination. + - Supports reading from stdout and stderr simultaneously. + - Support sending data to stdin of child process. + - Streaming reads from external commands work without blocking. + +2.1-762 | 2013-07-03 16:33:22 -0700 + + * Fix to correct support for TLS 1.2. Addresses #1020. (Seth Hall, + with help from Rafal Lesniak). + +2.1-760 | 2013-07-03 16:31:36 -0700 + + * Teach broxygen to generate protocol analyzer plugin reference. + (Jon Siwek) + + * Adding 'const' to a number of C++ methods. (Jon Siwek) + +2.1-757 | 2013-07-03 16:28:10 -0700 + + * Fix redef of table index from clearing table. + + `redef foo["x"] = 1` now acts like `redef foo += { ["x"] = 1 }` + instead of `redef foo = { ["x"] = 1 }`. + + Addresses #1013. (Jon Siwek) + + +2.1-755 | 2013-07-03 16:22:43 -0700 + + * Add a general file analysis overview/how-to document. (Jon Siwek) + + * Improve file analysis doxygen comments. (Jon Siwek) + + * Improve tracking of HTTP file extraction. http.log now has files + taken from request and response bodies in different fields for + each, and can now track multiple files per body. That is, the + "extraction_file" field is now "extracted_request_files" and + "extracted_response_files". Addresses #988. (Jon Siwek) + + * Fix HTTP multipart body file analysis. Each part now gets assigned + a different file handle/id. (Jon Siwek) + + * Remove logging of analyzers field of FileAnalysis::Info. (Jon + Siwek) + + * Remove extraction counter in default file extraction scripts. (Jon + Siwek) + + * Remove FileAnalysis::postpone_timeout. + FileAnalysis::set_timeout_interval can now perform same function. + (Jon Siwek) + + * Make default get_file_handle handlers &priority=5 so they're + easier to override. (Jon Siwek) + + * Add input interface to forward data for file analysis. The new + Input::add_analysis function is used to automatically forward + input data on to the file analysis framework. (Jon Siwek) + + * File analysis framework interface simplifications. (Jon Siwek) + + - Remove script-layer data input interface (will be managed directly + by input framework later). + + - Only track files internally by file id hash. Chance of collision + too small to justify also tracking unique file string. + + +2.1-741 | 2013-06-07 17:28:50 -0700 + + * Fixing typo that could cause an assertion to falsely trigger. + (Robin Sommer) + +2.1-740 | 2013-06-07 16:37:32 -0700 + + * Fix for CMake 2.6.x. (Robin Sommer) + +2.1-738 | 2013-06-07 08:38:13 -0700 + + * Remove invalid free on non-allocated pointer in hash function + object. Addresses #1018. (Matthias Vallentin) + 2.1-736 | 2013-06-06 10:05:20 -0700 * New "magic constants" @DIR and @FILENAME that expand to the diff --git a/NEWS b/NEWS index f0d302156b..1fce6b1d9d 100644 --- a/NEWS +++ b/NEWS @@ -73,10 +73,12 @@ New Functionality script file name without path, respectively. (Jon Siwek) - The new file analysis framework moves most of the processing of file - content from script-land into the core, where it belongs. Much of - this is an internal change, the framework comes with the following - user-visibible functionality (some of that was already available - before, but done differently): + content from script-land into the core, where it belongs. See + doc/file-analysis.rst for more information. + + Much of this is an internal change, but the framework also comes + with the following user-visibible functionality (some of that was + already available before, but done differently): [TODO: This will probably change with further script updates.] @@ -102,6 +104,10 @@ New Functionality - IRC DCC transfers: Record to disk. +- New packet filter framework supports BPF-based load-balancing, + shunting, and sampling; plus plugin support to customize filters + dynamically. + Changed Functionality ~~~~~~~~~~~~~~~~~~~~~ @@ -180,6 +186,12 @@ Changed Functionality - The SSH::Login notice has been superseded by an corresponding intelligence framework observation (SSH::SUCCESSFUL_LOGIN). +- PacketFilter::all_packets has been replaced with + PacketFilter::enable_auto_protocol_capture_filters. + +- We removed the BitTorrent DPD signatures pending further updates to + that analyzer. + Bro 2.1 ------- diff --git a/VERSION b/VERSION index 8fc9f0438e..d35eaf1454 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-736 +2.1-824 diff --git a/aux/broctl b/aux/broctl index a1aaa1608e..0cd102805e 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit a1aaa1608ef08761a211b1e251449d796ba5e4a0 +Subproject commit 0cd102805e73343cab3f9fd4a76552e13940dad9 diff --git a/aux/btest b/aux/btest index d5b8df42cb..ce366206e3 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit d5b8df42cb9c398142e02d4bf8ede835fd0227f4 +Subproject commit ce366206e3407e534a786ad572c342e9f9fef26b diff --git a/doc/ext/bro.py b/doc/ext/bro.py index 9bdd86bd9a..6ef11c37f6 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -82,7 +82,8 @@ class BroGeneric(ObjectDescription): objects = self.env.domaindata['bro']['objects'] key = (self.objtype, name) - if key in objects: + if ( key in objects and self.objtype != "id" and + self.objtype != "type" ): self.env.warn(self.env.docname, 'duplicate description of %s %s, ' % (self.objtype, name) + @@ -150,6 +151,12 @@ class BroEnum(BroGeneric): #self.indexnode['entries'].append(('single', indextext, # targetname, targetname)) m = sig.split() + + if len(m) < 2: + self.env.warn(self.env.docname, + "bro:enum directive missing argument(s)") + return + if m[1] == "Notice::Type": if 'notices' not in self.env.domaindata['bro']: self.env.domaindata['bro']['notices'] = [] diff --git a/doc/file-analysis.rst b/doc/file-analysis.rst new file mode 100644 index 0000000000..f312e06471 --- /dev/null +++ b/doc/file-analysis.rst @@ -0,0 +1,184 @@ +============= +File Analysis +============= + +.. rst-class:: opening + + In the past, writing Bro scripts with the intent of analyzing file + content could be cumbersome because of the fact that the content + would be presented in different ways, via events, at the + script-layer depending on which network protocol was involved in the + file transfer. Scripts written to analyze files over one protocol + would have to be copied and modified to fit other protocols. The + file analysis framework (FAF) instead provides a generalized + presentation of file-related information. The information regarding + the protocol involved in transporting a file over the network is + still available, but it no longer has to dictate how one organizes + their scripting logic to handle it. A goal of the FAF is to + provide analysis specifically for files that is analogous to the + analysis Bro provides for network connections. + +.. contents:: + +File Lifecycle Events +===================== + +The key events that may occur during the lifetime of a file are: +:bro:see:`file_new`, :bro:see:`file_over_new_connection`, +:bro:see:`file_timeout`, :bro:see:`file_gap`, and +:bro:see:`file_state_remove`. Handling any of these events provides +some information about the file such as which network +:bro:see:`connection` and protocol are transporting the file, how many +bytes have been transferred so far, and its MIME type. + +.. code:: bro + + event connection_state_remove(c: connection) + { + print "connection_state_remove"; + print c$uid; + print c$id; + for ( s in c$service ) + print s; + } + + event file_state_remove(f: fa_file) + { + print "file_state_remove"; + print f$id; + for ( cid in f$conns ) + { + print f$conns[cid]$uid; + print cid; + } + print f$source; + } + +might give output like:: + + file_state_remove + Cx92a0ym5R8 + REs2LQfVW2j + [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] + HTTP + connection_state_remove + REs2LQfVW2j + [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] + HTTP + +This doesn't perform any interesting analysis yet, but does highlight +the similarity between analysis of connections and files. Connections +are identified by the usual 5-tuple or a convenient UID string while +files are identified just by a string of the same format as the +connection UID. So there's unique ways to identify both files and +connections and files hold references to a connection (or connections) +that transported it. + +Adding Analysis +=============== + +There are builtin file analyzers which can be attached to files. Once +attached, they start receiving the contents of the file as Bro extracts +it from an ongoing network connection. What they do with the file +contents is up to the particular file analyzer implementation, but +they'll typically either report further information about the file via +events (e.g. :bro:see:`FileAnalysis::ANALYZER_MD5` will report the +file's MD5 checksum via :bro:see:`file_hash` once calculated) or they'll +have some side effect (e.g. :bro:see:`FileAnalysis::ANALYZER_EXTRACT` +will write the contents of the file out to the local file system). + +In the future there may be file analyzers that automatically attach to +files based on heuristics, similar to the Dynamic Protocol Detection +(DPD) framework for connections, but many will always require an +explicit attachment decision: + +.. code:: bro + + event file_new(f: fa_file) + { + print "new file", f$id; + if ( f?$mime_type && f$mime_type == "text/plain" ) + FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); + } + + event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } + +this script calculates MD5s for all plain text files and might give +output:: + + new file, Cx92a0ym5R8 + file_hash, Cx92a0ym5R8, md5, 397168fd09991a0e712254df7bc639ac + +Some file analyzers might have tunable parameters that need to be +specified in the call to :bro:see:`FileAnalysis::add_analyzer`: + +.. code:: bro + + event file_new(f: fa_file) + { + FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, + $extract_filename="./myfile"]); + } + +In this case, the file extraction analyzer doesn't generate any further +events, but does have the side effect of writing out the file contents +to the local file system at the specified location of ``./myfile``. Of +course, for a network with more than a single file being transferred, +it's probably preferable to specify a different extraction path for each +file, unlike this example. + +Regardless of which file analyzers end up acting on a file, general +information about the file (e.g. size, time of last data transferred, +MIME type, etc.) are logged in ``file_analysis.log``. + +Input Framework Integration +=========================== + +The FAF comes with a simple way to integrate with the :doc:`Input +Framework `, so that Bro can analyze files from external sources +in the same way it analyzes files that it sees coming over traffic from +a network interface it's monitoring. It only requires a call to +:bro:see:`Input::add_analysis`: + +.. code:: bro + + redef exit_only_after_terminate = T; + + event file_new(f: fa_file) + { + print "new file", f$id; + FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); + } + + event file_state_remove(f: fa_file) + { + Input::remove(f$source); + terminate(); + } + + event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } + + event bro_init() + { + local source: string = "./myfile"; + Input::add_analysis([$source=source, $name=source]); + } + +Note that the "source" field of :bro:see:`fa_file` corresponds to the +"name" field of :bro:see:`Input::AnalysisDescription` since that is what +the input framework uses to uniquely identify an input stream. + +The output of the above script may be:: + + new file, G1fS2xthS4l + file_hash, G1fS2xthS4l, md5, 54098b367d2e87b078671fad4afb9dbb + +Nothing that special, but it at least verifies the MD5 file analyzer +saw all the bytes of the input file and calculated the checksum +correctly! diff --git a/doc/index.rst b/doc/index.rst index 29b29541b4..ad05f7bf82 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -25,6 +25,7 @@ Frameworks notice logging input + file-analysis cluster signatures @@ -45,7 +46,7 @@ Script Reference scripts/packages scripts/index scripts/builtins - scripts/bifs + scripts/proto-analyzers Other Bro Components -------------------- diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 64c3de92eb..ddb09bb29c 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -15,11 +15,11 @@ endif () # # srcDir: the directory which contains broInput # broInput: the file name of a bro policy script, any path prefix of this -# argument will be used to derive what path under policy/ the generated +# argument will be used to derive what path under scripts/ the generated # documentation will be placed. # group: optional name of group that the script documentation will belong to. -# If this is not given, .bif files automatically get their own group or -# the group is automatically by any path portion of the broInput argument. +# If this is not given, the group is automatically set to any path portion +# of the broInput argument. # # In addition to adding the makefile target, several CMake variables are set: # @@ -64,8 +64,6 @@ macro(REST_TARGET srcDir broInput) if (NOT "${ARGN}" STREQUAL "") set(group ${ARGN}) - elseif (${broInput} MATCHES "\\.bif\\.bro$") - set(group bifs) elseif (relDstDir) set(group ${relDstDir}/index) # add package index to master package list if not already in it @@ -126,6 +124,29 @@ endmacro(REST_TARGET) # Schedule Bro scripts for which to generate documentation. include(DocSourcesList.cmake) +# This reST target is independent of a particular Bro script... +add_custom_command(OUTPUT proto-analyzers.rst + # delete any leftover state from previous bro runs + COMMAND "${CMAKE_COMMAND}" + ARGS -E remove_directory .state + # generate the reST documentation using bro + COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic ${CMAKE_BINARY_DIR}/src/bro + ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1) + # move generated doc into a new directory tree that + # defines the final structure of documents + COMMAND "${CMAKE_COMMAND}" + ARGS -E make_directory ${dstDir} + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy proto-analyzers.rst ${dstDir} + # clean up the build directory + COMMAND rm + ARGS -rf .state *.log *.rst + DEPENDS bro + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "[Bro] Generating reST docs for proto-analyzers.rst" +) +list(APPEND ALL_REST_OUTPUTS proto-analyzers.rst) + # create temporary list of all docs to include in the master policy/index file file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 0b077c2c50..529b03ca83 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -34,6 +34,7 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro) @@ -111,6 +112,7 @@ rest_target(${psd} base/frameworks/notice/non-cluster.bro) rest_target(${psd} base/frameworks/notice/weird.bro) rest_target(${psd} base/frameworks/packet-filter/main.bro) rest_target(${psd} base/frameworks/packet-filter/netstats.bro) +rest_target(${psd} base/frameworks/packet-filter/utils.bro) rest_target(${psd} base/frameworks/reporter/main.bro) rest_target(${psd} base/frameworks/signatures/main.bro) rest_target(${psd} base/frameworks/software/main.bro) @@ -189,6 +191,7 @@ rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro) rest_target(${psd} policy/frameworks/intel/smtp.bro) rest_target(${psd} policy/frameworks/intel/ssl.bro) rest_target(${psd} policy/frameworks/intel/where-locations.bro) +rest_target(${psd} policy/frameworks/packet-filter/shunt.bro) rest_target(${psd} policy/frameworks/software/version-changes.bro) rest_target(${psd} policy/frameworks/software/vulnerable.bro) rest_target(${psd} policy/integration/barnyard2/main.bro) @@ -197,6 +200,7 @@ rest_target(${psd} policy/integration/collective-intel/main.bro) rest_target(${psd} policy/misc/app-metrics.bro) rest_target(${psd} policy/misc/capture-loss.bro) rest_target(${psd} policy/misc/detect-traceroute/main.bro) +rest_target(${psd} policy/misc/load-balancing.bro) rest_target(${psd} policy/misc/loaded-scripts.bro) rest_target(${psd} policy/misc/profiling.bro) rest_target(${psd} policy/misc/scan.bro) diff --git a/doc/scripts/bifs.rst b/doc/scripts/bifs.rst deleted file mode 100644 index eaae0e13b8..0000000000 --- a/doc/scripts/bifs.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which broxygen appends during the build process - -Built-In Functions (BIFs) -========================= - diff --git a/scripts/base/frameworks/analyzer/main.bro b/scripts/base/frameworks/analyzer/main.bro index e2dcf151c7..c4ee5c943b 100644 --- a/scripts/base/frameworks/analyzer/main.bro +++ b/scripts/base/frameworks/analyzer/main.bro @@ -9,9 +9,9 @@ ##! :bro:enum:`Analyzer::ANALYZER_HTTP`. These tags are defined internally by ##! the analyzers themselves, and documented in their analyzer-specific ##! description along with the events that they generate. -##! -##! .. todo: ``The ANALYZER_*`` are in fact not yet documented, we need to -##! add that to Broxygen. + +@load base/frameworks/packet-filter/utils + module Analyzer; export { @@ -98,7 +98,21 @@ export { ## ## Returns: True if succesful. global schedule_analyzer: function(orig: addr, resp: addr, resp_p: port, - analyzer: Analyzer::Tag, tout: interval) : bool; + analyzer: Analyzer::Tag, tout: interval) : bool; + + ## Automatically creates a BPF filter for the specified protocol based + ## on the data supplied for the protocol through the + ## :bro:see:`Analyzer::register_for_ports` function. + ## + ## tag: The analyzer tag. + ## + ## Returns: BPF filter string. + global analyzer_to_bpf: function(tag: Analyzer::Tag): string; + + ## Create a BPF filter which matches all of the ports defined + ## by the various protocol analysis scripts as "registered ports" + ## for the protocol. + global get_bpf: function(): string; ## A set of analyzers to disable by default at startup. The default set ## contains legacy analyzers that are no longer supported. @@ -179,3 +193,25 @@ function schedule_analyzer(orig: addr, resp: addr, resp_p: port, return __schedule_analyzer(orig, resp, resp_p, analyzer, tout); } +function analyzer_to_bpf(tag: Analyzer::Tag): string + { + # Return an empty string if an undefined analyzer was given. + if ( tag !in ports ) + return ""; + + local output = ""; + for ( p in ports[tag] ) + output = PacketFilter::combine_filters(output, "or", PacketFilter::port_to_bpf(p)); + return output; + } + +function get_bpf(): string + { + local output = ""; + for ( tag in ports ) + { + output = PacketFilter::combine_filters(output, "or", analyzer_to_bpf(tag)); + } + return output; + } + diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 7ded67688a..caf5119d9d 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -216,12 +216,9 @@ function setup_peer(p: event_peer, node: Node) request_remote_events(p, node$events); } - if ( node?$capture_filter ) + if ( node?$capture_filter && node$capture_filter != "" ) { local filter = node$capture_filter; - if ( filter == "" ) - filter = PacketFilter::default_filter; - do_script_log(p, fmt("sending capture_filter: %s", filter)); send_capture_filter(p, filter); } diff --git a/scripts/base/frameworks/dpd/dpd.sig b/scripts/base/frameworks/dpd/dpd.sig deleted file mode 100644 index 49e24cefc6..0000000000 --- a/scripts/base/frameworks/dpd/dpd.sig +++ /dev/null @@ -1,212 +0,0 @@ -# Signatures to initiate dynamic protocol detection. - -signature dpd_ftp_client { - ip-proto == tcp - payload /(|.*[\n\r]) *[uU][sS][eE][rR] / - tcp-state originator -} - -# Match for server greeting (220, 120) and for login or passwd -# required (230, 331). -signature dpd_ftp_server { - ip-proto == tcp - payload /[\n\r ]*(120|220)[^0-9].*[\n\r] *(230|331)[^0-9]/ - tcp-state responder - requires-reverse-signature dpd_ftp_client - enable "ftp" -} - -signature dpd_http_client { - ip-proto == tcp - payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/ - tcp-state originator -} - -signature dpd_http_server { - ip-proto == tcp - payload /^HTTP\/[0-9]/ - tcp-state responder - requires-reverse-signature dpd_http_client - enable "http" -} - -signature dpd_bittorrenttracker_client { - ip-proto == tcp - payload /^.*\/announce\?.*info_hash/ - tcp-state originator -} - -signature dpd_bittorrenttracker_server { - ip-proto == tcp - payload /^HTTP\/[0-9]/ - tcp-state responder - requires-reverse-signature dpd_bittorrenttracker_client - enable "bittorrenttracker" -} - -signature dpd_bittorrent_peer1 { - ip-proto == tcp - payload /^\x13BitTorrent protocol/ - tcp-state originator -} - -signature dpd_bittorrent_peer2 { - ip-proto == tcp - payload /^\x13BitTorrent protocol/ - tcp-state responder - requires-reverse-signature dpd_bittorrent_peer1 - enable "bittorrent" -} - -signature irc_client1 { - ip-proto == tcp - payload /(|.*[\r\n]) *[Uu][Ss][Ee][Rr] +.+[\n\r]+ *[Nn][Ii][Cc][Kk] +.*[\r\n]/ - requires-reverse-signature irc_server_reply - tcp-state originator - enable "irc" -} - -signature irc_client2 { - ip-proto == tcp - payload /(|.*[\r\n]) *[Nn][Ii][Cc][Kk] +.+[\r\n]+ *[Uu][Ss][Ee][Rr] +.+[\r\n]/ - requires-reverse-signature irc_server_reply - tcp-state originator - enable "irc" -} - -signature irc_server_reply { - ip-proto == tcp - payload /^(|.*[\n\r])(:[^ \n\r]+ )?[0-9][0-9][0-9] / - tcp-state responder -} - -signature irc_server_to_server1 { - ip-proto == tcp - payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/ -} - -signature irc_server_to_server2 { - ip-proto == tcp - payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/ - requires-reverse-signature irc_server_to_server1 - enable "irc" -} - -signature dpd_smtp_client { - ip-proto == tcp - payload /(|.*[\n\r])[[:space:]]*([hH][eE][lL][oO]|[eE][hH][lL][oO])/ - requires-reverse-signature dpd_smtp_server - enable "smtp" - tcp-state originator -} - -signature dpd_smtp_server { - ip-proto == tcp - payload /^[[:space:]]*220[[:space:]-]/ - tcp-state responder -} - -signature dpd_ssh_client { - ip-proto == tcp - payload /^[sS][sS][hH]-/ - requires-reverse-signature dpd_ssh_server - enable "ssh" - tcp-state originator -} - -signature dpd_ssh_server { - ip-proto == tcp - payload /^[sS][sS][hH]-/ - tcp-state responder -} - -signature dpd_pop3_server { - ip-proto == tcp - payload /^\+OK/ - requires-reverse-signature dpd_pop3_client - enable "pop3" - tcp-state responder -} - -signature dpd_pop3_client { - ip-proto == tcp - payload /(|.*[\r\n])[[:space:]]*([uU][sS][eE][rR][[:space:]]|[aA][pP][oO][pP][[:space:]]|[cC][aA][pP][aA]|[aA][uU][tT][hH])/ - tcp-state originator -} - -signature dpd_ssl_server { - ip-proto == tcp - # Server hello. - payload /^(\x16\x03[\x00\x01\x02]..\x02...\x03[\x00\x01\x02]|...?\x04..\x00\x02).*/ - requires-reverse-signature dpd_ssl_client - enable "ssl" - tcp-state responder -} - -signature dpd_ssl_client { - ip-proto == tcp - # Client hello. - payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/ - tcp-state originator -} - -signature dpd_ayiya { - ip-proto = udp - payload /^..\x11\x29/ - enable "ayiya" -} - -signature dpd_teredo { - ip-proto = udp - payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/ - enable "teredo" -} - -signature dpd_socks4_client { - ip-proto == tcp - # '32' is a rather arbitrary max length for the user name. - payload /^\x04[\x01\x02].{0,32}\x00/ - tcp-state originator -} - -signature dpd_socks4_server { - ip-proto == tcp - requires-reverse-signature dpd_socks4_client - payload /^\x00[\x5a\x5b\x5c\x5d]/ - tcp-state responder - enable "socks" -} - -signature dpd_socks4_reverse_client { - ip-proto == tcp - # '32' is a rather arbitrary max length for the user name. - payload /^\x04[\x01\x02].{0,32}\x00/ - tcp-state responder -} - -signature dpd_socks4_reverse_server { - ip-proto == tcp - requires-reverse-signature dpd_socks4_reverse_client - payload /^\x00[\x5a\x5b\x5c\x5d]/ - tcp-state originator - enable "socks" -} - -signature dpd_socks5_client { - ip-proto == tcp - # Watch for a few authentication methods to reduce false positives. - payload /^\x05.[\x00\x01\x02]/ - tcp-state originator -} - -signature dpd_socks5_server { - ip-proto == tcp - requires-reverse-signature dpd_socks5_client - # Watch for a single authentication method to be chosen by the server or - # the server to indicate the no authentication is required. - payload /^\x05(\x00|\x01[\x00\x01\x02])/ - tcp-state responder - enable "socks" -} - - diff --git a/scripts/base/frameworks/dpd/main.bro b/scripts/base/frameworks/dpd/main.bro index c3282a1da4..9df8a45e5e 100644 --- a/scripts/base/frameworks/dpd/main.bro +++ b/scripts/base/frameworks/dpd/main.bro @@ -3,8 +3,6 @@ module DPD; -@load-sigs ./dpd.sig - export { ## Add the DPD logging stream identifier. redef enum Log::ID += { LOG }; diff --git a/scripts/base/frameworks/file-analysis/main.bro b/scripts/base/frameworks/file-analysis/main.bro index 0ed66464fe..3352787cba 100644 --- a/scripts/base/frameworks/file-analysis/main.bro +++ b/scripts/base/frameworks/file-analysis/main.bro @@ -15,18 +15,20 @@ export { ## A structure which represents a desired type of file analysis. type AnalyzerArgs: record { ## The type of analysis. - tag: Analyzer; + tag: FileAnalysis::Tag; ## The local filename to which to write an extracted file. Must be ## set when *tag* is :bro:see:`FileAnalysis::ANALYZER_EXTRACT`. extract_filename: string &optional; ## An event which will be generated for all new file contents, - ## chunk-wise. + ## chunk-wise. Used when *tag* is + ## :bro:see:`FileAnalysis::ANALYZER_DATA_EVENT`. chunk_event: event(f: fa_file, data: string, off: count) &optional; ## An event which will be generated for all new file contents, - ## stream-wise. + ## stream-wise. Used when *tag* is + ## :bro:see:`FileAnalysis::ANALYZER_DATA_EVENT`. stream_event: event(f: fa_file, data: string) &optional; } &redef; @@ -87,7 +89,7 @@ export { conn_uids: set[string] &log; ## A set of analysis types done during the file analysis. - analyzers: set[Analyzer] &log; + analyzers: set[FileAnalysis::Tag]; ## Local filenames of extracted files. extracted_files: set[string] &log; @@ -120,7 +122,9 @@ export { ## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is ## used to determine the length of inactivity that is allowed for a file - ## before internal state related to it is cleaned up. + ## before internal state related to it is cleaned up. When used within a + ## :bro:see:`file_timeout` handler, the analysis will delay timing out + ## again for the period specified by *t*. ## ## f: the file. ## @@ -130,18 +134,6 @@ export { ## for the *id* isn't currently active. global set_timeout_interval: function(f: fa_file, t: interval): bool; - ## Postpones the timeout of file analysis for a given file. - ## When used within a :bro:see:`file_timeout` handler for, the analysis - ## the analysis will delay timing out for the period of time indicated by - ## the *timeout_interval* field of :bro:see:`fa_file`, which can be set - ## with :bro:see:`FileAnalysis::set_timeout_interval`. - ## - ## f: the file. - ## - ## Returns: true if the timeout will be postponed, or false if analysis - ## for the *id* isn't currently active. - global postpone_timeout: function(f: fa_file): bool; - ## Adds an analyzer to the analysis of a given file. ## ## f: the file. @@ -171,58 +163,6 @@ export { ## rest of it's contents, or false if analysis for the *id* ## isn't currently active. global stop: function(f: fa_file): bool; - - ## Sends a sequential stream of data in for file analysis. - ## Meant for use when providing external file analysis input (e.g. - ## from the input framework). - ## - ## source: a string that uniquely identifies the logical file that the - ## data is a part of and describes its source. - ## - ## data: bytestring contents of the file to analyze. - global data_stream: function(source: string, data: string); - - ## Sends a non-sequential chunk of data in for file analysis. - ## Meant for use when providing external file analysis input (e.g. - ## from the input framework). - ## - ## source: a string that uniquely identifies the logical file that the - ## data is a part of and describes its source. - ## - ## data: bytestring contents of the file to analyze. - ## - ## offset: the offset within the file that this chunk starts. - global data_chunk: function(source: string, data: string, offset: count); - - ## Signals a content gap in the file bytestream. - ## Meant for use when providing external file analysis input (e.g. - ## from the input framework). - ## - ## source: a string that uniquely identifies the logical file that the - ## data is a part of and describes its source. - ## - ## offset: the offset within the file that this gap starts. - ## - ## len: the number of bytes that are missing. - global gap: function(source: string, offset: count, len: count); - - ## Signals the total size of a file. - ## Meant for use when providing external file analysis input (e.g. - ## from the input framework). - ## - ## source: a string that uniquely identifies the logical file that the - ## data is a part of and describes its source. - ## - ## size: the number of bytes that comprise the full file. - global set_size: function(source: string, size: count); - - ## Signals the end of a file. - ## Meant for use when providing external file analysis input (e.g. - ## from the input framework). - ## - ## source: a string that uniquely identifies the logical file that the - ## data is a part of and describes its source. - global eof: function(source: string); } redef record fa_file += { @@ -259,11 +199,6 @@ function set_timeout_interval(f: fa_file, t: interval): bool return __set_timeout_interval(f$id, t); } -function postpone_timeout(f: fa_file): bool - { - return __postpone_timeout(f$id); - } - function add_analyzer(f: fa_file, args: AnalyzerArgs): bool { if ( ! __add_analyzer(f$id, args) ) return F; @@ -287,31 +222,6 @@ function stop(f: fa_file): bool return __stop(f$id); } -function data_stream(source: string, data: string) - { - __data_stream(source, data); - } - -function data_chunk(source: string, data: string, offset: count) - { - __data_chunk(source, data, offset); - } - -function gap(source: string, offset: count, len: count) - { - __gap(source, offset, len); - } - -function set_size(source: string, size: count) - { - __set_size(source, size); - } - -function eof(source: string) - { - __eof(source); - } - event bro_init() &priority=5 { Log::create_stream(FileAnalysis::LOG, diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4de98ea0f2..e5d74cbc36 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -122,6 +122,34 @@ export { config: table[string] of string &default=table(); }; + ## A file analyis input stream type used to forward input data to the + ## file analysis framework. + type AnalysisDescription: record { + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. + source: string; + + ## Reader to use for this steam. Compatible readers must be + ## able to accept a filter of a single string type (i.e. + ## they read a byte stream). + reader: Reader &default=Input::READER_BINARY; + + ## Read mode to use for this stream + mode: Mode &default=default_mode; + + ## Descriptive name that uniquely identifies the input source. + ## Can be used used to remove a stream at a later time. + ## This will also be used for the unique *source* field of + ## :bro:see:`fa_file`. Most of the time, the best choice for this + ## field will be the same value as the *source* field. + name: string; + + ## A key/value table that will be passed on the reader. + ## Interpretation of the values is left to the writer, but + ## usually they will be used for configuration purposes. + config: table[string] of string &default=table(); + }; + ## Create a new table input from a given source. Returns true on success. ## ## description: `TableDescription` record describing the source. @@ -132,6 +160,14 @@ export { ## description: `TableDescription` record describing the source. global add_event: function(description: Input::EventDescription) : bool; + ## Create a new file analysis input from a given source. Data read from + ## the source is automatically forwarded to the file analysis framework. + ## + ## description: A record describing the source + ## + ## Returns: true on sucess. + global add_analysis: function(description: Input::AnalysisDescription) : bool; + ## Remove a input stream. Returns true on success and false if the named stream was ## not found. ## @@ -164,6 +200,11 @@ function add_event(description: Input::EventDescription) : bool return __create_event_stream(description); } +function add_analysis(description: Input::AnalysisDescription) : bool + { + return __create_analysis_stream(description); + } + function remove(id: string) : bool { return __remove_stream(id); diff --git a/scripts/base/frameworks/input/readers/raw.bro b/scripts/base/frameworks/input/readers/raw.bro index 45deed3eda..9fa7d1a2fb 100644 --- a/scripts/base/frameworks/input/readers/raw.bro +++ b/scripts/base/frameworks/input/readers/raw.bro @@ -6,4 +6,12 @@ export { ## Separator between input records. ## Please note that the separator has to be exactly one character long const record_separator = "\n" &redef; + + ## Event that is called when a process created by the raw reader exits. + ## + ## name: name of the input stream + ## source: source of the input stream + ## exit_code: exit code of the program, or number of the signal that forced the program to exit + ## signal_exit: false when program exitted normally, true when program was forced to exit by a signal + global process_finished: event(name: string, source:string, exit_code:count, signal_exit:bool); } diff --git a/scripts/base/frameworks/packet-filter/__load__.bro b/scripts/base/frameworks/packet-filter/__load__.bro index 1d72e1ebe0..011885e8b7 100644 --- a/scripts/base/frameworks/packet-filter/__load__.bro +++ b/scripts/base/frameworks/packet-filter/__load__.bro @@ -1,2 +1,3 @@ +@load ./utils @load ./main @load ./netstats diff --git a/scripts/base/frameworks/packet-filter/main.bro b/scripts/base/frameworks/packet-filter/main.bro index afcb5bd700..72b2b62f34 100644 --- a/scripts/base/frameworks/packet-filter/main.bro +++ b/scripts/base/frameworks/packet-filter/main.bro @@ -1,10 +1,12 @@ ##! This script supports how Bro sets it's BPF capture filter. By default -##! Bro sets an unrestricted filter that allows all traffic. If a filter +##! Bro sets a capture filter that allows all traffic. If a filter ##! is set on the command line, that filter takes precedence over the default ##! open filter and all filters defined in Bro scripts with the ##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables. @load base/frameworks/notice +@load base/frameworks/analyzer +@load ./utils module PacketFilter; @@ -14,11 +16,14 @@ export { ## Add notice types related to packet filter errors. redef enum Notice::Type += { - ## This notice is generated if a packet filter is unable to be compiled. + ## This notice is generated if a packet filter cannot be compiled. Compile_Failure, - ## This notice is generated if a packet filter is fails to install. + ## Generated if a packet filter is fails to install. Install_Failure, + + ## Generated when a notice takes too long to compile. + Too_Long_To_Compile_Filter }; ## The record type defining columns to be logged in the packet filter @@ -42,83 +47,248 @@ export { success: bool &log &default=T; }; - ## By default, Bro will examine all packets. If this is set to false, - ## it will dynamically build a BPF filter that only select protocols - ## for which the user has loaded a corresponding analysis script. - ## The latter used to be default for Bro versions < 2.0. That has now - ## changed however to enable port-independent protocol analysis. - const all_packets = T &redef; + ## The BPF filter that is used by default to define what traffic should + ## be captured. Filters defined in :bro:id:`restrict_filters` will still + ## be applied to reduce the captured traffic. + const default_capture_filter = "ip or not ip" &redef; ## Filter string which is unconditionally or'ed to the beginning of every ## dynamically built filter. const unrestricted_filter = "" &redef; + ## Filter string which is unconditionally and'ed to the beginning of every + ## dynamically built filter. This is mostly used when a custom filter is being + ## used but MPLS or VLAN tags are on the traffic. + const restricted_filter = "" &redef; + + ## The maximum amount of time that you'd like to allow for BPF filters to compile. + ## If this time is exceeded, compensation measures may be taken by the framework + ## to reduce the filter size. This threshold being crossed also results in + ## the :bro:see:`PacketFilter::Too_Long_To_Compile_Filter` notice. + const max_filter_compile_time = 100msec &redef; + + ## Install a BPF filter to exclude some traffic. The filter should positively + ## match what is to be excluded, it will be wrapped in a "not". + ## + ## filter_id: An arbitrary string that can be used to identify + ## the filter. + ## + ## filter: A BPF expression of traffic that should be excluded. + ## + ## Returns: A boolean value to indicate if the filter was successfully + ## installed or not. + global exclude: function(filter_id: string, filter: string): bool; + + ## Install a temporary filter to traffic which should not be passed through + ## the BPF filter. The filter should match the traffic you don't want + ## to see (it will be wrapped in a "not" condition). + ## + ## filter_id: An arbitrary string that can be used to identify + ## the filter. + ## + ## filter: A BPF expression of traffic that should be excluded. + ## + ## length: The duration for which this filter should be put in place. + ## + ## Returns: A boolean value to indicate if the filter was successfully + ## installed or not. + global exclude_for: function(filter_id: string, filter: string, span: interval): bool; + ## Call this function to build and install a new dynamically built ## packet filter. - global install: function(); + global install: function(): bool; + + ## A data structure to represent filter generating plugins. + type FilterPlugin: record { + ## A function that is directly called when generating the complete filter. + func : function(); + }; + + ## API function to register a new plugin for dynamic restriction filters. + global register_filter_plugin: function(fp: FilterPlugin); + + ## Enables the old filtering approach of "only watch common ports for + ## analyzed protocols". + ## + ## Unless you know what you are doing, leave this set to F. + const enable_auto_protocol_capture_filters = F &redef; ## This is where the default packet filter is stored and it should not ## normally be modified by users. - global default_filter = ""; + global current_filter = ""; } +global dynamic_restrict_filters: table[string] of string = {}; + +# Track if a filter is currently building so functions that would ultimately +# install a filter immediately can still be used but they won't try to build or +# install the filter. +global currently_building = F; + +# Internal tracking for if the the filter being built has possibly been changed. +global filter_changed = F; + +global filter_plugins: set[FilterPlugin] = {}; + redef enum PcapFilterID += { DefaultPcapFilter, + FilterTester, }; -function combine_filters(lfilter: string, rfilter: string, op: string): string +function test_filter(filter: string): bool { - if ( lfilter == "" && rfilter == "" ) - return ""; - else if ( lfilter == "" ) - return rfilter; - else if ( rfilter == "" ) - return lfilter; - else - return fmt("(%s) %s (%s)", lfilter, op, rfilter); + if ( ! precompile_pcap_filter(FilterTester, filter) ) + { + # The given filter was invalid + # TODO: generate a notice. + return F; + } + return T; } -function build_default_filter(): string +# This tracks any changes for filtering mechanisms that play along nice +# and set filter_changed to T. +event filter_change_tracking() + { + if ( filter_changed ) + install(); + + schedule 5min { filter_change_tracking() }; + } + +event bro_init() &priority=5 + { + Log::create_stream(PacketFilter::LOG, [$columns=Info]); + + # Preverify the capture and restrict filters to give more granular failure messages. + for ( id in capture_filters ) + { + if ( ! test_filter(capture_filters[id]) ) + Reporter::fatal(fmt("Invalid capture_filter named '%s' - '%s'", id, capture_filters[id])); + } + + for ( id in restrict_filters ) + { + if ( ! test_filter(restrict_filters[id]) ) + Reporter::fatal(fmt("Invalid restrict filter named '%s' - '%s'", id, restrict_filters[id])); + } + } + +event bro_init() &priority=-5 + { + install(); + + event filter_change_tracking(); + } + +function register_filter_plugin(fp: FilterPlugin) + { + add filter_plugins[fp]; + } + +event remove_dynamic_filter(filter_id: string) + { + if ( filter_id in dynamic_restrict_filters ) + { + delete dynamic_restrict_filters[filter_id]; + install(); + } + } + +function exclude(filter_id: string, filter: string): bool + { + if ( ! test_filter(filter) ) + return F; + + dynamic_restrict_filters[filter_id] = filter; + install(); + return T; + } + +function exclude_for(filter_id: string, filter: string, span: interval): bool + { + if ( exclude(filter_id, filter) ) + { + schedule span { remove_dynamic_filter(filter_id) }; + return T; + } + return F; + } + +function build(): string { if ( cmd_line_bpf_filter != "" ) # Return what the user specified on the command line; return cmd_line_bpf_filter; - if ( all_packets ) - # Return an "always true" filter. - return "ip or not ip"; + currently_building = T; - # Build filter dynamically. + # Generate all of the plugin based filters. + for ( plugin in filter_plugins ) + { + plugin$func(); + } - # First the capture_filter. local cfilter = ""; - for ( id in capture_filters ) - cfilter = combine_filters(cfilter, capture_filters[id], "or"); + if ( |capture_filters| == 0 && ! enable_auto_protocol_capture_filters ) + cfilter = default_capture_filter; - # Then the restrict_filter. + for ( id in capture_filters ) + cfilter = combine_filters(cfilter, "or", capture_filters[id]); + + if ( enable_auto_protocol_capture_filters ) + cfilter = combine_filters(cfilter, "or", Analyzer::get_bpf()); + + # Apply the restriction filters. local rfilter = ""; for ( id in restrict_filters ) - rfilter = combine_filters(rfilter, restrict_filters[id], "and"); + rfilter = combine_filters(rfilter, "and", restrict_filters[id]); + + # Apply the dynamic restriction filters. + for ( filt in dynamic_restrict_filters ) + rfilter = combine_filters(rfilter, "and", string_cat("not (", dynamic_restrict_filters[filt], ")")); # Finally, join them into one filter. - local filter = combine_filters(rfilter, cfilter, "and"); - if ( unrestricted_filter != "" ) - filter = combine_filters(unrestricted_filter, filter, "or"); + local filter = combine_filters(cfilter, "and", rfilter); + if ( unrestricted_filter != "" ) + filter = combine_filters(unrestricted_filter, "or", filter); + if ( restricted_filter != "" ) + filter = combine_filters(restricted_filter, "and", filter); + + currently_building = F; return filter; } -function install() +function install(): bool { - default_filter = build_default_filter(); + if ( currently_building ) + return F; - if ( ! precompile_pcap_filter(DefaultPcapFilter, default_filter) ) + local tmp_filter = build(); + + # No need to proceed if the filter hasn't changed. + if ( tmp_filter == current_filter ) + return F; + + local ts = current_time(); + if ( ! precompile_pcap_filter(DefaultPcapFilter, tmp_filter) ) { NOTICE([$note=Compile_Failure, $msg=fmt("Compiling packet filter failed"), - $sub=default_filter]); - Reporter::fatal(fmt("Bad pcap filter '%s'", default_filter)); + $sub=tmp_filter]); + if ( network_time() == 0.0 ) + Reporter::fatal(fmt("Bad pcap filter '%s'", tmp_filter)); + else + Reporter::warning(fmt("Bad pcap filter '%s'", tmp_filter)); } + local diff = current_time()-ts; + if ( diff > max_filter_compile_time ) + NOTICE([$note=Too_Long_To_Compile_Filter, + $msg=fmt("A BPF filter is taking longer than %0.1f seconds to compile", diff)]); + + # Set it to the current filter if it passed precompiling + current_filter = tmp_filter; # Do an audit log for the packet filter. local info: Info; @@ -129,7 +299,7 @@ function install() info$ts = current_time(); info$init = T; } - info$filter = default_filter; + info$filter = current_filter; if ( ! install_pcap_filter(DefaultPcapFilter) ) { @@ -137,15 +307,13 @@ function install() info$success = F; NOTICE([$note=Install_Failure, $msg=fmt("Installing packet filter failed"), - $sub=default_filter]); + $sub=current_filter]); } if ( reading_live_traffic() || reading_traces() ) Log::write(PacketFilter::LOG, info); - } -event bro_init() &priority=10 - { - Log::create_stream(PacketFilter::LOG, [$columns=Info]); - PacketFilter::install(); + # Update the filter change tracking + filter_changed = F; + return T; } diff --git a/scripts/base/frameworks/packet-filter/netstats.bro b/scripts/base/frameworks/packet-filter/netstats.bro index 9fbaa5cd1d..b5ffe24f54 100644 --- a/scripts/base/frameworks/packet-filter/netstats.bro +++ b/scripts/base/frameworks/packet-filter/netstats.bro @@ -13,7 +13,7 @@ export { }; ## This is the interval between individual statistics collection. - const stats_collection_interval = 10secs; + const stats_collection_interval = 5min; } event net_stats_update(last_stat: NetStats) diff --git a/scripts/base/frameworks/packet-filter/utils.bro b/scripts/base/frameworks/packet-filter/utils.bro new file mode 100644 index 0000000000..7728ebf9f9 --- /dev/null +++ b/scripts/base/frameworks/packet-filter/utils.bro @@ -0,0 +1,58 @@ +module PacketFilter; + +export { + ## Takes a :bro:type:`port` and returns a BPF expression which will + ## match the port. + ## + ## p: The port. + ## + ## Returns: A valid BPF filter string for matching the port. + global port_to_bpf: function(p: port): string; + + ## Create a BPF filter to sample IPv4 and IPv6 traffic. + ## + ## num_parts: The number of parts the traffic should be split into. + ## + ## this_part: The part of the traffic this filter will accept. 0-based. + global sampling_filter: function(num_parts: count, this_part: count): string; + + ## Combines two valid BPF filter strings with a string based operator + ## to form a new filter. + ## + ## lfilter: Filter which will go on the left side. + ## + ## op: Operation being applied (typically "or" or "and"). + ## + ## rfilter: Filter which will go on the right side. + ## + ## Returns: A new string representing the two filters combined with + ## the operator. Either filter being an empty string will + ## still result in a valid filter. + global combine_filters: function(lfilter: string, op: string, rfilter: string): string; +} + +function port_to_bpf(p: port): string + { + local tp = get_port_transport_proto(p); + return cat(tp, " and ", fmt("port %d", p)); + } + +function combine_filters(lfilter: string, op: string, rfilter: string): string + { + if ( lfilter == "" && rfilter == "" ) + return ""; + else if ( lfilter == "" ) + return rfilter; + else if ( rfilter == "" ) + return lfilter; + else + return fmt("(%s) %s (%s)", lfilter, op, rfilter); + } + +function sampling_filter(num_parts: count, this_part: count): string + { + local v4_filter = fmt("ip and ((ip[14:2]+ip[18:2]) - (%d*((ip[14:2]+ip[18:2])/%d)) == %d)", num_parts, num_parts, this_part); + # TODO: this is probably a fairly suboptimal filter, but it should work for now. + local v6_filter = fmt("ip6 and ((ip6[22:2]+ip6[38:2]) - (%d*((ip6[22:2]+ip6[38:2])/%d)) == %d)", num_parts, num_parts, this_part); + return combine_filters(v4_filter, "or", v6_filter); + } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 2110110a40..60ed0d2fd1 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -222,17 +222,6 @@ type endpoint_stats: record { endian_type: count; }; -## A unique analyzer instance ID. Each time instantiates a protocol analyzers -## for a connection, it assigns it a unique ID that can be used to reference -## that instance. -## -## .. bro:see:: Analyzer::name Analyzer::disable_analyzer protocol_confirmation -## protocol_violation -## -## .. todo::While we declare an alias for the type here, the events/functions still -## use ``count``. That should be changed. -type AnalyzerID: count; - module Tunnel; export { ## Records the identity of an encapsulating parent of a tunneled connection. @@ -777,19 +766,6 @@ global signature_files = "" &add_func = add_signature_file; ## ``p0f`` fingerprint file to use. Will be searched relative to ``BROPATH``. const passive_fingerprint_file = "base/misc/p0f.fp" &redef; -# todo::testing to see if I can remove these without causing problems. -#const ftp = 21/tcp; -#const ssh = 22/tcp; -#const telnet = 23/tcp; -#const smtp = 25/tcp; -#const domain = 53/tcp; # note, doesn't include UDP version -#const gopher = 70/tcp; -#const finger = 79/tcp; -#const http = 80/tcp; -#const ident = 113/tcp; -#const bgp = 179/tcp; -#const rlogin = 513/tcp; - # TCP values for :bro:see:`endpoint` *state* field. # todo::these should go into an enum to make them autodoc'able. const TCP_INACTIVE = 0; ##< Endpoint is still inactive. @@ -3065,12 +3041,12 @@ module GLOBAL; ## Number of bytes per packet to capture from live interfaces. const snaplen = 8192 &redef; +# Load BiFs defined by plugins. +@load base/bif/plugins + # Load these frameworks here because they use fairly deep integration with # BiFs and script-land defined types. @load base/frameworks/logging @load base/frameworks/input @load base/frameworks/analyzer @load base/frameworks/file-analysis - -# Load BiFs defined by plugins. -@load base/bif/plugins diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 9c3995673c..6aa8ff5e26 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -41,10 +41,12 @@ @load base/protocols/http @load base/protocols/irc @load base/protocols/modbus +@load base/protocols/pop3 @load base/protocols/smtp @load base/protocols/socks @load base/protocols/ssh @load base/protocols/ssl @load base/protocols/syslog +@load base/protocols/tunnels @load base/misc/find-checksum-offloading diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index 15da9aa7b7..bf47519cd8 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -122,14 +122,6 @@ redef record connection += { dns_state: State &optional; }; -# DPD configuration. -redef capture_filters += { - ["dns"] = "port 53", - ["mdns"] = "udp and port 5353", - ["llmns"] = "udp and port 5355", - ["netbios-ns"] = "udp port 137", -}; - const ports = { 53/udp, 53/tcp, 137/udp, 5353/udp, 5355/udp }; redef likely_server_ports += { ports }; @@ -215,6 +207,11 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) { if ( ans$answer_type == DNS_ANS ) { + if ( ! c?$dns ) + { + event conn_weird("dns_unmatched_reply", c, ""); + hook set_session(c, msg, F); + } c$dns$AA = msg$AA; c$dns$RA = msg$RA; diff --git a/scripts/base/protocols/ftp/__load__.bro b/scripts/base/protocols/ftp/__load__.bro index 464571dc7d..f3226de69d 100644 --- a/scripts/base/protocols/ftp/__load__.bro +++ b/scripts/base/protocols/ftp/__load__.bro @@ -3,3 +3,5 @@ @load ./file-analysis @load ./file-extract @load ./gridftp + +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/ftp/dpd.sig b/scripts/base/protocols/ftp/dpd.sig new file mode 100644 index 0000000000..3a6ceadd18 --- /dev/null +++ b/scripts/base/protocols/ftp/dpd.sig @@ -0,0 +1,15 @@ +signature dpd_ftp_client { + ip-proto == tcp + payload /(|.*[\n\r]) *[uU][sS][eE][rR] / + tcp-state originator +} + +# Match for server greeting (220, 120) and for login or passwd +# required (230, 331). +signature dpd_ftp_server { + ip-proto == tcp + payload /[\n\r ]*(120|220)[^0-9].*[\n\r] *(230|331)[^0-9]/ + tcp-state responder + requires-reverse-signature dpd_ftp_client + enable "ftp" +} diff --git a/scripts/base/protocols/ftp/file-analysis.bro b/scripts/base/protocols/ftp/file-analysis.bro index 2096af9a75..2d7609197a 100644 --- a/scripts/base/protocols/ftp/file-analysis.bro +++ b/scripts/base/protocols/ftp/file-analysis.bro @@ -41,6 +41,7 @@ function get_file_handle(c: connection, is_orig: bool): string module GLOBAL; event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) + &priority=5 { if ( tag != Analyzer::ANALYZER_FTP_DATA ) return; set_file_handle(FTP::get_file_handle(c, is_orig)); diff --git a/scripts/base/protocols/ftp/file-extract.bro b/scripts/base/protocols/ftp/file-extract.bro index f14839b616..2b7bb8cd50 100644 --- a/scripts/base/protocols/ftp/file-extract.bro +++ b/scripts/base/protocols/ftp/file-extract.bro @@ -13,8 +13,6 @@ export { const extraction_prefix = "ftp-item" &redef; } -global extract_count: count = 0; - redef record Info += { ## On disk file where it was extracted to. extraction_file: string &log &optional; @@ -26,8 +24,7 @@ redef record Info += { function get_extraction_name(f: fa_file): string { - local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count); - ++extract_count; + local r = fmt("%s-%s.dat", extraction_prefix, f$id); return r; } diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 88e1fbeeb8..7bf9d6cc4c 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -110,21 +110,18 @@ redef record connection += { ftp_data_reuse: bool &default=F; }; -# Configure DPD -redef capture_filters += { ["ftp"] = "port 21 and port 2811" }; - const ports = { 21/tcp, 2811/tcp }; redef likely_server_ports += { ports }; -# Establish the variable for tracking expected connections. -global ftp_data_expected: table[addr, port] of Info &read_expire=5mins; - event bro_init() &priority=5 { Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp]); Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, ports); } +# Establish the variable for tracking expected connections. +global ftp_data_expected: table[addr, port] of Info &read_expire=5mins; + ## A set of commands where the argument can be expected to refer ## to a file or directory. const file_cmds = { diff --git a/scripts/base/protocols/http/__load__.bro b/scripts/base/protocols/http/__load__.bro index 58618dedc7..8f426c1521 100644 --- a/scripts/base/protocols/http/__load__.bro +++ b/scripts/base/protocols/http/__load__.bro @@ -4,3 +4,5 @@ @load ./file-ident @load ./file-hash @load ./file-extract + +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/http/dpd.sig b/scripts/base/protocols/http/dpd.sig new file mode 100644 index 0000000000..13470f4e95 --- /dev/null +++ b/scripts/base/protocols/http/dpd.sig @@ -0,0 +1,13 @@ +signature dpd_http_client { + ip-proto == tcp + payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/ + tcp-state originator +} + +signature dpd_http_server { + ip-proto == tcp + payload /^HTTP\/[0-9]/ + tcp-state responder + requires-reverse-signature dpd_http_client + enable "http" +} diff --git a/scripts/base/protocols/http/file-analysis.bro b/scripts/base/protocols/http/file-analysis.bro index 51b3ea8dd5..d6da8c4f69 100644 --- a/scripts/base/protocols/http/file-analysis.bro +++ b/scripts/base/protocols/http/file-analysis.bro @@ -6,25 +6,48 @@ module HTTP; export { + redef record HTTP::Info += { + ## Number of MIME entities in the HTTP request message body so far. + request_mime_level: count &default=0; + ## Number of MIME entities in the HTTP response message body so far. + response_mime_level: count &default=0; + }; + ## Default file handle provider for HTTP. global get_file_handle: function(c: connection, is_orig: bool): string; } +event http_begin_entity(c: connection, is_orig: bool) &priority=5 + { + if ( ! c?$http ) + return; + + if ( is_orig ) + ++c$http$request_mime_level; + else + ++c$http$response_mime_level; + } + function get_file_handle(c: connection, is_orig: bool): string { if ( ! c?$http ) return ""; + local mime_level: count = + is_orig ? c$http$request_mime_level : c$http$response_mime_level; + local mime_level_str: string = mime_level > 1 ? cat(mime_level) : ""; + if ( c$http$range_request ) return cat(Analyzer::ANALYZER_HTTP, " ", is_orig, " ", c$id$orig_h, " ", build_url(c$http)); return cat(Analyzer::ANALYZER_HTTP, " ", c$start_time, " ", is_orig, " ", - c$http$trans_depth, " ", id_string(c$id)); + c$http$trans_depth, mime_level_str, " ", id_string(c$id)); } module GLOBAL; event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) + &priority=5 { if ( tag != Analyzer::ANALYZER_HTTP ) return; set_file_handle(HTTP::get_file_handle(c, is_orig)); diff --git a/scripts/base/protocols/http/file-extract.bro b/scripts/base/protocols/http/file-extract.bro index 9c0899b2b6..a8c6039395 100644 --- a/scripts/base/protocols/http/file-extract.bro +++ b/scripts/base/protocols/http/file-extract.bro @@ -14,8 +14,11 @@ export { const extraction_prefix = "http-item" &redef; redef record Info += { - ## On-disk file where the response body was extracted to. - extraction_file: string &log &optional; + ## On-disk location where files in request body were extracted. + extracted_request_files: vector of string &log &optional; + + ## On-disk location where files in response body were extracted. + extracted_response_files: vector of string &log &optional; ## Indicates if the response body is to be extracted or not. Must be ## set before or by the first :bro:see:`file_new` for the file content. @@ -23,15 +26,28 @@ export { }; } -global extract_count: count = 0; - function get_extraction_name(f: fa_file): string { - local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count); - ++extract_count; + local r = fmt("%s-%s.dat", extraction_prefix, f$id); return r; } +function add_extraction_file(c: connection, is_orig: bool, fn: string) + { + if ( is_orig ) + { + if ( ! c$http?$extracted_request_files ) + c$http$extracted_request_files = vector(); + c$http$extracted_request_files[|c$http$extracted_request_files|] = fn; + } + else + { + if ( ! c$http?$extracted_response_files ) + c$http$extracted_response_files = vector(); + c$http$extracted_response_files[|c$http$extracted_response_files|] = fn; + } + } + event file_new(f: fa_file) &priority=5 { if ( ! f?$source ) return; @@ -51,7 +67,7 @@ event file_new(f: fa_file) &priority=5 { c = f$conns[cid]; if ( ! c?$http ) next; - c$http$extraction_file = fname; + add_extraction_file(c, f$is_orig, fname); } return; @@ -79,6 +95,6 @@ event file_new(f: fa_file) &priority=5 { c = f$conns[cid]; if ( ! c?$http ) next; - c$http$extraction_file = fname; + add_extraction_file(c, f$is_orig, fname); } } diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 1c9c1cad2d..6d06376183 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -123,19 +123,12 @@ redef record connection += { http_state: State &optional; }; -# DPD configuration. -redef capture_filters += { - ["http"] = "tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888)" -}; - const ports = { 80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3128/tcp, 8000/tcp, 8080/tcp, 8888/tcp, }; - redef likely_server_ports += { ports }; - # Initialize the HTTP logging stream and ports. event bro_init() &priority=5 { diff --git a/scripts/base/protocols/irc/__load__.bro b/scripts/base/protocols/irc/__load__.bro index 5123385b0c..2e60cda0a6 100644 --- a/scripts/base/protocols/irc/__load__.bro +++ b/scripts/base/protocols/irc/__load__.bro @@ -1,3 +1,5 @@ @load ./main @load ./dcc-send @load ./file-analysis + +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/irc/dcc-send.bro b/scripts/base/protocols/irc/dcc-send.bro index 0e1d52af59..3194766946 100644 --- a/scripts/base/protocols/irc/dcc-send.bro +++ b/scripts/base/protocols/irc/dcc-send.bro @@ -39,8 +39,6 @@ export { global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins; -global extract_count: count = 0; - function set_dcc_mime(f: fa_file) { if ( ! f?$conns ) return; @@ -75,8 +73,7 @@ function set_dcc_extraction_file(f: fa_file, filename: string) function get_extraction_name(f: fa_file): string { - local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count); - ++extract_count; + local r = fmt("%s-%s.dat", extraction_prefix, f$id); return r; } @@ -188,5 +185,6 @@ event expected_connection_seen(c: connection, a: Analyzer::Tag) &priority=10 event connection_state_remove(c: connection) &priority=-5 { - delete dcc_expected_transfers[c$id$resp_h, c$id$resp_p]; + if ( [c$id$resp_h, c$id$resp_p] in dcc_expected_transfers ) + delete dcc_expected_transfers[c$id$resp_h, c$id$resp_p]; } diff --git a/scripts/base/protocols/irc/dpd.sig b/scripts/base/protocols/irc/dpd.sig new file mode 100644 index 0000000000..308358d619 --- /dev/null +++ b/scripts/base/protocols/irc/dpd.sig @@ -0,0 +1,33 @@ +signature irc_client1 { + ip-proto == tcp + payload /(|.*[\r\n]) *[Uu][Ss][Ee][Rr] +.+[\n\r]+ *[Nn][Ii][Cc][Kk] +.*[\r\n]/ + requires-reverse-signature irc_server_reply + tcp-state originator + enable "irc" +} + +signature irc_client2 { + ip-proto == tcp + payload /(|.*[\r\n]) *[Nn][Ii][Cc][Kk] +.+[\r\n]+ *[Uu][Ss][Ee][Rr] +.+[\r\n]/ + requires-reverse-signature irc_server_reply + tcp-state originator + enable "irc" +} + +signature irc_server_reply { + ip-proto == tcp + payload /^(|.*[\n\r])(:[^ \n\r]+ )?[0-9][0-9][0-9] / + tcp-state responder +} + +signature irc_server_to_server1 { + ip-proto == tcp + payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/ +} + +signature irc_server_to_server2 { + ip-proto == tcp + payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/ + requires-reverse-signature irc_server_to_server1 + enable "irc" +} diff --git a/scripts/base/protocols/irc/file-analysis.bro b/scripts/base/protocols/irc/file-analysis.bro index e1fdc9c484..89cbe7990c 100644 --- a/scripts/base/protocols/irc/file-analysis.bro +++ b/scripts/base/protocols/irc/file-analysis.bro @@ -18,6 +18,7 @@ function get_file_handle(c: connection, is_orig: bool): string module GLOBAL; event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) + &priority=5 { if ( tag != Analyzer::ANALYZER_IRC_DATA ) return; set_file_handle(IRC::get_file_handle(c, is_orig)); diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro index 490c39f54f..a57fc95448 100644 --- a/scripts/base/protocols/irc/main.bro +++ b/scripts/base/protocols/irc/main.bro @@ -38,13 +38,6 @@ redef record connection += { irc: Info &optional; }; -# Some common IRC ports. -redef capture_filters += { ["irc-6666"] = "port 6666" }; -redef capture_filters += { ["irc-6667"] = "port 6667" }; -redef capture_filters += { ["irc-6668"] = "port 6668" }; -redef capture_filters += { ["irc-6669"] = "port 6669" }; - -# DPD configuration. const ports = { 6666/tcp, 6667/tcp, 6668/tcp, 6669/tcp }; redef likely_server_ports += { ports }; diff --git a/scripts/base/protocols/modbus/main.bro b/scripts/base/protocols/modbus/main.bro index a418873306..d484e7582b 100644 --- a/scripts/base/protocols/modbus/main.bro +++ b/scripts/base/protocols/modbus/main.bro @@ -29,9 +29,6 @@ redef record connection += { modbus: Info &optional; }; -# Configure DPD and the packet filter. -redef capture_filters += { ["modbus"] = "tcp port 502" }; - const ports = { 502/tcp }; redef likely_server_ports += { ports }; diff --git a/scripts/base/protocols/pop3/__load__.bro b/scripts/base/protocols/pop3/__load__.bro new file mode 100644 index 0000000000..c5ddf0e788 --- /dev/null +++ b/scripts/base/protocols/pop3/__load__.bro @@ -0,0 +1,2 @@ + +@load-sigs ./dpd.sig diff --git a/scripts/base/protocols/pop3/dpd.sig b/scripts/base/protocols/pop3/dpd.sig new file mode 100644 index 0000000000..8d7e3567da --- /dev/null +++ b/scripts/base/protocols/pop3/dpd.sig @@ -0,0 +1,13 @@ +signature dpd_pop3_server { + ip-proto == tcp + payload /^\+OK/ + requires-reverse-signature dpd_pop3_client + enable "pop3" + tcp-state responder +} + +signature dpd_pop3_client { + ip-proto == tcp + payload /(|.*[\r\n])[[:space:]]*([uU][sS][eE][rR][[:space:]]|[aA][pP][oO][pP][[:space:]]|[cC][aA][pP][aA]|[aA][uU][tT][hH])/ + tcp-state originator +} diff --git a/scripts/base/protocols/smtp/__load__.bro b/scripts/base/protocols/smtp/__load__.bro index bac9cc118f..3e3fde6947 100644 --- a/scripts/base/protocols/smtp/__load__.bro +++ b/scripts/base/protocols/smtp/__load__.bro @@ -2,3 +2,5 @@ @load ./entities @load ./entities-excerpt @load ./file-analysis + +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/smtp/dpd.sig b/scripts/base/protocols/smtp/dpd.sig new file mode 100644 index 0000000000..6fbde59059 --- /dev/null +++ b/scripts/base/protocols/smtp/dpd.sig @@ -0,0 +1,13 @@ +signature dpd_smtp_client { + ip-proto == tcp + payload /(|.*[\n\r])[[:space:]]*([hH][eE][lL][oO]|[eE][hH][lL][oO])/ + requires-reverse-signature dpd_smtp_server + enable "smtp" + tcp-state originator +} + +signature dpd_smtp_server { + ip-proto == tcp + payload /^[[:space:]]*220[[:space:]-]/ + tcp-state responder +} \ No newline at end of file diff --git a/scripts/base/protocols/smtp/entities.bro b/scripts/base/protocols/smtp/entities.bro index 19cca30db1..b58766e51d 100644 --- a/scripts/base/protocols/smtp/entities.bro +++ b/scripts/base/protocols/smtp/entities.bro @@ -66,8 +66,6 @@ export { global log_mime: event(rec: EntityInfo); } -global extract_count: count = 0; - event bro_init() &priority=5 { Log::create_stream(SMTP::ENTITIES_LOG, [$columns=EntityInfo, $ev=log_mime]); @@ -90,8 +88,7 @@ function set_session(c: connection, new_entity: bool) function get_extraction_name(f: fa_file): string { - local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count); - ++extract_count; + local r = fmt("%s-%s.dat", extraction_prefix, f$id); return r; } @@ -127,7 +124,6 @@ event file_new(f: fa_file) &priority=5 [$tag=FileAnalysis::ANALYZER_EXTRACT, $extract_filename=fname]); extracting = T; - ++extract_count; } c$smtp$current_entity$extraction_file = fname; diff --git a/scripts/base/protocols/smtp/file-analysis.bro b/scripts/base/protocols/smtp/file-analysis.bro index 17f9a32498..68ec6390dd 100644 --- a/scripts/base/protocols/smtp/file-analysis.bro +++ b/scripts/base/protocols/smtp/file-analysis.bro @@ -20,6 +20,7 @@ function get_file_handle(c: connection, is_orig: bool): string module GLOBAL; event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) + &priority=5 { if ( tag != Analyzer::ANALYZER_SMTP ) return; set_file_handle(SMTP::get_file_handle(c, is_orig)); diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index c7b3a452d2..d53128b06c 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -81,9 +81,6 @@ redef record connection += { smtp_state: State &optional; }; -# Configure DPD -redef capture_filters += { ["smtp"] = "tcp port 25 or tcp port 587" }; - const ports = { 25/tcp, 587/tcp }; redef likely_server_ports += { ports }; diff --git a/scripts/base/protocols/socks/__load__.bro b/scripts/base/protocols/socks/__load__.bro index 0098b81a7a..80193afb6f 100644 --- a/scripts/base/protocols/socks/__load__.bro +++ b/scripts/base/protocols/socks/__load__.bro @@ -1,2 +1,4 @@ @load ./consts -@load ./main \ No newline at end of file +@load ./main + +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/socks/dpd.sig b/scripts/base/protocols/socks/dpd.sig new file mode 100644 index 0000000000..3dcd7a945a --- /dev/null +++ b/scripts/base/protocols/socks/dpd.sig @@ -0,0 +1,48 @@ +signature dpd_socks4_client { + ip-proto == tcp + # '32' is a rather arbitrary max length for the user name. + payload /^\x04[\x01\x02].{0,32}\x00/ + tcp-state originator +} + +signature dpd_socks4_server { + ip-proto == tcp + requires-reverse-signature dpd_socks4_client + payload /^\x00[\x5a\x5b\x5c\x5d]/ + tcp-state responder + enable "socks" +} + +signature dpd_socks4_reverse_client { + ip-proto == tcp + # '32' is a rather arbitrary max length for the user name. + payload /^\x04[\x01\x02].{0,32}\x00/ + tcp-state responder +} + +signature dpd_socks4_reverse_server { + ip-proto == tcp + requires-reverse-signature dpd_socks4_reverse_client + payload /^\x00[\x5a\x5b\x5c\x5d]/ + tcp-state originator + enable "socks" +} + +signature dpd_socks5_client { + ip-proto == tcp + # Watch for a few authentication methods to reduce false positives. + payload /^\x05.[\x00\x01\x02]/ + tcp-state originator +} + +signature dpd_socks5_server { + ip-proto == tcp + requires-reverse-signature dpd_socks5_client + # Watch for a single authentication method to be chosen by the server or + # the server to indicate the no authentication is required. + payload /^\x05(\x00|\x01[\x00\x01\x02])/ + tcp-state responder + enable "socks" +} + + diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index a188646515..f697b355c1 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -47,10 +47,6 @@ redef record connection += { socks: SOCKS::Info &optional; }; -# Configure DPD -redef capture_filters += { ["socks"] = "tcp port 1080" }; -redef likely_server_ports += { 1080/tcp }; - function set_session(c: connection, version: count) { if ( ! c?$socks ) diff --git a/scripts/base/protocols/ssh/__load__.bro b/scripts/base/protocols/ssh/__load__.bro index d551be57d3..0f3cb011f8 100644 --- a/scripts/base/protocols/ssh/__load__.bro +++ b/scripts/base/protocols/ssh/__load__.bro @@ -1 +1,3 @@ -@load ./main \ No newline at end of file +@load ./main + +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/ssh/dpd.sig b/scripts/base/protocols/ssh/dpd.sig new file mode 100644 index 0000000000..95e22908ab --- /dev/null +++ b/scripts/base/protocols/ssh/dpd.sig @@ -0,0 +1,13 @@ +signature dpd_ssh_client { + ip-proto == tcp + payload /^[sS][sS][hH]-/ + requires-reverse-signature dpd_ssh_server + enable "ssh" + tcp-state originator +} + +signature dpd_ssh_server { + ip-proto == tcp + payload /^[sS][sS][hH]-/ + tcp-state responder +} diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index 8e1c5515b5..53b61f00d8 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -70,17 +70,13 @@ export { global log_ssh: event(rec: Info); } -# Configure DPD and the packet filter - -const ports = { 22/tcp }; - -redef capture_filters += { ["ssh"] = "tcp port 22" }; -redef likely_server_ports += { ports }; - redef record connection += { ssh: Info &optional; }; +const ports = { 22/tcp }; +redef likely_server_ports += { ports }; + event bro_init() &priority=5 { Log::create_stream(SSH::LOG, [$columns=Info, $ev=log_ssh]); @@ -118,7 +114,7 @@ function check_ssh_connection(c: connection, done: bool) # Responder must have sent fewer than 40 packets. c$resp$num_pkts < 40 && # If there was a content gap we can't reliably do this heuristic. - c?$conn && c$conn$missed_bytes == 0)# && + c?$conn && c$conn$missed_bytes == 0 )# && # Only "normal" connections can count. #c$conn?$conn_state && c$conn$conn_state in valid_states ) { @@ -178,6 +174,7 @@ event ssh_watcher(c: connection) if ( ! connection_exists(id) ) return; + lookup_connection(c$id); check_ssh_connection(c, F); if ( ! c$ssh$done ) schedule +15secs { ssh_watcher(c) }; diff --git a/scripts/base/protocols/ssl/__load__.bro b/scripts/base/protocols/ssl/__load__.bro index 239438047c..80cb4e216a 100644 --- a/scripts/base/protocols/ssl/__load__.bro +++ b/scripts/base/protocols/ssl/__load__.bro @@ -1,3 +1,5 @@ @load ./consts @load ./main @load ./mozilla-ca-list + +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/ssl/dpd.sig b/scripts/base/protocols/ssl/dpd.sig new file mode 100644 index 0000000000..b36b9a5aa5 --- /dev/null +++ b/scripts/base/protocols/ssl/dpd.sig @@ -0,0 +1,15 @@ +signature dpd_ssl_server { + ip-proto == tcp + # Server hello. + payload /^(\x16\x03[\x00\x01\x02]..\x02...\x03[\x00\x01\x02]|...?\x04..\x00\x02).*/ + requires-reverse-signature dpd_ssl_client + enable "ssl" + tcp-state responder +} + +signature dpd_ssl_client { + ip-proto == tcp + # Client hello. + payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/ + tcp-state originator +} diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 36d0c3f54d..65526182ac 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -94,35 +94,12 @@ redef record Info += { delay_tokens: set[string] &optional; }; -redef capture_filters += { - ["ssl"] = "tcp port 443", - ["nntps"] = "tcp port 563", - ["imap4-ssl"] = "tcp port 585", - ["sshell"] = "tcp port 614", - ["ldaps"] = "tcp port 636", - ["ftps-data"] = "tcp port 989", - ["ftps"] = "tcp port 990", - ["telnets"] = "tcp port 992", - ["imaps"] = "tcp port 993", - ["ircs"] = "tcp port 994", - ["pop3s"] = "tcp port 995", - ["xmpps"] = "tcp port 5223", -}; - const ports = { 443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp, 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp -} &redef; - +}; redef likely_server_ports += { ports }; -# A queue that buffers log records. -global log_delay_queue: table[count] of Info; -# The top queue index where records are added. -global log_delay_queue_head = 0; -# The bottom queue index that points to the next record to be flushed. -global log_delay_queue_tail = 0; - event bro_init() &priority=5 { Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]); @@ -138,26 +115,17 @@ function set_session(c: connection) function delay_log(info: Info, token: string) { - info$delay_tokens = set(); + if ( ! info?$delay_tokens ) + info$delay_tokens = set(); add info$delay_tokens[token]; - - log_delay_queue[log_delay_queue_head] = info; - ++log_delay_queue_head; } function undelay_log(info: Info, token: string) { - if ( token in info$delay_tokens ) + if ( info?$delay_tokens && token in info$delay_tokens ) delete info$delay_tokens[token]; } -global log_record: function(info: Info); - -event delay_logging(info: Info) - { - log_record(info); - } - function log_record(info: Info) { if ( ! info?$delay_tokens || |info$delay_tokens| == 0 ) @@ -166,26 +134,14 @@ function log_record(info: Info) } else { - for ( unused_index in log_delay_queue ) + when ( |info$delay_tokens| == 0 ) { - if ( log_delay_queue_head == log_delay_queue_tail ) - return; - if ( |log_delay_queue[log_delay_queue_tail]$delay_tokens| > 0 ) - { - if ( info$ts + max_log_delay > network_time() ) - { - schedule 1sec { delay_logging(info) }; - return; - } - else - { - Reporter::info(fmt("SSL delay tokens not released in time (%s)", - info$delay_tokens)); - } - } - Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]); - delete log_delay_queue[log_delay_queue_tail]; - ++log_delay_queue_tail; + log_record(info); + } + timeout SSL::max_log_delay + { + Reporter::info(fmt("SSL delay tokens not released in time (%s tokens remaining)", + |info$delay_tokens|)); } } } @@ -295,15 +251,3 @@ event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, if ( c?$ssl ) finish(c); } - -event bro_done() - { - if ( |log_delay_queue| == 0 ) - return; - for ( unused_index in log_delay_queue ) - { - Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]); - delete log_delay_queue[log_delay_queue_tail]; - ++log_delay_queue_tail; - } - } diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro index 7c15fb4fae..afe562c890 100644 --- a/scripts/base/protocols/syslog/main.bro +++ b/scripts/base/protocols/syslog/main.bro @@ -26,15 +26,13 @@ export { }; } -redef capture_filters += { ["syslog"] = "port 514" }; - -const ports = { 514/udp }; -redef likely_server_ports += { ports }; - redef record connection += { syslog: Info &optional; }; +const ports = { 514/udp }; +redef likely_server_ports += { ports }; + event bro_init() &priority=5 { Log::create_stream(Syslog::LOG, [$columns=Info]); diff --git a/scripts/base/protocols/tunnels/__load__.bro b/scripts/base/protocols/tunnels/__load__.bro new file mode 100644 index 0000000000..9de7b6ff19 --- /dev/null +++ b/scripts/base/protocols/tunnels/__load__.bro @@ -0,0 +1 @@ +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/tunnels/dpd.sig b/scripts/base/protocols/tunnels/dpd.sig new file mode 100644 index 0000000000..0c66775f5d --- /dev/null +++ b/scripts/base/protocols/tunnels/dpd.sig @@ -0,0 +1,14 @@ +# Provide DPD signatures for tunneling protocols that otherwise +# wouldn't be detected at all. + +signature dpd_ayiya { + ip-proto = udp + payload /^..\x11\x29/ + enable "ayiya" +} + +signature dpd_teredo { + ip-proto = udp + payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/ + enable "teredo" +} diff --git a/scripts/policy/frameworks/packet-filter/shunt.bro b/scripts/policy/frameworks/packet-filter/shunt.bro new file mode 100644 index 0000000000..b87369ee62 --- /dev/null +++ b/scripts/policy/frameworks/packet-filter/shunt.bro @@ -0,0 +1,169 @@ +@load base/frameworks/notice +@load base/frameworks/packet-filter + +module PacketFilter; + +export { + ## The maximum number of BPF based shunts that Bro is allowed to perform. + const max_bpf_shunts = 100 &redef; + + ## Call this function to use BPF to shunt a connection (to prevent the + ## data packets from reaching Bro). For TCP connections, control packets + ## are still allowed through so that Bro can continue logging the connection + ## and it can stop shunting once the connection ends. + global shunt_conn: function(id: conn_id): bool; + + ## This function will use a BPF expresssion to shunt traffic between + ## the two hosts given in the `conn_id` so that the traffic is never + ## exposed to Bro's traffic processing. + global shunt_host_pair: function(id: conn_id): bool; + + ## Remove shunting for a host pair given as a `conn_id`. The filter + ## is not immediately removed. It waits for the occassional filter + ## update done by the `PacketFilter` framework. + global unshunt_host_pair: function(id: conn_id): bool; + + ## Performs the same function as the `unshunt_host_pair` function, but + ## it forces an immediate filter update. + global force_unshunt_host_pair: function(id: conn_id): bool; + + ## Retrieve the currently shunted connections. + global current_shunted_conns: function(): set[conn_id]; + + ## Retrieve the currently shunted host pairs. + global current_shunted_host_pairs: function(): set[conn_id]; + + redef enum Notice::Type += { + ## Indicative that :bro:id:`max_bpf_shunts` connections are already + ## being shunted with BPF filters and no more are allowed. + No_More_Conn_Shunts_Available, + + ## Limitations in BPF make shunting some connections with BPF impossible. + ## This notice encompasses those various cases. + Cannot_BPF_Shunt_Conn, + }; +} + +global shunted_conns: set[conn_id]; +global shunted_host_pairs: set[conn_id]; + +function shunt_filters() + { + # NOTE: this could wrongly match if a connection happens with the ports reversed. + local tcp_filter = ""; + local udp_filter = ""; + for ( id in shunted_conns ) + { + local prot = get_port_transport_proto(id$resp_p); + + local filt = fmt("host %s and port %d and host %s and port %d", id$orig_h, id$orig_p, id$resp_h, id$resp_p); + if ( prot == udp ) + udp_filter = combine_filters(udp_filter, "and", filt); + else if ( prot == tcp ) + tcp_filter = combine_filters(tcp_filter, "and", filt); + } + if ( tcp_filter != "" ) + tcp_filter = combine_filters("tcp and tcp[tcpflags] & (tcp-syn|tcp-fin|tcp-rst) == 0", "and", tcp_filter); + local conn_shunt_filter = combine_filters(tcp_filter, "and", udp_filter); + + local hp_shunt_filter = ""; + for ( id in shunted_host_pairs ) + hp_shunt_filter = combine_filters(hp_shunt_filter, "and", fmt("host %s and host %s", id$orig_h, id$resp_h)); + + local filter = combine_filters(conn_shunt_filter, "and", hp_shunt_filter); + if ( filter != "" ) + PacketFilter::exclude("shunt_filters", filter); +} + +event bro_init() &priority=5 + { + register_filter_plugin([ + $func()={ return shunt_filters(); } + ]); + } + +function current_shunted_conns(): set[conn_id] + { + return shunted_conns; + } + +function current_shunted_host_pairs(): set[conn_id] + { + return shunted_host_pairs; + } + +function reached_max_shunts(): bool + { + if ( |shunted_conns| + |shunted_host_pairs| > max_bpf_shunts ) + { + NOTICE([$note=No_More_Conn_Shunts_Available, + $msg=fmt("%d BPF shunts are in place and no more will be added until space clears.", max_bpf_shunts)]); + return T; + } + else + return F; + } + +function shunt_host_pair(id: conn_id): bool + { + PacketFilter::filter_changed = T; + + if ( reached_max_shunts() ) + return F; + + add shunted_host_pairs[id]; + install(); + return T; + } + +function unshunt_host_pair(id: conn_id): bool + { + PacketFilter::filter_changed = T; + + if ( id in shunted_host_pairs ) + { + delete shunted_host_pairs[id]; + return T; + } + else + return F; + } + +function force_unshunt_host_pair(id: conn_id): bool + { + if ( unshunt_host_pair(id) ) + { + install(); + return T; + } + else + return F; + } + +function shunt_conn(id: conn_id): bool + { + if ( is_v6_addr(id$orig_h) ) + { + NOTICE([$note=Cannot_BPF_Shunt_Conn, + $msg="IPv6 connections can't be shunted with BPF due to limitations in BPF", + $sub="ipv6_conn", + $id=id, $identifier=cat(id)]); + return F; + } + + if ( reached_max_shunts() ) + return F; + + PacketFilter::filter_changed = T; + add shunted_conns[id]; + install(); + return T; + } + +event connection_state_remove(c: connection) &priority=-5 + { + # Don't rebuild the filter right away because the packet filter framework + # will check every few minutes and update the filter if things have changed. + if ( c$id in shunted_conns ) + delete shunted_conns[c$id]; + } diff --git a/scripts/policy/misc/load-balancing.bro b/scripts/policy/misc/load-balancing.bro new file mode 100644 index 0000000000..fe07dd64da --- /dev/null +++ b/scripts/policy/misc/load-balancing.bro @@ -0,0 +1,132 @@ +##! This script implements the "Bro side" of several load balancing +##! approaches for Bro clusters. + +@load base/frameworks/cluster +@load base/frameworks/packet-filter + +module LoadBalancing; + +export { + + type Method: enum { + ## Apply BPF filters to each worker in a way that causes them to + ## automatically flow balance traffic between them. + AUTO_BPF, + ## Load balance traffic across the workers by making each one apply + ## a restrict filter to only listen to a single MAC address. This + ## is a somewhat common deployment option for sites doing network + ## based load balancing with MAC address rewriting and passing the + ## traffic to a single interface. Multiple MAC addresses will show + ## up on the same interface and need filtered to a single address. + #MAC_ADDR_BPF, + }; + + ## Defines the method of load balancing to use. + const method = AUTO_BPF &redef; + + # Configure the cluster framework to enable the load balancing filter configuration. + #global send_filter: event(for_node: string, filter: string); + #global confirm_filter_installation: event(success: bool); + + redef record Cluster::Node += { + ## A BPF filter for load balancing traffic sniffed on a single interface + ## across a number of processes. In normal uses, this will be assigned + ## dynamically by the manager and installed by the workers. + lb_filter: string &optional; + }; +} + +#redef Cluster::manager2worker_events += /LoadBalancing::send_filter/; +#redef Cluster::worker2manager_events += /LoadBalancing::confirm_filter_installation/; + +@if ( Cluster::is_enabled() ) + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +event bro_init() &priority=5 + { + if ( method != AUTO_BPF ) + return; + + local worker_ip_interface: table[addr, string] of count = table(); + for ( n in Cluster::nodes ) + { + local this_node = Cluster::nodes[n]; + + # Only workers! + if ( this_node$node_type != Cluster::WORKER || + ! this_node?$interface ) + next; + + if ( [this_node$ip, this_node$interface] !in worker_ip_interface ) + worker_ip_interface[this_node$ip, this_node$interface] = 0; + ++worker_ip_interface[this_node$ip, this_node$interface]; + } + + # Now that we've counted up how many processes are running on an interface + # let's create the filters for each worker. + local lb_proc_track: table[addr, string] of count = table(); + for ( no in Cluster::nodes ) + { + local that_node = Cluster::nodes[no]; + if ( that_node$node_type == Cluster::WORKER && + that_node?$interface && [that_node$ip, that_node$interface] in worker_ip_interface ) + { + if ( [that_node$ip, that_node$interface] !in lb_proc_track ) + lb_proc_track[that_node$ip, that_node$interface] = 0; + + local this_lb_proc = lb_proc_track[that_node$ip, that_node$interface]; + local total_lb_procs = worker_ip_interface[that_node$ip, that_node$interface]; + + ++lb_proc_track[that_node$ip, that_node$interface]; + if ( total_lb_procs > 1 ) + { + that_node$lb_filter = PacketFilter::sample_filter(total_lb_procs, this_lb_proc); + Communication::nodes[no]$capture_filter = that_node$lb_filter; + } + } + } + } + +#event remote_connection_established(p: event_peer) &priority=-5 +# { +# if ( is_remote_event() ) +# return; +# +# local for_node = p$descr; +# # Send the filter to the peer. +# if ( for_node in Cluster::nodes && +# Cluster::nodes[for_node]?$lb_filter ) +# { +# local filter = Cluster::nodes[for_node]$lb_filter; +# event LoadBalancing::send_filter(for_node, filter); +# } +# } + +#event LoadBalancing::confirm_filter_installation(success: bool) +# { +# # This doesn't really matter yet since we aren't getting back a meaningful success response. +# } + +@endif + + +@if ( Cluster::local_node_type() == Cluster::WORKER ) + +#event LoadBalancing::send_filter(for_node: string, filter: string) +event remote_capture_filter(p: event_peer, filter: string) + { + #if ( for_node !in Cluster::nodes ) + # return; + # + #if ( Cluster::node == for_node ) + # { + restrict_filters["lb_filter"] = filter; + PacketFilter::install(); + #event LoadBalancing::confirm_filter_installation(T); + # } + } + +@endif + +@endif diff --git a/scripts/policy/tuning/defaults/packet-fragments.bro b/scripts/policy/tuning/defaults/packet-fragments.bro index 30d7e23729..24b18d5917 100644 --- a/scripts/policy/tuning/defaults/packet-fragments.bro +++ b/scripts/policy/tuning/defaults/packet-fragments.bro @@ -3,7 +3,9 @@ ## This normally isn't used because of the default open packet filter ## but we set it anyway in case the user is using a packet filter. -redef capture_filters += { ["frag"] = "(ip[6:2] & 0x3fff != 0) and tcp" }; +## Note: This was removed because the default model now is to have a wide +## open packet filter. +#redef capture_filters += { ["frag"] = "(ip[6:2] & 0x3fff != 0) and tcp" }; ## Shorten the fragment timeout from never expiring to expiring fragments after ## five minutes. diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index daad03d9b6..1fd34d6f2f 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -24,6 +24,7 @@ @load frameworks/intel/smtp.bro @load frameworks/intel/ssl.bro @load frameworks/intel/where-locations.bro +@load frameworks/packet-filter/shunt.bro @load frameworks/software/version-changes.bro @load frameworks/software/vulnerable.bro @load integration/barnyard2/__load__.bro @@ -35,6 +36,7 @@ @load misc/capture-loss.bro @load misc/detect-traceroute/__load__.bro @load misc/detect-traceroute/main.bro +@load misc/load-balancing.bro @load misc/loaded-scripts.bro @load misc/profiling.bro @load misc/scan.bro diff --git a/src/3rdparty/sqlite3.c b/src/3rdparty/sqlite3.c index ba6a30e132..8d473d32b7 100644 --- a/src/3rdparty/sqlite3.c +++ b/src/3rdparty/sqlite3.c @@ -1,9 +1,6 @@ -# define SQLITE_THREADSAFE 2 -# define SQLITE_DEFAULT_MEMSTATUS 0 - /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.16.2. By combining all the individual C code files into this +** version 3.7.17. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -365,11 +362,11 @@ ** We support that for legacy. */ #if !defined(SQLITE_THREADSAFE) -#if defined(THREADSAFE) -# define SQLITE_THREADSAFE THREADSAFE -#else -# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ -#endif +# if defined(THREADSAFE) +# define SQLITE_THREADSAFE THREADSAFE +# else +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ +# endif #endif /* @@ -681,9 +678,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.16.2" -#define SQLITE_VERSION_NUMBER 3007016 -#define SQLITE_SOURCE_ID "2013-04-12 11:52:43 cbea02d93865ce0e06789db95fd9168ebac970c7" +#define SQLITE_VERSION "3.7.17" +#define SQLITE_VERSION_NUMBER 3007017 +#define SQLITE_SOURCE_ID "2013-05-20 00:56:22 118a3b35693b134d56ebd780123b7fd6f1497668" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -999,6 +996,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ @@ -1049,6 +1048,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) @@ -1068,6 +1068,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) #define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) #define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -1307,6 +1309,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; @@ -1443,7 +1448,8 @@ struct sqlite3_io_methods { ** it is able to override built-in [PRAGMA] statements. ** **
  • [[SQLITE_FCNTL_BUSYHANDLER]] -** ^This file-control may be invoked by SQLite on the database file handle +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle ** shortly after it is opened in order to provide a custom VFS with access ** to the connections busy-handler callback. The argument is of type (void **) ** - an array of two (void *) values. The first (void *) actually points @@ -1454,13 +1460,24 @@ struct sqlite3_io_methods { ** current operation. ** **
  • [[SQLITE_FCNTL_TEMPFILENAME]] -** ^Application can invoke this file-control to have SQLite generate a +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a ** temporary filename using the same algorithm that is followed to generate ** temporary filenames for TEMP tables and other internal uses. The ** argument should be a char** which will be filled with the filename ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1479,6 +1496,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle @@ -2145,7 +2163,9 @@ struct sqlite3_mem_methods { ** page cache implementation into that object.)^ ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    -**
    ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +**
    The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -2191,12 +2211,12 @@ struct sqlite3_mem_methods { **
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE **
    These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. -** +**
    ** ** [[SQLITE_CONFIG_SQLLOG]] **
    SQLITE_CONFIG_SQLLOG **
    This option is only available if sqlite is compiled with the -** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should ** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). ** The second should be of type (void*). The callback is invoked by the library ** in three separate circumstances, identified by the value passed as the @@ -2206,7 +2226,23 @@ struct sqlite3_mem_methods { ** fourth parameter is 1, then the SQL statement that the third parameter ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The -** third parameter is passed NULL In this case. +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option. +** If either argument to this option is negative, then that argument is +** changed to its compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2230,6 +2266,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options @@ -3063,6 +3100,9 @@ SQLITE_API int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -3601,7 +3641,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. **
    • ** **
  • @@ -3805,6 +3846,9 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the @@ -4761,7 +4805,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -5560,11 +5604,20 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -5590,11 +5643,11 @@ SQLITE_API int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -5606,7 +5659,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes @@ -7386,10 +7439,25 @@ SQLITE_API int sqlite3_unlock_notify( SQLITE_API int sqlite3_stricmp(const char *, const char *); SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + /* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. @@ -8074,6 +8142,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); */ #ifndef SQLITE_TEMP_STORE # define SQLITE_TEMP_STORE 1 +# define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif /* @@ -8221,6 +8290,49 @@ SQLITE_PRIVATE const int sqlite3one; # define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) #endif +/* +** Disable MMAP on platforms where it is known to not work +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +#endif + +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +# define SQLITE_MAX_MMAP_SIZE_xc 1 /* exclude from ctime.c */ +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +# define SQLITE_DEFAULT_MMAP_SIZE_xc 1 /* Exclude from ctime.c */ +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif /* ** An instance of the following structure is used to store the busy-handler @@ -8442,6 +8554,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); @@ -8518,6 +8631,7 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); #define BTREE_TEXT_ENCODING 5 #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 /* ** Values that may be OR'd together to form the second argument of an @@ -9142,6 +9256,12 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +/* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_ACQUIRE_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_ACQUIRE_READONLY 0x02 /* Read-only page is acceptable */ + /* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for @@ -9166,6 +9286,7 @@ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); +SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); SQLITE_PRIVATE void sqlite3PagerShrink(Pager*); SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int); @@ -9312,6 +9433,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void); SQLITE_PRIVATE void sqlite3PcacheShutdown(void); @@ -9523,14 +9646,6 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); # define SQLITE_OS_WINRT 0 #endif -/* -** When compiled for WinCE or WinRT, there is no concept of the current -** directory. - */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_CURDIR 1 -#endif - /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op */ @@ -9683,6 +9798,8 @@ SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* @@ -9922,6 +10039,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 szMmap; /* Default mmap_size setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -11158,6 +11276,8 @@ struct NameContext { #define NC_HasAgg 0x02 /* One or more aggregate functions seen */ #define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ +#define NC_AsMaybe 0x10 /* Resolve to AS terms of the result set only + ** if no other resolution is available */ /* ** An instance of the following structure contains all information @@ -11593,6 +11713,8 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 szMmap; /* mmap() space per open file */ + sqlite3_int64 mxMmap; /* Maximum value for szMmap */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ @@ -11627,6 +11749,7 @@ struct Walker { int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 bSelectDepthFirst; /* Do subqueries first */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ @@ -12130,6 +12253,12 @@ SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) +SQLITE_PRIVATE const char *sqlite3ErrName(int); +#endif + SQLITE_PRIVATE const char *sqlite3ErrStr(int); SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); @@ -12614,6 +12743,8 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ @@ -12737,15 +12868,15 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#ifdef SQLITE_CURDIR - "CURDIR", -#endif #ifdef SQLITE_DEBUG "DEBUG", #endif #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif +#if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), +#endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif @@ -12836,6 +12967,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif +#if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif #ifdef SQLITE_MAX_SCHEMA_RETRY "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), #endif @@ -12893,11 +13027,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_CHECK "OMIT_CHECK", #endif -/* // redundant -** #ifdef SQLITE_OMIT_COMPILEOPTION_DIAGS -** "OMIT_COMPILEOPTION_DIAGS", -** #endif -*/ #ifdef SQLITE_OMIT_COMPLETE "OMIT_COMPLETE", #endif @@ -13039,13 +13168,13 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_TCL "TCL", #endif -#ifdef SQLITE_TEMP_STORE +#if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), #endif #ifdef SQLITE_TEST "TEST", #endif -#ifdef SQLITE_THREADSAFE +#if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif #ifdef SQLITE_USE_ALLOCA @@ -13071,8 +13200,11 @@ SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ /* Since ArraySize(azCompileOpt) is normally in single digits, a ** linear search is adequate. No need for a binary search. */ for(i=0; ipMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +#if SQLITE_MAX_MMAP_SIZE>0 +/* The real implementation of xFetch and xUnfetch */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); +} +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif + /* ** The next group of routines are convenience wrappers around the ** VFS methods. @@ -22851,7 +23011,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* #include */ #include #include -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 #include #endif @@ -22950,6 +23110,11 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ @@ -22974,7 +23139,9 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ + #endif + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -22998,6 +23165,7 @@ struct unixFile { #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ /* ** Include code that is common to all os_*.c files @@ -23239,6 +23407,17 @@ SQLITE_API int sqlite3_open_file_count = 0; #define threadid 0 #endif +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -23263,7 +23442,7 @@ static int posixFchown(int fd, uid_t uid, gid_t gid){ /* Forward reference */ static int openDirectory(const char*, int*); -/* Fix for "error: 'fchmod' undeclared here (not in a function)" on FreeBSD 9 */ +/* fix compile on FreeBSD, not sure why needed... */ int fchmod(int, mode_t); /* @@ -23373,6 +23552,19 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if HAVE_MREMAP + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -23704,7 +23896,6 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { } - /****************************************************************************** ****************** Begin Unique File ID Utility Used By VxWorks *************** ** @@ -24040,7 +24231,6 @@ static int unixLogErrorAtLine( zErr = strerror(iErrno); #endif - assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; sqlite3_log(errcode, "os_unix.c:%d: (%d) %s(%s) - %s", @@ -24206,6 +24396,50 @@ static int findInodeInfo( } +/* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + if( pFile->ctrlFlags & UNIXFILE_WARNED ){ + /* One or more of the following warnings have already been issued. Do not + ** repeat them so as not to clutter the error log */ + return; + } + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( pFile->pInode!=0 + && ((rc = osStat(pFile->zPath, &buf))!=0 + || buf.st_ino!=pFile->pInode->fileId.ino) + ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } +} + + /* ** This routine checks if there is a RESERVED lock held on the specified ** file by this or any other process. If such a lock is held, set *pResOut @@ -24736,9 +24970,13 @@ end_unlock: ** the requested locking level, this routine is a no-op. */ static int unixUnlock(sqlite3_file *id, int eFileLock){ + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -24751,6 +24989,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -24777,6 +25016,7 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + verifyDbFile(pFile); unixUnlock(id, NO_LOCK); unixEnterMutex(); @@ -26008,6 +26248,8 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); + assert( offset>=0 ); + assert( amt>0 ); /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -26018,6 +26260,23 @@ static int unixRead( ); #endif +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -26032,6 +26291,51 @@ static int unixRead( } } +/* +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). +*/ +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + nBuf &= 0x1ffff; + TIMER_START; + +#if defined(USE_PREAD) + do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); +#elif defined(USE_PREAD64) + do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); +#else + do{ + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek-- ); + + if( iSeek!=iOff ){ + if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); + return -1; + } + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); +#endif + + TIMER_END; + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); + + if( rc<0 && piErrno ) *piErrno = errno; + return rc; +} + + /* ** Seek to the offset in id->offset then read cnt bytes into pBuf. ** Return the number of bytes actually read. Update the offset. @@ -26040,39 +26344,7 @@ static int unixRead( ** is set before returning. */ static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ - int got; -#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) - i64 newOffset; -#endif - assert( cnt==(cnt&0x1ffff) ); - cnt &= 0x1ffff; - TIMER_START; -#if defined(USE_PREAD) - do{ got = osPwrite(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); -#elif defined(USE_PREAD64) - do{ got = osPwrite64(id->h, pBuf, cnt, offset);}while( got<0 && errno==EINTR); -#else - do{ - newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; - } - return -1; - } - got = osWrite(id->h, pBuf, cnt); - }while( got<0 && errno==EINTR ); -#endif - TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; - } - - OSTRACE(("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); } @@ -26122,6 +26394,23 @@ static int unixWrite( } #endif +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -26404,6 +26693,14 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ } #endif + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } + return SQLITE_OK; } } @@ -26492,6 +26789,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -26559,6 +26869,18 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ){ + pFile->mmapSizeMax = newLimit; + if( newLimitmmapSize ) pFile->mmapSize = newLimit; + } + return SQLITE_OK; + } #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -26871,7 +27193,7 @@ static void unixShmPurge(unixFile *pFd){ sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ - munmap(p->apRegion[i], p->szRegion); + osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } @@ -27111,24 +27433,32 @@ static int unixShmMap( if( sStat.st_sizeh, sStat.st_size, nByte)!=0 ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "fallocate", - pShmNode->zFilename); + if( !bExtend ){ goto shmpage_out; } -#else - if( robust_ftruncate(pShmNode->h, nByte) ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", - pShmNode->zFilename); - goto shmpage_out; + + /* Alternatively, if bExtend is true, extend the file. Do this by + ** writing a single byte to the end of each (OS) page being + ** allocated or extended. Technically, we need only write to the + ** last page in order to extend the file. But writing to all new + ** pages forces the OS to allocate them immediately, which reduces + ** the chances of SIGBUS while accessing the mapped region later on. + */ + else{ + static const int pgsz = 4096; + int iPg; + + /* Write to the last byte of each newly allocated or extended page */ + assert( (nByte % pgsz)==0 ); + for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } } -#endif } } @@ -27144,7 +27474,7 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, + pMem = osMmap(0, szRegion, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -27361,6 +27691,236 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** If it is currently memory mapped, unmap file pFd. +*/ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->pMapRegion ){ + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapSizeActual = 0; + } +#endif +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if HAVE_MREMAP + return 512; +#elif defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapSizeActual +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + const char *zErr = "mmap"; + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapSizeMax ); + assert( nNew>0 ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ + const int szSyspage = unixGetPagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + } + + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapSizeMax = 0; + } + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; +} +#endif + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int unixMapfile(unixFile *pFd, i64 nByte){ +#if SQLITE_MAX_MMAP_SIZE>0 + i64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; + } + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + + if( nMap!=pFd->mmapSize ){ + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ + unixUnmapfile(pFd); + } + } +#endif + + return SQLITE_OK; +} + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif + *pp = 0; + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -27419,7 +27979,9 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -27436,7 +27998,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ @@ -27687,6 +28249,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; @@ -27822,15 +28385,15 @@ static int fillInUnixFile( if( h>=0 ) robust_close(pNew, h, __LINE__); h = -1; osUnlink(zFilename); - isDelete = 0; + pNew->ctrlFlags |= UNIXFILE_DELETE; } - if( isDelete ) pNew->ctrlFlags |= UNIXFILE_DELETE; #endif if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); }else{ pNew->pMethod = pLockingStyle; OpenCounter(+1); + verifyDbFile(pNew); } return rc; } @@ -29924,7 +30487,7 @@ SQLITE_API int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==21 ); + assert( ArraySize(aSyscall)==24 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ @@ -30307,11 +30870,20 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif }; /* ** Allowed values for winFile.ctrlFlags */ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ @@ -31671,7 +32243,7 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ } #endif if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", lastErrno, lastErrno); + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%lx (%lu)", lastErrno, lastErrno); }else{ /* copy a maximum of nBuf chars to output buffer */ sqlite3_snprintf(nBuf, zBuf, "%s", zOut); @@ -31714,7 +32286,7 @@ static int winLogErrorAtLine( for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} zMsg[i] = 0; sqlite3_log(errcode, - "os_win.c:%d: (%d) %s(%s) - %s", + "os_win.c:%d: (%lu) %s(%s) - %s", iLine, lastErrno, zFunc, zPath, zMsg ); @@ -32175,6 +32747,8 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ DWORD dwRet; /* Value returned by SetFilePointer() */ DWORD lastErrno; /* Value returned by GetLastError() */ + OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); @@ -32192,9 +32766,11 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = lastErrno; winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #else /* @@ -32211,13 +32787,20 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #endif } +#if SQLITE_MAX_MMAP_SIZE>0 +/* Forward references to VFS methods */ +static int winUnmapfile(winFile*); +#endif + /* ** Close a file. ** @@ -32237,8 +32820,14 @@ static int winClose(sqlite3_file *id){ #ifndef SQLITE_OMIT_WAL assert( pFile->pShm==0 ); #endif - OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + OSTRACE(("CLOSE file=%p\n", pFile->h)); + +#if SQLITE_MAX_MMAP_SIZE>0 + rc = winUnmapfile(pFile); + if( rc!=SQLITE_OK ) return rc; +#endif + do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ @@ -32258,11 +32847,11 @@ static int winClose(sqlite3_file *id){ sqlite3_free(pFile->zDeleteOnClose); } #endif - OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); if( rc ){ pFile->h = NULL; } OpenCounter(-1); + OSTRACE(("CLOSE file=%p, rc=%s\n", pFile->h, rc ? "ok" : "failed")); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), "winClose", pFile->zPath); @@ -32287,11 +32876,33 @@ static int winRead( int nRetry = 0; /* Number of retrys */ assert( id!=0 ); + assert( amt>0 ); + assert( offset>=0 ); SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + OSTRACE(("READ-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ + OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ @@ -32305,6 +32916,7 @@ static int winRead( DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, "winRead", pFile->zPath); } @@ -32312,9 +32924,11 @@ static int winRead( if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_SHORT_READ\n", pFile->h)); return SQLITE_IOERR_SHORT_READ; } + OSTRACE(("READ file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -32337,7 +32951,26 @@ static int winWrite( SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); - OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("WRITE file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + OSTRACE(("WRITE-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); @@ -32390,13 +33023,16 @@ static int winWrite( if( rc ){ if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) || ( pFile->lastErrno==ERROR_DISK_FULL )){ + OSTRACE(("WRITE file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } + OSTRACE(("WRITE file=%p, rc=SQLITE_IOERR_WRITE\n", pFile->h)); return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, "winWrite", pFile->zPath); }else{ logIoerr(nRetry); } + OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -32406,11 +33042,12 @@ static int winWrite( static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; assert( pFile ); - - OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); + OSTRACE(("TRUNCATE file=%p, size=%lld, lock=%d\n", + pFile->h, nByte, pFile->locktype)); /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the @@ -32424,14 +33061,25 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate1", pFile->zPath); - }else if( 0==osSetEndOfFile(pFile->h) ){ - pFile->lastErrno = osGetLastError(); + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate2", pFile->zPath); + "winTruncate2", pFile->zPath); } - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); +#if SQLITE_MAX_MMAP_SIZE>0 + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; + } +#endif + + OSTRACE(("TRUNCATE file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } @@ -32471,13 +33119,14 @@ static int winSync(sqlite3_file *id, int flags){ || (flags&0x0F)==SQLITE_SYNC_FULL ); - OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); - /* Unix cannot, but some systems may return SQLITE_FULL from here. This ** line is to test that doing so does not cause any problems. */ SimulateDiskfullError( return SQLITE_FULL ); + OSTRACE(("SYNC file=%p, flags=%x, lock=%d\n", + pFile->h, flags, pFile->locktype)); + #ifndef SQLITE_TEST UNUSED_PARAMETER(flags); #else @@ -32496,9 +33145,11 @@ static int winSync(sqlite3_file *id, int flags){ rc = osFlushFileBuffers(pFile->h); SimulateIOError( rc=FALSE ); if( rc ){ + OSTRACE(("SYNC file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; }else{ pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC file=%p, rc=SQLITE_IOERR_FSYNC\n", pFile->h)); return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, "winSync", pFile->zPath); } @@ -32513,7 +33164,10 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ int rc = SQLITE_OK; assert( id!=0 ); + assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); + OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + #if SQLITE_OS_WINRT { FILE_STANDARD_INFO info; @@ -32542,6 +33196,8 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ } } #endif + OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", + pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; } @@ -32583,6 +33239,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ */ static int getReadLock(winFile *pFile){ int res; + OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ #if SQLITE_OS_WINCE /* @@ -32608,6 +33265,7 @@ static int getReadLock(winFile *pFile){ pFile->lastErrno = osGetLastError(); /* No need to log a failure to lock */ } + OSTRACE(("READ-LOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -32617,6 +33275,7 @@ static int getReadLock(winFile *pFile){ static int unlockReadLock(winFile *pFile){ int res; DWORD lastErrno; + OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); } @@ -32630,6 +33289,7 @@ static int unlockReadLock(winFile *pFile){ winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, "unlockReadLock", pFile->zPath); } + OSTRACE(("READ-UNLOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -32668,14 +33328,15 @@ static int winLock(sqlite3_file *id, int locktype){ DWORD lastErrno = NO_ERROR; assert( id!=0 ); - OSTRACE(("LOCK %d %d was %d(%d)\n", - pFile->h, locktype, pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); /* If there is already a lock of this type or more restrictive on the ** OsFile, do nothing. Don't use the end_lock: exit path, as ** sqlite3OsEnterMutex() hasn't been called yet. */ if( pFile->locktype>=locktype ){ + OSTRACE(("LOCK-HELD file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -32703,7 +33364,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** If you are using this code as a model for alternative VFSes, do not ** copy this retry logic. It is a hack intended for Windows only. */ - OSTRACE(("could not get a PENDING lock. cnt=%d\n", cnt)); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(res))); if( cnt ) sqlite3_win32_sleep(1); } gotPendingLock = res; @@ -32748,14 +33410,12 @@ static int winLock(sqlite3_file *id, int locktype){ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); res = unlockReadLock(pFile); - OSTRACE(("unreadlock = %d\n", res)); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - OSTRACE(("error-code = %d\n", lastErrno)); getReadLock(pFile); } } @@ -32773,12 +33433,14 @@ static int winLock(sqlite3_file *id, int locktype){ if( res ){ rc = SQLITE_OK; }else{ - OSTRACE(("LOCK FAILED %d trying for %d but got %d\n", pFile->h, - locktype, newLocktype)); + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); pFile->lastErrno = lastErrno; rc = SQLITE_BUSY; } pFile->locktype = (u8)newLocktype; + OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -32792,20 +33454,23 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ winFile *pFile = (winFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); assert( id!=0 ); if( pFile->locktype>=RESERVED_LOCK ){ rc = 1; - OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); }else{ rc = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); if( rc ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } rc = !rc; - OSTRACE(("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (remote)\n", pFile->h, rc)); } *pResOut = rc; + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + pFile->h, pResOut, *pResOut)); return SQLITE_OK; } @@ -32826,8 +33491,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ int rc = SQLITE_OK; assert( pFile!=0 ); assert( locktype<=SHARED_LOCK ); - OSTRACE(("UNLOCK %d to %d was %d(%d)\n", pFile->h, locktype, - pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); @@ -32848,6 +33513,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); } pFile->locktype = (u8)locktype; + OSTRACE(("UNLOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -32875,17 +33542,21 @@ static int getTempname(int nBuf, char *zBuf); */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ winFile *pFile = (winFile*)id; + OSTRACE(("FCNTL file=%p, op=%d, pArg=%p\n", pFile->h, op, pArg)); switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->locktype; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { pFile->szChunk = *(int *)pArg; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { @@ -32900,20 +33571,25 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ SimulateIOErrorBenign(0); } } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_VFSNAME: { *(char**)pArg = sqlite3_mprintf("win32"); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_WIN32_AV_RETRY: { @@ -32928,6 +33604,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ }else{ a[1] = win32IoerrRetryDelay; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_TEMPFILENAME: { @@ -32936,9 +33613,23 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ getTempname(pFile->pVfs->mxPathname, zTFile); *(char**)pArg = zTFile; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#endif } + OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; } @@ -32966,8 +33657,6 @@ static int winDeviceCharacteristics(sqlite3_file *id){ ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } -#ifndef SQLITE_OMIT_WAL - /* ** Windows will only let you create file view mappings ** on allocation size granularity boundaries. @@ -32976,6 +33665,8 @@ static int winDeviceCharacteristics(sqlite3_file *id){ */ SYSTEM_INFO winSysInfo; +#ifndef SQLITE_OMIT_WAL + /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the winLockInfo objects used by @@ -33099,6 +33790,9 @@ static int winShmSystemLock( /* Access to the winShmNode object is serialized by the caller */ assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", + pFile->hFile.h, lockType, ofst, nByte)); + /* Release/Acquire the system-level lock */ if( lockType==_SHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); @@ -33116,11 +33810,9 @@ static int winShmSystemLock( rc = SQLITE_BUSY; } - OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", - pFile->hFile.h, - rc==SQLITE_OK ? "ok" : "failed", - lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", - pFile->lastErrno)); + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", + pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; } @@ -33140,6 +33832,8 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode *p; BOOL bRc; assert( winShmMutexHeld() ); + OSTRACE(("SHM-PURGE pid=%lu, deleteFlag=%d\n", + osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ if( p->nRef==0 ){ @@ -33147,13 +33841,11 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ bRc = osUnmapViewOfFile(p->aRegion[i].pMap); - OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-UNMAP pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); bRc = osCloseHandle(p->aRegion[i].hMap); - OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-CLOSE pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); } if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ SimulateIOErrorBenign(1); @@ -33432,9 +34124,9 @@ static int winShmLock( } } sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", - p->id, (int)osGetCurrentProcessId(), p->sharedMask, p->exclMask, - rc ? "failed" : "ok")); + OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc))); return rc; } @@ -33555,8 +34247,8 @@ static int winShmMap( NULL, PAGE_READWRITE, 0, nByte, NULL ); #endif - OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, nByte, + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; @@ -33570,8 +34262,8 @@ static int winShmMap( 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif - OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); } if( !pMap ){ @@ -33608,6 +34300,230 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Cleans up the mapped region of the specified file, if any. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +static int winUnmapfile(winFile *pFile){ + assert( pFile!=0 ); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " + "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, + pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), pFile, + pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFile, pFile->hMap)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap2", pFile->zPath); + } + pFile->hMap = NULL; + } + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile)); + return SQLITE_OK; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, size=%lld\n", + osGetCurrentProcessId(), pFd, nByte)); + + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_IOERR_FSTAT; + } + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#endif + if( pFd->hMap==NULL ){ + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-CREATE pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; + } + assert( (nMap % winSysInfo.dwPageSize)==0 ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); +#else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif + if( pNew==NULL ){ + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; + pFd->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + OSTRACE(("MAP-FILE-MAP pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapSizeActual = nMap; + } + + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling winUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif + *pp = 0; + + OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ){ + OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return rc; + } + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + + OSTRACE(("FETCH pid=%lu, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd, pp, *pp)); + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to winFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the winFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, offset=%lld, p=%p\n", + osGetCurrentProcessId(), pFd, iOff, p)); + + if( p ){ + pFd->nFetchOut--; + }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#endif + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd)); + return SQLITE_OK; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -33619,7 +34535,7 @@ shmpage_out: ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ + 3, /* iVersion */ winClose, /* xClose */ winRead, /* xRead */ winWrite, /* xWrite */ @@ -33635,7 +34551,9 @@ static const sqlite3_io_methods winIoMethod = { winShmMap, /* xShmMap */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ }; /**************************************************************************** @@ -33699,6 +34617,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); sqlite3_free(zMulti); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -33712,6 +34631,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); sqlite3_free(zUtf8); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -33724,6 +34644,7 @@ static int getTempname(int nBuf, char *zBuf){ nTempPath = sqlite3Strlen30(zTempPath); if( (nTempPath + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 18) >= nBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return SQLITE_ERROR; } @@ -33741,8 +34662,8 @@ static int getTempname(int nBuf, char *zBuf){ zBuf[j] = 0; zBuf[j+1] = 0; - OSTRACE(("TEMP FILENAME: %s\n", zBuf)); - return SQLITE_OK; + OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); + return SQLITE_OK; } /* @@ -33811,9 +34732,7 @@ static int winOpen( int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif int isReadWrite = (flags & SQLITE_OPEN_READWRITE); #ifndef NDEBUG @@ -33824,6 +34743,9 @@ static int winOpen( )); #endif + OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", + zUtf8Name, id, flags, pOutFlags)); + /* Check the following statements are true: ** ** (a) Exactly one of the READWRITE and READONLY flags must be set, and @@ -33869,6 +34791,7 @@ static int winOpen( memset(zTmpname, 0, MAX_PATH+2); rc = getTempname(MAX_PATH+2, zTmpname); if( rc!=SQLITE_OK ){ + OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); return rc; } zUtf8Name = zTmpname; @@ -33884,11 +34807,13 @@ static int winOpen( /* Convert the filename to the system encoding. */ zConverted = convertUtf8Filename(zUtf8Name); if( zConverted==0 ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); return SQLITE_IOERR_NOMEM; } if( winIsDir(zConverted) ){ sqlite3_free(zConverted); + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); return SQLITE_CANTOPEN_ISDIR; } @@ -33979,9 +34904,8 @@ static int winOpen( #endif logIoerr(cnt); - OSTRACE(("OPEN %d %s 0x%lx %s\n", - h, zName, dwDesiredAccess, - h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, + dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = lastErrno; @@ -34005,12 +34929,17 @@ static int winOpen( } } + OSTRACE(("OPEN file=%p, name=%s, access=%lx, pOutFlags=%p, *pOutFlags=%d, " + "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? + *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + #if SQLITE_OS_WINCE if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK ){ osCloseHandle(h); sqlite3_free(zConverted); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); return rc; } if( isTemp ){ @@ -34024,11 +34953,21 @@ static int winOpen( pFile->pMethod = &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif OpenCounter(+1); return rc; @@ -34060,6 +34999,8 @@ static int winDelete( UNUSED_PARAMETER(syncDir); SimulateIOError(return SQLITE_IOERR_DELETE); + OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM; @@ -34145,7 +35086,7 @@ static int winDelete( logIoerr(cnt); } sqlite3_free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); + OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); return rc; } @@ -34165,8 +35106,12 @@ static int winAccess( UNUSED_PARAMETER(pVfs); SimulateIOError( return SQLITE_IOERR_ACCESS; ); + OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", + zFilename, flags, pResOut)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ + OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); return SQLITE_IOERR_NOMEM; } if( isNT() ){ @@ -34217,6 +35162,8 @@ static int winAccess( assert(!"Invalid flags argument"); } *pResOut = rc; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); return SQLITE_OK; } @@ -34657,7 +35604,6 @@ SQLITE_API int sqlite3_os_init(void){ ** correctly. See ticket [bb3a86e890c8e96ab] */ assert( ArraySize(aSyscall)==74 ); -#ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); #if SQLITE_OS_WINRT @@ -34665,8 +35611,8 @@ SQLITE_API int sqlite3_os_init(void){ #else osGetSystemInfo(&winSysInfo); #endif - assert(winSysInfo.dwAllocationGranularity > 0); -#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; @@ -37303,7 +38249,6 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 i # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 @@ -37316,6 +38261,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 i # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 +# define sqlite3WalFindFrame(x,y,z) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -37343,7 +38289,8 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *); SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -SQLITE_PRIVATE int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); +SQLITE_PRIVATE int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +SQLITE_PRIVATE int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal); @@ -38043,6 +38990,11 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + + u8 bUseFetch; /* True to use xFetch() */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -38153,6 +39105,16 @@ static const unsigned char aJournalMagic[] = { # define MEMDB pPager->memDb #endif +/* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +# define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 +#endif + /* ** The maximum legal page number is (2^31 - 1). */ @@ -39640,7 +40602,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -40031,6 +40993,7 @@ static int pager_playback(Pager *pPager, int isHot){ int res = 1; /* Value returned by sqlite3OsAccess() */ char *zMaster = 0; /* Name of master journal file if any */ int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ /* Figure out how many records are in the journal. Abort early if ** the journal is empty. @@ -40131,7 +41094,9 @@ static int pager_playback(Pager *pPager, int isHot){ needPagerReset = 0; } rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0); - if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ if( rc==SQLITE_DONE ){ pPager->journalOff = szJ; break; @@ -40201,6 +41166,10 @@ end_playback: rc = pager_delmaster(pPager, zMaster); testcase( rc!=SQLITE_OK ); } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", + nPlayback, pPager->zJournal); + } /* The Pager.sectorSize variable may have been updated while rolling ** back a journal created by a process with a different sector size @@ -40222,11 +41191,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -40238,11 +41206,13 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ +#ifndef SQLITE_OMIT_WAL + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else +#endif + { i64 iOffset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ @@ -40321,12 +41291,17 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ Pager *pPager = (Pager *)pCtx; PgHdr *pPg; + assert( pagerUseWal(pPager) ); pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -40470,6 +41445,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -40731,6 +41707,29 @@ SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } +/* +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. +*/ +static void pagerFixMaplimit(Pager *pPager){ +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + sqlite3_int64 sz; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->szMmap>0; + sz = pPager->szMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); + } +#endif +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; + pagerFixMaplimit(pPager); +} + /* ** Free as much memory as possible from the pager. */ @@ -40966,6 +41965,7 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -41191,6 +42191,81 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ + PgHdr *p; /* Memory mapped page to return */ + + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); + return SQLITE_NOMEM; + } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pgno = pgno; + p->pData = pData; + pPager->nMmapOut++; + + return SQLITE_OK; +} + +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); +} + +/* +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. +*/ +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pMmapFreelist; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + /* ** Shutdown the page cache. Free all memory and close all files. ** @@ -41211,6 +42286,7 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); + pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -41472,7 +42548,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; @@ -42026,6 +43104,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; @@ -42317,9 +43396,11 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || USEFETCH(pPager) + )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -42345,7 +43426,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } }else{ @@ -42354,6 +43435,16 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + if( USEFETCH(pPager) ){ + sqlite3OsUnfetch(pPager->fd, 0, 0); + } } } @@ -42395,7 +43486,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } @@ -42454,13 +43545,27 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ + int flags /* PAGER_ACQUIRE_XXX flags */ ){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) +#ifdef SQLITE_HAS_CODEC + && pPager->xCodec==0 +#endif + ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -42471,6 +43576,39 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -42529,9 +43667,13 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -42584,7 +43726,11 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ if( pPg ){ Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } pagerUnlockIfUnused(pPager); } } @@ -42919,6 +44065,7 @@ SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); @@ -43118,6 +44265,11 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); pPager->changeCountDone = 1; } }else{ @@ -43475,7 +44627,7 @@ SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager @@ -44209,11 +45361,12 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -44304,6 +45457,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; @@ -45552,8 +46706,9 @@ finished: ** checkpointing the log file. */ if( pWal->hdr.nPage ){ - sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", - pWal->hdr.nPage, pWal->zWalName + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered %d frames from WAL file %s", + pWal->hdr.mxFrame, pWal->zWalName ); } } @@ -46067,8 +47222,8 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); @@ -46078,6 +47233,7 @@ static int walCheckpoint( } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -46632,19 +47788,17 @@ SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -SQLITE_PRIVATE int sqlite3WalRead( +SQLITE_PRIVATE int sqlite3WalFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ @@ -46660,7 +47814,7 @@ SQLITE_PRIVATE int sqlite3WalRead( ** WAL were empty. */ if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; + *piRead = 0; return SQLITE_OK; } @@ -46731,26 +47885,31 @@ SQLITE_PRIVATE int sqlite3WalRead( } #endif - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); - } - - *pInWal = 0; + *piRead = iRead; return SQLITE_OK; } +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} /* ** Return the size of the database in pages (or zero, if unknown). @@ -47297,6 +48456,9 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } /* Copy data from the log to the database file. */ @@ -49968,13 +51130,17 @@ static int btreeGetPage( BtShared *pBt, /* The btree */ Pgno pgno, /* Number of the page to fetch */ MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ + int noContent, /* Do not load page content if true */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; DbPage *pDbPage; + int flags = (noContent ? PAGER_ACQUIRE_NOCONTENT : 0) + | (bReadonly ? PAGER_ACQUIRE_READONLY : 0); + assert( noContent==0 || bReadonly==0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -50017,9 +51183,10 @@ SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); @@ -50027,7 +51194,7 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); + rc = btreeGetPage(pBt, pgno, ppPage, 0, bReadonly); if( rc==SQLITE_OK ){ rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ @@ -50258,6 +51425,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ @@ -50524,6 +51692,19 @@ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + /* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power @@ -50749,7 +51930,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->pPage1==0 ); rc = sqlite3PagerSharedLock(pBt->pPager); if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); + rc = btreeGetPage(pBt, 1, &pPage1, 0, 0); if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is @@ -50885,6 +52066,29 @@ page1_init_failed: return rc; } +#ifndef NDEBUG +/* +** Return the number of cursors open on pBt. This is for use +** in assert() expressions, so it is only compiled if NDEBUG is not +** defined. +** +** Only write cursors are counted if wrOnly is true. If wrOnly is +** false then all cursors are counted. +** +** For the purposes of this routine, a cursor is any cursor that +** is capable of reading or writing to the databse. Cursors that +** have been tripped into the CURSOR_FAULT state are not counted. +*/ +static int countValidCursors(BtShared *pBt, int wrOnly){ + BtCursor *pCur; + int r = 0; + for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ + if( (wrOnly==0 || pCur->wrFlag) && pCur->eState!=CURSOR_FAULT ) r++; + } + return r; +} +#endif + /* ** If there are no outstanding cursors and we are not in the middle ** of a transaction but there is a read lock on the database, then @@ -50895,7 +52099,7 @@ page1_init_failed: */ static void unlockBtreeIfUnused(BtShared *pBt){ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pBt->pCursor==0 || pBt->inTransaction>TRANS_NONE ); + assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ assert( pBt->pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); @@ -51308,7 +52512,7 @@ static int relocatePage( ** iPtrPage. */ if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -51392,7 +52596,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -51484,8 +52688,11 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ if( nOrig0 ){ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, nFin, nOrig, 0); + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[28], pBt->nPage); @@ -51533,7 +52740,9 @@ static int autoVacuumCommit(BtShared *pBt){ nFree = get4byte(&pBt->pPage1->aData[36]); nFin = finalDbSize(pBt, nOrig, nFree); if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; - + if( nFinnFin && rc==SQLITE_OK; iFree--){ rc = incrVacuumStep(pBt, nFin, iFree, 1); } @@ -51550,7 +52759,7 @@ static int autoVacuumCommit(BtShared *pBt){ } } - assert( nRef==sqlite3PagerRefcount(pPager) ); + assert( nRef>=sqlite3PagerRefcount(pPager) ); return rc; } @@ -51618,7 +52827,6 @@ static void btreeEndTransaction(Btree *p){ #ifndef SQLITE_OMIT_AUTOVACUUM pBt->bDoTruncate = 0; #endif - btreeClearHasContent(pBt); if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){ /* If there are other active statements that belong to this database ** handle, downgrade to a read-only transaction. The other statements @@ -51693,6 +52901,7 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ return rc; } pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); } btreeEndTransaction(p); @@ -51714,27 +52923,6 @@ SQLITE_PRIVATE int sqlite3BtreeCommit(Btree *p){ return rc; } -#ifndef NDEBUG -/* -** Return the number of write-cursors open on this handle. This is for use -** in assert() expressions, so it is only compiled if NDEBUG is not -** defined. -** -** For the purposes of this routine, a write-cursor is any cursor that -** is capable of writing to the databse. That means the cursor was -** originally opened for writing and the cursor has not be disabled -** by having its state changed to CURSOR_FAULT. -*/ -static int countWriteCursors(BtShared *pBt){ - BtCursor *pCur; - int r = 0; - for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ - if( pCur->wrFlag && pCur->eState!=CURSOR_FAULT ) r++; - } - return r; -} -#endif - /* ** This routine sets the state to CURSOR_FAULT and the error ** code to errCode for every cursor on BtShared that pBtree @@ -51806,7 +52994,7 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode){ /* The rollback may have destroyed the pPage1->aData value. So ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + if( btreeGetPage(pBt, 1, &pPage1, 0, 0)==SQLITE_OK ){ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); @@ -51814,8 +53002,9 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode){ pBt->nPage = nPage; releasePage(pPage1); } - assert( countWriteCursors(pBt)==0 ); + assert( countValidCursors(pBt, 1)==0 ); pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); } btreeEndTransaction(p); @@ -52240,7 +53429,7 @@ static int getOverflowPage( assert( next==0 || rc==SQLITE_DONE ); if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); + rc = btreeGetPage(pBt, ovfl, &pPage, 0, (ppPage==0)); assert( rc==SQLITE_OK || pPage==0 ); if( rc==SQLITE_OK ){ next = get4byte(pPage->aData); @@ -52461,7 +53650,9 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_ACQUIRE_READONLY : 0) + ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); @@ -52640,10 +53831,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + rc = getAndInitPage(pBt, newPgno, &pNewPage, (pCur->wrFlag==0)); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; @@ -52760,7 +53952,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], pCur->wrFlag==0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -53374,7 +54566,7 @@ static int allocateBtreePage( if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); } if( rc ){ pTrunk = 0; @@ -53438,7 +54630,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -53518,7 +54710,7 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno); - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, noContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -53566,7 +54758,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -53580,7 +54772,7 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent, 0); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -53648,7 +54840,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0, 0))!=0) ) || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ goto freepage_out; @@ -53675,7 +54867,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -53721,7 +54913,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0, 0)) ){ goto freepage_out; } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -54522,7 +55714,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -55610,10 +56802,17 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ u8 eType = 0; Pgno iPtrPage = 0; + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); releasePage(pPageMove); + if( rc!=SQLITE_OK ){ + return rc; + } /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -55634,7 +56833,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -55710,7 +56909,7 @@ static int clearDatabasePage( return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -55812,7 +57011,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -55847,7 +57046,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ */ MemPage *pMove; releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -55857,7 +57056,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -56269,7 +57468,7 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0, 0))!=0 ){ checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; @@ -56741,6 +57940,17 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void return SQLITE_ABORT; } + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. + */ + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, @@ -57222,7 +58432,8 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_ACQUIRE_READONLY); if( rc==SQLITE_OK ){ rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); @@ -62445,14 +63656,6 @@ end_of_step: return (rc&db->errMask); } -/* -** The maximum number of times that a statement will try to reparse -** itself before giving up and returning SQLITE_SCHEMA. -*/ -#ifndef SQLITE_MAX_SCHEMA_RETRY -# define SQLITE_MAX_SCHEMA_RETRY 5 -#endif - /* ** This is the top-level implementation of sqlite3_step(). Call ** sqlite3Step() to do most of the work. If a schema error occurs, @@ -63356,6 +64559,11 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ ** then the returned string holds a copy of zRawSql with "-- " prepended ** to each line of text. ** +** If the SQLITE_TRACE_SIZE_LIMIT macro is defined to an integer, then +** then long strings and blobs are truncated to that many bytes. This +** can be used to prevent unreasonably large trace strings when dealing +** with large (multi-megabyte) strings and blobs. +** ** The calling function is responsible for making sure the memory returned ** is eventually freed. ** @@ -63426,30 +64634,49 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Real ){ sqlite3XPrintf(&out, "%!.15g", pVar->r); }else if( pVar->flags & MEM_Str ){ + int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); + Mem utf8; if( enc!=SQLITE_UTF8 ){ - Mem utf8; memset(&utf8, 0, sizeof(utf8)); utf8.db = db; sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); - sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z); - sqlite3VdbeMemRelease(&utf8); - }else -#endif - { - sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z); + pVar = &utf8; } +#endif + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ){ + nOut = SQLITE_TRACE_SIZE_LIMIT; + while( nOutn && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } + } +#endif + sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOutn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); +#endif +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); +#endif }else if( pVar->flags & MEM_Zero ){ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ + int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); sqlite3StrAccumAppend(&out, "x'", 2); - for(i=0; in; i++){ + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; +#endif + for(i=0; iz[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOutn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); +#endif } } } @@ -67666,7 +68893,7 @@ case OP_SeekGt: { /* jump, in3 */ ** u.bc.r.flags = 0; ** } */ - u.bc.r.flags = (u16)(UNPACKED_INCRKEY * (1 & (u.bc.oc - OP_SeekLt))); + u.bc.r.flags = (u8)(UNPACKED_INCRKEY * (1 & (u.bc.oc - OP_SeekLt))); assert( u.bc.oc!=OP_SeekGt || u.bc.r.flags==UNPACKED_INCRKEY ); assert( u.bc.oc!=OP_SeekLe || u.bc.r.flags==UNPACKED_INCRKEY ); assert( u.bc.oc!=OP_SeekGe || u.bc.r.flags==0 ); @@ -70791,7 +72018,7 @@ SQLITE_API int sqlite3_blob_open( } sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); - } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); + } while( (++nAttempt)mallocFailed==0 ){ @@ -72476,7 +73703,9 @@ static const struct sqlite3_io_methods MemJournalMethods = { 0, /* xShmMap */ 0, /* xShmLock */ 0, /* xShmBarrier */ - 0 /* xShmUnlock */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ }; /* @@ -72620,7 +73849,9 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. +** on the compound select chain, p->pPrior. Invoke the xSelectCallback() +** either before or after the walk of expressions and FROM clause, depending +** on whether pWalker->bSelectDepthFirst is false or true, respectively. ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. @@ -72634,14 +73865,23 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ rc = WRC_Continue; pWalker->walkerDepth++; while( p ){ - rc = pWalker->xSelectCallback(pWalker, p); - if( rc ) break; + if( !pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) break; + } if( sqlite3WalkSelectExpr(pWalker, p) || sqlite3WalkSelectFrom(pWalker, p) ){ pWalker->walkerDepth--; return WRC_Abort; } + if( pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + /* Depth-first search is currently only used for + ** selectAddSubqueryTypeInfo() and that routine always returns + ** WRC_Continue (0). So the following branch is never taken. */ + if( NEVER(rc) ) break; + } p = p->pPrior; } pWalker->walkerDepth--; @@ -73039,7 +74279,10 @@ static int lookupName( ** Note that the expression in the result set should have already been ** resolved by the time the WHERE clause is resolved. */ - if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ + if( (pEList = pNC->pEList)!=0 + && zTab==0 + && ((pNC->ncFlags & NC_AsMaybe)==0 || cnt==0) + ){ for(j=0; jnExpr; j++){ char *zAs = pEList->a[j].zName; if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ @@ -73130,7 +74373,9 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + if( pExpr->op!=TK_AS ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ @@ -73805,11 +75050,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** re-evaluated for each reference to it. */ sNC.pEList = p->pEList; - if( sqlite3ResolveExprNames(&sNC, p->pWhere) || - sqlite3ResolveExprNames(&sNC, p->pHaving) - ){ - return WRC_Abort; - } + sNC.ncFlags |= NC_AsMaybe; + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AsMaybe; /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries @@ -73930,6 +75174,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( #endif savedHasAgg = pNC->ncFlags & NC_HasAgg; pNC->ncFlags &= ~NC_HasAgg; + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pNC->pParse; @@ -73970,6 +75215,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelectNames( Walker w; assert( p!=0 ); + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pParse; @@ -74096,12 +75342,7 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ } assert( op!=TK_REGISTER || p->op2!=TK_COLLATE ); if( op==TK_COLLATE ){ - if( db->init.busy ){ - /* Do not report errors when parsing while the schema */ - pColl = sqlite3FindCollSeq(db, ENC(db), p->u.zToken, 0); - }else{ - pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); - } + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); break; } if( p->pTab!=0 @@ -75194,6 +76435,7 @@ static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ } static int exprIsConst(Expr *p, int initFlag){ Walker w; + memset(&w, 0, sizeof(w)); w.u.i = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; @@ -77408,8 +78650,8 @@ SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; if( pParse->cookieGoto ) return; if( OptimizationDisabled(pParse->db, SQLITE_FactorOutConst) ) return; + memset(&w, 0, sizeof(w)); w.xExprCallback = evalConstExpr; - w.xSelectCallback = 0; w.pParse = pParse; sqlite3WalkExpr(&w, pExpr); } @@ -83601,10 +84843,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; if( pExpr ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( pColl ){ - nExtra += (1 + sqlite3Strlen30(pColl->zName)); - } + assert( pExpr->op==TK_COLLATE ); + nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -83665,7 +84905,6 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( const char *zColName = pListItem->zName; Column *pTabCol; int requestedSortOrder; - CollSeq *pColl; /* Collating sequence */ char *zColl; /* Collation sequence name */ for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ @@ -83678,11 +84917,10 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( goto exit_create_index; } pIndex->aiColumn[i] = j; - if( pListItem->pExpr - && (pColl = sqlite3ExprCollSeq(pParse, pListItem->pExpr))!=0 - ){ + if( pListItem->pExpr ){ int nColl; - zColl = pColl->zName; + assert( pListItem->pExpr->op==TK_COLLATE ); + zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); memcpy(zExtra, zColl, nColl); @@ -83691,9 +84929,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( nExtra -= nColl; }else{ zColl = pTab->aCol[j].zColl; - if( !zColl ){ - zColl = "BINARY"; - } + if( !zColl ) zColl = "BINARY"; } if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; @@ -86612,6 +87848,13 @@ static int patternCompare( return *zString==0; } +/* +** The sqlite3_strglob() interface. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; +} + /* ** Count the number of times that the LIKE operator (or GLOB which is ** just a variation of LIKE) gets called. This is used for testing @@ -90812,7 +92055,6 @@ SQLITE_API int sqlite3_exec( const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3_stmt *pStmt = 0; /* The current SQL statement */ char **azCols = 0; /* Names of result columns */ - int nRetry = 0; /* Number of retry attempts */ int callbackIsInit; /* True if callback data is initialized */ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; @@ -90820,12 +92062,12 @@ SQLITE_API int sqlite3_exec( sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK, 0); - while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ + while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; pStmt = 0; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); assert( rc==SQLITE_OK || pStmt==0 ); if( rc!=SQLITE_OK ){ continue; @@ -90882,11 +92124,8 @@ SQLITE_API int sqlite3_exec( if( rc!=SQLITE_ROW ){ rc = sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - if( rc!=SQLITE_SCHEMA ){ - nRetry = 0; - zSql = zLeftover; - while( sqlite3Isspace(zSql[0]) ) zSql++; - } + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; break; } } @@ -91410,8 +92649,17 @@ struct sqlite3_api_routines { #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 #endif /* SQLITE_CORE */ -#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; -#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +#endif #endif /* _SQLITE3EXT_H_ */ @@ -91814,8 +93062,23 @@ static int sqlite3LoadExtension( void *handle; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; void **aHandle; int nMsg = 300 + sqlite3Strlen30(zFile); + int ii; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + if( pzErrMsg ) *pzErrMsg = 0; @@ -91832,11 +93095,17 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } - if( zProc==0 ){ - zProc = "sqlite3_extension_init"; - } + zEntry = zProc ? zProc : "sqlite3_extension_init"; handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zEntry); + } if( xInit==0 ){ if( pzErrMsg ){ - nMsg += sqlite3Strlen30(zProc); + nMsg += sqlite3Strlen30(zEntry); *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, - "no entry point [%s] in shared library [%s]", zProc,zFile); + "no entry point [%s] in shared library [%s]", zEntry, zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } - sqlite3OsDlClose(pVfs, handle); } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); return SQLITE_ERROR; - }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + } + sqlite3_free(zAltEntry); + if( xInit(db, &zErrmsg, &sqlite3Apis) ){ if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } @@ -92391,7 +93697,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); @@ -92474,11 +93780,12 @@ SQLITE_PRIVATE void sqlite3Pragma( static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 2, 0}, { OP_Subtract, 1, 2, 1}, - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; int addr; @@ -92816,6 +94123,43 @@ SQLITE_PRIVATE void sqlite3Pragma( } }else + /* + ** PRAGMA [database.]mmap_size(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + sqlite3_int64 sz; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3Atoi64(zRight, &sz, 1000, SQLITE_UTF8); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); + } + } + } + sz = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_SIZE,&sz)==SQLITE_OK ){ +#if SQLITE_MAX_MMAP_SIZE==0 + sz = 0; +#endif + returnSingleInt(pParse, "mmap_size", sz); + } + }else + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" @@ -93601,6 +94945,11 @@ SQLITE_PRIVATE void sqlite3Pragma( ** PRAGMA [database.]user_version ** PRAGMA [database.]user_version = ** + ** PRAGMA [database.]freelist_count = + ** + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = + ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both ** the schema-version and the user-version are 32-bit signed integers @@ -93622,10 +94971,14 @@ SQLITE_PRIVATE void sqlite3Pragma( if( sqlite3StrICmp(zLeft, "schema_version")==0 || sqlite3StrICmp(zLeft, "user_version")==0 || sqlite3StrICmp(zLeft, "freelist_count")==0 + || sqlite3StrICmp(zLeft, "application_id")==0 ){ int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ sqlite3VdbeUsesBtree(v, iDb); switch( zLeft[0] ){ + case 'a': case 'A': + iCookie = BTREE_APPLICATION_ID; + break; case 'f': case 'F': iCookie = BTREE_FREE_PAGE_COUNT; break; @@ -94506,7 +95859,6 @@ static int sqlite3Prepare( } #endif - assert( db->init.busy==0 || saveSqlFlag==0 ); if( db->init.busy==0 ){ Vdbe *pVdbe = pParse->pVdbe; sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); @@ -97982,6 +99334,69 @@ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pF } return SQLITE_OK; } +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + pNew->pLimit = 0; + pNew->pOffset = 0; + return WRC_Continue; +} /* ** This routine is a Walker callback for "expanding" a SELECT statement. @@ -98298,10 +99713,13 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ */ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; - w.xSelectCallback = selectExpander; + memset(&w, 0, sizeof(w)); + w.xSelectCallback = convertCompoundSelectToSubquery; w.xExprCallback = exprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); + w.xSelectCallback = selectExpander; + sqlite3WalkSelect(&w, pSelect); } @@ -98356,9 +99774,11 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; + memset(&w, 0, sizeof(w)); w.xSelectCallback = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; + w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } @@ -98769,7 +100189,7 @@ SQLITE_PRIVATE int sqlite3Select( pItem->addrFillSub = topAddr+1; VdbeNoopComment((v, "materialize %s", pItem->pTab->zName)); if( pItem->isCorrelated==0 ){ - /* If the subquery is no correlated and if we are not inside of + /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ onceAddr = sqlite3CodeOnce(pParse); @@ -101035,6 +102455,7 @@ SQLITE_PRIVATE void sqlite3Update( } if( j>=pTab->nCol ){ if( sqlite3IsRowid(pChanges->a[i].zName) ){ + j = -1; chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; }else{ @@ -101047,7 +102468,8 @@ SQLITE_PRIVATE void sqlite3Update( { int rc; rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, - pTab->aCol[j].zName, db->aDb[iDb].zName); + j<0 ? "ROWID" : pTab->aCol[j].zName, + db->aDb[iDb].zName); if( rc==SQLITE_DENY ){ goto update_cleanup; }else if( rc==SQLITE_IGNORE ){ @@ -101790,6 +103212,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); @@ -103657,7 +105080,7 @@ static WhereTerm *findTerm( continue; } } - if( pTerm->prereqRight==0 ){ + if( pTerm->prereqRight==0 && (pTerm->eOperator&WO_EQ)!=0 ){ pResult = pTerm; goto findTerm_success; }else if( pResult==0 ){ @@ -105227,9 +106650,8 @@ static void bestVirtualIndex(WhereBestIdx *p){ struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_constraint_usage *pUsage; WhereTerm *pTerm; - int i, j, k; + int i, j; int nOrderBy; - int sortOrder; /* Sort order for IN clauses */ int bAllowIN; /* Allow IN optimizations */ double rCost; @@ -105328,7 +106750,6 @@ static void bestVirtualIndex(WhereBestIdx *p){ return; } - sortOrder = SQLITE_SO_ASC; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pIdxCons++){ if( pUsage[i].argvIndex>0 ){ @@ -105343,17 +106764,28 @@ static void bestVirtualIndex(WhereBestIdx *p){ ** repeated in the output. */ break; } - for(k=0; knOrderBy; k++){ - if( pIdxInfo->aOrderBy[k].iColumn==pIdxCons->iColumn ){ - sortOrder = pIdxInfo->aOrderBy[k].desc; - break; - } - } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; } } } if( i>=pIdxInfo->nConstraint ) break; } + + /* The orderByConsumed signal is only valid if all outer loops collectively + ** generate just a single row of output. + */ + if( pIdxInfo->orderByConsumed ){ + for(i=0; ii; i++){ + if( (p->aLevel[i].plan.wsFlags & WHERE_UNIQUE)==0 ){ + pIdxInfo->orderByConsumed = 0; + } + } + } /* If there is an ORDER BY clause, and the selected virtual table index ** does not satisfy it, increase the cost of the scan accordingly. This @@ -105378,8 +106810,7 @@ static void bestVirtualIndex(WhereBestIdx *p){ } p->cost.plan.u.pVtabIdx = pIdxInfo; if( pIdxInfo->orderByConsumed ){ - assert( sortOrder==0 || sortOrder==1 ); - p->cost.plan.wsFlags |= WHERE_ORDERED + sortOrder*WHERE_REVERSE; + p->cost.plan.wsFlags |= WHERE_ORDERED; p->cost.plan.nOBSat = nOrderBy; }else{ p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; @@ -107116,6 +108547,7 @@ static Bitmask codeOneLoopStart( int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ + Bitmask newNotReady; /* Return value */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -107126,6 +108558,7 @@ static Bitmask codeOneLoopStart( bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 && (wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeNoopComment((v, "Begin Join Loop %d", iLevel)); /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -107668,6 +109101,10 @@ static Bitmask codeOneLoopStart( ** the "interesting" terms of z - terms that did not originate in the ** ON or USING clause of a LEFT JOIN, and terms that are usable as ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 */ if( pWC->nTerm>1 ){ int iTerm; @@ -107689,7 +109126,7 @@ static Bitmask codeOneLoopStart( if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; - if( pAndExpr ){ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } @@ -107776,7 +109213,7 @@ static Bitmask codeOneLoopStart( pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } - notReady &= ~getMask(pWC->pMaskSet, iCur); + newNotReady = notReady & ~getMask(pWC->pMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. @@ -107790,7 +109227,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ testcase( pWInfo->untestedTerms==0 && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); pWInfo->untestedTerms = 1; @@ -107805,6 +109242,33 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + WhereTerm *pAlt; + Expr sEq; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & newNotReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + VdbeNoopComment((v, "begin transitive constraint")); + sEq = *pAlt->pExpr; + sEq.pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, &sEq, addrCont, SQLITE_JUMPIFNULL); + } + /* For a LEFT OUTER JOIN, generate code that will record the fact that ** at least one row of the right table has matched the left table. */ @@ -107817,7 +109281,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ assert( pWInfo->untestedTerms ); continue; } @@ -107828,7 +109292,7 @@ static Bitmask codeOneLoopStart( } sqlite3ReleaseTempReg(pParse, iReleaseReg); - return notReady; + return newNotReady; } #if defined(SQLITE_TEST) @@ -111146,7 +112610,9 @@ static void yy_reduce( struct SrcList_item *pOld = yymsp[-4].minor.yy347->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; } sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy347); }else{ @@ -113814,6 +115280,19 @@ SQLITE_API int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_MMAP_SIZE: { + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + sqlite3GlobalConfig.mxMmap = mxMmap; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + default: { rc = SQLITE_ERROR; break; @@ -114207,6 +115686,12 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ ** go ahead and free all resources. */ + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + /* Free any outstanding Savepoint structures. */ sqlite3CloseSavepoints(db); @@ -114307,6 +115792,15 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int inTrans = 0; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; if( p ){ @@ -114324,6 +115818,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); } + sqlite3BtreeLeaveAll(db); /* Any deferred constraint violations have now been resolved. */ db->nDeferredCons = 0; @@ -114334,6 +115829,110 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ } } +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + /* ** Return a static string that describes the kind of error specified in the ** argument. @@ -115634,6 +117233,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 @@ -117950,7 +119550,7 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const /* fts3_expr.c */ SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, - char **, int, int, int, const char *, int, Fts3Expr ** + char **, int, int, int, const char *, int, Fts3Expr **, char ** ); SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST @@ -117975,6 +119575,9 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iC SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); +/* fts3_tokenize_vtab.c */ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); + /* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifdef SQLITE_ENABLE_FTS4_UNICODE61 SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); @@ -120671,14 +122274,12 @@ static int fts3FilterMethod( pCsr->iLangid = 0; if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); + assert( p->base.zErrMsg==0 ); rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg ); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_ERROR ){ - static const char *zErr = "malformed MATCH expression: [%s]"; - p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); - } return rc; } @@ -121342,9 +122943,13 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ db, "fts4", &fts3Module, (void *)pHash, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3InitTok(db, (void *)pHash); + } return rc; } + /* An error has occurred. Delete the hash table and return the error code. */ assert( rc!=SQLITE_OK ); if( pHash ){ @@ -123118,17 +124723,26 @@ static int fts3auxConnectMethod( UNUSED_PARAMETER(pUnused); - /* The user should specify a single argument - the name of an fts3 table. */ - if( argc!=4 ){ - *pzErr = sqlite3_mprintf( - "wrong number of arguments to fts4aux constructor" - ); - return SQLITE_ERROR; - } + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; zDb = argv[1]; nDb = (int)strlen(zDb); - zFts3 = argv[3]; + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } nFts3 = (int)strlen(zFts3); rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); @@ -123151,6 +124765,10 @@ static int fts3auxConnectMethod( *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; + + bad_args: + *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; } /* @@ -124164,8 +125782,10 @@ static int fts3ExprParse( } pNot->eType = FTSQUERY_NOT; pNot->pRight = p; + p->pParent = pNot; if( pNotBranch ){ pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; } pNotBranch = pNot; p = pPrev; @@ -124253,6 +125873,7 @@ static int fts3ExprParse( pIter = pIter->pLeft; } pIter->pLeft = pRet; + pRet->pParent = pIter; pRet = pNotBranch; } } @@ -124269,6 +125890,223 @@ exprparse_out: return rc; } +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + /* ** Parameters z and n contain a pointer to and length of a buffer containing ** an fts3 query expression, respectively. This function attempts to parse the @@ -124301,49 +126139,74 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse( int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr /* OUT: Parsed query structure */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ - int nParsed; - int rc; - ParseContext sParse; - - memset(&sParse, 0, sizeof(ParseContext)); - sParse.pTokenizer = pTokenizer; - sParse.iLangid = iLangid; - sParse.azCol = (const char **)azCol; - sParse.nCol = nCol; - sParse.iDefaultCol = iDefaultCol; - sParse.bFts4 = bFts4; - if( z==0 ){ - *ppExpr = 0; - return SQLITE_OK; + static const int MAX_EXPR_DEPTH = 12; + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, MAX_EXPR_DEPTH); + } } - if( n<0 ){ - n = (int)strlen(z); - } - rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); - /* Check for mismatched parenthesis */ - if( rc==SQLITE_OK && sParse.nNest ){ - rc = SQLITE_ERROR; + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(*ppExpr); *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + *pzErr = sqlite3_mprintf( + "FTS expression tree is too large (maximum depth %d)", MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + *pzErr = sqlite3_mprintf("malformed MATCH expression: [%s]", z); + } } return rc; } /* -** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** Free a single node of an expression tree. */ -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){ - if( p ){ - assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); - sqlite3Fts3ExprFree(p->pLeft); - sqlite3Fts3ExprFree(p->pRight); - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->aMI); - sqlite3_free(p); +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } } } @@ -124395,6 +126258,9 @@ static int queryTestTokenizer( ** the returned expression text and then freed using sqlite3_free(). */ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } switch( pExpr->eType ){ case FTSQUERY_PHRASE: { Fts3Phrase *pPhrase = pExpr->pPhrase; @@ -124502,10 +126368,21 @@ static void fts3ExprTest( azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]); } - rc = sqlite3Fts3ExprParse( - pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr - ); + if( sqlite3_user_data(context) ){ + char *zDummy = 0; + rc = sqlite3Fts3ExprParse( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy + ); + assert( rc==SQLITE_OK || pExpr==0 ); + sqlite3_free(zDummy); + }else{ + rc = fts3ExprParseUnbalanced( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + ); + } + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ + sqlite3Fts3ExprFree(pExpr); sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); @@ -124528,9 +126405,15 @@ exprtest_out: ** with database connection db. */ SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ - return sqlite3_create_function( + int rc = sqlite3_create_function( db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 + ); + } + return rc; } #endif @@ -126293,6 +128176,462 @@ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ /************** End of fts3_tokenizer1.c *************************************/ +/************** Begin file fts3_tokenize_vtab.c ******************************/ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); + + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } + + *pp = p; + return SQLITE_OK; +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenize_vtab.c **********************************/ /************** Begin file fts3_write.c **************************************/ /* ** 2009 Oct 23 diff --git a/src/3rdparty/sqlite3.h b/src/3rdparty/sqlite3.h index 69b4586a3f..e398838287 100644 --- a/src/3rdparty/sqlite3.h +++ b/src/3rdparty/sqlite3.h @@ -107,9 +107,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.16.2" -#define SQLITE_VERSION_NUMBER 3007016 -#define SQLITE_SOURCE_ID "2013-04-12 11:52:43 cbea02d93865ce0e06789db95fd9168ebac970c7" +#define SQLITE_VERSION "3.7.17" +#define SQLITE_VERSION_NUMBER 3007017 +#define SQLITE_SOURCE_ID "2013-05-20 00:56:22 118a3b35693b134d56ebd780123b7fd6f1497668" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -425,6 +425,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ @@ -475,6 +477,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) @@ -494,6 +497,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) #define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) #define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -733,6 +738,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; @@ -869,7 +877,8 @@ struct sqlite3_io_methods { ** it is able to override built-in [PRAGMA] statements. ** **
  • [[SQLITE_FCNTL_BUSYHANDLER]] -** ^This file-control may be invoked by SQLite on the database file handle +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle ** shortly after it is opened in order to provide a custom VFS with access ** to the connections busy-handler callback. The argument is of type (void **) ** - an array of two (void *) values. The first (void *) actually points @@ -880,13 +889,24 @@ struct sqlite3_io_methods { ** current operation. ** **
  • [[SQLITE_FCNTL_TEMPFILENAME]] -** ^Application can invoke this file-control to have SQLite generate a +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a ** temporary filename using the same algorithm that is followed to generate ** temporary filenames for TEMP tables and other internal uses. The ** argument should be a char** which will be filled with the filename ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -905,6 +925,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle @@ -1571,7 +1592,9 @@ struct sqlite3_mem_methods { ** page cache implementation into that object.)^
    • ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    -**
    ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +**
    The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -1617,12 +1640,12 @@ struct sqlite3_mem_methods { **
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE **
    These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. -** +**
    ** ** [[SQLITE_CONFIG_SQLLOG]] **
    SQLITE_CONFIG_SQLLOG **
    This option is only available if sqlite is compiled with the -** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should ** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). ** The second should be of type (void*). The callback is invoked by the library ** in three separate circumstances, identified by the value passed as the @@ -1632,7 +1655,23 @@ struct sqlite3_mem_methods { ** fourth parameter is 1, then the SQL statement that the third parameter ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The -** third parameter is passed NULL In this case. +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option. +** If either argument to this option is negative, then that argument is +** changed to its compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1656,6 +1695,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2489,6 +2529,9 @@ SQLITE_API int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -3027,7 +3070,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. **
    • ** **
  • @@ -3231,6 +3275,9 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the @@ -4187,7 +4234,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -4986,11 +5033,20 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -5016,11 +5072,11 @@ SQLITE_API int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -5032,7 +5088,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes @@ -6812,10 +6868,25 @@ SQLITE_API int sqlite3_unlock_notify( SQLITE_API int sqlite3_stricmp(const char *, const char *); SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + /* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 984bdc90a4..c04cd92eca 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -8,6 +8,9 @@ #include "BroDoc.h" #include "BroDocObj.h" #include "util.h" +#include "plugin/Manager.h" +#include "analyzer/Manager.h" +#include "analyzer/Component.h" BroDoc::BroDoc(const std::string& rel, const std::string& abs) { @@ -164,84 +167,77 @@ void BroDoc::SetPacketFilter(const std::string& s) packet_filter.clear(); } -void BroDoc::AddPortAnalysis(const std::string& analyzer, - const std::string& ports) - { - std::string reST_string = analyzer + "::\n" + ports + "\n\n"; - port_analysis.push_back(reST_string); - } - void BroDoc::WriteDocFile() const { - WriteToDoc(".. Automatically generated. Do not edit.\n\n"); + WriteToDoc(reST_file, ".. Automatically generated. Do not edit.\n\n"); - WriteToDoc(":tocdepth: 3\n\n"); + WriteToDoc(reST_file, ":tocdepth: 3\n\n"); - WriteSectionHeading(doc_title.c_str(), '='); + WriteSectionHeading(reST_file, doc_title.c_str(), '='); - WriteStringList(".. bro:namespace:: %s\n", modules); + WriteStringList(reST_file, ".. bro:namespace:: %s\n", modules); - WriteToDoc("\n"); + WriteToDoc(reST_file, "\n"); - // WriteSectionHeading("Overview", '-'); - WriteStringList("%s\n", summary); + // WriteSectionHeading(reST_file, "Overview", '-'); + WriteStringList(reST_file, "%s\n", summary); - WriteToDoc("\n"); + WriteToDoc(reST_file, "\n"); if ( ! modules.empty() ) { - WriteToDoc(":Namespace%s: ", (modules.size() > 1 ? "s" : "")); - // WriteStringList(":bro:namespace:`%s`", modules); - WriteStringList("``%s``, ", "``%s``", modules); - WriteToDoc("\n"); + WriteToDoc(reST_file, ":Namespace%s: ", (modules.size() > 1 ? "s" : "")); + // WriteStringList(reST_file, ":bro:namespace:`%s`", modules); + WriteStringList(reST_file, "``%s``, ", "``%s``", modules); + WriteToDoc(reST_file, "\n"); } if ( ! imports.empty() ) { - WriteToDoc(":Imports: "); + WriteToDoc(reST_file, ":Imports: "); std::list::const_iterator it; for ( it = imports.begin(); it != imports.end(); ++it ) { if ( it != imports.begin() ) - WriteToDoc(", "); + WriteToDoc(reST_file, ", "); string pretty(*it); size_t pos = pretty.find("/index"); if ( pos != std::string::npos && pos + 6 == pretty.size() ) pretty = pretty.substr(0, pos); - WriteToDoc(":doc:`%s `", pretty.c_str(), it->c_str()); + WriteToDoc(reST_file, ":doc:`%s `", pretty.c_str(), it->c_str()); } - WriteToDoc("\n"); + WriteToDoc(reST_file, "\n"); } - WriteToDoc(":Source File: :download:`%s`\n", + WriteToDoc(reST_file, ":Source File: :download:`%s`\n", downloadable_filename.c_str()); - WriteToDoc("\n"); + WriteToDoc(reST_file, "\n"); WriteInterface("Summary", '~', '#', true, true); if ( ! notices.empty() ) - WriteBroDocObjList(notices, "Notices", '#'); + WriteBroDocObjList(reST_file, notices, "Notices", '#'); if ( port_analysis.size() || packet_filter.size() ) - WriteSectionHeading("Configuration Changes", '#'); + WriteSectionHeading(reST_file, "Configuration Changes", '#'); if ( ! port_analysis.empty() ) { - WriteSectionHeading("Port Analysis", '^'); - WriteToDoc("Loading this script makes the following changes to " + WriteSectionHeading(reST_file, "Port Analysis", '^'); + WriteToDoc(reST_file, "Loading this script makes the following changes to " ":bro:see:`dpd_config`.\n\n"); - WriteStringList("%s, ", "%s", port_analysis); + WriteStringList(reST_file, "%s, ", "%s", port_analysis); } if ( ! packet_filter.empty() ) { - WriteSectionHeading("Packet Filter", '^'); - WriteToDoc("Loading this script makes the following changes to " + WriteSectionHeading(reST_file, "Packet Filter", '^'); + WriteToDoc(reST_file, "Loading this script makes the following changes to " ":bro:see:`capture_filters`.\n\n"); - WriteToDoc("Filters added::\n\n"); - WriteToDoc("%s\n", packet_filter.c_str()); + WriteToDoc(reST_file, "Filters added::\n\n"); + WriteToDoc(reST_file, "%s\n", packet_filter.c_str()); } WriteInterface("Detailed Interface", '~', '#', true, false); @@ -267,23 +263,23 @@ void BroDoc::WriteDocFile() const void BroDoc::WriteInterface(const char* heading, char underline, char sub, bool isPublic, bool isShort) const { - WriteSectionHeading(heading, underline); - WriteBroDocObjList(options, isPublic, "Options", sub, isShort); - WriteBroDocObjList(constants, isPublic, "Constants", sub, isShort); - WriteBroDocObjList(state_vars, isPublic, "State Variables", sub, isShort); - WriteBroDocObjList(types, isPublic, "Types", sub, isShort); - WriteBroDocObjList(events, isPublic, "Events", sub, isShort); - WriteBroDocObjList(hooks, isPublic, "Hooks", sub, isShort); - WriteBroDocObjList(functions, isPublic, "Functions", sub, isShort); - WriteBroDocObjList(redefs, isPublic, "Redefinitions", sub, isShort); + WriteSectionHeading(reST_file, heading, underline); + WriteBroDocObjList(reST_file, options, isPublic, "Options", sub, isShort); + WriteBroDocObjList(reST_file, constants, isPublic, "Constants", sub, isShort); + WriteBroDocObjList(reST_file, state_vars, isPublic, "State Variables", sub, isShort); + WriteBroDocObjList(reST_file, types, isPublic, "Types", sub, isShort); + WriteBroDocObjList(reST_file, events, isPublic, "Events", sub, isShort); + WriteBroDocObjList(reST_file, hooks, isPublic, "Hooks", sub, isShort); + WriteBroDocObjList(reST_file, functions, isPublic, "Functions", sub, isShort); + WriteBroDocObjList(reST_file, redefs, isPublic, "Redefinitions", sub, isShort); } -void BroDoc::WriteStringList(const char* format, const char* last_format, - const std::list& l) const +void BroDoc::WriteStringList(FILE* f, const char* format, const char* last_format, + const std::list& l) { if ( l.empty() ) { - WriteToDoc("\n"); + WriteToDoc(f, "\n"); return; } @@ -292,12 +288,12 @@ void BroDoc::WriteStringList(const char* format, const char* last_format, last--; for ( it = l.begin(); it != last; ++it ) - WriteToDoc(format, it->c_str()); + WriteToDoc(f, format, it->c_str()); - WriteToDoc(last_format, last->c_str()); + WriteToDoc(f, last_format, last->c_str()); } -void BroDoc::WriteBroDocObjTable(const BroDocObjList& l) const +void BroDoc::WriteBroDocObjTable(FILE* f, const BroDocObjList& l) { int max_id_col = 0; int max_com_col = 0; @@ -317,38 +313,38 @@ void BroDoc::WriteBroDocObjTable(const BroDocObjList& l) const } // Start table. - WriteRepeatedChar('=', max_id_col); - WriteToDoc(" "); + WriteRepeatedChar(f, '=', max_id_col); + WriteToDoc(f, " "); if ( max_com_col == 0 ) - WriteToDoc("="); + WriteToDoc(f, "="); else - WriteRepeatedChar('=', max_com_col); + WriteRepeatedChar(f, '=', max_com_col); - WriteToDoc("\n"); + WriteToDoc(f, "\n"); for ( it = l.begin(); it != l.end(); ++it ) { if ( it != l.begin() ) - WriteToDoc("\n\n"); - (*it)->WriteReSTCompact(reST_file, max_id_col); + WriteToDoc(f, "\n\n"); + (*it)->WriteReSTCompact(f, max_id_col); } // End table. - WriteToDoc("\n"); - WriteRepeatedChar('=', max_id_col); - WriteToDoc(" "); + WriteToDoc(f, "\n"); + WriteRepeatedChar(f, '=', max_id_col); + WriteToDoc(f, " "); if ( max_com_col == 0 ) - WriteToDoc("="); + WriteToDoc(f, "="); else - WriteRepeatedChar('=', max_com_col); + WriteRepeatedChar(f, '=', max_com_col); - WriteToDoc("\n\n"); + WriteToDoc(f, "\n\n"); } -void BroDoc::WriteBroDocObjList(const BroDocObjList& l, bool wantPublic, - const char* heading, char underline, bool isShort) const +void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l, bool wantPublic, + const char* heading, char underline, bool isShort) { if ( l.empty() ) return; @@ -366,7 +362,7 @@ void BroDoc::WriteBroDocObjList(const BroDocObjList& l, bool wantPublic, if ( it == l.end() ) return; - WriteSectionHeading(heading, underline); + WriteSectionHeading(f, heading, underline); BroDocObjList filtered_list; @@ -377,13 +373,13 @@ void BroDoc::WriteBroDocObjList(const BroDocObjList& l, bool wantPublic, } if ( isShort ) - WriteBroDocObjTable(filtered_list); + WriteBroDocObjTable(f, filtered_list); else - WriteBroDocObjList(filtered_list); + WriteBroDocObjList(f, filtered_list); } -void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic, - const char* heading, char underline, bool isShort) const +void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjMap& m, bool wantPublic, + const char* heading, char underline, bool isShort) { BroDocObjMap::const_iterator it; BroDocObjList l; @@ -391,24 +387,24 @@ void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic, for ( it = m.begin(); it != m.end(); ++it ) l.push_back(it->second); - WriteBroDocObjList(l, wantPublic, heading, underline, isShort); + WriteBroDocObjList(f, l, wantPublic, heading, underline, isShort); } -void BroDoc::WriteBroDocObjList(const BroDocObjList& l, const char* heading, - char underline) const +void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l, const char* heading, + char underline) { - WriteSectionHeading(heading, underline); - WriteBroDocObjList(l); + WriteSectionHeading(f, heading, underline); + WriteBroDocObjList(f, l); } -void BroDoc::WriteBroDocObjList(const BroDocObjList& l) const +void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l) { for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it ) - (*it)->WriteReST(reST_file); + (*it)->WriteReST(f); } -void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, const char* heading, - char underline) const +void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjMap& m, const char* heading, + char underline) { BroDocObjMap::const_iterator it; BroDocObjList l; @@ -416,28 +412,28 @@ void BroDoc::WriteBroDocObjList(const BroDocObjMap& m, const char* heading, for ( it = m.begin(); it != m.end(); ++it ) l.push_back(it->second); - WriteBroDocObjList(l, heading, underline); + WriteBroDocObjList(f, l, heading, underline); } -void BroDoc::WriteToDoc(const char* format, ...) const +void BroDoc::WriteToDoc(FILE* f, const char* format, ...) { va_list argp; va_start(argp, format); - vfprintf(reST_file, format, argp); + vfprintf(f, format, argp); va_end(argp); } -void BroDoc::WriteSectionHeading(const char* heading, char underline) const +void BroDoc::WriteSectionHeading(FILE* f, const char* heading, char underline) { - WriteToDoc("%s\n", heading); - WriteRepeatedChar(underline, strlen(heading)); - WriteToDoc("\n"); + WriteToDoc(f, "%s\n", heading); + WriteRepeatedChar(f, underline, strlen(heading)); + WriteToDoc(f, "\n"); } -void BroDoc::WriteRepeatedChar(char c, size_t n) const +void BroDoc::WriteRepeatedChar(FILE* f, char c, size_t n) { for ( size_t i = 0; i < n; ++i ) - WriteToDoc("%c", c); + WriteToDoc(f, "%c", c); } void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l) @@ -459,3 +455,143 @@ void BroDoc::AddFunction(BroDocObj* o) else functions[o->Name()]->Combine(o); } + +static void WritePluginSectionHeading(FILE* f, const plugin::Plugin* p) + { + string name = p->Name(); + + fprintf(f, "%s\n", name.c_str()); + for ( size_t i = 0; i < name.size(); ++i ) + fprintf(f, "-"); + fprintf(f, "\n\n"); + + fprintf(f, "%s\n\n", p->Description()); + } + +static void WriteAnalyzerComponent(FILE* f, const analyzer::Component* c) + { + EnumType* atag = analyzer_mgr->GetTagEnumType(); + string tag = fmt("ANALYZER_%s", c->CanonicalName()); + + if ( atag->Lookup("Analyzer", tag.c_str()) < 0 ) + reporter->InternalError("missing analyzer tag for %s", tag.c_str()); + + fprintf(f, ":bro:enum:`Analyzer::%s`\n\n", tag.c_str()); + } + +static void WritePluginComponents(FILE* f, const plugin::Plugin* p) + { + plugin::Plugin::component_list components = p->Components(); + plugin::Plugin::component_list::const_iterator it; + + fprintf(f, "Components\n"); + fprintf(f, "++++++++++\n\n"); + + for ( it = components.begin(); it != components.end(); ++it ) + { + switch ( (*it)->Type() ) { + case plugin::component::ANALYZER: + WriteAnalyzerComponent(f, + dynamic_cast(*it)); + break; + case plugin::component::READER: + reporter->InternalError("docs for READER component unimplemented"); + case plugin::component::WRITER: + reporter->InternalError("docs for WRITER component unimplemented"); + default: + reporter->InternalError("docs for unknown component unimplemented"); + } + } + } + +static void WritePluginBifItems(FILE* f, const plugin::Plugin* p, + plugin::BifItem::Type t, const string& heading) + { + plugin::Plugin::bif_item_list bifitems = p->BifItems(); + plugin::Plugin::bif_item_list::iterator it = bifitems.begin(); + + while ( it != bifitems.end() ) + { + if ( it->GetType() != t ) + it = bifitems.erase(it); + else + ++it; + } + + if ( bifitems.empty() ) + return; + + fprintf(f, "%s\n", heading.c_str()); + for ( size_t i = 0; i < heading.size(); ++i ) + fprintf(f, "+"); + fprintf(f, "\n\n"); + + for ( it = bifitems.begin(); it != bifitems.end(); ++it ) + { + BroDocObj* o = doc_ids[it->GetID()]; + + if ( o ) + o->WriteReST(f); + else + reporter->Warning("No docs for ID: %s\n", it->GetID()); + } + } + +static void WriteAnalyzerTagDefn(FILE* f, EnumType* e) + { + e = new CommentedEnumType(e); + e->SetTypeID(copy_string("Analyzer::Tag")); + + ID* dummy_id = new ID(copy_string("Analyzer::Tag"), SCOPE_GLOBAL, true); + dummy_id->SetType(e); + dummy_id->MakeType(); + + list* r = new list(); + r->push_back("Unique identifiers for protocol analyzers."); + + BroDocObj bdo(dummy_id, r, true); + + bdo.WriteReST(f); + } + +static bool IsAnalyzerPlugin(const plugin::Plugin* p) + { + plugin::Plugin::component_list components = p->Components(); + plugin::Plugin::component_list::const_iterator it; + + for ( it = components.begin(); it != components.end(); ++it ) + if ( (*it)->Type() != plugin::component::ANALYZER ) + return false; + + return true; + } + +void CreateProtoAnalyzerDoc(const char* filename) + { + FILE* f = fopen(filename, "w"); + + fprintf(f, "Protocol Analyzer Reference\n"); + fprintf(f, "===========================\n\n"); + + WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType()); + + plugin::Manager::plugin_list plugins = plugin_mgr->Plugins(); + plugin::Manager::plugin_list::const_iterator it; + + for ( it = plugins.begin(); it != plugins.end(); ++it ) + { + if ( ! IsAnalyzerPlugin(*it) ) + continue; + + WritePluginSectionHeading(f, *it); + WritePluginComponents(f, *it); + WritePluginBifItems(f, *it, plugin::BifItem::CONSTANT, + "Options/Constants"); + WritePluginBifItems(f, *it, plugin::BifItem::GLOBAL, "Globals"); + WritePluginBifItems(f, *it, plugin::BifItem::TYPE, "Types"); + WritePluginBifItems(f, *it, plugin::BifItem::EVENT, "Events"); + WritePluginBifItems(f, *it, plugin::BifItem::FUNCTION, "Functions"); + } + + fclose(f); + } diff --git a/src/BroDoc.h b/src/BroDoc.h index 79f02b7110..9f92f821f8 100644 --- a/src/BroDoc.h +++ b/src/BroDoc.h @@ -81,15 +81,6 @@ public: */ void SetPacketFilter(const std::string& s); - /** - * Schedules documentation of a given set of ports being associated - * with a particular analyzer as a result of the current script - * being loaded -- the way the "dpd_config" table is changed. - * @param analyzer An analyzer that changed the "dpd_config" table. - * @param ports The set of ports assigned to the analyzer in table. - */ - void AddPortAnalysis(const std::string& analyzer, const std::string& ports); - /** * Schedules documentation of a script option. An option is * defined as any variable in the script that is declared 'const' @@ -242,7 +233,115 @@ public: return reST_filename.c_str(); } -protected: + typedef std::list BroDocObjList; + typedef std::map BroDocObjMap; + + /** + * Writes out a table of BroDocObj's to the reST document + * @param f The file to write to. + * @param l A list of BroDocObj pointers + */ + static void WriteBroDocObjTable(FILE* f, const BroDocObjList& l); + + /** + * Writes out given number of characters to reST document + * @param f The file to write to. + * @param c the character to write + * @param n the number of characters to write + */ + static void WriteRepeatedChar(FILE* f, char c, size_t n); + + /** + * A wrapper to fprintf() that always uses the reST document + * for the FILE* argument. + * @param f The file to write to. + * @param format A printf style format string. + */ + static void WriteToDoc(FILE* f, const char* format, ...); + + /** + * Writes out a list of strings to the reST document. + * If the list is empty, prints a newline character. + * @param f The file to write to. + * @param format A printf style format string for elements of the list + * except for the last one in the list + * @param last_format A printf style format string to use for the last + * element of the list + * @param l A reference to a list of strings + */ + static void WriteStringList(FILE* f, const char* format, const char* last_format, + const std::list& l); + + /** + * @see WriteStringList(FILE* f, const char*, const char*, + * const std::list&>) + */ + static void WriteStringList(FILE* f, const char* format, + const std::list& l){ + WriteStringList(f, format, format, l); + } + + /** + * Writes out a list of BroDocObj objects to the reST document + * @param f The file to write to. + * @param l A list of BroDocObj pointers + * @param wantPublic If true, filter out objects that are not declared + * in the global scope. If false, filter out those that are in + * the global scope. + * @param heading The title of the section to create in the reST doc. + * @param underline The character to use to underline the reST + * section heading. + * @param isShort Whether to write the full documentation or a "short" + * version (a single sentence) + */ + static void WriteBroDocObjList(FILE* f, const BroDocObjList& l, bool wantPublic, + const char* heading, char underline, + bool isShort); + + /** + * Wraps the BroDocObjMap into a BroDocObjList and the writes that list + * to the reST document + * @see WriteBroDocObjList(FILE* f, const BroDocObjList&, bool, const char*, char, + bool) + */ + static void WriteBroDocObjList(FILE* f, const BroDocObjMap& m, bool wantPublic, + const char* heading, char underline, + bool isShort); + + /** + * Writes out a list of BroDocObj objects to the reST document + * @param l A list of BroDocObj pointers + * @param heading The title of the section to create in the reST doc. + * @param underline The character to use to underline the reST + * section heading. + */ + static void WriteBroDocObjList(FILE* f, const BroDocObjList& l, const char* heading, + char underline); + + /** + * Writes out a list of BroDocObj objects to the reST document + * @param l A list of BroDocObj pointers + */ + static void WriteBroDocObjList(FILE* f, const BroDocObjList& l); + + /** + * Wraps the BroDocObjMap into a BroDocObjList and the writes that list + * to the reST document + * @see WriteBroDocObjList(FILE* f, const BroDocObjList&, const char*, char) + */ + static void WriteBroDocObjList(FILE* f, const BroDocObjMap& m, const char* heading, + char underline); + + /** + * Writes out a reST section heading + * @param f The file to write to. + * @param heading The title of the heading to create + * @param underline The character to use to underline the section title + * within the reST document + */ + static void WriteSectionHeading(FILE* f, const char* heading, char underline); + +private: FILE* reST_file; std::string reST_filename; std::string source_filename; // points to the basename of source file @@ -255,9 +354,6 @@ protected: std::list imports; std::list port_analysis; - typedef std::list BroDocObjList; - typedef std::map BroDocObjMap; - BroDocObjList options; BroDocObjList constants; BroDocObjList state_vars; @@ -272,107 +368,6 @@ protected: BroDocObjList all; - /** - * Writes out a list of strings to the reST document. - * If the list is empty, prints a newline character. - * @param format A printf style format string for elements of the list - * except for the last one in the list - * @param last_format A printf style format string to use for the last - * element of the list - * @param l A reference to a list of strings - */ - void WriteStringList(const char* format, const char* last_format, - const std::list& l) const; - - /** - * @see WriteStringList(const char*, const char*, - * const std::list&>) - */ - void WriteStringList(const char* format, - const std::list& l) const - { - WriteStringList(format, format, l); - } - - - /** - * Writes out a table of BroDocObj's to the reST document - * @param l A list of BroDocObj pointers - */ - void WriteBroDocObjTable(const BroDocObjList& l) const; - - /** - * Writes out a list of BroDocObj objects to the reST document - * @param l A list of BroDocObj pointers - * @param wantPublic If true, filter out objects that are not declared - * in the global scope. If false, filter out those that are in - * the global scope. - * @param heading The title of the section to create in the reST doc. - * @param underline The character to use to underline the reST - * section heading. - * @param isShort Whether to write the full documentation or a "short" - * version (a single sentence) - */ - void WriteBroDocObjList(const BroDocObjList& l, bool wantPublic, - const char* heading, char underline, - bool isShort) const; - - /** - * Wraps the BroDocObjMap into a BroDocObjList and the writes that list - * to the reST document - * @see WriteBroDocObjList(const BroDocObjList&, bool, const char*, char, - bool) - */ - void WriteBroDocObjList(const BroDocObjMap& m, bool wantPublic, - const char* heading, char underline, - bool isShort) const; - - /** - * Writes out a list of BroDocObj objects to the reST document - * @param l A list of BroDocObj pointers - * @param heading The title of the section to create in the reST doc. - * @param underline The character to use to underline the reST - * section heading. - */ - void WriteBroDocObjList(const BroDocObjList& l, const char* heading, - char underline) const; - - /** - * Writes out a list of BroDocObj objects to the reST document - * @param l A list of BroDocObj pointers - */ - void WriteBroDocObjList(const BroDocObjList& l) const; - - /** - * Wraps the BroDocObjMap into a BroDocObjList and the writes that list - * to the reST document - * @see WriteBroDocObjList(const BroDocObjList&, const char*, char) - */ - void WriteBroDocObjList(const BroDocObjMap& m, const char* heading, - char underline) const; - - /** - * A wrapper to fprintf() that always uses the reST document - * for the FILE* argument. - * @param format A printf style format string. - */ - void WriteToDoc(const char* format, ...) const; - - /** - * Writes out a reST section heading - * @param heading The title of the heading to create - * @param underline The character to use to underline the section title - * within the reST document - */ - void WriteSectionHeading(const char* heading, char underline) const; - - /** - * Writes out given number of characters to reST document - * @param c the character to write - * @param n the number of characters to write - */ - void WriteRepeatedChar(char c, size_t n) const; - /** * Writes out the reST for either the script's public or private interface * @param heading The title of the interfaces section heading @@ -387,7 +382,6 @@ protected: */ void WriteInterface(const char* heading, char underline, char subunderline, bool isPublic, bool isShort) const; -private: /** * Frees memory allocated to BroDocObj's objects in a given list. @@ -413,4 +407,10 @@ private: }; }; +/** + * Writes out plugin index documentation for all analyzer plugins. + * @param filename the name of the file to write. + */ +void CreateProtoAnalyzerDoc(const char* filename); + #endif diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index 12753ea15d..4316b3113a 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -4,6 +4,8 @@ #include "ID.h" #include "BroDocObj.h" +map doc_ids = map(); + BroDocObj* BroDocObj::last = 0; BroDocObj::BroDocObj(const ID* id, std::list*& reST, @@ -16,6 +18,7 @@ BroDocObj::BroDocObj(const ID* id, std::list*& reST, is_fake_id = is_fake; use_role = 0; FormulateShortDesc(); + doc_ids[id->Name()] = this; } BroDocObj::~BroDocObj() diff --git a/src/BroDocObj.h b/src/BroDocObj.h index cb512f8cda..ab42dc3c94 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "ID.h" @@ -134,4 +135,9 @@ protected: private: }; +/** + * Map identifiers to their broxygen documentation objects. + */ +extern map doc_ids; + #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 87a3db3b62..9b1a55a31c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -114,7 +114,6 @@ set(BIF_SRCS logging.bif input.bif event.bif - file_analysis.bif const.bif types.bif strings.bif @@ -150,6 +149,7 @@ set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE) set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE) add_subdirectory(analyzer) +add_subdirectory(file_analysis) set(bro_SUBDIRS ${bro_SUBDIR_LIBS} @@ -359,21 +359,12 @@ set(bro_SRCS input/readers/Binary.cc input/readers/SQLite.cc - file_analysis/Manager.cc - file_analysis/File.cc - file_analysis/FileTimer.cc - file_analysis/FileID.h - file_analysis/Analyzer.h - file_analysis/AnalyzerSet.cc - file_analysis/Extract.cc - file_analysis/Hash.cc - file_analysis/DataEvent.cc - 3rdparty/sqlite3.c plugin/Component.cc plugin/Manager.cc plugin/Plugin.cc + plugin/Macros.h nb_dns.c digest.h @@ -395,7 +386,8 @@ set(BRO_EXE bro CACHE STRING "Bro executable binary" FORCE) # Target to create all the autogenerated files. -add_custom_target(generate_outputs DEPENDS ${bro_ALL_GENERATED_OUTPUTS}) +add_custom_target(generate_outputs) +add_dependencies(generate_outputs ${bro_ALL_GENERATED_OUTPUTS}) # Build __load__.bro files for plugins/*.bif.bro. bro_bif_create_loader(bif_loader_plugins ${CMAKE_BINARY_DIR}/scripts/base/bif/plugins) diff --git a/src/Func.cc b/src/Func.cc index 97d84013e6..f3718fe231 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -553,14 +553,12 @@ void builtin_error(const char* msg, BroObj* arg) #include "input.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" -#include "file_analysis.bif.func_h" #include "bro.bif.func_def" #include "logging.bif.func_def" #include "input.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" -#include "file_analysis.bif.func_def" void init_builtin_funcs() { @@ -575,7 +573,6 @@ void init_builtin_funcs() #include "input.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" -#include "file_analysis.bif.func_init" did_builtin_init = true; } diff --git a/src/NetVar.cc b/src/NetVar.cc index b12c5f1420..388aa46f10 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -250,7 +250,6 @@ OpaqueType* bloomfilter_type; #include "logging.bif.netvar_def" #include "input.bif.netvar_def" #include "reporter.bif.netvar_def" -#include "file_analysis.bif.netvar_def" void init_event_handlers() { @@ -319,7 +318,6 @@ void init_net_var() #include "logging.bif.netvar_init" #include "input.bif.netvar_init" #include "reporter.bif.netvar_init" -#include "file_analysis.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); endpoint = internal_type("endpoint")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index cdf9ad9f2a..7ce33d1a1a 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -261,6 +261,5 @@ extern void init_net_var(); #include "logging.bif.netvar_h" #include "input.bif.netvar_h" #include "reporter.bif.netvar_h" -#include "file_analysis.bif.netvar_h" #endif diff --git a/src/Type.cc b/src/Type.cc index f19de461cd..57d9d0e6e5 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1334,6 +1334,16 @@ EnumType::EnumType(const string& arg_name) counter = 0; } +EnumType::EnumType(EnumType* e) +: BroType(TYPE_ENUM) + { + name = e->name; + counter = e->counter; + + for ( NameMap::iterator it = e->names.begin(); it != e->names.end(); ++it ) + names[copy_string(it->first)] = it->second; + } + EnumType::~EnumType() { for ( NameMap::iterator iter = names.begin(); iter != names.end(); ++iter ) diff --git a/src/Type.h b/src/Type.h index bad51776d9..b10e249745 100644 --- a/src/Type.h +++ b/src/Type.h @@ -523,6 +523,7 @@ protected: class EnumType : public BroType { public: EnumType(const string& arg_name); + EnumType(EnumType* e); ~EnumType(); // The value of this name is next internal counter value, starting @@ -567,6 +568,7 @@ protected: class CommentedEnumType: public EnumType { public: CommentedEnumType(const string& arg_name) : EnumType(arg_name) {} + CommentedEnumType(EnumType* e) : EnumType(e) {} ~CommentedEnumType(); void DescribeReST(ODesc* d) const; diff --git a/src/Var.cc b/src/Var.cc index b7f2c77203..e85b1ba124 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -156,6 +156,12 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init, if ( do_init ) { + if ( c == INIT_NONE && dt == VAR_REDEF && t->IsTable() && + init && init->Tag() == EXPR_ASSIGN ) + // e.g. 'redef foo["x"] = 1' is missing an init class, but the + // intention clearly isn't to overwrite entire existing table val. + c = INIT_EXTRA; + if ( (c == INIT_EXTRA && id->FindAttr(ATTR_ADD_FUNC)) || (c == INIT_REMOVE && id->FindAttr(ATTR_DEL_FUNC)) ) // Just apply the function. diff --git a/src/analyzer/Component.cc b/src/analyzer/Component.cc index 5844da848f..cbb0f40c20 100644 --- a/src/analyzer/Component.cc +++ b/src/analyzer/Component.cc @@ -4,26 +4,12 @@ #include "Manager.h" #include "../Desc.h" +#include "../util.h" using namespace analyzer; Tag::type_t Component::type_counter = 0; -static const char* canonify_name(const char* name) - { - unsigned int len = strlen(name); - char* nname = new char[len + 1]; - - for ( unsigned int i = 0; i < len; i++ ) - { - char c = isalnum(name[i]) ? name[i] : '_'; - nname[i] = toupper(c); - } - - nname[len] = '\0'; - return nname; - } - Component::Component(const char* arg_name, factory_callback arg_factory, Tag::subtype_t arg_subtype, bool arg_enabled, bool arg_partial) : plugin::Component(plugin::component::ANALYZER) { @@ -58,7 +44,7 @@ analyzer::Tag Component::Tag() const return tag; } -void Component::Describe(ODesc* d) +void Component::Describe(ODesc* d) const { plugin::Component::Describe(d); d->Add(name); diff --git a/src/analyzer/Component.h b/src/analyzer/Component.h index b766c2fe82..9e12ed347e 100644 --- a/src/analyzer/Component.h +++ b/src/analyzer/Component.h @@ -23,7 +23,6 @@ class Analyzer; */ class Component : public plugin::Component { public: - typedef bool (*available_callback)(); typedef Analyzer* (*factory_callback)(Connection* conn); /** @@ -73,7 +72,7 @@ public: * from what's passed to the constructor but upper-cased and * canonified to allow being part of a script-level ID. */ - const char* Name() const { return name; } + virtual const char* Name() const { return name; } /** * Returns a canonocalized version of the analyzer's name. The @@ -120,7 +119,7 @@ public: * Generates a human-readable description of the component's main * parameters. This goes into the output of \c "bro -NN". */ - virtual void Describe(ODesc* d); + virtual void Describe(ODesc* d) const; Component& operator=(const Component& other); diff --git a/src/analyzer/Tag.cc b/src/analyzer/Tag.cc index 469b61a6c5..2f04ff17da 100644 --- a/src/analyzer/Tag.cc +++ b/src/analyzer/Tag.cc @@ -24,7 +24,7 @@ Tag::Tag(type_t arg_type, subtype_t arg_subtype) Tag::Tag(EnumVal* arg_val) { - assert(val); + assert(arg_val); val = arg_val; Ref(val); diff --git a/src/analyzer/Tag.h b/src/analyzer/Tag.h index cf33dca41c..edb0ade8a7 100644 --- a/src/analyzer/Tag.h +++ b/src/analyzer/Tag.h @@ -8,6 +8,11 @@ class EnumVal; +namespace file_analysis { +class Manager; +class Component; +} + namespace analyzer { class Manager; @@ -24,7 +29,7 @@ class Component; * subtype form an analyzer "tag". Each unique tag corresponds to a single * "analyzer" from the user's perspective. At the script layer, these tags * are mapped into enums of type \c Analyzer::Tag. Internally, the - * analyzer::Mangager maintains the mapping of tag to analyzer (and it also + * analyzer::Manager maintains the mapping of tag to analyzer (and it also * assigns them their main types), and analyzer::Component creates new * tags. * @@ -121,9 +126,11 @@ public: protected: friend class analyzer::Manager; friend class analyzer::Component; + friend class file_analysis::Manager; + friend class file_analysis::Component; /** - * Constructor. Note + * Constructor. * * @param type The main type. Note that the \a analyzer::Manager * manages the value space internally, so noone else should assign diff --git a/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc b/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc index b32db9a8bd..98adcaa610 100644 --- a/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc +++ b/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc @@ -22,7 +22,7 @@ static RecordType* bittorrent_benc_value; static TableType* bittorrent_benc_dir; BitTorrentTracker_Analyzer::BitTorrentTracker_Analyzer(Connection* c) -: tcp::TCP_ApplicationAnalyzer("BITTORRENT", c) +: tcp::TCP_ApplicationAnalyzer("BITTORRENTTRACKER", c) { if ( ! bt_tracker_headers ) { diff --git a/src/analyzer/protocol/bittorrent/Plugin.cc b/src/analyzer/protocol/bittorrent/Plugin.cc index 2da9972d0d..7fea68bf07 100644 --- a/src/analyzer/protocol/bittorrent/Plugin.cc +++ b/src/analyzer/protocol/bittorrent/Plugin.cc @@ -7,6 +7,6 @@ BRO_PLUGIN_BEGIN(Bro, BitTorrent) BRO_PLUGIN_DESCRIPTION("BitTorrent Analyzer"); BRO_PLUGIN_ANALYZER("BitTorrent", bittorrent::BitTorrent_Analyzer); - BRO_PLUGIN_ANALYZER("BitTorrentTracker", bittorrent::BitTorrent_Analyzer); + BRO_PLUGIN_ANALYZER("BitTorrentTracker", bittorrent::BitTorrentTracker_Analyzer); BRO_PLUGIN_BIF_FILE(events); BRO_PLUGIN_END diff --git a/src/analyzer/protocol/ssl/ssl-protocol.pac b/src/analyzer/protocol/ssl/ssl-protocol.pac index 0019478518..b35d07f18b 100644 --- a/src/analyzer/protocol/ssl/ssl-protocol.pac +++ b/src/analyzer/protocol/ssl/ssl-protocol.pac @@ -693,7 +693,7 @@ refine connection SSL_Conn += { head2 : uint8) : int %{ if ( head0 >= 20 && head0 <= 23 && - head1 == 0x03 && head2 < 0x03 ) + head1 == 0x03 && head2 <= 0x03 ) // This is most probably SSL version 3. return (head1 << 8) | head2; diff --git a/src/const.bif b/src/const.bif index 31e6ccee1a..ea84b3363d 100644 --- a/src/const.bif +++ b/src/const.bif @@ -23,5 +23,3 @@ const Tunnel::delay_gtp_confirmation: bool; const Tunnel::ip_tunnel_timeout: interval; const Threading::heartbeat_interval: interval; - -const FileAnalysis::salt: string; diff --git a/src/event.bif b/src/event.bif index f7fd9b4147..6f363cb961 100644 --- a/src/event.bif +++ b/src/event.bif @@ -920,7 +920,7 @@ event file_over_new_connection%(f: fa_file, c: connection%); ## f: The file. ## ## .. bro:see:: file_new file_over_new_connection file_gap file_state_remove -## default_file_timeout_interval FileAnalysis::postpone_timeout +## default_file_timeout_interval FileAnalysis::set_timeout_interval ## FileAnalysis::set_timeout_interval event file_timeout%(f: fa_file%); @@ -942,19 +942,6 @@ event file_gap%(f: fa_file, offset: count, len: count%); ## .. bro:see:: file_new file_over_new_connection file_timeout file_gap event file_state_remove%(f: fa_file%); -## This event is generated each time file analysis generates a digest of the -## file contents. -## -## f: The file. -## -## kind: The type of digest algorithm. -## -## hash: The result of the hashing. -## -## .. bro:see:: FileAnalysis::add_analyzer FileAnalysis::ANALYZER_MD5 -## FileAnalysis::ANALYZER_SHA1 FileAnalysis::ANALYZER_SHA256 -event file_hash%(f: fa_file, kind: string, hash: string%); - ## Generated when an internal DNS lookup produces the same result as last time. ## Bro keeps an internal DNS cache for host names and IP addresses it has ## already resolved. This event is generated when a subsequent lookup returns diff --git a/src/file_analysis.bif b/src/file_analysis.bif deleted file mode 100644 index cdece0d350..0000000000 --- a/src/file_analysis.bif +++ /dev/null @@ -1,127 +0,0 @@ -##! Internal functions and types used by the logging framework. - -module FileAnalysis; - -%%{ -#include "file_analysis/Manager.h" -%%} - -type AnalyzerArgs: record; - -## An enumeration of various file analysis actions that can be taken. -enum Analyzer %{ - - ## Extract a file to local filesystem - ANALYZER_EXTRACT, - - ## Calculate an MD5 digest of the file's contents. - ANALYZER_MD5, - - ## Calculate an SHA1 digest of the file's contents. - ANALYZER_SHA1, - - ## Calculate an SHA256 digest of the file's contents. - ANALYZER_SHA256, - - ## Deliver the file contents to the script-layer in an event. - ANALYZER_DATA_EVENT, -%} - -## :bro:see:`FileAnalysis::postpone_timeout`. -function FileAnalysis::__postpone_timeout%(file_id: string%): bool - %{ - using file_analysis::FileID; - bool result = file_mgr->PostponeTimeout(FileID(file_id->CheckString())); - return new Val(result, TYPE_BOOL); - %} - -## :bro:see:`FileAnalysis::set_timeout_interval`. -function FileAnalysis::__set_timeout_interval%(file_id: string, t: interval%): bool - %{ - using file_analysis::FileID; - bool result = file_mgr->SetTimeoutInterval(FileID(file_id->CheckString()), - t); - return new Val(result, TYPE_BOOL); - %} - -## :bro:see:`FileAnalysis::add_analyzer`. -function FileAnalysis::__add_analyzer%(file_id: string, args: any%): bool - %{ - using file_analysis::FileID; - using BifType::Record::FileAnalysis::AnalyzerArgs; - RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); - bool result = file_mgr->AddAnalyzer(FileID(file_id->CheckString()), rv); - Unref(rv); - return new Val(result, TYPE_BOOL); - %} - -## :bro:see:`FileAnalysis::remove_analyzer`. -function FileAnalysis::__remove_analyzer%(file_id: string, args: any%): bool - %{ - using file_analysis::FileID; - using BifType::Record::FileAnalysis::AnalyzerArgs; - RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); - bool result = file_mgr->RemoveAnalyzer(FileID(file_id->CheckString()), rv); - Unref(rv); - return new Val(result, TYPE_BOOL); - %} - -## :bro:see:`FileAnalysis::stop`. -function FileAnalysis::__stop%(file_id: string%): bool - %{ - using file_analysis::FileID; - bool result = file_mgr->IgnoreFile(FileID(file_id->CheckString())); - return new Val(result, TYPE_BOOL); - %} - -## :bro:see:`FileAnalysis::data_stream`. -function FileAnalysis::__data_stream%(source: string, data: string%): any - %{ - file_mgr->DataIn(data->Bytes(), data->Len(), source->CheckString()); - return 0; - %} - -## :bro:see:`FileAnalysis::data_chunk`. -function FileAnalysis::__data_chunk%(source: string, data: string, - offset: count%): any - %{ - file_mgr->DataIn(data->Bytes(), data->Len(), offset, source->CheckString()); - return 0; - %} - -## :bro:see:`FileAnalysis::gap`. -function FileAnalysis::__gap%(source: string, offset: count, len: count%): any - %{ - file_mgr->Gap(offset, len, source->CheckString()); - return 0; - %} - -## :bro:see:`FileAnalysis::set_size`. -function FileAnalysis::__set_size%(source: string, size: count%): any - %{ - file_mgr->SetSize(size, source->CheckString()); - return 0; - %} - -## :bro:see:`FileAnalysis::eof`. -function FileAnalysis::__eof%(source: string%): any - %{ - file_mgr->EndOfFile(source->CheckString()); - return 0; - %} - -module GLOBAL; - -## For use within a :bro:see:`get_file_handle` handler to set a unique -## identifier to associate with the current input to the file analysis -## framework. Using an empty string for the handle signifies that the -## input will be ignored/discarded. -## -## handle: A string that uniquely identifies a file. -## -## .. bro:see:: get_file_handle -function set_file_handle%(handle: string%): any - %{ - file_mgr->SetHandle(handle->CheckString()); - return 0; - %} diff --git a/src/file_analysis/Analyzer.h b/src/file_analysis/Analyzer.h index 6ba76317a7..2589dfbe35 100644 --- a/src/file_analysis/Analyzer.h +++ b/src/file_analysis/Analyzer.h @@ -5,10 +5,13 @@ #include "Val.h" #include "NetVar.h" +#include "analyzer/Tag.h" + +#include "file_analysis/file_analysis.bif.h" namespace file_analysis { -typedef BifEnum::FileAnalysis::Analyzer FA_Tag; +typedef int FA_Tag; class File; @@ -17,6 +20,11 @@ class File; */ class Analyzer { public: + + /** + * Destructor. Nothing special about it. Virtual since we definitely expect + * to delete instances of derived classes via pointers to this class. + */ virtual ~Analyzer() { DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %d", tag); @@ -24,7 +32,10 @@ public: } /** - * Subclasses may override this to receive file data non-sequentially. + * Subclasses may override this metod to receive file data non-sequentially. + * @param data points to start of a chunk of file data. + * @param len length in bytes of the chunk of data pointed to by \a data. + * @param offset the byte offset within full file that data chunk starts. * @return true if the analyzer is still in a valid state to continue * receiving data/events or false if it's essentially "done". */ @@ -32,7 +43,9 @@ public: { return true; } /** - * Subclasses may override this to receive file sequentially. + * Subclasses may override this method to receive file sequentially. + * @param data points to start of the next chunk of file data. + * @param len length in bytes of the chunk of data pointed to by \a data. * @return true if the analyzer is still in a valid state to continue * receiving data/events or false if it's essentially "done". */ @@ -40,7 +53,7 @@ public: { return true; } /** - * Subclasses may override this to specifically handle an EOF signal, + * Subclasses may override this method to specifically handle an EOF signal, * which means no more data is going to be incoming and the analyzer * may be deleted/cleaned up soon. * @return true if the analyzer is still in a valid state to continue @@ -50,7 +63,10 @@ public: { return true; } /** - * Subclasses may override this to handle missing data in a file stream. + * Subclasses may override this method to handle missing data in a file. + * @param offset the byte offset within full file at which the missing + * data chunk occurs. + * @param len the number of missing bytes. * @return true if the analyzer is still in a valid state to continue * receiving data/events or false if it's essentially "done". */ @@ -73,17 +89,25 @@ public: File* GetFile() const { return file; } /** + * Retrieves an analyzer tag field from full analyzer argument record. + * @param args an \c AnalyzerArgs (script-layer type) value. * @return the analyzer tag equivalent of the 'tag' field from the - * AnalyzerArgs value \a args. + * \c AnalyzerArgs value \a args. */ static FA_Tag ArgsTag(const RecordVal* args) { using BifType::Record::FileAnalysis::AnalyzerArgs; - return static_cast( - args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum()); + return args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum(); } protected: + + /** + * Constructor. Only derived classes are meant to be instantiated. + * @param arg_args an \c AnalyzerArgs (script-layer type) value specifiying + * tunable options, if any, related to a particular analyzer type. + * @param arg_file the file to which the the analyzer is being attached. + */ Analyzer(RecordVal* arg_args, File* arg_file) : tag(file_analysis::Analyzer::ArgsTag(arg_args)), args(arg_args->Ref()->AsRecordVal()), @@ -91,13 +115,11 @@ protected: {} private: - FA_Tag tag; - RecordVal* args; - File* file; -}; -typedef file_analysis::Analyzer* (*AnalyzerInstantiator)(RecordVal* args, - File* file); + FA_Tag tag; /**< The particular analyzer type of the analyzer instance. */ + RecordVal* args; /**< \c AnalyzerArgs val gives tunable analyzer params. */ + File* file; /**< The file to which the analyzer is attached. */ +}; } // namespace file_analysis diff --git a/src/file_analysis/AnalyzerSet.cc b/src/file_analysis/AnalyzerSet.cc index 83c60d9abe..e350e8b0d8 100644 --- a/src/file_analysis/AnalyzerSet.cc +++ b/src/file_analysis/AnalyzerSet.cc @@ -3,21 +3,10 @@ #include "AnalyzerSet.h" #include "File.h" #include "Analyzer.h" -#include "Extract.h" -#include "DataEvent.h" -#include "Hash.h" +#include "Manager.h" using namespace file_analysis; -// keep in order w/ declared enum values in file_analysis.bif -static AnalyzerInstantiator analyzer_factory[] = { - file_analysis::Extract::Instantiate, - file_analysis::MD5::Instantiate, - file_analysis::SHA1::Instantiate, - file_analysis::SHA256::Instantiate, - file_analysis::DataEvent::Instantiate, -}; - static void analyzer_del_func(void* v) { delete (file_analysis::Analyzer*) v; @@ -154,14 +143,13 @@ HashKey* AnalyzerSet::GetKey(const RecordVal* args) const file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(RecordVal* args) const { - file_analysis::Analyzer* a = - analyzer_factory[file_analysis::Analyzer::ArgsTag(args)](args, file); + FA_Tag tag = file_analysis::Analyzer::ArgsTag(args); + file_analysis::Analyzer* a = file_mgr->InstantiateAnalyzer(tag, args, file); if ( ! a ) { - DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %d failed for file id", - " %s", file_analysis::Analyzer::ArgsTag(args), - file->GetID().c_str()); + reporter->Error("Failed file analyzer %s instantiation for file id %s", + file_mgr->GetAnalyzerName(tag), file->GetID().c_str()); return 0; } diff --git a/src/file_analysis/AnalyzerSet.h b/src/file_analysis/AnalyzerSet.h index e982cc9f8f..6f14149e30 100644 --- a/src/file_analysis/AnalyzerSet.h +++ b/src/file_analysis/AnalyzerSet.h @@ -16,67 +16,144 @@ class File; declare(PDict,Analyzer); /** - * A set of file analysis analyzers indexed by AnalyzerArgs. Allows queueing - * of addition/removals so that those modifications can happen at well-defined - * times (e.g. to make sure a loop iterator isn't invalidated). + * A set of file analysis analyzers indexed by an \c AnalyzerArgs (script-layer + * type) value. Allows queueing of addition/removals so that those + * modifications can happen at well-defined times (e.g. to make sure a loop + * iterator isn't invalidated). */ class AnalyzerSet { public: + + /** + * Constructor. Nothing special. + * @param arg_file the file to which all analyzers in the set are attached. + */ AnalyzerSet(File* arg_file); + /** + * Destructor. Any queued analyzer additions/removals are aborted and + * will not occur. + */ ~AnalyzerSet(); /** + * Attach an analyzer to #file immediately. + * @param args an \c AnalyzerArgs value which specifies an analyzer. * @return true if analyzer was instantiated/attached, else false. */ bool Add(RecordVal* args); /** + * Queue the attachment of an analyzer to #file. + * @param args an \c AnalyzerArgs value which specifies an analyzer. * @return true if analyzer was able to be instantiated, else false. */ bool QueueAdd(RecordVal* args); /** + * Remove an analyzer from #file immediately. + * @param args an \c AnalyzerArgs value which specifies an analyzer. * @return false if analyzer didn't exist and so wasn't removed, else true. */ bool Remove(const RecordVal* args); /** + * Queue the removal of an analyzer from #file. + * @param args an \c AnalyzerArgs value which specifies an analyzer. * @return true if analyzer exists at time of call, else false; */ bool QueueRemove(const RecordVal* args); /** - * Perform all queued modifications to the currently active analyzers. + * Perform all queued modifications to the current analyzer set. */ void DrainModifications(); + /** + * Prepare the analyzer set to be iterated over. + * @see Dictionary#InitForIteration + * @return an iterator that may be used to loop over analyzers in the set. + */ IterCookie* InitForIteration() const { return analyzer_map.InitForIteration(); } + /** + * Get next entry in the analyzer set. + * @see Dictionary#NextEntry + * @param c a set iterator. + * @return the next analyzer in the set or a null pointer if there is no + * more left (in that case the cookie is also deleted). + */ file_analysis::Analyzer* NextEntry(IterCookie* c) { return analyzer_map.NextEntry(c); } protected: + + /** + * Get a hash key which represents an analyzer instance. + * @param args an \c AnalyzerArgs value which specifies an analyzer. + * @return the hash key calculated from \a args + */ HashKey* GetKey(const RecordVal* args) const; + + /** + * Create an instance of a file analyzer. + * @param args an \c AnalyzerArgs value which specifies an analyzer. + * @return a new file analyzer instance. + */ file_analysis::Analyzer* InstantiateAnalyzer(RecordVal* args) const; + + /** + * Insert an analyzer instance in to the set. + * @param a an analyzer instance. + * @param key the hash key which represents the analyzer's \c AnalyzerArgs. + */ void Insert(file_analysis::Analyzer* a, HashKey* key); + + /** + * Remove an analyzer instance from the set. + * @param tag enumarator which specifies type of the analyzer to remove, + * just used for debugging messages. + * @param key the hash key which represents the analyzer's \c AnalyzerArgs. + */ bool Remove(FA_Tag tag, HashKey* key); private: - File* file; + + File* file; /**< File which owns the set */ CompositeHash* analyzer_hash; /**< AnalyzerArgs hashes. */ PDict(file_analysis::Analyzer) analyzer_map; /**< Indexed by AnalyzerArgs. */ + /** + * Abstract base class for analyzer set modifications. + */ class Modification { public: virtual ~Modification() {} + + /** + * Perform the modification on an analyzer set. + * @param set the analyzer set on which the modification will happen. + * @return true if the modification altered \a set. + */ virtual bool Perform(AnalyzerSet* set) = 0; + + /** + * Don't perform the modification on the analyzer set and clean up. + */ virtual void Abort() = 0; }; + /** + * Represents a request to add an analyzer to an analyzer set. + */ class AddMod : public Modification { public: + /** + * Construct request which can add an analyzer to an analyzer set. + * @param arg_a an analyzer instance to add to an analyzer set. + * @param arg_key hash key representing the analyzer's \c AnalyzerArgs. + */ AddMod(file_analysis::Analyzer* arg_a, HashKey* arg_key) : Modification(), a(arg_a), key(arg_key) {} virtual ~AddMod() {} @@ -88,8 +165,16 @@ private: HashKey* key; }; + /** + * Represents a request to remove an analyzer from an analyzer set. + */ class RemoveMod : public Modification { public: + /** + * Construct request which can remove an analyzer from an analyzer set. + * @param arg_a an analyzer instance to add to an analyzer set. + * @param arg_key hash key representing the analyzer's \c AnalyzerArgs. + */ RemoveMod(FA_Tag arg_tag, HashKey* arg_key) : Modification(), tag(arg_tag), key(arg_key) {} virtual ~RemoveMod() {} @@ -102,7 +187,7 @@ private: }; typedef queue ModQueue; - ModQueue mod_queue; + ModQueue mod_queue; /**< A queue of analyzer additions/removals requests. */ }; } // namespace file_analysiss diff --git a/src/file_analysis/CMakeLists.txt b/src/file_analysis/CMakeLists.txt new file mode 100644 index 0000000000..f22c293cc4 --- /dev/null +++ b/src/file_analysis/CMakeLists.txt @@ -0,0 +1,22 @@ +include(BroSubdir) + +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_subdirectory(analyzer) + +set(file_analysis_SRCS + Manager.cc + File.cc + FileTimer.cc + Analyzer.h + AnalyzerSet.cc + Component.cc +) + +bif_target(file_analysis.bif) + +bro_add_subdir_library(file_analysis ${file_analysis_SRCS} ${BIF_OUTPUT_CC}) +add_dependencies(bro_file_analysis generate_outputs) diff --git a/src/file_analysis/Component.cc b/src/file_analysis/Component.cc new file mode 100644 index 0000000000..99531e40f5 --- /dev/null +++ b/src/file_analysis/Component.cc @@ -0,0 +1,69 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Component.h" +#include "Manager.h" + +#include "../Desc.h" +#include "../util.h" + +using namespace file_analysis; + +analyzer::Tag::type_t Component::type_counter = 0; + +Component::Component(const char* arg_name, factory_callback arg_factory, + analyzer::Tag::subtype_t arg_subtype) + : plugin::Component(plugin::component::FILE_ANALYZER) + { + name = copy_string(arg_name); + canon_name = canonify_name(arg_name); + factory = arg_factory; + + tag = analyzer::Tag(++type_counter, arg_subtype); + } + +Component::Component(const Component& other) + : plugin::Component(Type()) + { + name = copy_string(other.name); + canon_name = copy_string(other.canon_name); + factory = other.factory; + tag = other.tag; + } + +Component::~Component() + { + delete [] name; + delete [] canon_name; + } + +analyzer::Tag Component::Tag() const + { + return tag; + } + +void Component::Describe(ODesc* d) const + { + plugin::Component::Describe(d); + d->Add(name); + d->Add(" ("); + + if ( factory ) + { + d->Add("ANALYZER_"); + d->Add(canon_name); + } + + d->Add(")"); + } + +Component& Component::operator=(const Component& other) + { + if ( &other != this ) + { + name = copy_string(other.name); + factory = other.factory; + tag = other.tag; + } + + return *this; + } diff --git a/src/file_analysis/Component.h b/src/file_analysis/Component.h new file mode 100644 index 0000000000..3cdc69efdf --- /dev/null +++ b/src/file_analysis/Component.h @@ -0,0 +1,109 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef FILE_ANALYZER_PLUGIN_COMPONENT_H +#define FILE_ANALYZER_PLUGIN_COMPONENT_H + +#include "analyzer/Tag.h" +#include "plugin/Component.h" + +#include "Val.h" + +#include "../config.h" +#include "../util.h" + +namespace file_analysis { + +class File; +class Analyzer; + +/** + * Component description for plugins providing file analyzers. + * + * A plugin can provide a specific file analyzer by registering this + * analyzer component, describing the analyzer. + */ +class Component : public plugin::Component { +public: + typedef Analyzer* (*factory_callback)(RecordVal* args, File* file); + + /** + * Constructor. + * + * @param name The name of the provided analyzer. This name is used + * across the system to identify the analyzer, e.g., when calling + * file_analysis::Manager::InstantiateAnalyzer with a name. + * + * @param factory A factory function to instantiate instances of the + * analyzer's class, which must be derived directly or indirectly + * from file_analysis::Analyzer. This is typically a static \c + * Instatiate() method inside the class that just allocates and + * returns a new instance. + * + * @param subtype A subtype associated with this component that + * further distinguishes it. The subtype will be integrated into + * the analyzer::Tag that the manager associates with this analyzer, + * and analyzer instances can accordingly access it via analyzer::Tag(). + * If not used, leave at zero. + */ + Component(const char* name, factory_callback factory, + analyzer::Tag::subtype_t subtype = 0); + + /** + * Copy constructor. + */ + Component(const Component& other); + + /** + * Destructor. + */ + ~Component(); + + /** + * Returns the name of the analyzer. This name is unique across all + * analyzers and used to identify it. The returned name is derived + * from what's passed to the constructor but upper-cased and + * canonified to allow being part of a script-level ID. + */ + virtual const char* Name() const { return name; } + + /** + * Returns a canonocalized version of the analyzer's name. The + * returned name is derived from what's passed to the constructor but + * upper-cased and transformed to allow being part of a script-level + * ID. + */ + const char* CanonicalName() const { return canon_name; } + + /** + * Returns the analyzer's factory function. + */ + factory_callback Factory() const { return factory; } + + /** + * Returns the analyzer's tag. Note that this is automatically + * generated for each new Components, and hence unique across all of + * them. + */ + analyzer::Tag Tag() const; + + /** + * Generates a human-readable description of the component's main + * parameters. This goes into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + + Component& operator=(const Component& other); + +private: + const char* name; // The analyzer's name. + const char* canon_name; // The analyzer's canonical name. + factory_callback factory; // The analyzer's factory callback. + analyzer::Tag tag; // The automatically assigned analyzer tag. + + // Global counter used to generate unique tags. + static analyzer::Tag::type_t type_counter; +}; + +} + +#endif diff --git a/src/file_analysis/DataEvent.h b/src/file_analysis/DataEvent.h deleted file mode 100644 index 40a7f5971f..0000000000 --- a/src/file_analysis/DataEvent.h +++ /dev/null @@ -1,36 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef FILE_ANALYSIS_DATAEVENT_H -#define FILE_ANALYSIS_DATAEVENT_H - -#include - -#include "Val.h" -#include "File.h" -#include "Analyzer.h" - -namespace file_analysis { - -/** - * An analyzer to send file data to script-layer events. - */ -class DataEvent : public file_analysis::Analyzer { -public: - virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset); - - virtual bool DeliverStream(const u_char* data, uint64 len); - - static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file); - -protected: - DataEvent(RecordVal* args, File* file, - EventHandlerPtr ce, EventHandlerPtr se); - -private: - EventHandlerPtr chunk_event; - EventHandlerPtr stream_event; -}; - -} // namespace file_analysis - -#endif diff --git a/src/file_analysis/Extract.h b/src/file_analysis/Extract.h deleted file mode 100644 index 1f5ee3a185..0000000000 --- a/src/file_analysis/Extract.h +++ /dev/null @@ -1,35 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef FILE_ANALYSIS_EXTRACT_H -#define FILE_ANALYSIS_EXTRACT_H - -#include - -#include "Val.h" -#include "File.h" -#include "Analyzer.h" - -namespace file_analysis { - -/** - * An analyzer to extract files to disk. - */ -class Extract : public file_analysis::Analyzer { -public: - virtual ~Extract(); - - virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset); - - static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file); - -protected: - Extract(RecordVal* args, File* file, const string& arg_filename); - -private: - string filename; - int fd; -}; - -} // namespace file_analysis - -#endif diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index bc40eb058c..ee590a23a7 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -1,11 +1,9 @@ // See the file "COPYING" in the main distribution directory for copyright. #include -#include #include "File.h" #include "FileTimer.h" -#include "FileID.h" #include "Analyzer.h" #include "Manager.h" #include "Reporter.h" @@ -53,8 +51,6 @@ int File::bof_buffer_size_idx = -1; int File::bof_buffer_idx = -1; int File::mime_type_idx = -1; -string File::salt; - void File::StaticInit() { if ( id_idx != -1 ) @@ -74,42 +70,27 @@ void File::StaticInit() bof_buffer_size_idx = Idx("bof_buffer_size"); bof_buffer_idx = Idx("bof_buffer"); mime_type_idx = Idx("mime_type"); - - salt = BifConst::FileAnalysis::salt->CheckString(); } -File::File(const string& unique, Connection* conn, analyzer::Tag tag, +File::File(const string& file_id, Connection* conn, analyzer::Tag tag, bool is_orig) - : id(""), unique(unique), val(0), postpone_timeout(false), - first_chunk(true), missed_bof(false), need_reassembly(false), done(false), - analyzers(this) + : id(file_id), val(0), postpone_timeout(false), first_chunk(true), + missed_bof(false), need_reassembly(false), done(false), analyzers(this) { StaticInit(); - char tmp[20]; - uint64 hash[2]; - string msg(unique + salt); - MD5(reinterpret_cast(msg.data()), msg.size(), - reinterpret_cast(hash)); - uitoa_n(hash[0], tmp, sizeof(tmp), 62); - - DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s (%s)", tmp, - unique.c_str()); + DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s", file_id.c_str()); val = new RecordVal(fa_file_type); - val->Assign(id_idx, new StringVal(tmp)); - id = FileID(tmp); + val->Assign(id_idx, new StringVal(file_id.c_str())); if ( conn ) { // add source, connection, is_orig fields - val->Assign(source_idx, new StringVal(analyzer_mgr->GetAnalyzerName(tag))); + SetSource(analyzer_mgr->GetAnalyzerName(tag)); val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL)); UpdateConnectionFields(conn); } - else - // use the unique file handle as source - val->Assign(source_idx, new StringVal(unique.c_str())); UpdateLastActivityTime(); } @@ -189,6 +170,18 @@ int File::Idx(const string& field) return rval; } +string File::GetSource() const + { + Val* v = val->Lookup(source_idx); + + return v ? v->AsString()->CheckString() : string(); + } + +void File::SetSource(const string& source) + { + val->Assign(source_idx, new StringVal(source.c_str())); + } + double File::GetTimeoutInterval() const { return LookupFieldDefaultInterval(timeout_interval_idx); @@ -425,7 +418,7 @@ void File::Gap(uint64 offset, uint64 len) bool File::FileEventAvailable(EventHandlerPtr h) { - return h && ! file_mgr->IsIgnored(unique); + return h && ! file_mgr->IsIgnored(id); } void File::FileEvent(EventHandlerPtr h) diff --git a/src/file_analysis/File.h b/src/file_analysis/File.h index 40446934e1..ac54c75bc5 100644 --- a/src/file_analysis/File.h +++ b/src/file_analysis/File.h @@ -9,7 +9,6 @@ #include "Conn.h" #include "Val.h" #include "AnalyzerSet.h" -#include "FileID.h" #include "BroString.h" namespace file_analysis { @@ -19,13 +18,30 @@ namespace file_analysis { */ class File { public: + + /** + * Destructor. Nothing fancy, releases a reference to the wrapped + * \c fa_file value. + */ ~File(); /** - * @return the #val record. + * @return the wrapped \c fa_file record value, #val. */ RecordVal* GetVal() const { return val; } + /** + * @return the value of the "source" field from #val record or an empty + * string if it's not initialized. + */ + string GetSource() const; + + /** + * Set the "source" field from #val record to \a source. + * @param source the new value of the "source" field. + */ + void SetSource(const string& source); + /** * @return value (seconds) of the "timeout_interval" field from #val record. */ @@ -33,18 +49,14 @@ public: /** * Set the "timeout_interval" field from #val record to \a interval seconds. + * @param interval the new value of the "timeout_interval" field. */ void SetTimeoutInterval(double interval); /** * @return value of the "id" field from #val record. */ - FileID GetID() const { return id; } - - /** - * @return the string which uniquely identifies the file. - */ - string GetUnique() const { return unique; } + string GetID() const { return id; } /** * @return value of "last_active" field in #val record; @@ -58,13 +70,15 @@ public: /** * Set "total_bytes" field of #val record to \a size. + * @param size the new value of the "total_bytes" field. */ void SetTotalBytes(uint64 size); /** - * Compares "seen_bytes" field to "total_bytes" field of #val record - * and returns true if the comparison indicates the full file was seen. - * If "total_bytes" hasn't been set yet, it returns false. + * Compares "seen_bytes" field to "total_bytes" field of #val record to + * determine if the full file has been seen. + * @return false if "total_bytes" hasn't been set yet or "seen_bytes" is + * less than it, else true. */ bool IsComplete() const; @@ -78,23 +92,30 @@ public: /** * Queues attaching an analyzer. Only one analyzer per type can be attached * at a time unless the arguments differ. + * @param args an \c AnalyzerArgs value representing a file analyzer. * @return false if analyzer can't be instantiated, else true. */ bool AddAnalyzer(RecordVal* args); /** * Queues removal of an analyzer. + * @param args an \c AnalyzerArgs value representing a file analyzer. * @return true if analyzer was active at time of call, else false. */ bool RemoveAnalyzer(const RecordVal* args); /** * Pass in non-sequential data and deliver to attached analyzers. + * @param data pointer to start of a chunk of file data. + * @param len number of bytes in the data chunk. + * @param offset number of bytes from start of file at which chunk occurs. */ void DataIn(const u_char* data, uint64 len, uint64 offset); /** * Pass in sequential data and deliver to attached analyzers. + * @param data pointer to start of a chunk of file data. + * @param len number of bytes in the data chunk. */ void DataIn(const u_char* data, uint64 len); @@ -105,10 +126,13 @@ public: /** * Inform attached analyzers about a gap in file stream. + * @param offset number of bytes in to file at which missing chunk starts. + * @param len length in bytes of the missing chunk of file data. */ void Gap(uint64 offset, uint64 len); /** + * @param h pointer to an event handler. * @return true if event has a handler and the file isn't ignored. */ bool FileEventAvailable(EventHandlerPtr h); @@ -116,11 +140,14 @@ public: /** * Raises an event related to the file's life-cycle, the only parameter * to that event is the \c fa_file record.. + * @param h pointer to an event handler. */ void FileEvent(EventHandlerPtr h); /** * Raises an event related to the file's life-cycle. + * @param h pointer to an event handler. + * @param vl list of argument values to pass to event call. */ void FileEvent(EventHandlerPtr h, val_list* vl); @@ -129,35 +156,51 @@ protected: /** * Constructor; only file_analysis::Manager should be creating these. + * @param file_id an identifier string for the file in pretty hash form + * (similar to connection uids). + * @param conn a network connection over which the file is transferred. + * @param tag the network protocol over which the file is transferred. + * @param is_orig true if the file is being transferred from the originator + * of the connection to the responder. False indicates the other + * direction. */ - File(const string& unique, Connection* conn = 0, + File(const string& file_id, Connection* conn = 0, analyzer::Tag tag = analyzer::Tag::Error, bool is_orig = false); /** * Updates the "conn_ids" and "conn_uids" fields in #val record with the * \c conn_id and UID taken from \a conn. + * @param conn the connection over which a part of the file has been seen. */ void UpdateConnectionFields(Connection* conn); /** * Increment a byte count field of #val record by \a size. + * @param size number of bytes by which to increment. + * @param field_idx the index of the field in \c fa_file to increment. */ void IncrementByteCount(uint64 size, int field_idx); /** * Wrapper to RecordVal::LookupWithDefault for the field in #val at index * \a idx which automatically unrefs the Val and returns a converted value. + * @param idx the index of a field of type "count" in \c fa_file. + * @return the value of the field, which may be it &default. */ uint64 LookupFieldDefaultCount(int idx) const; /** * Wrapper to RecordVal::LookupWithDefault for the field in #val at index * \a idx which automatically unrefs the Val and returns a converted value. + * @param idx the index of a field of type "interval" in \c fa_file. + * @return the value of the field, which may be it &default. */ double LookupFieldDefaultInterval(int idx) const; /** * Buffers incoming data at the beginning of a file. + * @param data pointer to a data chunk to buffer. + * @param len number of bytes in the data chunk. * @return true if buffering is still required, else false */ bool BufferBOF(const u_char* data, uint64 len); @@ -170,11 +213,15 @@ protected: /** * Does mime type detection and assigns type (if available) to \c mime_type * field in #val. + * @param data pointer to a chunk of file data. + * @param len number of bytes in the data chunk. * @return whether mime type was available. */ bool DetectMIME(const u_char* data, uint64 len); /** + * Lookup a record field index/offset by name. + * @param field_name the name of the \c fa_file record field. * @return the field offset in #val record corresponding to \a field_name. */ static int Idx(const string& field_name); @@ -185,15 +232,14 @@ protected: static void StaticInit(); private: - FileID id; /**< A pretty hash that likely identifies file */ - string unique; /**< A string that uniquely identifies file */ + string id; /**< A pretty hash that likely identifies file */ RecordVal* val; /**< \c fa_file from script layer. */ bool postpone_timeout; /**< Whether postponing timeout is requested. */ bool first_chunk; /**< Track first non-linear chunk. */ bool missed_bof; /**< Flags that we missed start of file. */ bool need_reassembly; /**< Whether file stream reassembly is needed. */ bool done; /**< If this object is about to be deleted. */ - AnalyzerSet analyzers; + AnalyzerSet analyzers; /**< A set of attached file analyzer. */ struct BOF_Buffer { BOF_Buffer() : full(false), replayed(false), size(0) {} @@ -206,8 +252,6 @@ private: BroString::CVec chunks; } bof_buffer; /**< Beginning of file buffer. */ - static string salt; - static int id_idx; static int parent_id_idx; static int source_idx; diff --git a/src/file_analysis/FileID.h b/src/file_analysis/FileID.h deleted file mode 100644 index 9816437214..0000000000 --- a/src/file_analysis/FileID.h +++ /dev/null @@ -1,34 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef FILE_ANALYSIS_FILEID_H -#define FILE_ANALYSIS_FILEID_H - -namespace file_analysis { - -/** - * A simple string wrapper class to help enforce some type safety between - * methods of FileAnalysis::Manager, some of which use a unique string to - * identify files, and others which use a pretty hash (the FileID) to identify - * files. A FileID is primarily used in methods which interface with the - * script-layer, while the unique strings are used for methods which interface - * with protocol analyzers or anything that sends data to the file analysis - * framework. - */ -struct FileID { - string id; - - explicit FileID(const string arg_id) : id(arg_id) {} - FileID(const FileID& other) : id(other.id) {} - - const char* c_str() const { return id.c_str(); } - - bool operator==(const FileID& rhs) const { return id == rhs.id; } - bool operator<(const FileID& rhs) const { return id < rhs.id; } - - FileID& operator=(const FileID& rhs) { id = rhs.id; return *this; } - FileID& operator=(const string& rhs) { id = rhs; return *this; } -}; - -} // namespace file_analysis - -#endif diff --git a/src/file_analysis/FileTimer.cc b/src/file_analysis/FileTimer.cc index 84d4138616..575857fd15 100644 --- a/src/file_analysis/FileTimer.cc +++ b/src/file_analysis/FileTimer.cc @@ -5,7 +5,7 @@ using namespace file_analysis; -FileTimer::FileTimer(double t, const FileID& id, double interval) +FileTimer::FileTimer(double t, const string& id, double interval) : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id) { DBG_LOG(DBG_FILE_ANALYSIS, "New %f second timeout timer for %s", diff --git a/src/file_analysis/FileTimer.h b/src/file_analysis/FileTimer.h index 6ab2638e5f..bdfd1fe165 100644 --- a/src/file_analysis/FileTimer.h +++ b/src/file_analysis/FileTimer.h @@ -5,7 +5,6 @@ #include #include "Timer.h" -#include "FileID.h" namespace file_analysis { @@ -14,16 +13,25 @@ namespace file_analysis { */ class FileTimer : public Timer { public: - FileTimer(double t, const FileID& id, double interval); + + /** + * Constructor, nothing interesting about it. + * @param t unix time at which the timer should start ticking. + * @param id the file identifier which will be checked for inactivity. + * @param interval amount of time after \a t to check for inactivity. + */ + FileTimer(double t, const string& id, double interval); /** * Check inactivity of file_analysis::File corresponding to #file_id, * reschedule if active, else call file_analysis::Manager::Timeout. + * @param t current unix time + * @param is_expire true if all pending timers are being expired. */ void Dispatch(double t, int is_expire); private: - FileID file_id; + string file_id; }; } // namespace file_analysis diff --git a/src/file_analysis/Hash.h b/src/file_analysis/Hash.h deleted file mode 100644 index e4bc8f1747..0000000000 --- a/src/file_analysis/Hash.h +++ /dev/null @@ -1,74 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef FILE_ANALYSIS_HASH_H -#define FILE_ANALYSIS_HASH_H - -#include - -#include "Val.h" -#include "OpaqueVal.h" -#include "File.h" -#include "Analyzer.h" - -namespace file_analysis { - -/** - * An analyzer to produce a hash of file contents. - */ -class Hash : public file_analysis::Analyzer { -public: - virtual ~Hash(); - - virtual bool DeliverStream(const u_char* data, uint64 len); - - virtual bool EndOfFile(); - - virtual bool Undelivered(uint64 offset, uint64 len); - -protected: - Hash(RecordVal* args, File* file, HashVal* hv, const char* kind); - - void Finalize(); - -private: - HashVal* hash; - bool fed; - const char* kind; -}; - -class MD5 : public Hash { -public: - static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file) - { return file_hash ? new MD5(args, file) : 0; } - -protected: - MD5(RecordVal* args, File* file) - : Hash(args, file, new MD5Val(), "md5") - {} -}; - -class SHA1 : public Hash { -public: - static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file) - { return file_hash ? new SHA1(args, file) : 0; } - -protected: - SHA1(RecordVal* args, File* file) - : Hash(args, file, new SHA1Val(), "sha1") - {} -}; - -class SHA256 : public Hash { -public: - static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file) - { return file_hash ? new SHA256(args, file) : 0; } - -protected: - SHA256(RecordVal* args, File* file) - : Hash(args, file, new SHA256Val(), "sha256") - {} -}; - -} // namespace file_analysis - -#endif diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index 5ddfd085b3..ea1ed954ed 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -2,6 +2,7 @@ #include #include +#include #include "Manager.h" #include "File.h" @@ -9,12 +10,18 @@ #include "Var.h" #include "Event.h" +#include "plugin/Manager.h" + using namespace file_analysis; TableVal* Manager::disabled = 0; +string Manager::salt; Manager::Manager() { + tag_enum_type = new EnumType("FileAnalysis::Tag"); + ::ID* id = install_ID("Tag", "FileAnalysis", true, true); + add_type(id, tag_enum_type, 0, 0); } Manager::~Manager() @@ -22,9 +29,44 @@ Manager::~Manager() Terminate(); } +void Manager::InitPreScript() + { + std::list analyzers = plugin_mgr->Components(); + + for ( std::list::const_iterator i = analyzers.begin(); + i != analyzers.end(); ++i ) + RegisterAnalyzerComponent(*i); + } + +void Manager::RegisterAnalyzerComponent(Component* component) + { + const char* cname = component->CanonicalName(); + + if ( tag_enum_type->Lookup("FileAnalysis", cname) != -1 ) + reporter->FatalError("File Analyzer %s defined more than once", cname); + + DBG_LOG(DBG_FILE_ANALYSIS, "Registering analyzer %s (tag %s)", + component->Name(), component->Tag().AsString().c_str()); + + analyzers_by_name.insert(std::make_pair(cname, component)); + analyzers_by_tag.insert(std::make_pair(component->Tag(), component)); + analyzers_by_val.insert(std::make_pair( + component->Tag().AsEnumVal()->InternalInt(), component)); + + string id = fmt("ANALYZER_%s", cname); + tag_enum_type->AddName("FileAnalysis", id.c_str(), + component->Tag().AsEnumVal()->InternalInt(), true); + } + +void Manager::InitPostScript() + { + #include "file_analysis.bif.init.cc" + } + void Manager::Terminate() { - vector keys; + vector keys; + for ( IDMap::iterator it = id_map.begin(); it != id_map.end(); ++it ) keys.push_back(it->first); @@ -32,66 +74,77 @@ void Manager::Terminate() Timeout(keys[i], true); } +string Manager::HashHandle(const string& handle) const + { + if ( salt.empty() ) + salt = BifConst::FileAnalysis::salt->CheckString(); + + char tmp[20]; + uint64 hash[2]; + string msg(handle + salt); + + MD5(reinterpret_cast(msg.data()), msg.size(), + reinterpret_cast(hash)); + uitoa_n(hash[0], tmp, sizeof(tmp), 62); + + return tmp; + } + void Manager::SetHandle(const string& handle) { - current_handle = handle; + if ( handle.empty() ) + return; + + current_file_id = HashHandle(handle); } void Manager::DataIn(const u_char* data, uint64 len, uint64 offset, analyzer::Tag tag, Connection* conn, bool is_orig) { - if ( IsDisabled(tag) ) - return; - GetFileHandle(tag, conn, is_orig); - DataIn(data, len, offset, GetFile(current_handle, conn, tag, is_orig)); - } + File* file = GetFile(current_file_id, conn, tag, is_orig); -void Manager::DataIn(const u_char* data, uint64 len, uint64 offset, - const string& unique) - { - DataIn(data, len, offset, GetFile(unique)); - } - -void Manager::DataIn(const u_char* data, uint64 len, uint64 offset, - File* file) - { if ( ! file ) return; file->DataIn(data, len, offset); if ( file->IsComplete() ) - RemoveFile(file->GetUnique()); + RemoveFile(file->GetID()); } void Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag, Connection* conn, bool is_orig) { - if ( IsDisabled(tag) ) - return; - GetFileHandle(tag, conn, is_orig); - // Sequential data input shouldn't be going over multiple conns, so don't // do the check to update connection set. - DataIn(data, len, GetFile(current_handle, conn, tag, is_orig, false)); - } + File* file = GetFile(current_file_id, conn, tag, is_orig, false); -void Manager::DataIn(const u_char* data, uint64 len, const string& unique) - { - DataIn(data, len, GetFile(unique)); - } - -void Manager::DataIn(const u_char* data, uint64 len, File* file) - { if ( ! file ) return; file->DataIn(data, len); if ( file->IsComplete() ) - RemoveFile(file->GetUnique()); + RemoveFile(file->GetID()); + } + +void Manager::DataIn(const u_char* data, uint64 len, const string& file_id, + const string& source) + { + File* file = GetFile(file_id); + + if ( ! file ) + return; + + if ( file->GetSource().empty() ) + file->SetSource(source); + + file->DataIn(data, len); + + if ( file->IsComplete() ) + RemoveFile(file->GetID()); } void Manager::EndOfFile(analyzer::Tag tag, Connection* conn) @@ -102,35 +155,22 @@ void Manager::EndOfFile(analyzer::Tag tag, Connection* conn) void Manager::EndOfFile(analyzer::Tag tag, Connection* conn, bool is_orig) { - if ( IsDisabled(tag) ) - return; - + // Don't need to create a file if we're just going to remove it right away. GetFileHandle(tag, conn, is_orig); - EndOfFile(current_handle); + RemoveFile(current_file_id); } -void Manager::EndOfFile(const string& unique) +void Manager::EndOfFile(const string& file_id) { - RemoveFile(unique); + RemoveFile(file_id); } void Manager::Gap(uint64 offset, uint64 len, analyzer::Tag tag, Connection* conn, bool is_orig) { - if ( IsDisabled(tag) ) - return; - GetFileHandle(tag, conn, is_orig); - Gap(offset, len, GetFile(current_handle, conn, tag, is_orig)); - } + File* file = GetFile(current_file_id, conn, tag, is_orig); -void Manager::Gap(uint64 offset, uint64 len, const string& unique) - { - Gap(offset, len, GetFile(unique)); - } - -void Manager::Gap(uint64 offset, uint64 len, File* file) - { if ( ! file ) return; @@ -140,52 +180,33 @@ void Manager::Gap(uint64 offset, uint64 len, File* file) void Manager::SetSize(uint64 size, analyzer::Tag tag, Connection* conn, bool is_orig) { - if ( IsDisabled(tag) ) - return; - GetFileHandle(tag, conn, is_orig); - SetSize(size, GetFile(current_handle, conn, tag, is_orig)); - } + File* file = GetFile(current_file_id, conn, tag, is_orig); -void Manager::SetSize(uint64 size, const string& unique) - { - SetSize(size, GetFile(unique)); - } - -void Manager::SetSize(uint64 size, File* file) - { if ( ! file ) return; file->SetTotalBytes(size); if ( file->IsComplete() ) - RemoveFile(file->GetUnique()); + RemoveFile(file->GetID()); } -bool Manager::PostponeTimeout(const FileID& file_id) const +bool Manager::SetTimeoutInterval(const string& file_id, double interval) const { File* file = Lookup(file_id); if ( ! file ) return false; - file->postpone_timeout = true; - return true; - } - -bool Manager::SetTimeoutInterval(const FileID& file_id, double interval) const - { - File* file = Lookup(file_id); - - if ( ! file ) - return false; + if ( interval > 0 ) + file->postpone_timeout = true; file->SetTimeoutInterval(interval); return true; } -bool Manager::AddAnalyzer(const FileID& file_id, RecordVal* args) const +bool Manager::AddAnalyzer(const string& file_id, RecordVal* args) const { File* file = Lookup(file_id); @@ -195,7 +216,7 @@ bool Manager::AddAnalyzer(const FileID& file_id, RecordVal* args) const return file->AddAnalyzer(args); } -bool Manager::RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const +bool Manager::RemoveAnalyzer(const string& file_id, const RecordVal* args) const { File* file = Lookup(file_id); @@ -205,32 +226,23 @@ bool Manager::RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const return file->RemoveAnalyzer(args); } -File* Manager::GetFile(const string& unique, Connection* conn, +File* Manager::GetFile(const string& file_id, Connection* conn, analyzer::Tag tag, bool is_orig, bool update_conn) { - if ( unique.empty() ) + if ( file_id.empty() ) return 0; - if ( IsIgnored(unique) ) + if ( IsIgnored(file_id) ) return 0; - File* rval = str_map[unique]; + File* rval = id_map[file_id]; if ( ! rval ) { - rval = str_map[unique] = new File(unique, conn, tag, is_orig); - FileID id = rval->GetID(); - - if ( id_map[id] ) - { - reporter->Error("Evicted duplicate file ID: %s", id.c_str()); - RemoveFile(unique); - } - - id_map[id] = rval; + rval = id_map[file_id] = new File(file_id, conn, tag, is_orig); rval->ScheduleInactivityTimer(); - if ( IsIgnored(unique) ) + if ( IsIgnored(file_id) ) return 0; } else @@ -244,7 +256,7 @@ File* Manager::GetFile(const string& unique, Connection* conn, return rval; } -File* Manager::Lookup(const FileID& file_id) const +File* Manager::Lookup(const string& file_id) const { IDMap::const_iterator it = id_map.find(file_id); @@ -254,7 +266,7 @@ File* Manager::Lookup(const FileID& file_id) const return it->second; } -void Manager::Timeout(const FileID& file_id, bool is_terminating) +void Manager::Timeout(const string& file_id, bool is_terminating) { File* file = Lookup(file_id); @@ -277,53 +289,50 @@ void Manager::Timeout(const FileID& file_id, bool is_terminating) DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s", file->GetID().c_str()); - RemoveFile(file->GetUnique()); + RemoveFile(file->GetID()); } -bool Manager::IgnoreFile(const FileID& file_id) +bool Manager::IgnoreFile(const string& file_id) + { + if ( id_map.find(file_id) == id_map.end() ) + return false; + + DBG_LOG(DBG_FILE_ANALYSIS, "Ignore FileID %s", file_id.c_str()); + + ignored.insert(file_id); + + return true; + } + +bool Manager::RemoveFile(const string& file_id) { IDMap::iterator it = id_map.find(file_id); if ( it == id_map.end() ) return false; - DBG_LOG(DBG_FILE_ANALYSIS, "Ignore FileID %s", file_id.c_str()); - - ignored.insert(it->second->GetUnique()); - - return true; - } - -bool Manager::RemoveFile(const string& unique) - { - StrMap::iterator it = str_map.find(unique); - - if ( it == str_map.end() ) - return false; + DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", file_id.c_str()); it->second->EndOfFile(); - FileID id = it->second->GetID(); - - DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", id.c_str()); - - if ( ! id_map.erase(id) ) - reporter->Error("No mapping for fileID %s", id.c_str()); - - ignored.erase(unique); delete it->second; - str_map.erase(unique); + id_map.erase(file_id); + ignored.erase(file_id); + return true; } -bool Manager::IsIgnored(const string& unique) +bool Manager::IsIgnored(const string& file_id) { - return ignored.find(unique) != ignored.end(); + return ignored.find(file_id) != ignored.end(); } void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig) { - current_handle.clear(); + current_file_id.clear(); + + if ( IsDisabled(tag) ) + return; if ( ! get_file_handle ) return; @@ -357,3 +366,31 @@ bool Manager::IsDisabled(analyzer::Tag tag) return rval; } + +Analyzer* Manager::InstantiateAnalyzer(int tag, RecordVal* args, File* f) const + { + analyzer_map_by_val::const_iterator it = analyzers_by_val.find(tag); + + if ( it == analyzers_by_val.end() ) + reporter->InternalError("cannot instantiate unknown file analyzer: %d", + tag); + + Component* c = it->second; + + if ( ! c->Factory() ) + reporter->InternalError("file analyzer %s cannot be instantiated " + "dynamically", c->CanonicalName()); + + return c->Factory()(args, f); + } + +const char* Manager::GetAnalyzerName(int tag) const + { + analyzer_map_by_val::const_iterator it = analyzers_by_val.find(tag); + + if ( it == analyzers_by_val.end() ) + reporter->InternalError("cannot get name of unknown file analyzer: %d", + tag); + + return it->second->CanonicalName(); + } diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 99121b8575..84b606173d 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -17,10 +17,12 @@ #include "File.h" #include "FileTimer.h" -#include "FileID.h" +#include "Component.h" #include "analyzer/Tag.h" +#include "file_analysis/file_analysis.bif.h" + namespace file_analysis { /** @@ -28,152 +30,280 @@ namespace file_analysis { */ class Manager { public: + + /** + * Constructor. + */ Manager(); + + /** + * Destructor. Times out any currently active file analyses. + */ ~Manager(); + /** + * First-stage initializion of the manager. This is called early on + * during Bro's initialization, before any scripts are processed. + */ + void InitPreScript(); + + /** + * Second-stage initialization of the manager. This is called late + * during Bro's initialization after any scripts are processed. + */ + void InitPostScript(); + /** * Times out any active file analysis to prepare for shutdown. */ void Terminate(); /** - * Take in a unique file handle string to identifiy incoming file data. + * Creates a file identifier from a unique file handle string. + * @param handle a unique string which identifies a single file. + * @return a prettified MD5 hash of \a handle, truncated to 64-bits. + */ + string HashHandle(const string& handle) const; + + /** + * Take in a unique file handle string to identify next piece of + * incoming file data/information. + * @param handle a unique string which identifies a single file. */ void SetHandle(const string& handle); /** * Pass in non-sequential file data. + * @param data pointer to start of a chunk of file data. + * @param len number of bytes in the data chunk. + * @param offset number of bytes from start of file that data chunk occurs. + * @param tag network protocol over which the file data is transferred. + * @param conn network connection over which the file data is transferred. + * @param is_orig true if the file is being sent from connection originator + * or false if is being sent in the opposite direction. */ void DataIn(const u_char* data, uint64 len, uint64 offset, analyzer::Tag tag, Connection* conn, bool is_orig); - void DataIn(const u_char* data, uint64 len, uint64 offset, - const string& unique); - void DataIn(const u_char* data, uint64 len, uint64 offset, - File* file); /** * Pass in sequential file data. + * @param data pointer to start of a chunk of file data. + * @param len number of bytes in the data chunk. + * @param tag network protocol over which the file data is transferred. + * @param conn network connection over which the file data is transferred. + * @param is_orig true if the file is being sent from connection originator + * or false if is being sent in the opposite direction. */ void DataIn(const u_char* data, uint64 len, analyzer::Tag tag, Connection* conn, bool is_orig); - void DataIn(const u_char* data, uint64 len, const string& unique); - void DataIn(const u_char* data, uint64 len, File* file); /** - * Signal the end of file data. + * Pass in sequential file data from external source (e.g. input framework). + * @param data pointer to start of a chunk of file data. + * @param len number of bytes in the data chunk. + * @param file_id an identifier for the file (usually a hash of \a source). + * @param source uniquely identifies the file and should also describe + * in human-readable form where the file input is coming from (e.g. + * a local file path). + */ + void DataIn(const u_char* data, uint64 len, const string& file_id, + const string& source); + + /** + * Signal the end of file data regardless of which direction it is being + * sent over the connection. + * @param tag network protocol over which the file data is transferred. + * @param conn network connection over which the file data is transferred. */ void EndOfFile(analyzer::Tag tag, Connection* conn); + + /** + * Signal the end of file data being transferred over a connection in + * a particular direction. + * @param tag network protocol over which the file data is transferred. + * @param conn network connection over which the file data is transferred. + */ void EndOfFile(analyzer::Tag tag, Connection* conn, bool is_orig); - void EndOfFile(const string& unique); + + /** + * Signal the end of file data being transferred using the file identifier. + * @param file_id the file identifier/hash. + */ + void EndOfFile(const string& file_id); /** * Signal a gap in the file data stream. + * @param offset number of bytes in to file at which missing chunk starts. + * @param len length in bytes of the missing chunk of file data. + * @param tag network protocol over which the file data is transferred. + * @param conn network connection over which the file data is transferred. + * @param is_orig true if the file is being sent from connection originator + * or false if is being sent in the opposite direction. */ void Gap(uint64 offset, uint64 len, analyzer::Tag tag, Connection* conn, bool is_orig); - void Gap(uint64 offset, uint64 len, const string& unique); - void Gap(uint64 offset, uint64 len, File* file); /** * Provide the expected number of bytes that comprise a file. + * @param size the number of bytes in the full file. + * @param tag network protocol over which the file data is transferred. + * @param conn network connection over which the file data is transferred. + * @param is_orig true if the file is being sent from connection originator + * or false if is being sent in the opposite direction. */ void SetSize(uint64 size, analyzer::Tag tag, Connection* conn, bool is_orig); - void SetSize(uint64 size, const string& unique); - void SetSize(uint64 size, File* file); /** * Starts ignoring a file, which will finally be removed from internal * mappings on EOF or TIMEOUT. + * @param file_id the file identifier/hash. * @return false if file identifier did not map to anything, else true. */ - bool IgnoreFile(const FileID& file_id); - - /** - * If called during a \c file_timeout event handler, requests deferral of - * analysis timeout. - */ - bool PostponeTimeout(const FileID& file_id) const; + bool IgnoreFile(const string& file_id); /** * Set's an inactivity threshold for the file. + * @param file_id the file identifier/hash. + * @param interval the amount of time in which no activity is seen for + * the file identified by \a file_id that will cause the file + * to be considered stale, timed out, and then resource reclaimed. + * @return false if file identifier did not map to anything, else true. */ - bool SetTimeoutInterval(const FileID& file_id, double interval) const; + bool SetTimeoutInterval(const string& file_id, double interval) const; /** * Queue attachment of an analzer to the file identifier. Multiple * analyzers of a given type can be attached per file identifier at a time * as long as the arguments differ. + * @param file_id the file identifier/hash. + * @param args a \c AnalyzerArgs value which describes a file analyzer. * @return false if the analyzer failed to be instantiated, else true. */ - bool AddAnalyzer(const FileID& file_id, RecordVal* args) const; + bool AddAnalyzer(const string& file_id, RecordVal* args) const; /** * Queue removal of an analyzer for a given file identifier. + * @param file_id the file identifier/hash. + * @param args a \c AnalyzerArgs value which describes a file analyzer. * @return true if the analyzer is active at the time of call, else false. */ - bool RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const; + bool RemoveAnalyzer(const string& file_id, const RecordVal* args) const; /** - * @return whether the file mapped to \a unique is being ignored. + * Tells whether analysis for a file is active or ignored. + * @param file_id the file identifier/hash. + * @return whether the file mapped to \a file_id is being ignored. */ - bool IsIgnored(const string& unique); + bool IsIgnored(const string& file_id); + + /** + * Instantiates a new file analyzer instance for the file. + * @param tag The file analyzer's tag. + * @param args The file analzer argument/option values. + * @param f The file analzer is to be associated with. + * @return The new analyzer instance or null if tag is invalid. + */ + Analyzer* InstantiateAnalyzer(int tag, RecordVal* args, File* f) const; + + /** + * Translates a script-level file analyzer tag in to corresponding file + * analyzer name. + * @param tag The enum val of a file analyzer. + * @return The human-readable name of the file analyzer. + */ + const char* GetAnalyzerName(int tag) const; protected: friend class FileTimer; - typedef map StrMap; - typedef set StrSet; - typedef map IDMap; + typedef set IDSet; + typedef map IDMap; /** - * @return the File object mapped to \a unique or a null pointer if analysis - * is being ignored for the associated file. An File object may be - * created if a mapping doesn't exist, and if it did exist, the - * activity time is refreshed along with any connection-related - * fields. + * Create a new file to be analyzed or retrieve an existing one. + * @param file_id the file identifier/hash. + * @param conn network connection, if any, over which the file is + * transferred. + * @param tag network protocol, if any, over which the file is transferred. + * @param is_orig true if the file is being sent from connection originator + * or false if is being sent in the opposite direction (or if it + * this file isn't related to a connection). + * @param update_conn whether we need to update connection-related field + * in the \c fa_file record value associated with the file. + * @return the File object mapped to \a file_id or a null pointer if + * analysis is being ignored for the associated file. An File + * object may be created if a mapping doesn't exist, and if it did + * exist, the activity time is refreshed along with any + * connection-related fields. */ - File* GetFile(const string& unique, Connection* conn = 0, + File* GetFile(const string& file_id, Connection* conn = 0, analyzer::Tag tag = analyzer::Tag::Error, bool is_orig = false, bool update_conn = true); /** + * Try to retrieve a file that's being analyzed, using its identifier/hash. + * @param file_id the file identifier/hash. * @return the File object mapped to \a file_id, or a null pointer if no * mapping exists. */ - File* Lookup(const FileID& file_id) const; + File* Lookup(const string& file_id) const; /** * Evaluate timeout policy for a file and remove the File object mapped to * \a file_id if needed. + * @param file_id the file identifier/hash. + * @param is_termination whether the Manager (and probably Bro) is in a + * terminating state. If true, then the timeout cannot be postponed. */ - void Timeout(const FileID& file_id, bool is_terminating = ::terminating); + void Timeout(const string& file_id, bool is_terminating = ::terminating); /** - * Immediately remove file_analysis::File object associated with \a unique. - * @return false if file string did not map to anything, else true. + * Immediately remove file_analysis::File object associated with \a file_id. + * @param file_id the file identifier/hash. + * @return false if file id string did not map to anything, else true. */ - bool RemoveFile(const string& unique); + bool RemoveFile(const string& file_id); /** - * Sets #current_handle to a unique file handle string based on what the - * \c get_file_handle event derives from the connection params. The - * event queue is flushed so that we can get the handle value immediately. + * Sets #current_file_id to a hash of a unique file handle string based on + * what the \c get_file_handle event derives from the connection params. + * Event queue is flushed so that we can get the handle value immediately. + * @param tag network protocol over which the file is transferred. + * @param conn network connection over which the file is transferred. + * @param is_orig true if the file is being sent from connection originator + * or false if is being sent in the opposite direction. */ void GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig); /** - * @return whether file analysis is disabled for the given analyzer. + * Check if analysis is available for files transferred over a given + * network protocol. + * @param tag the network protocol over which files can be transferred and + * analyzed by the file analysis framework. + * @return whether file analysis is disabled for the analyzer given by + * \a tag. */ static bool IsDisabled(analyzer::Tag tag); private: - StrMap str_map; /**< Map unique string to file_analysis::File. */ + typedef map analyzer_map_by_name; + typedef map analyzer_map_by_tag; + typedef map analyzer_map_by_val; + + void RegisterAnalyzerComponent(Component* component); + IDMap id_map; /**< Map file ID to file_analysis::File records. */ - StrSet ignored; /**< Ignored files. Will be finally removed on EOF. */ - string current_handle; /**< Last file handle set by get_file_handle event.*/ + IDSet ignored; /**< Ignored files. Will be finally removed on EOF. */ + string current_file_id; /**< Hash of what get_file_handle event sets. */ + EnumType* tag_enum_type; /**< File analyzer tag type. */ + + analyzer_map_by_name analyzers_by_name; + analyzer_map_by_tag analyzers_by_tag; + analyzer_map_by_val analyzers_by_val; static TableVal* disabled; /**< Table of disabled analyzers. */ + static string salt; /**< A salt added to file handles before hashing. */ }; } // namespace file_analysis diff --git a/src/file_analysis/analyzer/CMakeLists.txt b/src/file_analysis/analyzer/CMakeLists.txt new file mode 100644 index 0000000000..bfafcd2894 --- /dev/null +++ b/src/file_analysis/analyzer/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(data_event) +add_subdirectory(extract) +add_subdirectory(hash) diff --git a/src/file_analysis/analyzer/data_event/CMakeLists.txt b/src/file_analysis/analyzer/data_event/CMakeLists.txt new file mode 100644 index 0000000000..81551feda2 --- /dev/null +++ b/src/file_analysis/analyzer/data_event/CMakeLists.txt @@ -0,0 +1,8 @@ +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro FileDataEvent) +bro_plugin_cc(DataEvent.cc Plugin.cc) +bro_plugin_end() diff --git a/src/file_analysis/DataEvent.cc b/src/file_analysis/analyzer/data_event/DataEvent.cc similarity index 100% rename from src/file_analysis/DataEvent.cc rename to src/file_analysis/analyzer/data_event/DataEvent.cc diff --git a/src/file_analysis/analyzer/data_event/DataEvent.h b/src/file_analysis/analyzer/data_event/DataEvent.h new file mode 100644 index 0000000000..60b0487a6f --- /dev/null +++ b/src/file_analysis/analyzer/data_event/DataEvent.h @@ -0,0 +1,69 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef FILE_ANALYSIS_DATAEVENT_H +#define FILE_ANALYSIS_DATAEVENT_H + +#include + +#include "Val.h" +#include "File.h" +#include "Analyzer.h" + +namespace file_analysis { + +/** + * An analyzer to send file data to script-layer via events. + */ +class DataEvent : public file_analysis::Analyzer { +public: + + /** + * Generates the event, if any, specified by the "chunk_event" field of this + * analyzer's \c AnalyzerArgs. This is for non-sequential file data input. + * @param data pointer to start of file data chunk. + * @param len number of bytes in the data chunk. + * @param offset number of bytes from start of file at which chunk occurs. + * @return always true + */ + virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset); + + /** + * Generates the event, if any, specified by the "stream_event" field of + * this analyzer's \c AnalyzerArgs. This is for sequential file data input. + * @param data pointer to start of file data chunk. + * @param len number of bytes in the data chunk. + * @return always true + */ + virtual bool DeliverStream(const u_char* data, uint64 len); + + /** + * Create a new instance of a DataEvent analyzer. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @return the new DataEvent analyzer instance or a null pointer if + * no "chunk_event" or "stream_event" field was specfied in \a args. + */ + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file); + +protected: + + /** + * Constructor. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @param ce pointer to event handler which will be called to receive + * non-sequential file data. + * @param se pointer to event handler which will be called to receive + * sequential file data. + */ + DataEvent(RecordVal* args, File* file, + EventHandlerPtr ce, EventHandlerPtr se); + +private: + EventHandlerPtr chunk_event; + EventHandlerPtr stream_event; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/analyzer/data_event/Plugin.cc b/src/file_analysis/analyzer/data_event/Plugin.cc new file mode 100644 index 0000000000..7eb637f3a5 --- /dev/null +++ b/src/file_analysis/analyzer/data_event/Plugin.cc @@ -0,0 +1,26 @@ +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" + +#include "DataEvent.h" + +namespace plugin { namespace Bro_FileDataEvent { + +class Plugin : public plugin::Plugin { +protected: + void InitPreScript() + { + SetName("Bro::FileDataEvent"); + SetVersion(-1); + SetAPIVersion(BRO_PLUGIN_API_VERSION); + SetDynamicPlugin(false); + + SetDescription("Delivers file content via events"); + + AddComponent(new ::file_analysis::Component("DATA_EVENT", + ::file_analysis::DataEvent::Instantiate)); + } +}; + +Plugin __plugin; + +} } diff --git a/src/file_analysis/analyzer/extract/CMakeLists.txt b/src/file_analysis/analyzer/extract/CMakeLists.txt new file mode 100644 index 0000000000..df3fa2646d --- /dev/null +++ b/src/file_analysis/analyzer/extract/CMakeLists.txt @@ -0,0 +1,8 @@ +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro FileExtract) +bro_plugin_cc(Extract.cc Plugin.cc) +bro_plugin_end() diff --git a/src/file_analysis/Extract.cc b/src/file_analysis/analyzer/extract/Extract.cc similarity index 100% rename from src/file_analysis/Extract.cc rename to src/file_analysis/analyzer/extract/Extract.cc diff --git a/src/file_analysis/analyzer/extract/Extract.h b/src/file_analysis/analyzer/extract/Extract.h new file mode 100644 index 0000000000..85d2a9e7a8 --- /dev/null +++ b/src/file_analysis/analyzer/extract/Extract.h @@ -0,0 +1,62 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef FILE_ANALYSIS_EXTRACT_H +#define FILE_ANALYSIS_EXTRACT_H + +#include + +#include "Val.h" +#include "File.h" +#include "Analyzer.h" + +namespace file_analysis { + +/** + * An analyzer to extract content of files to local disk. + */ +class Extract : public file_analysis::Analyzer { +public: + + /** + * Destructor. Will close the file that was used for data extraction. + */ + virtual ~Extract(); + + /** + * Write a chunk of file data to the local extraction file. + * @param data pointer to a chunk of file data. + * @param len number of bytes in the data chunk. + * @param offset number of bytes from start of file at which chunk starts. + * @return false if there was no extraction file open and the data couldn't + * be written, else true. + */ + virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset); + + /** + * Create a new instance of an Extract analyzer. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @return the new Extract analyzer instance or a null pointer if the + * the "extraction_file" field of \a args wasn't set. + */ + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file); + +protected: + + /** + * Constructor. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @param arg_filename a file system path which specifies the local file + * to which the contents of the file will be extracted/written. + */ + Extract(RecordVal* args, File* file, const string& arg_filename); + +private: + string filename; + int fd; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/analyzer/extract/Plugin.cc b/src/file_analysis/analyzer/extract/Plugin.cc new file mode 100644 index 0000000000..f6cde57f03 --- /dev/null +++ b/src/file_analysis/analyzer/extract/Plugin.cc @@ -0,0 +1,26 @@ +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" + +#include "Extract.h" + +namespace plugin { namespace Bro_FileExtract { + +class Plugin : public plugin::Plugin { +protected: + void InitPreScript() + { + SetName("Bro::FileExtract"); + SetVersion(-1); + SetAPIVersion(BRO_PLUGIN_API_VERSION); + SetDynamicPlugin(false); + + SetDescription("Extract file content to local file system"); + + AddComponent(new ::file_analysis::Component("EXTRACT", + ::file_analysis::Extract::Instantiate)); + } +}; + +Plugin __plugin; + +} } diff --git a/src/file_analysis/analyzer/hash/CMakeLists.txt b/src/file_analysis/analyzer/hash/CMakeLists.txt new file mode 100644 index 0000000000..5734740198 --- /dev/null +++ b/src/file_analysis/analyzer/hash/CMakeLists.txt @@ -0,0 +1,9 @@ +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro FileHash) +bro_plugin_cc(Hash.cc Plugin.cc) +bro_plugin_bif(events.bif) +bro_plugin_end() diff --git a/src/file_analysis/Hash.cc b/src/file_analysis/analyzer/hash/Hash.cc similarity index 100% rename from src/file_analysis/Hash.cc rename to src/file_analysis/analyzer/hash/Hash.cc diff --git a/src/file_analysis/analyzer/hash/Hash.h b/src/file_analysis/analyzer/hash/Hash.h new file mode 100644 index 0000000000..13303e21fc --- /dev/null +++ b/src/file_analysis/analyzer/hash/Hash.h @@ -0,0 +1,160 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef FILE_ANALYSIS_HASH_H +#define FILE_ANALYSIS_HASH_H + +#include + +#include "Val.h" +#include "OpaqueVal.h" +#include "File.h" +#include "Analyzer.h" + +#include "events.bif.h" + +namespace file_analysis { + +/** + * An analyzer to produce a hash of file contents. + */ +class Hash : public file_analysis::Analyzer { +public: + + /** + * Destructor. + */ + virtual ~Hash(); + + /** + * Incrementally hash next chunk of file contents. + * @param data pointer to start of a chunk of a file data. + * @param len number of bytes in the data chunk. + * @return false if the digest is in an invalid state, else true. + */ + virtual bool DeliverStream(const u_char* data, uint64 len); + + /** + * Finalizes the hash and raises a "file_hash" event. + * @return always false so analyze will be deteched from file. + */ + virtual bool EndOfFile(); + + /** + * Missing data can't be handled, so just indicate the this analyzer should + * be removed from receiving further data. The hash will not be finalized. + * @param offset byte offset in file at which missing chunk starts. + * @param len number of missing bytes. + * @return always false so analyzer will detach from file. + */ + virtual bool Undelivered(uint64 offset, uint64 len); + +protected: + + /** + * Constructor. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @param hv specific hash calculator object. + * @param kind human readable name of the hash algorithm to use. + */ + Hash(RecordVal* args, File* file, HashVal* hv, const char* kind); + + /** + * If some file contents have been seen, finalizes the hash of them and + * raises the "file_hash" event with the results. + */ + void Finalize(); + +private: + HashVal* hash; + bool fed; + const char* kind; +}; + +/** + * An analyzer to produce an MD5 hash of file contents. + */ +class MD5 : public Hash { +public: + + /** + * Create a new instance of the MD5 hashing file analyzer. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @return the new MD5 analyzer instance or a null pointer if there's no + * handler for the "file_hash" event. + */ + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file) + { return file_hash ? new MD5(args, file) : 0; } + +protected: + + /** + * Constructor. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + */ + MD5(RecordVal* args, File* file) + : Hash(args, file, new MD5Val(), "md5") + {} +}; + +/** + * An analyzer to produce a SHA1 hash of file contents. + */ +class SHA1 : public Hash { +public: + + /** + * Create a new instance of the SHA1 hashing file analyzer. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @return the new MD5 analyzer instance or a null pointer if there's no + * handler for the "file_hash" event. + */ + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file) + { return file_hash ? new SHA1(args, file) : 0; } + +protected: + + /** + * Constructor. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + */ + SHA1(RecordVal* args, File* file) + : Hash(args, file, new SHA1Val(), "sha1") + {} +}; + +/** + * An analyzer to produce a SHA256 hash of file contents. + */ +class SHA256 : public Hash { +public: + + /** + * Create a new instance of the SHA256 hashing file analyzer. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @return the new MD5 analyzer instance or a null pointer if there's no + * handler for the "file_hash" event. + */ + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file) + { return file_hash ? new SHA256(args, file) : 0; } + +protected: + + /** + * Constructor. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + */ + SHA256(RecordVal* args, File* file) + : Hash(args, file, new SHA256Val(), "sha256") + {} +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/analyzer/hash/Plugin.cc b/src/file_analysis/analyzer/hash/Plugin.cc new file mode 100644 index 0000000000..1a7254105e --- /dev/null +++ b/src/file_analysis/analyzer/hash/Plugin.cc @@ -0,0 +1,33 @@ +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" + +#include "Hash.h" + +namespace plugin { namespace Bro_FileHash { + +class Plugin : public plugin::Plugin { +protected: + void InitPreScript() + { + SetName("Bro::FileHash"); + SetVersion(-1); + SetAPIVersion(BRO_PLUGIN_API_VERSION); + SetDynamicPlugin(false); + + SetDescription("Hash file content"); + + AddComponent(new ::file_analysis::Component("MD5", + ::file_analysis::MD5::Instantiate)); + AddComponent(new ::file_analysis::Component("SHA1", + ::file_analysis::SHA1::Instantiate)); + AddComponent(new ::file_analysis::Component("SHA256", + ::file_analysis::SHA256::Instantiate)); + + extern std::list > __bif_events_init(); + AddBifInitFunction(&__bif_events_init); + } +}; + +Plugin __plugin; + +} } diff --git a/src/file_analysis/analyzer/hash/events.bif b/src/file_analysis/analyzer/hash/events.bif new file mode 100644 index 0000000000..b4a8de1c74 --- /dev/null +++ b/src/file_analysis/analyzer/hash/events.bif @@ -0,0 +1,12 @@ +## This event is generated each time file analysis generates a digest of the +## file contents. +## +## f: The file. +## +## kind: The type of digest algorithm. +## +## hash: The result of the hashing. +## +## .. bro:see:: FileAnalysis::add_analyzer FileAnalysis::ANALYZER_MD5 +## FileAnalysis::ANALYZER_SHA1 FileAnalysis::ANALYZER_SHA256 +event file_hash%(f: fa_file, kind: string, hash: string%); diff --git a/src/file_analysis/file_analysis.bif b/src/file_analysis/file_analysis.bif new file mode 100644 index 0000000000..06ae9450dd --- /dev/null +++ b/src/file_analysis/file_analysis.bif @@ -0,0 +1,61 @@ +##! Internal functions and types used by the file analysis framework. + +module FileAnalysis; + +%%{ +#include "file_analysis/Manager.h" +%%} + +type AnalyzerArgs: record; + +## :bro:see:`FileAnalysis::set_timeout_interval`. +function FileAnalysis::__set_timeout_interval%(file_id: string, t: interval%): bool + %{ + bool result = file_mgr->SetTimeoutInterval(file_id->CheckString(), t); + return new Val(result, TYPE_BOOL); + %} + +## :bro:see:`FileAnalysis::add_analyzer`. +function FileAnalysis::__add_analyzer%(file_id: string, args: any%): bool + %{ + using BifType::Record::FileAnalysis::AnalyzerArgs; + RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); + bool result = file_mgr->AddAnalyzer(file_id->CheckString(), rv); + Unref(rv); + return new Val(result, TYPE_BOOL); + %} + +## :bro:see:`FileAnalysis::remove_analyzer`. +function FileAnalysis::__remove_analyzer%(file_id: string, args: any%): bool + %{ + using BifType::Record::FileAnalysis::AnalyzerArgs; + RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); + bool result = file_mgr->RemoveAnalyzer(file_id->CheckString(), rv); + Unref(rv); + return new Val(result, TYPE_BOOL); + %} + +## :bro:see:`FileAnalysis::stop`. +function FileAnalysis::__stop%(file_id: string%): bool + %{ + bool result = file_mgr->IgnoreFile(file_id->CheckString()); + return new Val(result, TYPE_BOOL); + %} + +module GLOBAL; + +## For use within a :bro:see:`get_file_handle` handler to set a unique +## identifier to associate with the current input to the file analysis +## framework. Using an empty string for the handle signifies that the +## input will be ignored/discarded. +## +## handle: A string that uniquely identifies a file. +## +## .. bro:see:: get_file_handle +function set_file_handle%(handle: string%): any + %{ + file_mgr->SetHandle(handle->CheckString()); + return 0; + %} + +const FileAnalysis::salt: string; diff --git a/src/input.bif b/src/input.bif index 40d8225400..d6a880d9e9 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,6 +9,7 @@ module Input; type TableDescription: record; type EventDescription: record; +type AnalysisDescription: record; function Input::__create_table_stream%(description: Input::TableDescription%) : bool %{ @@ -22,6 +23,12 @@ function Input::__create_event_stream%(description: Input::EventDescription%) : return new Val(res, TYPE_BOOL); %} +function Input::__create_analysis_stream%(description: Input::AnalysisDescription%) : bool + %{ + bool res = input_mgr->CreateAnalysisStream(description->AsRecordVal()); + return new Val(res, TYPE_BOOL); + %} + function Input::__remove_stream%(id: string%) : bool %{ bool res = input_mgr->RemoveStream(id->AsString()->CheckString()); diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 933b0b594c..d467e32005 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -15,10 +15,9 @@ #include "EventHandler.h" #include "NetVar.h" #include "Net.h" - - #include "CompHash.h" +#include "../file_analysis/Manager.h" #include "../threading/SerialTypes.h" using namespace input; @@ -148,6 +147,14 @@ public: ~EventStream(); }; +class Manager::AnalysisStream: public Manager::Stream { +public: + string file_id; + + AnalysisStream(); + ~AnalysisStream(); +}; + Manager::TableStream::TableStream() : Manager::Stream::Stream() { stream_type = TABLE_STREAM; @@ -198,6 +205,15 @@ Manager::TableStream::~TableStream() } } +Manager::AnalysisStream::AnalysisStream() : Manager::Stream::Stream() + { + stream_type = ANALYSIS_STREAM; + } + +Manager::AnalysisStream::~AnalysisStream() + { + } + Manager::Manager() { end_of_data = internal_handler("Input::end_of_data"); @@ -274,7 +290,8 @@ bool Manager::CreateStream(Stream* info, RecordVal* description) RecordType* rtype = description->Type()->AsRecordType(); if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) - || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) + || same_type(rtype, BifType::Record::Input::EventDescription, 0) + || same_type(rtype, BifType::Record::Input::AnalysisDescription, 0) ) ) { reporter->Error("Streamdescription argument not of right type for new input stream"); return false; @@ -303,6 +320,7 @@ bool Manager::CreateStream(Stream* info, RecordVal* description) ReaderBackend::ReaderInfo* rinfo = new ReaderBackend::ReaderInfo(); rinfo->source = copy_string(source.c_str()); + rinfo->name = copy_string(name.c_str()); EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); switch ( mode->InternalInt() ) @@ -680,6 +698,40 @@ bool Manager::CreateTableStream(RecordVal* fval) return true; } +bool Manager::CreateAnalysisStream(RecordVal* fval) + { + RecordType* rtype = fval->Type()->AsRecordType(); + + if ( ! same_type(rtype, BifType::Record::Input::AnalysisDescription, 0) ) + { + reporter->Error("AnalysisDescription argument not of right type"); + return false; + } + + AnalysisStream* stream = new AnalysisStream(); + + if ( ! CreateStream(stream, fval) ) + { + delete stream; + return false; + } + + stream->file_id = file_mgr->HashHandle(stream->name); + + assert(stream->reader); + + // reader takes in a byte stream as the only field + Field** fields = new Field*[1]; + fields[0] = new Field("bytestream", 0, TYPE_STRING, TYPE_VOID, false); + stream->reader->Init(1, fields); + + readers[stream->reader] = stream; + + DBG_LOG(DBG_INPUT, "Successfully created analysis stream %s", + stream->name.c_str()); + + return true; + } bool Manager::IsCompatibleType(BroType* t, bool atomic_only) { @@ -966,6 +1018,15 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) readFields = SendEventStreamEvent(i, type, vals); } + else if ( i->stream_type == ANALYSIS_STREAM ) + { + readFields = 1; + assert(vals[0]->type == TYPE_STRING); + file_mgr->DataIn(reinterpret_cast(vals[0]->val.string_val.data), + vals[0]->val.string_val.length, + static_cast(i)->file_id, i->name); + } + else assert(false); @@ -1179,8 +1240,11 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", i->name.c_str()); #endif - if ( i->stream_type == EVENT_STREAM ) + if ( i->stream_type != TABLE_STREAM ) { +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "%s is event, sending end of data", i->name.c_str()); +#endif // just signal the end of the data source SendEndOfData(i); return; @@ -1285,9 +1349,17 @@ void Manager::SendEndOfData(ReaderFrontend* reader) SendEndOfData(i); } + void Manager::SendEndOfData(const Stream *i) { +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "SendEndOfData for stream %s", + i->name.c_str()); +#endif SendEvent(end_of_data, 2, new StringVal(i->name.c_str()), new StringVal(i->info->source)); + + if ( i->stream_type == ANALYSIS_STREAM ) + file_mgr->EndOfFile(static_cast(i)->file_id); } void Manager::Put(ReaderFrontend* reader, Value* *vals) @@ -1299,6 +1371,11 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) return; } +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Put for stream %s", + i->name.c_str()); +#endif + int readFields = 0; if ( i->stream_type == TABLE_STREAM ) @@ -1310,6 +1387,15 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) readFields = SendEventStreamEvent(i, type, vals); } + else if ( i->stream_type == ANALYSIS_STREAM ) + { + readFields = 1; + assert(vals[0]->type == TYPE_STRING); + file_mgr->DataIn(reinterpret_cast(vals[0]->val.string_val.data), + vals[0]->val.string_val.length, + static_cast(i)->file_id, i->name); + } + else assert(false); @@ -1577,6 +1663,12 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) success = true; } + else if ( i->stream_type == ANALYSIS_STREAM ) + { + // can't do anything + success = true; + } + else { assert(false); @@ -1622,6 +1714,11 @@ bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) return false; } +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "SendEvent for event %s with num_vals vals", + name.c_str(), num_vals); +#endif + RecordType *type = handler->FType()->Args(); int num_event_vals = type->NumFields(); if ( num_vals != num_event_vals ) @@ -1634,7 +1731,7 @@ bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) for ( int i = 0; i < num_vals; i++) vl->append(ValueToVal(vals[i], type->FieldType(i))); - mgr.Dispatch(new Event(handler, vl)); + mgr.QueueEvent(handler, vl, SOURCE_LOCAL); for ( int i = 0; i < num_vals; i++ ) delete vals[i]; @@ -1648,6 +1745,11 @@ void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) { val_list* vl = new val_list; +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "SendEvent with %d vals", + numvals); +#endif + va_list lP; va_start(lP, numvals); for ( int i = 0; i < numvals; i++ ) @@ -1662,6 +1764,11 @@ void Manager::SendEvent(EventHandlerPtr ev, list events) { val_list* vl = new val_list; +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "SendEvent with %d vals (list)", + events.size()); +#endif + for ( list::iterator i = events.begin(); i != events.end(); i++ ) { vl->append( *i ); @@ -2166,3 +2273,18 @@ Manager::Stream* Manager::FindStream(ReaderFrontend* reader) return 0; } + +// Function is called on Bro shutdown. +// Signal all frontends that they will cease operation. +void Manager::Terminate() + { + for ( map::iterator i = readers.begin(); i != readers.end(); ++i ) + { + if ( i->second->removed ) + continue; + + i->second->removed = true; + i->second->reader->Stop(); + } + + } diff --git a/src/input/Manager.h b/src/input/Manager.h index 633b20f8ed..8156ed5248 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -55,6 +55,18 @@ public: */ bool CreateEventStream(RecordVal* description); + /** + * Creates a new input stream which will forward the data from the data + * source on to the file analysis framework. The internal BiF defined + * in input.bif just forward here. For an input reader to be compatible + * with this method, it must be able to accept a filter of a single string + * type (i.e. they read a byte stream). + * + * @param description A record of the script type \c + * Input::AnalysisDescription + */ + bool CreateAnalysisStream(RecordVal* description); + /** * Force update on a input stream. Forces a re-read of the whole * input source. Usually used when an input stream is opened in @@ -79,6 +91,11 @@ public: */ bool RemoveStream(const string &id); + /** + * Signals the manager to shutdown at Bro's termination. + */ + void Terminate(); + protected: friend class ReaderFrontend; friend class PutMessage; @@ -138,6 +155,7 @@ private: class Stream; class TableStream; class EventStream; + class AnalysisStream; // Actual RemoveStream implementation -- the function's public and // protected definitions are wrappers around this function. @@ -202,7 +220,7 @@ private: Stream* FindStream(const string &name); Stream* FindStream(ReaderFrontend* reader); - enum StreamType { TABLE_STREAM, EVENT_STREAM }; + enum StreamType { TABLE_STREAM, EVENT_STREAM, ANALYSIS_STREAM }; map readers; diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 73e5475db6..5419879e13 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -85,6 +85,11 @@ public: */ const char* source; + /** + * The name of the input stream. + */ + const char* name; + /** * A map of key/value pairs corresponding to the relevant * filter's "config" table. @@ -99,12 +104,14 @@ public: ReaderInfo() { source = 0; + name = 0; mode = MODE_NONE; } ReaderInfo(const ReaderInfo& other) { source = other.source ? copy_string(other.source) : 0; + name = other.name ? copy_string(other.name) : 0; mode = other.mode; for ( config_map::const_iterator i = other.config.begin(); i != other.config.end(); i++ ) diff --git a/src/input/fdstream.h b/src/input/fdstream.h deleted file mode 100644 index cda767dd52..0000000000 --- a/src/input/fdstream.h +++ /dev/null @@ -1,189 +0,0 @@ -/* The following code declares classes to read from and write to - * file descriptore or file handles. - * - * See - * http://www.josuttis.com/cppcode - * for details and the latest version. - * - * - open: - * - integrating BUFSIZ on some systems? - * - optimized reading of multiple characters - * - stream for reading AND writing - * - i18n - * - * (C) Copyright Nicolai M. Josuttis 2001. - * Permission to copy, use, modify, sell and distribute this software - * is granted provided this copyright notice appears in all copies. - * This software is provided "as is" without express or implied - * warranty, and with no claim as to its suitability for any purpose. - * - * Version: Jul 28, 2002 - * History: - * Jul 28, 2002: bugfix memcpy() => memmove() - * fdinbuf::underflow(): cast for return statements - * Aug 05, 2001: first public version - */ -#ifndef BOOST_FDSTREAM_HPP -#define BOOST_FDSTREAM_HPP - -#include -#include -#include -// for EOF: -#include -// for memmove(): -#include - - - -// low-level read and write functions -#ifdef _MSC_VER -# include -#else -# include -# include -//extern "C" { -// int write (int fd, const char* buf, int num); -// int read (int fd, char* buf, int num); -//} -#endif - - -// BEGIN namespace BOOST -namespace boost { - - -/************************************************************ - * fdostream - * - a stream that writes on a file descriptor - ************************************************************/ - - -class fdoutbuf : public std::streambuf { - protected: - int fd; // file descriptor - public: - // constructor - fdoutbuf (int _fd) : fd(_fd) { - } - protected: - // write one character - virtual int_type overflow (int_type c) { - if (c != EOF) { - char z = c; - if (write (fd, &z, 1) != 1) { - return EOF; - } - } - return c; - } - // write multiple characters - virtual - std::streamsize xsputn (const char* s, - std::streamsize num) { - return write(fd,s,num); - } -}; - -class fdostream : public std::ostream { - protected: - fdoutbuf buf; - public: - fdostream (int fd) : std::ostream(0), buf(fd) { - rdbuf(&buf); - } -}; - - -/************************************************************ - * fdistream - * - a stream that reads on a file descriptor - ************************************************************/ - -class fdinbuf : public std::streambuf { - protected: - int fd; // file descriptor - protected: - /* data buffer: - * - at most, pbSize characters in putback area plus - * - at most, bufSize characters in ordinary read buffer - */ - static const int pbSize = 4; // size of putback area - static const int bufSize = 1024; // size of the data buffer - char buffer[bufSize+pbSize]; // data buffer - - public: - /* constructor - * - initialize file descriptor - * - initialize empty data buffer - * - no putback area - * => force underflow() - */ - fdinbuf (int _fd) : fd(_fd) { - setg (buffer+pbSize, // beginning of putback area - buffer+pbSize, // read position - buffer+pbSize); // end position - } - - protected: - // insert new characters into the buffer - virtual int_type underflow () { -#ifndef _MSC_VER - using std::memmove; -#endif - - // is read position before end of buffer? - if (gptr() < egptr()) { - return traits_type::to_int_type(*gptr()); - } - - /* process size of putback area - * - use number of characters read - * - but at most size of putback area - */ - int numPutback; - numPutback = gptr() - eback(); - if (numPutback > pbSize) { - numPutback = pbSize; - } - - /* copy up to pbSize characters previously read into - * the putback area - */ - memmove (buffer+(pbSize-numPutback), gptr()-numPutback, - numPutback); - - // read at most bufSize new characters - int num; - num = read (fd, buffer+pbSize, bufSize); - if ( num == EAGAIN ) { - return 0; - } - if (num <= 0) { - // ERROR or EOF - return EOF; - } - - // reset buffer pointers - setg (buffer+(pbSize-numPutback), // beginning of putback area - buffer+pbSize, // read position - buffer+pbSize+num); // end of buffer - - // return next character - return traits_type::to_int_type(*gptr()); - } -}; - -class fdistream : public std::istream { - protected: - fdinbuf buf; - public: - fdistream (int fd) : std::istream(0), buf(fd) { - rdbuf(&buf); - } -}; - - -} // END namespace boost - -#endif /*BOOST_FDSTREAM_HPP*/ diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index ac96e5c0f5..2820923a25 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -3,32 +3,46 @@ #include "Raw.h" #include "NetVar.h" -#include -#include - #include "../../threading/SerialTypes.h" -#include "../fdstream.h" #include +#include #include #include #include #include +#include +#include using namespace input::reader; using threading::Value; using threading::Field; +const int Raw::block_size = 4096; // how big do we expect our chunks of data to be. + + Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) { file = 0; - in = 0; - + stderrfile = 0; + forcekill = false; separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), BifConst::InputRaw::record_separator->Len()); - if ( separator.size() != 1 ) - Error("separator length has to be 1. Separator will be truncated."); + sep_length = BifConst::InputRaw::record_separator->Len(); + + buf = 0; + outbuf = 0; + bufpos = 0; + + stdin_fileno = fileno(stdin); + stdout_fileno = fileno(stdout); + stderr_fileno = fileno(stderr); + + childpid = -1; + + stdin_towrite = 0; // by default do not open stdin + use_stderr = false; } Raw::~Raw() @@ -40,41 +54,130 @@ void Raw::DoClose() { if ( file != 0 ) CloseInput(); + + if ( buf != 0 ) + { + // we still have output that has not been flushed. Throw away. + delete buf; + buf = 0; + } + + if ( execute && childpid > 0 && kill(childpid, 0) == 0 ) + { + // kill child process + kill(childpid, 15); // sigterm + + if ( forcekill ) + { + usleep(200); // 200 msecs should be enough for anyone ;) + + if ( kill(childpid, 0) == 0 ) // perhaps it is already gone + kill(childpid, 9); // TERMINATE + } + } + } + +bool Raw::Execute() + { + if ( pipe(pipes) != 0 || pipe(pipes+2) || pipe(pipes+4) ) + { + Error(Fmt("Could not open pipe: %d", errno)); + return false; + } + + childpid = fork(); + if ( childpid < 0 ) + { + Error(Fmt("Could not create child process: %d", errno)); + return false; + } + + else if ( childpid == 0 ) + { + // we are the child. + close(pipes[stdout_in]); + dup2(pipes[stdout_out], stdout_fileno); + + if ( stdin_towrite ) + { + close(pipes[stdin_out]); + dup2(pipes[stdin_in], stdin_fileno); + } + + if ( use_stderr ) + { + close(pipes[stderr_in]); + dup2(pipes[stderr_out], stderr_fileno); + } + + execl("/bin/sh", "sh", "-c", fname.c_str(), NULL); + fprintf(stderr, "Exec failed :(......\n"); + exit(255); + } + else + { + // we are the parent + close(pipes[stdout_out]); + pipes[stdout_out] = -1; + + if ( Info().mode == MODE_STREAM ) + fcntl(pipes[stdout_in], F_SETFL, O_NONBLOCK); + + if ( stdin_towrite ) + { + close(pipes[stdin_in]); + pipes[stdin_in] = -1; + fcntl(pipes[stdin_out], F_SETFL, O_NONBLOCK); // ya, just always set this to nonblocking. we do not want to block on a program receiving data. + // note that there is a small gotcha with it. More data is queued when more data is read from the program output. Hence, when having + // a program in mode_manual where the first write cannot write everything, the rest will be stuck in a queue that is never emptied. + } + + if ( use_stderr ) + { + close(pipes[stderr_out]); + pipes[stderr_out] = -1; + fcntl(pipes[stderr_in], F_SETFL, O_NONBLOCK); // true for this too. + } + + file = fdopen(pipes[stdout_in], "r"); + pipes[stdout_in] = -1; // will be closed by fclose + + if ( use_stderr ) + stderrfile = fdopen(pipes[stderr_in], "r"); + pipes[stderr_in] = -1; // will be closed by fclose + if ( file == 0 || (stderrfile == 0 && use_stderr) ) + { + Error("Could not convert fileno to file"); + return false; + } + + + return true; + } } bool Raw::OpenInput() { if ( execute ) - { - file = popen(fname.c_str(), "r"); - if ( file == NULL ) - { - Error(Fmt("Could not execute command %s", fname.c_str())); - return false; - } - } + return Execute(); + else { file = fopen(fname.c_str(), "r"); - if ( file == NULL ) + if ( ! file ) { Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } + fcntl(fileno(file), F_SETFD, FD_CLOEXEC); } - // This is defined in input/fdstream.h - in = new boost::fdistream(fileno(file)); - - if ( execute && Info().mode == MODE_STREAM ) - fcntl(fileno(file), F_SETFL, O_NONBLOCK); - return true; } bool Raw::CloseInput() { - if ( file == NULL ) + if ( file == 0 ) { InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str())); return false; @@ -83,15 +186,20 @@ bool Raw::CloseInput() Debug(DBG_INPUT, "Raw reader starting close"); #endif - delete in; + fclose(file); - if ( execute ) - pclose(file); - else - fclose(file); + if ( use_stderr ) + fclose(stderrfile); - in = NULL; - file = NULL; + if ( execute ) // we do not care if any of those fails. They should all be defined. + { + for ( int i = 0; i < 6; i ++ ) + if ( pipes[i] != -1 ) + close(pipes[i]); + } + + file = 0; + stderrfile = 0; #ifdef DEBUG Debug(DBG_INPUT, "Raw reader finished close"); @@ -106,28 +214,9 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie mtime = 0; execute = false; firstrun = true; + int want_fields = 1; bool result; - if ( ! info.source || strlen(info.source) == 0 ) - { - Error("No source path provided"); - return false; - } - - if ( num_fields != 1 ) - { - Error("Filter for raw reader contains more than one field. " - "Filters for the raw reader may only contain exactly one string field. " - "Filter ignored."); - return false; - } - - if ( fields[0]->type != TYPE_STRING ) - { - Error("Filter for raw reader contains a field that is not of type string."); - return false; - } - // do Initialization string source = string(info.source); char last = info.source[source.length() - 1]; @@ -135,23 +224,63 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie { execute = true; fname = source.substr(0, fname.length() - 1); - - if ( (info.mode != MODE_MANUAL) ) - { - Error(Fmt("Unsupported read mode %d for source %s in execution mode", - info.mode, fname.c_str())); - return false; - } - - result = OpenInput(); - } - else + + if ( ! info.source || strlen(info.source) == 0 ) { - execute = false; - result = OpenInput(); + Error("No source path provided"); + return false; } + map::const_iterator it = info.config.find("stdin"); // data that is sent to the child process + if ( it != info.config.end() ) + { + stdin_string = it->second; + stdin_towrite = stdin_string.length(); + } + + it = info.config.find("read_stderr"); // we want to read stderr + if ( it != info.config.end() && execute ) + { + use_stderr = true; + want_fields = 2; + } + + it = info.config.find("force_kill"); // we want to be sure that our child is dead when we exit + if ( it != info.config.end() && execute ) + { + forcekill = true; + } + + if ( num_fields != want_fields ) + { + Error(Fmt("Filter for raw reader contains wrong number of fields -- got %d, expected %d. " + "Filters for the raw reader contain one string field when used in normal mode and one string and one bool fields when using execute mode with stderr capuring. " + "Filter ignored.", num_fields, want_fields)); + return false; + } + + if ( fields[0]->type != TYPE_STRING ) + { + Error("First field for raw reader always has to be of type string."); + return false; + } + if ( use_stderr && fields[1]->type != TYPE_BOOL ) + { + Error("Second field for raw reader always has to be of type bool."); + return false; + } + + if ( execute && Info().mode == MODE_REREAD ) + { + // for execs this makes no sense - would have to execute each heartbeat? + Error("Rereading only supported for files, not for executables."); + return false; + } + + + result = OpenInput(); + if ( result == false ) return result; @@ -168,18 +297,115 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie return true; } - -bool Raw::GetLine(string& str) +int64_t Raw::GetLine(FILE* arg_file) { - if ( in->peek() == std::iostream::traits_type::eof() ) - return false; + errno = 0; + int pos = 0; // strstr_n only works on ints - so no use to use something different here + int offset = 0; - if ( in->eofbit == true || in->failbit == true ) - return false; + if ( buf == 0 ) + buf = new char[block_size]; - return getline(*in, str, separator[0]); + int repeats = 1; + + for (;;) + { + size_t readbytes = fread(buf+bufpos+offset, 1, block_size-bufpos, arg_file); + pos += bufpos + readbytes; + //printf("Pos: %d\n", pos); + bufpos = offset = 0; // read full block size in next read... + + if ( pos == 0 && errno != 0 ) + break; + + // researching everything each time is a bit... cpu-intensive. But otherwhise we have + // to deal with situations where the separator is multi-character and split over multiple + // reads... + int found = strstr_n(pos, (unsigned char*) buf, separator.size(), (unsigned char*) separator.c_str()); + + if ( found == -1 ) + { + // we did not find it and have to search again in the next try. resize buffer.... + // but first check if we encountered the file end - because if we did this was it. + if ( feof(arg_file) != 0 ) + { + if ( pos == 0 ) + return -1; // signal EOF - and that we had no more data. + else + { + outbuf = buf; + buf = 0; + return pos; + } + } + + repeats++; + // bah, we cannot use realloc because we would have to change the delete in the manager to a free. + char * newbuf = new char[block_size*repeats]; + memcpy(newbuf, buf, block_size*(repeats-1)); + delete buf; + buf = newbuf; + offset = block_size*(repeats-1); + } + else + { + outbuf = buf; + buf = 0; + + if ( found < pos ) + { + // we have leftovers. copy them into the buffer for the next line + buf = new char[block_size]; + memcpy(buf, outbuf + found + sep_length, pos - found - sep_length); + bufpos = pos - found - sep_length; + } + + return found; + } + + } + + if ( errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR ) + return -2; + + else + { + // an error code we did no expect. This probably is bad. + Error(Fmt("Reader encountered unexpected error code %d", errno)); + return -3; + } + + InternalError("Internal control flow execution error in raw reader"); + assert(false); } +// write to the stdin of the child process +void Raw::WriteToStdin() + { + assert(stdin_towrite <= stdin_string.length()); + uint64_t pos = stdin_string.length() - stdin_towrite; + + errno = 0; + ssize_t written = write(pipes[stdin_out], stdin_string.c_str() + pos, stdin_towrite); + stdin_towrite -= written; + + if ( errno != 0 && errno != EAGAIN && errno != EWOULDBLOCK ) + { + Error(Fmt("Writing to child process stdin failed: %d. Stopping writing at position %d", errno, pos)); + stdin_towrite = 0; + close(pipes[stdin_out]); + } + + if ( stdin_towrite == 0 ) // send EOF when we are done. + close(pipes[stdin_out]); + + if ( Info().mode == MODE_MANUAL && stdin_towrite != 0 ) + { + Error(Fmt("Could not write whole string to stdin of child process in one go. Please use STREAM mode to pass more data to child.")); + } + } + + // read the entire file and send appropriate thingies back to InputMgr bool Raw::DoUpdate() { @@ -191,6 +417,7 @@ bool Raw::DoUpdate() switch ( Info().mode ) { case MODE_REREAD: { + assert(childpid == -1); // mode may not be used to execute child programs // check if the file has changed struct stat sb; if ( stat(fname.c_str(), &sb) == -1 ) @@ -211,10 +438,9 @@ bool Raw::DoUpdate() case MODE_MANUAL: case MODE_STREAM: - if ( Info().mode == MODE_STREAM && file != NULL && in != NULL ) + if ( Info().mode == MODE_STREAM && file != 0 ) { - //fpurge(file); - in->clear(); // remove end of file evil bits + clearerr(file); // remove end of file evil bits break; } @@ -230,21 +456,118 @@ bool Raw::DoUpdate() } string line; - while ( GetLine(line) ) + assert ( (NumFields() == 1 && !use_stderr) || (NumFields() == 2 && use_stderr)); + for ( ;; ) { - assert (NumFields() == 1); + if ( stdin_towrite > 0 ) + WriteToStdin(); - Value** fields = new Value*[1]; + int64_t length = GetLine(file); + //printf("Read %lld bytes\n", length); + + if ( length == -3 ) + return false; + + else if ( length == -2 || length == -1 ) + // no data ready or eof + break; + + Value** fields = new Value*[2]; // just always reserve 2. This means that our [] is too long by a count of 1 if not using stderr. But who cares... // filter has exactly one text field. convert to it. Value* val = new Value(TYPE_STRING, true); - val->val.string_val.data = copy_string(line.c_str()); - val->val.string_val.length = line.size(); + val->val.string_val.data = outbuf; + val->val.string_val.length = length; fields[0] = val; + if ( use_stderr ) + { + Value* bval = new Value(TYPE_BOOL, true); + bval->val.int_val = 0; + fields[1] = bval; + } + Put(fields); + + outbuf = 0; } + if ( use_stderr ) + { + for ( ;; ) + { + int64_t length = GetLine(stderrfile); + //printf("Read stderr %lld bytes\n", length); + if ( length == -3 ) + return false; + + else if ( length == -2 || length == -1 ) + break; + + Value** fields = new Value*[2]; + Value* val = new Value(TYPE_STRING, true); + val->val.string_val.data = outbuf; + val->val.string_val.length = length; + fields[0] = val; + Value* bval = new Value(TYPE_BOOL, true); + bval->val.int_val = 1; // yes, we are stderr + fields[1] = bval; + + Put(fields); + + outbuf = 0; + } + } + + if ( ( Info().mode == MODE_MANUAL ) || ( Info().mode == MODE_REREAD ) ) + // done with the current data source + EndCurrentSend(); + + // and let's check if the child process is still alive + int return_code; + if ( childpid != -1 && waitpid(childpid, &return_code, WNOHANG) != 0 ) + { + // child died + bool signal = false; + int code = 0; + if ( WIFEXITED(return_code) ) + { + code = WEXITSTATUS(return_code); + if ( code != 0 ) + Error(Fmt("Child process exited with non-zero return code %d", code)); + } + + else if ( WIFSIGNALED(return_code) ) + { + signal = false; + code = WTERMSIG(return_code); + Error(Fmt("Child process exited due to signal %d", code)); + } + + else + assert(false); + + Value** vals = new Value*[4]; + vals[0] = new Value(TYPE_STRING, true); + vals[0]->val.string_val.data = copy_string(Info().name); + vals[0]->val.string_val.length = strlen(Info().name); + vals[1] = new Value(TYPE_STRING, true); + vals[1]->val.string_val.data = copy_string(Info().source); + vals[1]->val.string_val.length = strlen(Info().source); + vals[2] = new Value(TYPE_COUNT, true); + vals[2]->val.int_val = code; + vals[3] = new Value(TYPE_BOOL, true); + vals[3]->val.int_val = signal; + + // and in this case we can signal end_of_data even for the streaming reader + if ( Info().mode == MODE_STREAM ) + EndCurrentSend(); + + SendEvent("InputRaw::process_finished", 4, vals); + } + + + #ifdef DEBUG Debug(DBG_INPUT, "DoUpdate finished successfully"); #endif diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index 48912b70a7..6dbae21002 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -3,7 +3,6 @@ #ifndef INPUT_READERS_RAW_H #define INPUT_READERS_RAW_H -#include #include #include "../ReaderBackend.h" @@ -30,17 +29,49 @@ protected: private: bool OpenInput(); bool CloseInput(); - bool GetLine(string& str); + int64_t GetLine(FILE* file); + bool Execute(); + void WriteToStdin(); string fname; // Source with a potential "|" removed. - istream* in; FILE* file; + FILE* stderrfile; bool execute; bool firstrun; time_t mtime; // options set from the script-level. string separator; + unsigned int sep_length; // length of the separator + + static const int block_size; + int bufpos; + char* buf; + char* outbuf; + + int stdin_fileno; + int stdout_fileno; + int stderr_fileno; + + string stdin_string; + uint64_t stdin_towrite; + + bool use_stderr; + + bool forcekill; + + int pipes[6]; + pid_t childpid; + + enum IoChannels { + stdout_in = 0, + stdout_out = 1, + stdin_in = 2, + stdin_out = 3, + stderr_in = 4, + stderr_out = 5 + }; + }; } diff --git a/src/main.cc b/src/main.cc index 491f8a732d..56193a935b 100644 --- a/src/main.cc +++ b/src/main.cc @@ -357,6 +357,7 @@ void terminate_bro() file_mgr->Terminate(); log_mgr->Terminate(); + input_mgr->Terminate(); thread_mgr->Terminate(); mgr.Drain(); @@ -834,6 +835,7 @@ int main(int argc, char** argv) plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); + file_mgr->InitPreScript(); if ( events_file ) event_player = new EventPlayer(events_file); @@ -855,6 +857,7 @@ int main(int argc, char** argv) plugin_mgr->InitPostScript(); analyzer_mgr->InitPostScript(); + file_mgr->InitPostScript(); if ( print_plugins ) { @@ -868,6 +871,8 @@ int main(int argc, char** argv) if ( generate_documentation ) { + CreateProtoAnalyzerDoc("proto-analyzers.rst"); + std::list::iterator it; for ( it = docs_generated.begin(); it != docs_generated.end(); ++it ) @@ -1109,6 +1114,7 @@ int main(int argc, char** argv) reporter->ReportViaEvents(true); + // Drain the event queue here to support the protocols framework configuring DPM mgr.Drain(); analyzer_mgr->DumpDebug(); diff --git a/src/plugin/Component.cc b/src/plugin/Component.cc index 7d2e69eb86..074f00c264 100644 --- a/src/plugin/Component.cc +++ b/src/plugin/Component.cc @@ -21,7 +21,7 @@ component::Type Component::Type() const return type; } -void Component::Describe(ODesc* d) +void Component::Describe(ODesc* d) const { d->Add(" "); d->Add("["); @@ -39,6 +39,10 @@ void Component::Describe(ODesc* d) d->Add("Analyzer"); break; + case component::FILE_ANALYZER: + d->Add("File Analyzer"); + break; + default: reporter->InternalError("unknown component type in plugin::Component::Describe"); } diff --git a/src/plugin/Component.h b/src/plugin/Component.h index fbeb70ebed..ad02dc7e4b 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -15,16 +15,11 @@ namespace component { enum Type { READER, /// An input reader (not currently used). WRITER, /// An logging writer (not currenly used). - ANALYZER /// A protocol analyzer. + ANALYZER, /// A protocol analyzer. + FILE_ANALYZER /// A file analyzer. }; } -#if 0 -namespace input { class PluginComponent; } -namespace logging { class PluginComponent; } -namespace analyzer { class PluginComponent; } -#endif - /** * Base class for plugin components. A component is a specific piece of * functionality that a plugin provides, such as a protocol analyzer or a log @@ -50,6 +45,12 @@ public: */ component::Type Type() const; + /** + * Returns a descriptive name for the analyzer. This name must be + * unique across all components of the same type. + */ + virtual const char* Name() const = 0; + /** * Returns a textual representation of the component. The default * version just output the type. Derived version should call the @@ -57,7 +58,7 @@ public: * * @param d The description object to use. */ - virtual void Describe(ODesc* d); + virtual void Describe(ODesc* d) const; private: component::Type type; diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 93ed3f2b97..67f4dea2bd 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -30,9 +30,18 @@ bool Manager::LoadPluginsFrom(const std::string& dir) return false; } +static bool plugin_cmp(const Plugin* a, const Plugin* b) + { + return a->Name() < b->Name(); + } + bool Manager::RegisterPlugin(Plugin *plugin) { Manager::PluginsInternal()->push_back(plugin); + + // Sort plugins by name to make sure we have a deterministic order. + PluginsInternal()->sort(plugin_cmp); + return true; } diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index 352aff6aed..eaac8a3b25 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -59,7 +59,7 @@ Plugin::~Plugin() delete [] description; } -const char* Plugin::Name() +const char* Plugin::Name() const { return name; } @@ -69,7 +69,7 @@ void Plugin::SetName(const char* arg_name) name = copy_string(arg_name); } -const char* Plugin::Description() +const char* Plugin::Description() const { return description; } @@ -79,7 +79,7 @@ void Plugin::SetDescription(const char* arg_description) description = copy_string(arg_description); } -int Plugin::Version() +int Plugin::Version() const { return dynamic ? version : 0; } @@ -89,12 +89,12 @@ void Plugin::SetVersion(int arg_version) version = arg_version; } -int Plugin::APIVersion() +int Plugin::APIVersion() const { return api_version; } -bool Plugin::DynamicPlugin() +bool Plugin::DynamicPlugin() const { return dynamic; } @@ -127,7 +127,7 @@ void Plugin::InitPostScript() } } -Plugin::bif_item_list Plugin::BifItems() +Plugin::bif_item_list Plugin::BifItems() const { bif_item_list l1 = bif_items; bif_item_list l2 = CustomBifItems(); @@ -138,7 +138,7 @@ Plugin::bif_item_list Plugin::BifItems() return l1; } -Plugin::bif_item_list Plugin::CustomBifItems() +Plugin::bif_item_list Plugin::CustomBifItems() const { return bif_item_list(); } @@ -151,14 +151,23 @@ void Plugin::Done() components.clear(); } -Plugin::component_list Plugin::Components() +Plugin::component_list Plugin::Components() const { return components; } +static bool component_cmp(const Component* a, const Component* b) + { + return a->Name() < b->Name(); + } + void Plugin::AddComponent(Component* c) { components.push_back(c); + + // Sort components by name to make sure we have a deterministic + // order. + components.sort(component_cmp); } void Plugin::AddBifInitFunction(bif_init_func c) @@ -166,7 +175,7 @@ void Plugin::AddBifInitFunction(bif_init_func c) bif_inits.push_back(c); } -void Plugin::Describe(ODesc* d) +void Plugin::Describe(ODesc* d) const { d->Add("Plugin: "); d->Add(name); diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index 6c6d89a4d1..4abd260550 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -102,24 +102,24 @@ public: /** * Returns the name of the plugin. */ - const char* Name(); + const char* Name() const; /** * Returns a short textual description of the plugin, if provided. */ - const char* Description(); + const char* Description() const; /** * Returns the version of the plugin. Version are only meaningful for * dynamically compiled plugins; for statically compiled ones, this * will always return 0. */ - int Version(); + int Version() const; /** * Returns true if this is a dynamically linked in plugin. */ - bool DynamicPlugin(); + bool DynamicPlugin() const; /** * Returns the internal API version that this plugin relies on. Only @@ -128,18 +128,18 @@ public: * dynamically loaded plugins may cause a mismatch if they were * compiled for a different Bro version. */ - int APIVersion(); + int APIVersion() const; /** * Returns a list of all components the plugin provides. */ - component_list Components(); + component_list Components() const; /** * Returns a list of all BiF items that the plugin provides. This * must be called only after InitBif() has been executed. */ - bif_item_list BifItems(); + bif_item_list BifItems() const; /** * First-stage initialization of the plugin called early during Bro's @@ -171,7 +171,7 @@ public: * is disabled, the rendering will include a list of all components * and BiF items. */ - void Describe(ODesc* d); + void Describe(ODesc* d) const; protected: typedef std::list > bif_init_func_result; @@ -225,7 +225,7 @@ protected: * for informational purpuses only and will show up in the result of * BifItems() as well as in the Describe() output. */ - virtual bif_item_list CustomBifItems() ; + virtual bif_item_list CustomBifItems() const; /** * Internal function adding an entry point for registering diff --git a/src/util.cc b/src/util.cc index cdd257d94f..81ec135f98 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1637,3 +1637,18 @@ const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length) return rval; } + +const char* canonify_name(const char* name) + { + unsigned int len = strlen(name); + char* nname = new char[len + 1]; + + for ( unsigned int i = 0; i < len; i++ ) + { + char c = isalnum(name[i]) ? name[i] : '_'; + nname[i] = toupper(c); + } + + nname[len] = '\0'; + return nname; + } diff --git a/src/util.h b/src/util.h index 0af401c668..5689253d95 100644 --- a/src/util.h +++ b/src/util.h @@ -391,4 +391,12 @@ extern magic_t magic_mime_cookie; void bro_init_magic(magic_t* cookie_ptr, int flags); const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length); +/** + * Canonicalizes a name by converting it to uppercase letters and replacing + * all non-alphanumeric characters with an underscore. + * @param name The string to canonicalize. + * @return The canonicalized version of \a name which caller may later delete[]. + */ +const char* canonify_name(const char* name); + #endif diff --git a/testing/btest/Baseline/core.check-unused-event-handlers/.stderr b/testing/btest/Baseline/core.check-unused-event-handlers/.stderr index 8d8bf1a85b..1a32ad442c 100644 --- a/testing/btest/Baseline/core.check-unused-event-handlers/.stderr +++ b/testing/btest/Baseline/core.check-unused-event-handlers/.stderr @@ -1 +1,2 @@ warning in , line 1: event handler never invoked: this_is_never_used +warning in , line 1: event handler never invoked: InputRaw::process_finished diff --git a/testing/btest/Baseline/core.print-bpf-filters/conn.log b/testing/btest/Baseline/core.print-bpf-filters/conn.log index 0fd86b8dc4..166286203e 100644 --- a/testing/btest/Baseline/core.print-bpf-filters/conn.log +++ b/testing/btest/Baseline/core.print-bpf-filters/conn.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path conn -#open 2005-10-07-23-23-57 +#open 2013-07-18-00-18-33 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool count string count count count count table[string] -1128727435.450898 UWkUyAuUGXf 141.42.64.125 56730 125.190.109.199 80 tcp http 1.733303 98 9417 SF - 0 ShADdFaf 12 730 10 9945 (empty) -#close 2005-10-07-23-23-57 +1278600802.069419 UWkUyAuUGXf 10.20.80.1 50343 10.0.0.15 80 tcp - 0.004152 9 3429 SF - 0 ShADadfF 7 381 7 3801 (empty) +#close 2013-07-18-00-18-33 diff --git a/testing/btest/Baseline/core.print-bpf-filters/output b/testing/btest/Baseline/core.print-bpf-filters/output index cadc8b22db..2f7a1d9386 100644 --- a/testing/btest/Baseline/core.print-bpf-filters/output +++ b/testing/btest/Baseline/core.print-bpf-filters/output @@ -3,38 +3,28 @@ #empty_field (empty) #unset_field - #path packet_filter -#open 2012-11-06-00-53-09 +#open 2013-07-19-02-54-13 #fields ts node filter init success #types time string string bool bool -1352163189.729807 - ip or not ip T T -#close 2012-11-06-00-53-09 +1374202453.158981 - ip or not ip T T +#close 2013-07-19-02-54-13 #separator \x09 #set_separator , #empty_field (empty) #unset_field - #path packet_filter -#open 2012-11-06-00-53-10 +#open 2013-07-19-02-54-13 #fields ts node filter init success #types time string string bool bool -1352163190.114261 - ((((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 502)) or (tcp port 995)) or (tcp port 22)) or (port 21 and port 2811)) or (tcp port 25 or tcp port 587)) or (tcp port 614)) or (tcp port 990)) or (port 6667)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666) T T -#close 2012-11-06-00-53-10 +1374202453.437816 - port 42 T T +#close 2013-07-19-02-54-13 #separator \x09 #set_separator , #empty_field (empty) #unset_field - #path packet_filter -#open 2012-11-06-00-53-10 +#open 2013-07-19-02-54-13 #fields ts node filter init success #types time string string bool bool -1352163190.484506 - port 42 T T -#close 2012-11-06-00-53-10 -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path packet_filter -#open 2012-11-06-00-53-10 -#fields ts node filter init success -#types time string string bool bool -1352163190.855090 - port 56730 T T -#close 2012-11-06-00-53-10 +1374202453.715717 - (vlan) and (ip or not ip) T T +#close 2013-07-19-02-54-13 diff --git a/testing/btest/Baseline/core.print-bpf-filters/output2 b/testing/btest/Baseline/core.print-bpf-filters/output2 new file mode 100644 index 0000000000..99ad929fbf --- /dev/null +++ b/testing/btest/Baseline/core.print-bpf-filters/output2 @@ -0,0 +1,43 @@ +2 1080 +1 137 +1 21 +1 2123 +1 2152 +1 22 +1 25 +1 2811 +1 3128 +1 3544 +1 443 +1 502 +1 5072 +1 514 +1 5223 +2 53 +1 5353 +1 5355 +1 563 +1 585 +1 587 +1 614 +1 631 +1 636 +1 6666 +1 6667 +1 6668 +1 6669 +1 80 +1 8000 +1 8080 +1 81 +1 8888 +1 989 +1 990 +1 992 +1 993 +1 995 +40 and +39 or +40 port +31 tcp +9 udp diff --git a/testing/btest/Baseline/core.tunnels.ayiya/http.log b/testing/btest/Baseline/core.tunnels.ayiya/http.log index cab51f8224..cd49c4cc89 100644 --- a/testing/btest/Baseline/core.tunnels.ayiya/http.log +++ b/testing/btest/Baseline/core.tunnels.ayiya/http.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-14-38-11 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html - - -1257655302.514424 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 2 GET ipv6.google.com /csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - -1257655303.603569 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 3 GET ipv6.google.com /gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - -#close 2013-03-22-14-38-11 +#open 2013-05-21-21-11-20 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html - - - +1257655302.514424 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 2 GET ipv6.google.com /csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - - +1257655303.603569 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 3 GET ipv6.google.com /gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - - +#close 2013-05-21-21-11-20 diff --git a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log index 51f3b28791..e88be88763 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-14-37-45 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - -1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - -#close 2013-03-22-14-37-45 +#open 2013-05-21-21-11-21 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - - +1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - - +#close 2013-05-21-21-11-21 diff --git a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log index 5067915aff..8f2893caa7 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-28-21-35-15 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - application/octet-stream - - -#close 2013-03-28-21-35-15 +#open 2013-05-21-21-11-22 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - application/octet-stream - - - +#close 2013-05-21-21-11-22 diff --git a/testing/btest/Baseline/core.tunnels.teredo/http.log b/testing/btest/Baseline/core.tunnels.teredo/http.log index f8be9be69b..4e3cdfd61d 100644 --- a/testing/btest/Baseline/core.tunnels.teredo/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo/http.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-14-37-44 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - text/plain - - -1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - -1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - -1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - application/xml - - -#close 2013-03-22-14-37-44 +#open 2013-05-21-21-11-21 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - text/plain - - - +1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - - +1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - - +1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - application/xml - - - +#close 2013-05-21-21-11-21 diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log index 4ad6d6cd60..65ec33186e 100644 --- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-14-37-44 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - -1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - -#close 2013-03-22-14-37-44 +#open 2013-05-21-21-11-22 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - - +1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - - +#close 2013-05-21-21-11-22 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 06652e37e7..b7585a1477 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-05-17-03-57-47 +#open 2013-07-05-05-20-50 #fields name #types string scripts/base/init-bare.bro @@ -13,31 +13,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/bro.bif.bro build/scripts/base/bif/reporter.bif.bro build/scripts/base/bif/event.bif.bro - scripts/base/frameworks/logging/__load__.bro - scripts/base/frameworks/logging/main.bro - build/scripts/base/bif/logging.bif.bro - scripts/base/frameworks/logging/postprocessors/__load__.bro - scripts/base/frameworks/logging/postprocessors/scp.bro - scripts/base/frameworks/logging/postprocessors/sftp.bro - scripts/base/frameworks/logging/writers/ascii.bro - scripts/base/frameworks/logging/writers/dataseries.bro - scripts/base/frameworks/logging/writers/sqlite.bro - scripts/base/frameworks/logging/writers/elasticsearch.bro - scripts/base/frameworks/logging/writers/none.bro - scripts/base/frameworks/input/__load__.bro - scripts/base/frameworks/input/main.bro - build/scripts/base/bif/input.bif.bro - scripts/base/frameworks/input/readers/ascii.bro - scripts/base/frameworks/input/readers/raw.bro - scripts/base/frameworks/input/readers/benchmark.bro - scripts/base/frameworks/input/readers/binary.bro - scripts/base/frameworks/input/readers/sqlite.bro - scripts/base/frameworks/analyzer/__load__.bro - scripts/base/frameworks/analyzer/main.bro - build/scripts/base/bif/analyzer.bif.bro - scripts/base/frameworks/file-analysis/__load__.bro - scripts/base/frameworks/file-analysis/main.bro - build/scripts/base/bif/file_analysis.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -50,6 +25,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro build/scripts/base/bif/plugins/Bro_FTP.functions.bif.bro build/scripts/base/bif/plugins/Bro_File.events.bif.bro + build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro build/scripts/base/bif/plugins/Bro_Finger.events.bif.bro build/scripts/base/bif/plugins/Bro_GTPv1.events.bif.bro build/scripts/base/bif/plugins/Bro_Gnutella.events.bif.bro @@ -85,6 +61,32 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro + scripts/base/frameworks/logging/__load__.bro + scripts/base/frameworks/logging/main.bro + build/scripts/base/bif/logging.bif.bro + scripts/base/frameworks/logging/postprocessors/__load__.bro + scripts/base/frameworks/logging/postprocessors/scp.bro + scripts/base/frameworks/logging/postprocessors/sftp.bro + scripts/base/frameworks/logging/writers/ascii.bro + scripts/base/frameworks/logging/writers/dataseries.bro + scripts/base/frameworks/logging/writers/sqlite.bro + scripts/base/frameworks/logging/writers/elasticsearch.bro + scripts/base/frameworks/logging/writers/none.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/main.bro + build/scripts/base/bif/input.bif.bro + scripts/base/frameworks/input/readers/ascii.bro + scripts/base/frameworks/input/readers/raw.bro + scripts/base/frameworks/input/readers/benchmark.bro + scripts/base/frameworks/input/readers/binary.bro + scripts/base/frameworks/input/readers/sqlite.bro + scripts/base/frameworks/analyzer/__load__.bro + scripts/base/frameworks/analyzer/main.bro + scripts/base/frameworks/packet-filter/utils.bro + build/scripts/base/bif/analyzer.bif.bro + scripts/base/frameworks/file-analysis/__load__.bro + scripts/base/frameworks/file-analysis/main.bro + build/scripts/base/bif/file_analysis.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2013-05-17-03-57-47 +#close 2013-07-05-05-20-50 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index cb92b663f0..999fd7c841 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-05-17-03-58-48 +#open 2013-07-10-21-18-31 #fields name #types string scripts/base/init-bare.bro @@ -13,31 +13,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/bro.bif.bro build/scripts/base/bif/reporter.bif.bro build/scripts/base/bif/event.bif.bro - scripts/base/frameworks/logging/__load__.bro - scripts/base/frameworks/logging/main.bro - build/scripts/base/bif/logging.bif.bro - scripts/base/frameworks/logging/postprocessors/__load__.bro - scripts/base/frameworks/logging/postprocessors/scp.bro - scripts/base/frameworks/logging/postprocessors/sftp.bro - scripts/base/frameworks/logging/writers/ascii.bro - scripts/base/frameworks/logging/writers/dataseries.bro - scripts/base/frameworks/logging/writers/sqlite.bro - scripts/base/frameworks/logging/writers/elasticsearch.bro - scripts/base/frameworks/logging/writers/none.bro - scripts/base/frameworks/input/__load__.bro - scripts/base/frameworks/input/main.bro - build/scripts/base/bif/input.bif.bro - scripts/base/frameworks/input/readers/ascii.bro - scripts/base/frameworks/input/readers/raw.bro - scripts/base/frameworks/input/readers/benchmark.bro - scripts/base/frameworks/input/readers/binary.bro - scripts/base/frameworks/input/readers/sqlite.bro - scripts/base/frameworks/analyzer/__load__.bro - scripts/base/frameworks/analyzer/main.bro - build/scripts/base/bif/analyzer.bif.bro - scripts/base/frameworks/file-analysis/__load__.bro - scripts/base/frameworks/file-analysis/main.bro - build/scripts/base/bif/file_analysis.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -50,6 +25,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro build/scripts/base/bif/plugins/Bro_FTP.functions.bif.bro build/scripts/base/bif/plugins/Bro_File.events.bif.bro + build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro build/scripts/base/bif/plugins/Bro_Finger.events.bif.bro build/scripts/base/bif/plugins/Bro_GTPv1.events.bif.bro build/scripts/base/bif/plugins/Bro_Gnutella.events.bif.bro @@ -85,6 +61,32 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro + scripts/base/frameworks/logging/__load__.bro + scripts/base/frameworks/logging/main.bro + build/scripts/base/bif/logging.bif.bro + scripts/base/frameworks/logging/postprocessors/__load__.bro + scripts/base/frameworks/logging/postprocessors/scp.bro + scripts/base/frameworks/logging/postprocessors/sftp.bro + scripts/base/frameworks/logging/writers/ascii.bro + scripts/base/frameworks/logging/writers/dataseries.bro + scripts/base/frameworks/logging/writers/sqlite.bro + scripts/base/frameworks/logging/writers/elasticsearch.bro + scripts/base/frameworks/logging/writers/none.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/main.bro + build/scripts/base/bif/input.bif.bro + scripts/base/frameworks/input/readers/ascii.bro + scripts/base/frameworks/input/readers/raw.bro + scripts/base/frameworks/input/readers/benchmark.bro + scripts/base/frameworks/input/readers/binary.bro + scripts/base/frameworks/input/readers/sqlite.bro + scripts/base/frameworks/analyzer/__load__.bro + scripts/base/frameworks/analyzer/main.bro + scripts/base/frameworks/packet-filter/utils.bro + build/scripts/base/bif/analyzer.bif.bro + scripts/base/frameworks/file-analysis/__load__.bro + scripts/base/frameworks/file-analysis/main.bro + build/scripts/base/bif/file_analysis.bif.bro scripts/base/init-default.bro scripts/base/utils/site.bro scripts/base/utils/patterns.bro @@ -176,6 +178,7 @@ scripts/base/init-default.bro scripts/base/protocols/modbus/__load__.bro scripts/base/protocols/modbus/consts.bro scripts/base/protocols/modbus/main.bro + scripts/base/protocols/pop3/__load__.bro scripts/base/protocols/smtp/__load__.bro scripts/base/protocols/smtp/main.bro scripts/base/protocols/smtp/entities.bro @@ -189,6 +192,7 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/__load__.bro scripts/base/protocols/syslog/consts.bro scripts/base/protocols/syslog/main.bro + scripts/base/protocols/tunnels/__load__.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-05-17-03-58-48 +#close 2013-07-10-21-18-31 diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index aa69373171..be7e6e5692 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-21-05-55 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1363986354.505533 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - -#close 2013-03-22-21-05-56 +#open 2013-05-21-21-11-32 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1369170691.550143 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-05-21-21-11-33 diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index 5ecca912f8..be7e6e5692 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-04-10-15-49-37 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1365608977.146651 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - -#close 2013-04-10-15-49-38 +#open 2013-05-21-21-11-32 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1369170691.550143 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-05-21-21-11-33 diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index 2531eb4bc0..ae693399c3 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-21-03-17 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1363986197.076696 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - -#close 2013-03-22-21-03-18 +#open 2013-05-21-21-11-40 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1369170699.511968 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-05-21-21-11-41 diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index e8f1872b95..ae693399c3 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-04-10-15-48-08 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1365608887.935644 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - -#close 2013-04-10-15-48-09 +#open 2013-05-21-21-11-40 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1369170699.511968 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-05-21-21-11-41 diff --git a/testing/btest/Baseline/language.table-redef/out b/testing/btest/Baseline/language.table-redef/out new file mode 100644 index 0000000000..fd1939df7e --- /dev/null +++ b/testing/btest/Baseline/language.table-redef/out @@ -0,0 +1,6 @@ +{ +[def] = 99.0, +[neat] = 1.0, +[cool] = 28.0, +[abc] = 8.0 +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out index 45756e5323..ddc3449a4c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out @@ -1,23 +1,23 @@ FILE_NEW -BYYd1GSNX5c, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain -file_stream, BYYd1GSNX5c, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea -file_chunk, BYYd1GSNX5c, 1500, 0, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea -file_stream, BYYd1GSNX5c, 1024, se script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices f -file_chunk, BYYd1GSNX5c, 1024, 1500, se script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices f -file_stream, BYYd1GSNX5c, 476, ormat for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tools/check-release to run before making releases.^J (Robin Sommer)^J^J * devel-tools/update-changes gets a new option -a to amend to^J previous commit if possible. Default is now not to (used to be the^J opposite). (Robin Sommer)^J^J * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J * Change distclean to only remove build dir. (Jon Siwek)^J^J * Make dist now cleans the -file_chunk, BYYd1GSNX5c, 476, 2524, ormat for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tools/check-release to run before making releases.^J (Robin Sommer)^J^J * devel-tools/update-changes gets a new option -a to amend to^J previous commit if possible. Default is now not to (used to be the^J opposite). (Robin Sommer)^J^J * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J * Change distclean to only remove build dir. (Jon Siwek)^J^J * Make dist now cleans the -file_stream, BYYd1GSNX5c, 1024, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP -file_chunk, BYYd1GSNX5c, 1024, 3000, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP -file_stream, BYYd1GSNX5c, 476, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J -file_chunk, BYYd1GSNX5c, 476, 4024, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J -file_stream, BYYd1GSNX5c, 205, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J -file_chunk, BYYd1GSNX5c, 205, 4500, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J +file_stream, file #0, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea +file_chunk, file #0, 1500, 0, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea +file_stream, file #0, 1024, se script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices f +file_chunk, file #0, 1024, 1500, se script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices f +file_stream, file #0, 476, ormat for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tools/check-release to run before making releases.^J (Robin Sommer)^J^J * devel-tools/update-changes gets a new option -a to amend to^J previous commit if possible. Default is now not to (used to be the^J opposite). (Robin Sommer)^J^J * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J * Change distclean to only remove build dir. (Jon Siwek)^J^J * Make dist now cleans the +file_chunk, file #0, 476, 2524, ormat for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tools/check-release to run before making releases.^J (Robin Sommer)^J^J * devel-tools/update-changes gets a new option -a to amend to^J previous commit if possible. Default is now not to (used to be the^J opposite). (Robin Sommer)^J^J * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J * Change distclean to only remove build dir. (Jon Siwek)^J^J * Make dist now cleans the +file_stream, file #0, 1024, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP +file_chunk, file #0, 1024, 3000, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP +file_stream, file #0, 476, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J +file_chunk, file #0, 476, 4024, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J +file_stream, file #0, 205, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J +file_chunk, file #0, 205, 4500, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J FILE_STATE_REMOVE -BYYd1GSNX5c, 4705, 0 +file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] total bytes: 4705 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out index ab736c80db..4b572d5df9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out @@ -1,11 +1,11 @@ FILE_NEW -BYYd1GSNX5c, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain FILE_STATE_REMOVE -BYYd1GSNX5c, 4705, 0 +file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] total bytes: 4705 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout similarity index 81% rename from testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout rename to testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout index 5ee716da85..160a51a543 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout @@ -1,20 +1,20 @@ FILE_NEW -Cvu8OAp0WEd, 0, 0 +file #0, 0, 0 MIME_TYPE application/x-dosexec FILE_STATE_REMOVE -Cvu8OAp0WEd, 1022920, 0 +file #0, 1022920, 0 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] total bytes: 1022920 source: HTTP FILE_NEW -Cvu8OAp0WEd, 0, 0 +file #1, 0, 0 MIME_TYPE application/octet-stream FILE_TIMEOUT FILE_TIMEOUT FILE_STATE_REMOVE -Cvu8OAp0WEd, 206024, 0 +file #1, 206024, 0 [orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp] total bytes: 1022920 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out index 006c66ebe1..f7182027aa 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out @@ -1,5 +1,5 @@ FILE_NEW -BYYd1GSNX5c, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out index a24c711b36..4463db6958 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out @@ -1,11 +1,11 @@ FILE_NEW -5LcdtqrLA97, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER The Nationa MIME_TYPE text/x-pascal FILE_STATE_REMOVE -5LcdtqrLA97, 16557, 0 +file #0, 16557, 0 [orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp] source: FTP_DATA MD5: 7192a8075196267203adb3dfaa5c908d diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out index 20474cab30..2b46d02042 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out @@ -1,11 +1,11 @@ FILE_NEW -FBfDYB0kA49, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER {^J "origin MIME_TYPE text/plain FILE_STATE_REMOVE -FBfDYB0kA49, 197, 0 +file #0, 197, 0 [orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp] source: HTTP MD5: 5baba7eea57bc8a42a92c817ed566d72 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out index 1668467841..bb2f622969 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out @@ -1,11 +1,11 @@ FILE_NEW -BYYd1GSNX5c, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain FILE_STATE_REMOVE -BYYd1GSNX5c, 4705, 0 +file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] total bytes: 4705 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/1-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/1-file new file mode 100644 index 0000000000..77356c3140 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/1-file @@ -0,0 +1 @@ +test diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/2-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/2-file new file mode 100644 index 0000000000..ac2a9e002d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/2-file @@ -0,0 +1 @@ +test2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/3-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/3-file new file mode 100644 index 0000000000..ae48ec8c20 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/3-file @@ -0,0 +1 @@ +test3 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/4-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/4-file new file mode 100644 index 0000000000..8f0eb247e3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/4-file @@ -0,0 +1,21 @@ +{ + "data": "", + "form": { + "example": "test", + "example2": "test2", + "example3": "test3" + }, + "origin": "141.142.228.5", + "json": null, + "url": "http://httpbin.org/post", + "args": {}, + "headers": { + "Content-Type": "multipart/form-data; boundary=----------------------------4ebf00fbcf09", + "User-Agent": "curl/7.30.0", + "Connection": "close", + "Accept": "*/*", + "Content-Length": "350", + "Host": "httpbin.org" + }, + "files": {} +} \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out new file mode 100644 index 0000000000..4b6fa76c0c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out @@ -0,0 +1,53 @@ +FILE_NEW +file #0, 0, 0 +FILE_BOF_BUFFER +test^M^J +MIME_TYPE +text/plain +FILE_STATE_REMOVE +file #0, 6, 0 +[orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] +source: HTTP +MD5: 9f06243abcb89c70e0c331c61d871fa7 +SHA1: fde773a18bb29f5ed65e6f0a7aa717fd1fa485d4 +SHA256: 837ccb607e312b170fac7383d7ccfd61fa5072793f19a25e75fbacb56539b86b +FILE_NEW +file #1, 0, 0 +FILE_BOF_BUFFER +test2^M^J +MIME_TYPE +text/plain +FILE_STATE_REMOVE +file #1, 7, 0 +[orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] +source: HTTP +MD5: d68af81ef370b3873d50f09140068810 +SHA1: 51a7b6f2d91f6a87822dc04560f2972bc14fc97e +SHA256: de0edd0ac4a705aff70f34734e90a1d0a1d8b76abe4bb53f3ea934bc105b3b17 +FILE_NEW +file #2, 0, 0 +FILE_BOF_BUFFER +test3^M^J +MIME_TYPE +text/plain +FILE_STATE_REMOVE +file #2, 7, 0 +[orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] +source: HTTP +MD5: 1a3d75d44753ad246f0bd333cdaf08b0 +SHA1: 4f98809ab09272dfcc58266e3f23ae2393f70e76 +SHA256: 018c67a2c30ed9977e1dddfe98cac542165dac355cf9764c91a362613e752933 +FILE_NEW +file #3, 0, 0 +FILE_BOF_BUFFER +{^J "data": +MIME_TYPE +text/plain +FILE_STATE_REMOVE +file #3, 465, 0 +[orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] +total bytes: 465 +source: HTTP +MD5: 226244811006caf4ac904344841168dd +SHA1: 7222902b8b8e68e25c0422e7f8bdf344efeda54d +SHA256: dd485ecf240e12807516b0a27718fc3ab9a17c1158a452967343c98cefba07a0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out index 50bacac717..f8f2538e92 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out @@ -1,10 +1,10 @@ FILE_NEW -1QXxzNpRT3h, 0, 0 +file #0, 0, 0 MIME_TYPE application/pdf FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE -1QXxzNpRT3h, 555523, 0 +file #0, 555523, 0 [orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp] [orig_h=10.101.84.70, orig_p=10977/tcp, resp_h=129.174.93.161, resp_p=80/tcp] total bytes: 555523 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out index 5b892c7e9a..b2a0cb66a2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out @@ -1,19 +1,19 @@ FILE_NEW -Cvu8OAp0WEd, 0, 0 +file #0, 0, 0 MIME_TYPE application/x-dosexec FILE_STATE_REMOVE -Cvu8OAp0WEd, 1022920, 0 +file #0, 1022920, 0 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] total bytes: 1022920 source: HTTP FILE_NEW -Cvu8OAp0WEd, 0, 0 +file #1, 0, 0 MIME_TYPE application/octet-stream FILE_TIMEOUT FILE_STATE_REMOVE -Cvu8OAp0WEd, 206024, 0 +file #1, 206024, 0 [orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp] total bytes: 1022920 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out index 886abee0f2..7c5e9dfeca 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out @@ -1,10 +1,10 @@ FILE_NEW -me4WAjZH0Ik, 0, 0 +file #0, 0, 0 MIME_TYPE application/octet-stream FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE -me4WAjZH0Ik, 498702, 0 +file #0, 498702, 0 [orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp] [orig_h=10.45.179.94, orig_p=19953/tcp, resp_h=129.174.93.170, resp_p=80/tcp] total bytes: 498668 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out index fb045a346c..02ac2f0a7e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out @@ -1,37 +1,37 @@ FILE_NEW -FiqZGsUZjXk, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER /*^J******** MIME_TYPE text/plain FILE_STATE_REMOVE -FiqZGsUZjXk, 2675, 0 +file #0, 2675, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] source: HTTP MD5: b932c3310ce47e158d1a5a42e0b01279 SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706 SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42 FILE_NEW -GU8RrggV4f5, 0, 0 +file #1, 0, 0 FILE_BOF_BUFFER //-- Google MIME_TYPE text/plain FILE_STATE_REMOVE -GU8RrggV4f5, 21421, 0 +file #1, 21421, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] source: HTTP MD5: e732f7bf1d7cb4eedcb1661697d7bc8c SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983 SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da FILE_NEW -0afVj9ZG1J9, 0, 0 +file #2, 0, 0 FILE_BOF_BUFFER GIF89a^D\0^D\0\xb3 MIME_TYPE image/gif FILE_STATE_REMOVE -0afVj9ZG1J9, 94, 0 +file #2, 94, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] total bytes: 94 source: HTTP @@ -39,13 +39,13 @@ MD5: d903de7e30db1691d3130ba5eae6b9a7 SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315 FILE_NEW -oMJlhgZt8Nh, 0, 0 +file #3, 0, 0 FILE_BOF_BUFFER \x89PNG^M^J^Z^J\0\0\0 MIME_TYPE image/png FILE_STATE_REMOVE -oMJlhgZt8Nh, 2349, 0 +file #3, 2349, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] total bytes: 2349 source: HTTP @@ -53,13 +53,13 @@ MD5: e0029eea80812e9a8e57b8d05d52938a SHA1: 560eab5a0177246827a94042dd103916d8765ac7 SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4 FILE_NEW -KajlXqmipId, 0, 0 +file #4, 0, 0 FILE_BOF_BUFFER \x89PNG^M^J^Z^J\0\0\0 MIME_TYPE image/png FILE_STATE_REMOVE -KajlXqmipId, 27579, 0 +file #4, 27579, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] total bytes: 27579 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out index 72868299f6..3103ecb39e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out @@ -1,11 +1,11 @@ FILE_NEW -1V1QkS1JR02, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER hello world MIME_TYPE text/plain FILE_STATE_REMOVE -1V1QkS1JR02, 11, 0 +file #0, 11, 0 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] total bytes: 11 source: HTTP @@ -13,13 +13,13 @@ MD5: 5eb63bbbe01eeed093cb22bb8f5acdc3 SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 FILE_NEW -IYuq13QwRPh, 0, 0 +file #1, 0, 0 FILE_BOF_BUFFER {^J "origin MIME_TYPE text/plain FILE_STATE_REMOVE -IYuq13QwRPh, 366, 0 +file #1, 366, 0 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] total bytes: 366 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout index a7d837475f..afeb32b334 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout @@ -1,11 +1,11 @@ FILE_NEW -nYgPNGLrZf9, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER #separator MIME_TYPE text/plain FILE_STATE_REMOVE -nYgPNGLrZf9, 311, 0 +file #0, 311, 0 source: ../input.log MD5: bf4dfa6169b74146da5236e918743599 SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out index 4d869319f7..36da7bdeed 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out @@ -1,11 +1,11 @@ FILE_NEW -A3OSdqG9zvk, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER PK^C^D^T\0\0\0^H\0\xae MIME_TYPE application/zip FILE_STATE_REMOVE -A3OSdqG9zvk, 42208, 0 +file #0, 42208, 0 [orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] source: IRC_DATA MD5: 8c0803242f549c2780cb88b9a9215c65 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log index 3dc2106d85..f95a70d50a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path file_analysis -#open 2013-05-17-00-55-16 -#fields id parent_id source is_orig last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size mime_type timedout conn_uids analyzers extracted_files md5 sha1 sha256 -#types string string string bool time count count count count interval count string bool table[string] table[enum] table[string] string string string -BYYd1GSNX5c - HTTP F 1362692527.009775 4705 4705 0 0 120.000000 1024 text/plain F UWkUyAuUGXf FileAnalysis::ANALYZER_SHA1,FileAnalysis::ANALYZER_EXTRACT,FileAnalysis::ANALYZER_DATA_EVENT,FileAnalysis::ANALYZER_MD5,FileAnalysis::ANALYZER_SHA256 BYYd1GSNX5c-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 -#close 2013-05-17-00-55-16 +#open 2013-06-07-18-51-45 +#fields id parent_id source is_orig last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size mime_type timedout conn_uids extracted_files md5 sha1 sha256 +#types string string string bool time count count count count interval count string bool table[string] table[string] string string string +BYYd1GSNX5c - HTTP F 1362692527.009775 4705 4705 0 0 120.000000 1024 text/plain F UWkUyAuUGXf BYYd1GSNX5c-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 +#close 2013-06-07-18-51-46 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out index eeb138cba7..ac4e6e50fa 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out @@ -1,37 +1,37 @@ FILE_NEW -mR3f2AAKo11, 0, 0 +file #0, 0, 0 FILE_BOF_BUFFER Hello^M^J^M^J ^M MIME_TYPE text/plain FILE_STATE_REMOVE -mR3f2AAKo11, 79, 0 +file #0, 79, 0 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] source: SMTP MD5: 92bca2e6cdcde73647125da7dccbdd07 SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13 SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5 FILE_NEW -svBvmJEWan2, 0, 0 +file #1, 0, 0 FILE_BOF_BUFFER , want_record=F, ev=line +{ +print outfile, A::description; +print outfile, A::tpe; +print outfile, A::s; +try = try + 1; +if (2 == try) +{ +Input::remove(input2); +close(outfile); +terminate(); +} + +}, config={ +[stdin] = hello^Jthere^A^B^C^D^E^A^B^Cyay +}] +Input::EVENT_NEW +hello +[source=cat |, reader=Input::READER_RAW, mode=Input::STREAM, name=input2, fields=, want_record=F, ev=line +{ +print outfile, A::description; +print outfile, A::tpe; +print outfile, A::s; +try = try + 1; +if (2 == try) +{ +Input::remove(input2); +close(outfile); +terminate(); +} + +}, config={ +[stdin] = hello^Jthere^A^B^C^D^E^A^B^Cyay +}] +Input::EVENT_NEW +there^A^B^C^D^E^A^B^Cyay diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/test.txt b/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/test.txt new file mode 100644 index 0000000000..0205cd7c3a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/test.txt @@ -0,0 +1,2 @@ +hello +thereyay \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestream/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestream/out new file mode 100644 index 0000000000..59a5f2c116 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.executestream/out @@ -0,0 +1,153 @@ +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (8 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +terminate(); +} + +}, config={ + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (8 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +terminate(); +} + +}, config={ + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (8 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +terminate(); +} + +}, config={ + +}] +Input::EVENT_NEW +q3r3057fdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (8 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +terminate(); +} + +}, config={ + +}] +Input::EVENT_NEW +sdfs\d +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (8 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +terminate(); +} + +}, config={ + +}] +Input::EVENT_NEW + +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (8 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +terminate(); +} + +}, config={ + +}] +Input::EVENT_NEW +dfsdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (8 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +terminate(); +} + +}, config={ + +}] +Input::EVENT_NEW +sdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (8 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +terminate(); +} + +}, config={ + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.long/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.long/out new file mode 100644 index 0000000000..fac8e79c0b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.long/out @@ -0,0 +1,2 @@ +Input::EVENT_NEW +8193 diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.rereadraw/out similarity index 100% rename from testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out rename to testing/btest/Baseline/scripts.base.frameworks.input.raw.rereadraw/out diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw.stderr/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.stderr/out new file mode 100644 index 0000000000..b7f857339d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw.stderr/out @@ -0,0 +1,27 @@ +Input::EVENT_NEW +..: +F +Input::EVENT_NEW +bro +F +Input::EVENT_NEW +out +F +Input::EVENT_NEW +stderr.bro +F +Input::EVENT_NEW +stderr output contained nonexistant +T +Input::EVENT_NEW +stderr output contained nonexistant +T +Input::EVENT_NEW +stderr output contained nonexistant +T +done +End of Data event +input +Process finished event +input +Exit code != 0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw.streamraw/out similarity index 100% rename from testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out rename to testing/btest/Baseline/scripts.base.frameworks.input.raw.streamraw/out diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log index 472dfcce39..026b25b161 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-14-38-21 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html - - -#close 2013-03-22-14-38-21 +#open 2013-05-21-21-11-23 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html - - - +#close 2013-05-21-21-11-23 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt index bcfad8fd80..e919233b79 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt @@ -34,7 +34,8 @@ - + + @@ -61,21 +62,22 @@ - + + # Extent, type='http' -ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.918358 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.952307 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.952296 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.954820 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.962687 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.975934 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.976436 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475168.979264 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 -1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.918358 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.952307 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.952296 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.954820 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.962687 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.975934 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.976436 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.979264 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.wikipedia/http.select b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.wikipedia/http.select index 2f3c305a39..a228fa2e11 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.wikipedia/http.select +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.wikipedia/http.select @@ -1,14 +1,14 @@ -1300475168.78402|j4u32Pc5bif|141.142.220.118|48649|208.80.152.118|80|1|GET|bits.wikimedia.org|/skins-1.5/monobook/main.css|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.91602|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/6/63/Wikipedia-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.91618|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.91836|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/b/bd/Bookshelf-40x201_6.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.9523|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.95231|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.95482|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.96269|i2rO3KD1Syg|141.142.220.118|35642|208.80.152.2|80|1|GET|meta.wikimedia.org|/images/wikimedia-button.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.97593|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.97644|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475168.97926|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475169.01459|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475169.01462|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| -1300475169.01493|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)|||||| +1300475168.78402|j4u32Pc5bif|141.142.220.118|48649|208.80.152.118|80|1|GET|bits.wikimedia.org|/skins-1.5/monobook/main.css|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.91602|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/6/63/Wikipedia-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.91618|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.91836|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/b/bd/Bookshelf-40x201_6.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.9523|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.95231|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.95482|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|1|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.96269|i2rO3KD1Syg|141.142.220.118|35642|208.80.152.2|80|1|GET|meta.wikimedia.org|/images/wikimedia-button.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.97593|VW0XPVINV8a|141.142.220.118|49997|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.97644|3PKsZ2Uye21|141.142.220.118|49996|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475168.97926|GSxOnSLghOa|141.142.220.118|49998|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475169.01459|P654jzLoe3a|141.142.220.118|49999|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475169.01462|Tw8jXtpTGu6|141.142.220.118|50000|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| +1300475169.01493|0Q4FH8sESw5|141.142.220.118|50001|208.80.152.3|80|2|GET|upload.wikimedia.org|/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png|http://www.wikipedia.org/|Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15|0|0|304|Not Modified||||(empty)||||||| diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log index 5d707d5cb8..6b7bea88c9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log @@ -3,21 +3,21 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-14-38-24 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.918358 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.952307 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.952296 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.954820 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.962687 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.975934 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.976436 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475168.979264 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -#close 2013-03-22-14-38-24 +#open 2013-05-21-21-11-23 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.918358 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.952307 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.952296 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.954820 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.962687 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.975934 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.976436 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475168.979264 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - +#close 2013-05-21-21-11-23 diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/dns.log b/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/dns.log new file mode 100644 index 0000000000..ca071ee8ef --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/dns.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path dns +#open 2013-07-18-13-21-52 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected +#types time string addr port addr port enum count string count string count string count string bool bool bool bool count vector[string] vector[interval] bool +1363716396.798072 UWkUyAuUGXf 55.247.223.174 27285 222.195.43.124 53 udp 21140 www.cmu.edu 1 C_INTERNET 1 A 0 NOERROR T F F F 1 www-cmu.andrew.cmu.edu,www-cmu-2.andrew.cmu.edu,128.2.10.163,www-cmu.andrew.cmu.edu 86400.000000,5.000000,21600.000000,86400.000000 F +1363716396.798374 UWkUyAuUGXf 55.247.223.174 27285 222.195.43.124 53 udp 21140 - - - - - 0 NOERROR T F F F 0 www-cmu-2.andrew.cmu.edu,128.2.10.163 5.000000,21600.000000 F +#close 2013-07-18-13-21-52 diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log b/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log new file mode 100644 index 0000000000..c7de92f894 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open 2013-07-18-13-21-52 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer +#types time string addr port addr port string string bool string +1363716396.798286 UWkUyAuUGXf 55.247.223.174 27285 222.195.43.124 53 DNS_RR_unknown_type - F bro +1363716396.798374 UWkUyAuUGXf 55.247.223.174 27285 222.195.43.124 53 dns_unmatched_reply - F bro +#close 2013-07-18-13-21-52 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/extractions b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/extractions new file mode 100644 index 0000000000..1933de9992 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/extractions @@ -0,0 +1,22 @@ +-rw-rw-r-- 1 600 netbsd 50158695 Feb 21 03:10 ls-lRA.gz +-rw-rw-r-- 1 600 netbsd 50158695 Feb 21 03:10 ls-lRA.gz +-rw-rw-r-- 1 root wheel 77 Aug 16 2009 robots.txt +-rw-rw-r-- 1 root wheel 77 Aug 16 2009 robots.txt +Disallow: *.bz2 +Disallow: *.bz2 +Disallow: *.gz +Disallow: *.gz +Disallow: *.tbz +Disallow: *.tbz +Disallow: *.tgz +Disallow: *.tgz +User-agent: * +User-agent: * +drwxr-x--x 3 root wheel 512 Aug 16 2009 etc +drwxr-x--x 3 root wheel 512 Aug 16 2009 etc +drwxr-xr-x 7 root wheel 512 Aug 20 2009 pub +drwxr-xr-x 7 root wheel 512 Aug 20 2009 pub +lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control +lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control +total 98028 +total 98028 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-0.dat b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-0.dat deleted file mode 100644 index 8bd2e31300..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-0.dat +++ /dev/null @@ -1,6 +0,0 @@ -total 98028 -lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control -drwxr-x--x 3 root wheel 512 Aug 16 2009 etc --rw-rw-r-- 1 600 netbsd 50158695 Feb 21 03:10 ls-lRA.gz -drwxr-xr-x 7 root wheel 512 Aug 20 2009 pub --rw-rw-r-- 1 root wheel 77 Aug 16 2009 robots.txt diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-1.dat b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-1.dat deleted file mode 100644 index a59965e6f6..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-1.dat +++ /dev/null @@ -1,5 +0,0 @@ -User-agent: * -Disallow: *.tgz -Disallow: *.gz -Disallow: *.tbz -Disallow: *.bz2 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-2.dat b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-2.dat deleted file mode 100644 index 8bd2e31300..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-2.dat +++ /dev/null @@ -1,6 +0,0 @@ -total 98028 -lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control -drwxr-x--x 3 root wheel 512 Aug 16 2009 etc --rw-rw-r-- 1 600 netbsd 50158695 Feb 21 03:10 ls-lRA.gz -drwxr-xr-x 7 root wheel 512 Aug 20 2009 pub --rw-rw-r-- 1 root wheel 77 Aug 16 2009 robots.txt diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-3.dat b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-3.dat deleted file mode 100644 index a59965e6f6..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp-item-3.dat +++ /dev/null @@ -1,5 +0,0 @@ -User-agent: * -Disallow: *.tgz -Disallow: *.gz -Disallow: *.tbz -Disallow: *.bz2 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log index 948d737979..e77f59dc44 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log @@ -3,19 +3,19 @@ #empty_field (empty) #unset_field - #path ftp -#open 2013-05-18-00-48-19 +#open 2013-06-07-18-57-22 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file #types time string addr port addr port string string string string string count count string table[string] bool addr addr port string 1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) (empty) T 141.142.220.235 199.233.217.249 56666 - 1329843175.791528 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test LIST - - - 226 Transfer complete. (empty) - - - - - 1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) (empty) T 141.142.220.235 199.233.217.249 56667 - -1329843193.984222 arKYeMETxOg 141.142.220.235 37604 199.233.217.249 56666 - - - - - - - (empty) - - - - ftp-item-pVhQhhFsB2b-0.dat -1329843193.984222 k6kgXLOoSKl 141.142.220.235 59378 199.233.217.249 56667 - - - - - - - (empty) - - - - ftp-item-fFCPkV1sEsc-1.dat +1329843193.984222 arKYeMETxOg 141.142.220.235 37604 199.233.217.249 56666 - - - - - - - (empty) - - - - ftp-item-pVhQhhFsB2b.dat +1329843193.984222 k6kgXLOoSKl 141.142.220.235 59378 199.233.217.249 56667 - - - - - - - (empty) - - - - ftp-item-fFCPkV1sEsc.dat 1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - 1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 33582 - 1329843194.095782 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test LIST - - - 226 Transfer complete. (empty) - - - - - 1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 37835 - -1329843199.968212 nQcgTWjvg4c 199.233.217.249 61920 141.142.220.235 33582 - - - - - - - (empty) - - - - ftp-item-g3zS3MuJFh-2.dat +1329843199.968212 nQcgTWjvg4c 199.233.217.249 61920 141.142.220.235 33582 - - - - - - - (empty) - - - - ftp-item-g3zS3MuJFh.dat 1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -1329843200.079930 j4u32Pc5bif 199.233.217.249 61918 141.142.220.235 37835 - - - - - - - (empty) - - - - ftp-item-lMf4UWRkEO5-3.dat -#close 2013-05-18-00-48-19 +1329843200.079930 j4u32Pc5bif 199.233.217.249 61918 141.142.220.235 37835 - - - - - - - (empty) - - - - ftp-item-lMf4UWRkEO5.dat +#close 2013-06-07-18-57-22 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index 8053b3a287..edbee28991 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-14-38-28 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - - -#close 2013-03-22-14-38-28 +#open 2013-05-21-21-11-24 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - - - +#close 2013-05-21-21-11-24 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log index 9c891f4c74..53b80e5e9e 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-17-23-19-09 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - http-item-54zlJFqn0x6-0.dat -#close 2013-05-17-23-19-09 +#open 2013-06-07-19-04-27 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - http-item-54zlJFqn0x6.dat +#close 2013-06-07-19-04-27 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log index 9dafcc74e0..54a75f4697 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log @@ -3,56 +3,56 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-25-20-20-22 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1354328870.191989 UWkUyAuUGXf 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - - 0 962 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328874.237327 arKYeMETxOg 128.2.6.136 46563 173.194.75.103 80 1 OPTIONS www.google.com HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328874.299063 k6kgXLOoSKl 128.2.6.136 46564 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328874.342591 nQcgTWjvg4c 128.2.6.136 46565 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328874.364020 j4u32Pc5bif 128.2.6.136 46566 173.194.75.103 80 1 GET www.google.com / - - 0 43911 200 OK - - - (empty) - - - text/html - - -1354328878.470424 TEfuqmmG4bh 128.2.6.136 46567 173.194.75.103 80 1 GET www.google.com / - - 0 43983 200 OK - - - (empty) - - - text/html - - -1354328882.575456 FrJExwHcSal 128.2.6.136 46568 173.194.75.103 80 1 GET www.google.com /HTTP/1.1 - - 0 1207 403 Forbidden - - - (empty) - - - text/html - - -1354328882.928027 5OKnoww6xl4 128.2.6.136 46569 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328882.968948 3PKsZ2Uye21 128.2.6.136 46570 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328882.990373 VW0XPVINV8a 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - - 0 43913 200 OK - - - (empty) - - - text/html - - -1354328887.114613 fRFu0wcOle6 128.2.6.136 46572 173.194.75.103 80 0 - - - - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328891.161077 qSsw6ESzHV4 128.2.6.136 46573 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328891.204740 iE6yhOq3SF 128.2.6.136 46574 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328891.245592 GSxOnSLghOa 128.2.6.136 46575 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328891.287655 qCaWGmzFtM5 128.2.6.136 46576 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328891.309065 70MGiRM1Qf4 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328895.355012 h5DsfNtYzi1 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328895.416133 P654jzLoe3a 128.2.6.136 46579 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328895.459490 Tw8jXtpTGu6 128.2.6.136 46580 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328895.480865 c4Zw9TmAE05 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328899.526682 EAr0uf4mhq 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328903.572533 GvmoxJFXdTa 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328903.634196 0Q4FH8sESw5 128.2.6.136 46584 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328903.676395 slFea8xwSmb 128.2.6.136 46585 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328903.697693 UfGkYA2HI2g 128.2.6.136 46586 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328907.743696 i2rO3KD1Syg 128.2.6.136 46587 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328911.790590 2cx26uAvUPl 128.2.6.136 46588 173.194.75.103 80 1 TRACE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328911.853464 BWaU4aSuwkc 128.2.6.136 46589 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328911.897044 10XodEwRycf 128.2.6.136 46590 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328911.918511 zno26fFZkrh 128.2.6.136 46591 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328915.964678 v5rgkJBig5l 128.2.6.136 46592 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328920.010458 eWZCH7OONC1 128.2.6.136 46593 173.194.75.103 80 1 DELETE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328920.072101 0Pwk3ntf8O3 128.2.6.136 46594 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328920.114526 0HKorjr8Zp7 128.2.6.136 46595 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328920.136714 yC2d6kVg709 128.2.6.136 46596 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - -1354328924.183211 VcgagLjnO92 128.2.6.136 46597 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.224567 bdRoHfaPBo3 128.2.6.136 46598 173.194.75.103 80 1 PUT www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.287402 zHqb7t7kv28 128.2.6.136 46599 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328924.328257 rrZWoMUQpv8 128.2.6.136 46600 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328924.350343 xNYSS2hJkle 128.2.6.136 46601 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.391728 vMVjlplKKbd 128.2.6.136 46602 173.194.75.103 80 1 POST www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.433150 3omNawSNrxj 128.2.6.136 46603 173.194.75.103 80 1 POST www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - -1354328924.496732 Rv8AJVfi9Zi 128.2.6.136 46604 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328924.537671 wEyF3OvvcQe 128.2.6.136 46605 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328924.559704 E490YZTUozc 128.2.6.136 46606 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - -1354328928.625437 YIeWJmXWNWj 128.2.6.136 46607 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - -1354328932.692706 ydiZblvsYri 128.2.6.136 46608 173.194.75.103 80 1 HEAD www.google.com /HTTP/1.1 - - 0 0 400 Bad Request - - - (empty) - - - - - - -1354328932.754657 HFYOnBqSE5e 128.2.6.136 46609 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -1354328932.796568 JcUvhfWUMgd 128.2.6.136 46610 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - -#close 2013-03-25-20-20-22 +#open 2013-05-21-21-11-25 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1354328870.191989 UWkUyAuUGXf 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - - 0 962 405 Method Not Allowed - - - (empty) - - - text/html - - - +1354328874.237327 arKYeMETxOg 128.2.6.136 46563 173.194.75.103 80 1 OPTIONS www.google.com HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328874.299063 k6kgXLOoSKl 128.2.6.136 46564 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328874.342591 nQcgTWjvg4c 128.2.6.136 46565 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328874.364020 j4u32Pc5bif 128.2.6.136 46566 173.194.75.103 80 1 GET www.google.com / - - 0 43911 200 OK - - - (empty) - - - text/html - - - +1354328878.470424 TEfuqmmG4bh 128.2.6.136 46567 173.194.75.103 80 1 GET www.google.com / - - 0 43983 200 OK - - - (empty) - - - text/html - - - +1354328882.575456 FrJExwHcSal 128.2.6.136 46568 173.194.75.103 80 1 GET www.google.com /HTTP/1.1 - - 0 1207 403 Forbidden - - - (empty) - - - text/html - - - +1354328882.928027 5OKnoww6xl4 128.2.6.136 46569 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328882.968948 3PKsZ2Uye21 128.2.6.136 46570 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328882.990373 VW0XPVINV8a 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - - 0 43913 200 OK - - - (empty) - - - text/html - - - +1354328887.114613 fRFu0wcOle6 128.2.6.136 46572 173.194.75.103 80 0 - - - - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - +1354328891.161077 qSsw6ESzHV4 128.2.6.136 46573 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328891.204740 iE6yhOq3SF 128.2.6.136 46574 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328891.245592 GSxOnSLghOa 128.2.6.136 46575 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328891.287655 qCaWGmzFtM5 128.2.6.136 46576 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328891.309065 70MGiRM1Qf4 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - - +1354328895.355012 h5DsfNtYzi1 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328895.416133 P654jzLoe3a 128.2.6.136 46579 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328895.459490 Tw8jXtpTGu6 128.2.6.136 46580 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328895.480865 c4Zw9TmAE05 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - - +1354328899.526682 EAr0uf4mhq 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328903.572533 GvmoxJFXdTa 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328903.634196 0Q4FH8sESw5 128.2.6.136 46584 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328903.676395 slFea8xwSmb 128.2.6.136 46585 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328903.697693 UfGkYA2HI2g 128.2.6.136 46586 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328907.743696 i2rO3KD1Syg 128.2.6.136 46587 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - - +1354328911.790590 2cx26uAvUPl 128.2.6.136 46588 173.194.75.103 80 1 TRACE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328911.853464 BWaU4aSuwkc 128.2.6.136 46589 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328911.897044 10XodEwRycf 128.2.6.136 46590 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328911.918511 zno26fFZkrh 128.2.6.136 46591 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - - +1354328915.964678 v5rgkJBig5l 128.2.6.136 46592 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - +1354328920.010458 eWZCH7OONC1 128.2.6.136 46593 173.194.75.103 80 1 DELETE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328920.072101 0Pwk3ntf8O3 128.2.6.136 46594 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328920.114526 0HKorjr8Zp7 128.2.6.136 46595 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328920.136714 yC2d6kVg709 128.2.6.136 46596 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - +1354328924.183211 VcgagLjnO92 128.2.6.136 46597 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - +1354328924.224567 bdRoHfaPBo3 128.2.6.136 46598 173.194.75.103 80 1 PUT www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - - +1354328924.287402 zHqb7t7kv28 128.2.6.136 46599 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328924.328257 rrZWoMUQpv8 128.2.6.136 46600 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328924.350343 xNYSS2hJkle 128.2.6.136 46601 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - +1354328924.391728 vMVjlplKKbd 128.2.6.136 46602 173.194.75.103 80 1 POST www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - +1354328924.433150 3omNawSNrxj 128.2.6.136 46603 173.194.75.103 80 1 POST www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - - +1354328924.496732 Rv8AJVfi9Zi 128.2.6.136 46604 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328924.537671 wEyF3OvvcQe 128.2.6.136 46605 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328924.559704 E490YZTUozc 128.2.6.136 46606 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - - +1354328928.625437 YIeWJmXWNWj 128.2.6.136 46607 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - - +1354328932.692706 ydiZblvsYri 128.2.6.136 46608 173.194.75.103 80 1 HEAD www.google.com /HTTP/1.1 - - 0 0 400 Bad Request - - - (empty) - - - - - - - +1354328932.754657 HFYOnBqSE5e 128.2.6.136 46609 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +1354328932.796568 JcUvhfWUMgd 128.2.6.136 46610 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +#close 2013-05-21-21-11-25 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log index 6073e9b563..97e797b4fb 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log @@ -3,12 +3,12 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-16-25-59 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - text/plain - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - text/plain - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - image/gif - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e - -#close 2013-03-22-16-25-59 +#open 2013-05-21-21-11-25 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - text/plain - - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - text/plain - - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - image/gif - - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a - - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e - - +#close 2013-05-21-21-11-25 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index d7791097a9..e22fb53103 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -3,12 +3,12 @@ #empty_field (empty) #unset_field - #path http -#open 2013-03-22-14-38-28 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - -#close 2013-03-22-14-38-28 +#open 2013-05-21-21-11-25 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string vector[string] vector[string] +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - - +#close 2013-05-21-21-11-25 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log b/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log new file mode 100644 index 0000000000..0bd15badef --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open 2013-06-07-19-57-15 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +1369159408.455878 UWkUyAuUGXf 141.142.228.5 57262 54.243.88.146 80 1 POST httpbin.org /post - curl/7.30.0 370 465 200 OK - - - (empty) - - - text/plain - http-item-lcf92jVphSl.dat,http-item-z8gOS6arddh.dat,http-item-tBYz7eElzTb.dat http-item-GVJrSB2Vxk6.dat +#close 2013-06-07-19-57-15 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log index 2d37e2626f..28ca448e05 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path irc -#open 2013-05-17-23-19-21 +#open 2013-06-07-19-08-42 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size dcc_mime_type extraction_file #types time string addr port addr port string string string string string string count string string 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item-A3OSdqG9zvk-0.dat -#close 2013-05-17-23-19-21 +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 application/zip irc-dcc-item-A3OSdqG9zvk.dat +#close 2013-06-07-19-08-42 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-1.dat b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions similarity index 96% rename from testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-1.dat rename to testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions index 9eb3055735..45d776a8e9 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-1.dat +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions @@ -1,264 +1,277 @@ -Version 4.9.9.1 -* Many bug fixes -* Improved editor -Version 4.9.9.0 -* Support for latest Mingw compiler system builds -* Bug fixes -Version 4.9.8.9 -* New code tooltip display -* Improved Indent/Unindent and Remove Comment -* Improved automatic indent -* Added support for the "interface" keyword -* WebUpdate should now report installation problems from PackMan -* New splash screen and association icons -* Improved installer -* Many bug fixes -Version 4.9.8.7 -* Added support for GCC > 3.2 -* Debug variables are now resent during next debug session -* Watched Variables not in correct context are now kept and updated when it is needed -* Added new compiler/linker options: 20 - - Strip executable - - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2) - - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow) -* "Default" button in Compiler Options is back -* Error messages parsing improved -* Bug fixes - -Version 4.9.8.5 -* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value") -* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features. -* Many bug fixes - -Version 4.9.8.4 -* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup -* Improved code completion cache -* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP -* Big speed up in function parameters listing while editing -* Bug fixes - -Version 4.9.8.3 -* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20 - include files can now be generated. -* Improved WebUpdate module -* Many bug fixes - -Version 4.9.8.2 -* New debug feature for DLLs: attach to a running process -* New project option: Use custom Makefile. 20 -* New WebUpdater module. -* Allow user to specify an alternate configuration file in Environment Options 20 + "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of + #included directly or indirectly)! + (available when right-clicking the class-browser (still can be overriden by using "-c" command line parameter). -* Lots of bug fixes. - -Version 4.9.8.1 -* When creating a DLL, the created static lib respects now the project-defined output directory - -Version 4.9.8.0 -* Changed position of compiler/linker parameters in Project Options. -* Improved help file -* Bug fixes - -Version 4.9.7.9 -* Resource errors are now reported in the Resource sheet -* Many bug fixes - -Version 4.9.7.8 -* Made whole bottom report control floating instead of only debug output. -* Many bug fixes - -Version 4.9.7.7 -* Printing settings are now saved -* New environment options : "watch variable under mouse" and "Report watch errors" -* Bug fixes - -Version 4.9.7.6 -* Debug variable browser -* Added possibility to include in a Template the Project's directories (include, libs and ressources) -* Changed tint of Class browser pictures colors to match the New Look style -* Bug fixes - -Version 4.9.7.5 -* Bug fixes - -Version 4.9.7.4 -* When compiling with debugging symbols, an extra definition is passed to the - compiler: -D__DEBUG__ -* Each project creates a _private.h file containing version - information definitions -* When compiling the current file only, no dependency checks are performed -* ~300% Speed-up in class parser -* Added "External programs" in Tools/Environment Options (for units "Open with") -* Added "Open with" in project units context menu -* Added "Classes" toolbar -* Fixed pre-compilation dependency checks to work correctly -* Added new file menu entry: Save Project As -* Bug-fix for double quotes in devcpp.cfg file read by vUpdate -* Other bug fixes - -Version 4.9.7.3 -* When adding debugging symbols on request, remove "-s" option from linker -* Compiling progress window -* Environment options : "Show progress window" and "Auto-close progress window" -* Bug fixes - -Version 4.9.7.2 -* Bug fixes - -Version 4.9.7.1 -* "Build priority" per-unit -* "Include file in linking process" per-unit -* New feature: compile current file only -* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744) -* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744) -* Necessary UI changes in Project Options -* Added display of project filename, project output and a summary of the project files in Project Options General tab. -* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0 -* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings) -* Resource files are treated as ordinary files now -* Updates in "Project Options/Files" code -* MSVC import now creates the folders structure of the original VC project -* Bug fixes - -Version 4.9.7.0 -* Allow customizing of per-unit compile command in projects -* Added two new macros: and -* Added support for macros in the "default source code" (Tools/Editor Options/Code) -* Separated layout info from project file. It is now kept in a different file (the same filename as the project's but with extension ".layout"). If you - have your project under CVS control, you ''ll know why this had to happen... -* Compiler settings per-project -* Compiler set per-project -* Implemented new compiler settings framework -* "Compile as C++" per-unit -* "Include file in compilation process" per-unit -* Project version info (creates the relevant VERSIONINFO struct in the private - resource) -* Support XP Themes (creates the CommonControls 6.0 manifest file and includes - it in the private resource) -* Added CVS "login" and "logout" commands -* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows. -* Added "Add Library" button in Project Options -* Bug fixes - -Version 4.9.6.9 -* Implemented search in help files for the word at cursor (context sensitive help) -* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2) -* Added "Files" tab in CVS form to allow selection of more than one file for - the requested CVS action + - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow) + - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20 + - Strip executable + -c 20 -Version 4.9.6.8 -* support for DLL application hosting, for debugging and executing DLLs under Dev-C++. -* New class browser option: "Show inherited members" -* Added support for the '::' member access operator in code-completion -* Added *working* function arguments hint -* Added bracket highlighting. When the caret is on a bracket, that bracket and + Instead open the file in an already launched Dev-C++. + It used to be a checkbox, allowing only two states (on or off), but there is + The user can define this in the class browser's context menu under "View mode". + Well, it adds caching to code-completion. Depending on the cache size, + a third relevant option now: "Project classes" so it didn't fit the purpose... + and selecting "View mode"). + cause of many errors (although it should be fixed by now), we are giving the + class inheritance and visibility (shows items only from files + code-completion and the user has all the commands (belonging to the files + compiler: -D__DEBUG__ + displayed in the editor when the mouse moves over a word. Since this was the + have your project under CVS control, you ''ll know why this had to happen... + he added in the cache) at his fingertips. If, for example, the user adds + include files can now be generated. + information definitions + it in the private resource) its counterpart are highlighted -* Nested folders in project view - -Version 4.9.6.7 -* XP Theme support + resource) + the program may take a bit longer to start-up, but provides very fast + the requested CVS action + then we even get a stack trace in the bug report! + user the option to disable this feature. + wxWindows! You get the picture... +* "Build priority" per-unit +* "Compile as C++" per-unit +* "Default" button in Compiler Options is back +* "Include file in compilation process" per-unit +* "Include file in linking process" per-unit +* Added "Add Library" button in Project Options +* Added "Classes" toolbar +* Added "External programs" in Tools/Environment Options (for units "Open with") +* Added "Files" tab in CVS form to allow selection of more than one file for +* Added "Open with" in project units context menu +* Added "Tip of the day" system. +* Added *working* function arguments hint +* Added CVS "login" and "logout" commands * Added CVS commands "Add" and "Remove" +* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory +* Added bracket highlighting. When the caret is on a bracket, that bracket and * Added configuration option for "Templates Directory" in "Environment Options" -* Code-completion updates -* Bug fixes - -Version 4.9.6.6 -* Editor colors are initialized properly on Dev-C++ first-run +* Added display of project filename, project output and a summary of the project files in Project Options General tab. * Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVariable wizards * Added file's date/time stamp in File/Properties window -* Current windows listing in Window menu -* Bug fixes - -Version 4.9.6.5 -* CVS support -* Window list (in Window menu) -* bug fixes - -version 4.9.6.4 -* added ENTER key for opening file in project browser, DEL to delete from the project. -* bug fixes - -version 4.9.6.3 -* Bug fixes - -version 4.9.6.2 -* Bug fixes - -version 4.9.6.1 -* New "Abort compilation" button -* Bug fixes -* Now checks for vRoach existance when sending a crash report - -Version 4.9.5.5 -* New option in Editor Options: Show editor hints. User can disable the hints - displayed in the editor when the mouse moves over a word. Since this was the - cause of many errors (although it should be fixed by now), we are giving the - user the option to disable this feature. -* New option in Editor Options (code-completion): Use code-completion cache. - Well, it adds caching to code-completion. Depending on the cache size, - the program may take a bit longer to start-up, but provides very fast - code-completion and the user has all the commands (belonging to the files - he added in the cache) at his fingertips. If, for example, the user adds - "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of - wxWindows! You get the picture... -* Removed "Only show classes from current file" option in class browser settings. - It used to be a checkbox, allowing only two states (on or off), but there is - a third relevant option now: "Project classes" so it didn't fit the purpose... - The user can define this in the class browser's context menu under "View mode". -* Fixed the dreaded "Clock skew detected" compiler warning! -* Fixed many class browser bugs, including some that had to do with class folders. - -Version 4.9.5.4 -* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data) - -Version 4.9.5.3 -* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory - then we even get a stack trace in the bug report! * Added new WebUpdate module (inactive temporarily). * Added new code for code-completion caching of files (disabled - work in progress). - -Version 4.9.5.2 +* Added new compiler/linker options: 20 +* Added new file menu entry: Save Project As * Added new option in class-browser: Use colors - (available when right-clicking the class-browser - and selecting "View mode"). -* Dev-C++ now traps access violation of your programs (and of itself too ;) - -Version 4.9.5.1 -* Implemented the "File/Export/Project to HTML" function. -* Added "Tip of the day" system. -* When running a source file in explorer, don't spawn new instance. - Instead open the file in an already launched Dev-C++. -* Class-parser speed-up (50% to 85% improvement timed!!!) -* Many code-completion updates. Now takes into account context, - class inheritance and visibility (shows items only from files - #included directly or indirectly)! -* Caching of result set of code-completion for speed-up. -* New option "Execution/Parameters" (and "Debug/Parameters"). - -Version 4.9.5.0 (5.0 beta 5): -* CPU Window (still in development) -* ToDo list +* Added possibility to include in a Template the Project's directories (include, libs and ressources) +* Added support for GCC > 3.2 +* Added support for macros in the "default source code" (Tools/Editor Options/Code) +* Added support for the "interface" keyword +* Added support for the '::' member access operator in code-completion +* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value") +* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup +* Added two new macros: and +* Allow customizing of per-unit compile command in projects +* Allow user to specify an alternate configuration file in Environment Options 20 * Backtrace in debugging -* Run to cursor +* Big speed up in function parameters listing while editing +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug fixes +* Bug-fix for double quotes in devcpp.cfg file read by vUpdate +* CPU Window (still in development) +* CVS support +* Caching of result set of code-completion for speed-up. +* Changed position of compiler/linker parameters in Project Options. +* Changed tint of Class browser pictures colors to match the New Look style +* Class-parser speed-up (50% to 85% improvement timed!!!) +* Code-completion updates +* Compiler set per-project +* Compiler settings per-project +* Compiling progress window +* Current windows listing in Window menu +* Debug variable browser +* Debug variables are now resent during next debug session +* Dev-C++ now traps access violation of your programs (and of itself too ;) +* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features. +* Each project creates a _private.h file containing version +* Editor colors are initialized properly on Dev-C++ first-run +* Environment options : "Show progress window" and "Auto-close progress window" +* Error messages parsing improved +* Fixed many class browser bugs, including some that had to do with class folders. +* Fixed pre-compilation dependency checks to work correctly +* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0 +* Fixed the dreaded "Clock skew detected" compiler warning! * Folders in Project and Class Browser -* Send custom commands to GDB -* Makefile can now be customized. -* Modified the behaviour of the -c param : 20 - -c -* Saving of custom syntax parameter group -* Possibility of changing compilers and tools filename. -* Many bug fixes - - -Version 4.9.4.1 (5.0 beta 4.1): - -* back to gcc 2.95.3 -* Profiling support -* new update/packages checker (vUpdate) +* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2) +* Implemented new compiler settings framework +* Implemented search in help files for the word at cursor (context sensitive help) +* Implemented the "File/Export/Project to HTML" function. +* Improved Indent/Unindent and Remove Comment +* Improved WebUpdate module +* Improved automatic indent +* Improved code completion cache +* Improved editor +* Improved help file +* Improved installer +* Lots of bug fixes. * Lots of bugfixes - +* MSVC import now creates the folders structure of the original VC project +* Made whole bottom report control floating instead of only debug output. +* Makefile can now be customized. +* Many bug fixes +* Many bug fixes +* Many bug fixes +* Many bug fixes +* Many bug fixes +* Many bug fixes +* Many bug fixes +* Many code-completion updates. Now takes into account context, +* Modified the behaviour of the -c param : 20 +* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings) +* Necessary UI changes in Project Options +* Nested folders in project view +* New "Abort compilation" button +* New WebUpdater module. +* New class browser option: "Show inherited members" +* New code tooltip display +* New debug feature for DLLs: attach to a running process +* New environment options : "watch variable under mouse" and "Report watch errors" +* New feature: compile current file only +* New option "Execution/Parameters" (and "Debug/Parameters"). +* New option in Editor Options (code-completion): Use code-completion cache. +* New option in Editor Options: Show editor hints. User can disable the hints +* New project option: Use custom Makefile. 20 +* New splash screen and association icons +* Now checks for vRoach existance when sending a crash report +* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20 +* Other bug fixes +* Possibility of changing compilers and tools filename. +* Printing settings are now saved +* Profiling support +* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows. +* Project version info (creates the relevant VERSIONINFO struct in the private +* Removed "Only show classes from current file" option in class browser settings. +* Resource errors are now reported in the Resource sheet +* Resource files are treated as ordinary files now +* Run to cursor +* Saving of custom syntax parameter group +* Send custom commands to GDB +* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744) +* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744) +* Separated layout info from project file. It is now kept in a different file +* Support XP Themes (creates the CommonControls 6.0 manifest file and includes +* Support for latest Mingw compiler system builds +* ToDo list +* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data) +* Updates in "Project Options/Files" code +* Watched Variables not in correct context are now kept and updated when it is needed +* WebUpdate should now report installation problems from PackMan +* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP +* When adding debugging symbols on request, remove "-s" option from linker +* When compiling the current file only, no dependency checks are performed +* When compiling with debugging symbols, an extra definition is passed to the +* When creating a DLL, the created static lib respects now the project-defined output directory +* When running a source file in explorer, don't spawn new instance. +* Window list (in Window menu) +* XP Theme support +* added ENTER key for opening file in project browser, DEL to delete from the project. +* back to gcc 2.95.3 +* bug fixes +* bug fixes +* new update/packages checker (vUpdate) +* support for DLL application hosting, for debugging and executing DLLs under Dev-C++. +* ~300% Speed-up in class parser +Find the attachment +GPS +Hello +I send u smtp pcap file +Version 4.9.4.1 (5.0 beta 4.1): +Version 4.9.5.0 (5.0 beta 5): +Version 4.9.5.1 +Version 4.9.5.2 +Version 4.9.5.3 +Version 4.9.5.4 +Version 4.9.5.5 +Version 4.9.6.5 +Version 4.9.6.6 +Version 4.9.6.7 +Version 4.9.6.8 +Version 4.9.6.9 +Version 4.9.7.0 +Version 4.9.7.1 +Version 4.9.7.2 +Version 4.9.7.3 +Version 4.9.7.4 +Version 4.9.7.5 +Version 4.9.7.6 +Version 4.9.7.7 +Version 4.9.7.8 +Version 4.9.7.9 +Version 4.9.8.0 +Version 4.9.8.1 +Version 4.9.8.2 +Version 4.9.8.3 +Version 4.9.8.4 +Version 4.9.8.5 +Version 4.9.8.7 +Version 4.9.8.9 +Version 4.9.9.0 +Version 4.9.9.1 +version 4.9.6.1 +version 4.9.6.2 +version 4.9.6.3 +version 4.9.6.4 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount new file mode 100644 index 0000000000..0cfbf08886 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount @@ -0,0 +1 @@ +2 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-0.dat b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-0.dat deleted file mode 100644 index f4dd7d22f4..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp-entity-0.dat +++ /dev/null @@ -1,13 +0,0 @@ -Hello - - - -I send u smtp pcap file - -Find the attachment - - - -GPS - - diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log index 039af42a2b..865694e8a2 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path smtp_entities -#open 2013-05-17-23-19-41 +#open 2013-06-07-19-32-56 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt #types time string addr port addr port count string count string string string string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 text/plain - smtp-entity-mR3f2AAKo11-0.dat (empty) +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 text/plain - smtp-entity-mR3f2AAKo11.dat (empty) 1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 text/html - - (empty) -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 text/plain - smtp-entity-ZNp0KBSLByc-1.dat (empty) -#close 2013-05-17-23-19-41 +1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 text/plain - smtp-entity-ZNp0KBSLByc.dat (empty) +#close 2013-06-07-19-32-56 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log new file mode 100644 index 0000000000..375c033c38 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2013-07-02-18-46-17 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher server_name session_id subject issuer_subject not_valid_before not_valid_after last_alert client_subject client_issuer_subject +#types time string addr port addr port string string string string string string time time string string string +1357328848.549370 UWkUyAuUGXf 10.0.0.80 56637 68.233.76.12 443 TLSv12 TLS_RSA_WITH_RC4_128_MD5 - - CN=*.taleo.net,OU=Comodo PremiumSSL Wildcard,OU=Web,O=Taleo Inc.,street=4140 Dublin Boulevard,street=Suite 400,L=Dublin,ST=CA,postalCode=94568,C=US CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB 1304467200.000000 1467676799.000000 - - - +#close 2013-07-02-18-46-17 diff --git a/testing/btest/Traces/dns-two-responses.trace b/testing/btest/Traces/dns-two-responses.trace new file mode 100644 index 0000000000..627b0d2ebe Binary files /dev/null and b/testing/btest/Traces/dns-two-responses.trace differ diff --git a/testing/btest/Traces/http/multipart.trace b/testing/btest/Traces/http/multipart.trace new file mode 100644 index 0000000000..5ce8b6e16f Binary files /dev/null and b/testing/btest/Traces/http/multipart.trace differ diff --git a/testing/btest/Traces/tls1.2.trace b/testing/btest/Traces/tls1.2.trace new file mode 100644 index 0000000000..87d50c277c Binary files /dev/null and b/testing/btest/Traces/tls1.2.trace differ diff --git a/testing/btest/core/print-bpf-filters.bro b/testing/btest/core/print-bpf-filters.bro index 6d9cef0220..6e4a4d5c30 100644 --- a/testing/btest/core/print-bpf-filters.bro +++ b/testing/btest/core/print-bpf-filters.bro @@ -1,10 +1,15 @@ -# @TEST-EXEC: bro -r $TRACES/empty.trace -e '' >output +# @TEST-EXEC: bro -r $TRACES/empty.trace >output # @TEST-EXEC: cat packet_filter.log >>output -# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::all_packets=F >>output +# @TEST-EXEC: bro -r $TRACES/empty.trace -f "port 42" >>output # @TEST-EXEC: cat packet_filter.log >>output -# @TEST-EXEC: bro -r $TRACES/empty.trace -f "port 42" -e '' >>output -# @TEST-EXEC: cat packet_filter.log >>output -# @TEST-EXEC: bro -r $TRACES/empty.trace -C -f "port 56730" -r $TRACES/mixed-vlan-mpls.trace >>output +# @TEST-EXEC: bro -r $TRACES/mixed-vlan-mpls.trace PacketFilter::restricted_filter="vlan" >>output # @TEST-EXEC: cat packet_filter.log >>output # @TEST-EXEC: btest-diff output # @TEST-EXEC: btest-diff conn.log +# +# The order in the output of enable_auto_protocol_capture_filters isn't +# stable, for reasons not clear. We canonify it first. +# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::enable_auto_protocol_capture_filters=T +# @TEST-EXEC: cat packet_filter.log | bro-cut filter | sed 's#[()]##g' | tr ' ' '\n' | sort | uniq -c | awk '{print $1, $2}' >output2 +# @TEST-EXEC: btest-diff output2 + diff --git a/testing/btest/core/tunnels/teredo-known-services.test b/testing/btest/core/tunnels/teredo-known-services.test index d03ef2ab71..da3a538515 100644 --- a/testing/btest/core/tunnels/teredo-known-services.test +++ b/testing/btest/core/tunnels/teredo-known-services.test @@ -1,6 +1,6 @@ -# @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}" +# @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd base/protocols/tunnels protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}" # @TEST-EXEC: test ! -e known_services.log -# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=F "Site::local_nets+={192.168.1.0/24}" +# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd base/protocols/tunnels protocols/conn/known-services Tunnel::delay_teredo_confirmation=F "Site::local_nets+={192.168.1.0/24}" # @TEST-EXEC: btest-diff known_services.log # The first case using Tunnel::delay_teredo_confirmation=T doesn't produce diff --git a/testing/btest/language/table-redef.bro b/testing/btest/language/table-redef.bro new file mode 100644 index 0000000000..290610499f --- /dev/null +++ b/testing/btest/language/table-redef.bro @@ -0,0 +1,26 @@ +# @TEST-EXEC: bro -b %INPUT > out +# @TEST-EXEC: btest-diff out + +const foo: table[string] of double &redef; + +# full (re)initialization +redef foo = { ["nope"] = 37.0 }; + +# full (re)initialization, discards "nope" index +redef foo = { ["abc"] = 42.0 }; + +# add elements +redef foo += { ["def"] = -42.0, ["ghi"] = 7.0 }; + +# remove elements from LHS based on indices shared with RHS +redef foo -= { ["ghi"] = 0.0 }; + +# RHS can be a table value +redef foo += table(["cool"] = 5.0, ["neat"] = 1.0); + +# Redef at a single index is allowed, same as += when RHS has overlapping index +redef foo["cool"] = 28.0; +redef foo["abc"] = 8.0; +redef foo += { ["def"] = 99.0 }; + +print foo; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro similarity index 90% rename from testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro rename to testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro index eddc933658..8ec4704cdb 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro @@ -20,7 +20,7 @@ redef default_file_timeout_interval = 2sec; event file_timeout(f: fa_file) { if ( timeout_cnt < 1 ) - FileAnalysis::postpone_timeout(f); + FileAnalysis::set_timeout_interval(f, f$timeout_interval); else terminate(); ++timeout_cnt; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/multipart.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/multipart.bro new file mode 100644 index 0000000000..57fe2348c2 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/multipart.bro @@ -0,0 +1,16 @@ +# @TEST-EXEC: bro -r $TRACES/http/multipart.trace $SCRIPTS/file-analysis-test.bro %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff 1-file +# @TEST-EXEC: btest-diff 2-file +# @TEST-EXEC: btest-diff 3-file +# @TEST-EXEC: btest-diff 4-file + +redef test_file_analysis_source = "HTTP"; + +global cnt: count = 0; + +redef test_get_file_name = function(f: fa_file): string + { + ++cnt; + return fmt("%d-file", cnt); + }; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro b/testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro index eedb56d359..f9ca9fb325 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro @@ -18,28 +18,12 @@ redef test_get_file_name = function(f: fa_file): string T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 @TEST-END-FILE -module A; - -type Val: record { - s: string; -}; - -event line(description: Input::EventDescription, tpe: Input::Event, s: string) - { - FileAnalysis::data_stream(description$source, s); - } - -event Input::end_of_data(name: string, source: string) - { - FileAnalysis::eof(source); - } - event bro_init() { - Input::add_event([$source="../input.log", $reader=Input::READER_BINARY, - $mode=Input::MANUAL, $name="input", $fields=Val, - $ev=line, $want_record=F]); - Input::remove("input"); + local source: string = "../input.log"; + Input::add_analysis([$source=source, $reader=Input::READER_BINARY, + $mode=Input::MANUAL, $name=source]); + Input::remove(source); } event file_state_remove(f: fa_file) &priority=-10 diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw/basic.bro similarity index 100% rename from testing/btest/scripts/base/frameworks/input/raw.bro rename to testing/btest/scripts/base/frameworks/input/raw/basic.bro diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/raw/execute.bro similarity index 100% rename from testing/btest/scripts/base/frameworks/input/executeraw.bro rename to testing/btest/scripts/base/frameworks/input/raw/execute.bro diff --git a/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro b/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro new file mode 100644 index 0000000000..729844e4b4 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro @@ -0,0 +1,44 @@ +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff test.txt +# @TEST-EXEC: btest-diff out + +redef exit_only_after_terminate = T; +@load base/frameworks/communication # let network-time run. otherwise there are no heartbeats... + +global outfile: file; +global try: count; + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + print outfile, description; + print outfile, tpe; + print outfile, s; + try = try + 1; + if ( try == 2 ) + { + Input::remove("input2"); + close(outfile); + terminate(); + } + } + +event bro_init() + { + local config_strings: table[string] of string = { + ["stdin"] = "hello\nthere\1\2\3\4\5\1\2\3yay" + #["stdin"] = "yay" + }; + + try = 0; + outfile = open("../out"); + Input::add_event([$source="cat > ../test.txt |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings]); + Input::remove("input"); + Input::add_event([$source="cat |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input2", $fields=Val, $ev=line, $want_record=F, $config=config_strings]); + } diff --git a/testing/btest/scripts/base/frameworks/input/raw/executestream.bro b/testing/btest/scripts/base/frameworks/input/raw/executestream.bro new file mode 100644 index 0000000000..ead33018dc --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw/executestream.bro @@ -0,0 +1,61 @@ +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +redef exit_only_after_terminate = T; + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +@load base/frameworks/communication # let network-time run + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + print outfile, description; + print outfile, tpe; + print outfile, s; + + try = try + 1; + if ( try == 8 ) + { + print outfile, "done"; + close(outfile); + Input::remove("input"); + terminate(); + } + } + +event bro_init() + { + outfile = open("../out"); + try = 0; + Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]); + } diff --git a/testing/btest/scripts/base/frameworks/input/raw/long.bro b/testing/btest/scripts/base/frameworks/input/raw/long.bro new file mode 100644 index 0000000000..ac07639f77 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw/long.bro @@ -0,0 +1,37 @@ +# @TEST-EXEC: dd if=/dev/zero of=input.log bs=8193 count=1 +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out +# +# this test should be longer than one block-size. to test behavior of input-reader if it has to re-allocate stuff. + +redef exit_only_after_terminate = T; + +global outfile: file; +global try: count; + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + print outfile, tpe; + print outfile, |s|; + try = try + 1; + if ( try == 1 ) + { + close(outfile); + terminate(); + } + } + +event bro_init() + { + try = 0; + outfile = open("../out"); + Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]); + Input::remove("input"); + } diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/raw/rereadraw.bro similarity index 100% rename from testing/btest/scripts/base/frameworks/input/rereadraw.bro rename to testing/btest/scripts/base/frameworks/input/raw/rereadraw.bro diff --git a/testing/btest/scripts/base/frameworks/input/raw/stderr.bro b/testing/btest/scripts/base/frameworks/input/raw/stderr.bro new file mode 100644 index 0000000000..e84ed048cd --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw/stderr.bro @@ -0,0 +1,66 @@ +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +redef exit_only_after_terminate = T; + +type Val: record { + s: string; + is_stderr: bool; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string, is_stderr: bool) + { + print outfile, tpe; + if ( is_stderr ) + { + # work around localized error messages. and if some localization does not include the filename... well... that would be bad :) + if ( strstr(s, "nonexistant") > 0 ) + { + print outfile, "stderr output contained nonexistant"; + } + } + else + { + print outfile, s; + } + print outfile, is_stderr; + + try = try + 1; + if ( try == 7 ) + { + print outfile, "done"; + Input::remove("input"); + } + } + +event Input::end_of_data(name: string, source:string) + { + print outfile, "End of Data event"; + print outfile, name; + terminate(); # due to the current design, end_of_data will be called after process_finshed and all line events. + # this could potentially change + } + +event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool) + { + print outfile, "Process finished event"; + print outfile, name; + if ( exit_code != 0 ) + print outfile, "Exit code != 0"; + } + +event bro_init() + { + + local config_strings: table[string] of string = { + ["read_stderr"] = "1" + }; + + outfile = open("../out"); + try = 0; + Input::add_event([$source="ls .. ../nonexistant ../nonexistant2 ../nonexistant3 |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings]); + } diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/raw/streamraw.bro similarity index 100% rename from testing/btest/scripts/base/frameworks/input/streamraw.bro rename to testing/btest/scripts/base/frameworks/input/raw/streamraw.bro diff --git a/testing/btest/scripts/base/protocols/dns/duplicate-reponses.bro b/testing/btest/scripts/base/protocols/dns/duplicate-reponses.bro new file mode 100644 index 0000000000..a16235b9a5 --- /dev/null +++ b/testing/btest/scripts/base/protocols/dns/duplicate-reponses.bro @@ -0,0 +1,5 @@ +# This tests the case where the DNS server responded with zero RRs. +# +# @TEST-EXEC: bro -r $TRACES/dns-two-responses.trace +# @TEST-EXEC: btest-diff dns.log +# @TEST-EXEC: btest-diff weird.log \ No newline at end of file diff --git a/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro b/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro index de1025ed82..8cbacdbf6f 100644 --- a/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro +++ b/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro @@ -3,14 +3,8 @@ # @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace %INPUT # @TEST-EXEC: btest-diff conn.log # @TEST-EXEC: btest-diff ftp.log -# @TEST-EXEC: mv ftp-item-*-0.dat ftp-item-0.dat -# @TEST-EXEC: mv ftp-item-*-1.dat ftp-item-1.dat -# @TEST-EXEC: mv ftp-item-*-2.dat ftp-item-2.dat -# @TEST-EXEC: mv ftp-item-*-3.dat ftp-item-3.dat -# @TEST-EXEC: btest-diff ftp-item-0.dat -# @TEST-EXEC: btest-diff ftp-item-1.dat -# @TEST-EXEC: btest-diff ftp-item-2.dat -# @TEST-EXEC: btest-diff ftp-item-3.dat +# @TEST-EXEC: cat ftp-item-*.dat | sort > extractions +# @TEST-EXEC: btest-diff extractions redef FTP::logged_commands += {"LIST"}; redef FTP::extract_file_types=/.*/; diff --git a/testing/btest/scripts/base/protocols/http/multipart-extract.bro b/testing/btest/scripts/base/protocols/http/multipart-extract.bro new file mode 100644 index 0000000000..c2789750a3 --- /dev/null +++ b/testing/btest/scripts/base/protocols/http/multipart-extract.bro @@ -0,0 +1,5 @@ +# @TEST-EXEC: bro -C -r $TRACES/http/multipart.trace %INPUT +# @TEST-EXEC: btest-diff http.log +# @TEST-EXEC: cat http-item-* | sort > extractions + +redef HTTP::extract_file_types += /.*/; diff --git a/testing/btest/scripts/base/protocols/irc/dcc-extract.test b/testing/btest/scripts/base/protocols/irc/dcc-extract.test index 71ab1b0900..cbfc6890da 100644 --- a/testing/btest/scripts/base/protocols/irc/dcc-extract.test +++ b/testing/btest/scripts/base/protocols/irc/dcc-extract.test @@ -1,27 +1,11 @@ # This tests that the contents of a DCC transfer negotiated with IRC can be -# correctly extracted. The mime type of the file transferred is normalized -# to prevent sensitivity to libmagic version being used. +# correctly extracted. # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT # @TEST-EXEC: btest-diff irc.log -# @TEST-EXEC: mv irc-dcc-item-*-0.dat irc-dcc-item.dat +# @TEST-EXEC: mv irc-dcc-item-*.dat irc-dcc-item.dat # @TEST-EXEC: btest-diff irc-dcc-item.dat # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test" -# @TEST-EXEC: test -e test-*-0.dat +# @TEST-EXEC: test -e test-*.dat redef IRC::extract_file_types=/.*/; - -event bro_init() - { - Log::remove_default_filter(IRC::LOG); - Log::add_filter(IRC::LOG, [$name="normalized-mime-types", - $pred=function(rec: IRC::Info): bool - { - if ( rec?$dcc_mime_type ) - { - rec$dcc_mime_type = "FAKE_MIME"; - } - return T; - } - ]); - } diff --git a/testing/btest/scripts/base/protocols/smtp/mime-extract.test b/testing/btest/scripts/base/protocols/smtp/mime-extract.test index 149fcf67c3..0caa5d530c 100644 --- a/testing/btest/scripts/base/protocols/smtp/mime-extract.test +++ b/testing/btest/scripts/base/protocols/smtp/mime-extract.test @@ -1,12 +1,10 @@ # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT # @TEST-EXEC: btest-diff smtp_entities.log -# @TEST-EXEC: mv smtp-entity-*-0.dat smtp-entity-0.dat -# @TEST-EXEC: mv smtp-entity-*-1.dat smtp-entity-1.dat -# @TEST-EXEC: btest-diff smtp-entity-0.dat -# @TEST-EXEC: btest-diff smtp-entity-1.dat +# @TEST-EXEC: cat smtp-entity-*.dat | sort > extractions +# @TEST-EXEC: btest-diff extractions # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test" -# @TEST-EXEC: test -e test-*-0.dat -# @TEST-EXEC: test -e test-*-1.dat +# @TEST-EXEC: cnt=0 && for f in test-*.dat; do cnt=$((cnt+1)); done && echo $cnt >filecount +# @TEST-EXEC: btest-diff filecount @load base/protocols/smtp diff --git a/testing/btest/scripts/base/protocols/ssl/tls-1.2.test b/testing/btest/scripts/base/protocols/ssl/tls-1.2.test new file mode 100644 index 0000000000..25b9083587 --- /dev/null +++ b/testing/btest/scripts/base/protocols/ssl/tls-1.2.test @@ -0,0 +1,2 @@ +# @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT +# @TEST-EXEC: btest-diff ssl.log diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index f4356154e4..37a51fa72f 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -2,10 +2,17 @@ # # Default canonifier used with the trace-based tests in testing/external/*. +addl="cat" + +if [ "$1" == "capture_loss.log" ]; then + addl="`dirname $0`/diff-remove-fractions" +fi + `dirname $0`/diff-remove-timestamps \ | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-file-ids \ | `dirname $0`/diff-remove-x509-names \ | `dirname $0`/diff-canon-notice-policy \ - | `dirname $0`/diff-sort + | `dirname $0`/diff-sort \ + | eval $addl diff --git a/testing/scripts/diff-remove-fractions b/testing/scripts/diff-remove-fractions new file mode 100755 index 0000000000..975157913c --- /dev/null +++ b/testing/scripts/diff-remove-fractions @@ -0,0 +1,6 @@ +#! /usr/bin/env bash +# +# Replace fractions of double value (i.e., 3.14 -> 3.x). + +sed 's/\.[0-9]\{1,\}/.X/g' + diff --git a/testing/scripts/file-analysis-test.bro b/testing/scripts/file-analysis-test.bro index 15929dd4f6..8b85ae2bd5 100644 --- a/testing/scripts/file-analysis-test.bro +++ b/testing/scripts/file-analysis-test.bro @@ -8,23 +8,35 @@ global test_get_file_name: function(f: fa_file): string = global test_print_file_data_events: bool = F &redef; +global file_count: count = 0; + +global file_map: table[string] of count; + +function canonical_file_name(f: fa_file): string + { + return fmt("file #%d", file_map[f$id]); + } + event file_chunk(f: fa_file, data: string, off: count) { if ( test_print_file_data_events ) - print "file_chunk", f$id, |data|, off, data; + print "file_chunk", canonical_file_name(f), |data|, off, data; } event file_stream(f: fa_file, data: string) { if ( test_print_file_data_events ) - print "file_stream", f$id, |data|, data; + print "file_stream", canonical_file_name(f), |data|, data; } event file_new(f: fa_file) { print "FILE_NEW"; - print f$id, f$seen_bytes, f$missing_bytes; + file_map[f$id] = file_count; + ++file_count; + + print canonical_file_name(f), f$seen_bytes, f$missing_bytes; if ( test_file_analysis_source == "" || f$source == test_file_analysis_source ) @@ -72,7 +84,7 @@ event file_gap(f: fa_file, offset: count, len: count) event file_state_remove(f: fa_file) { print "FILE_STATE_REMOVE"; - print f$id, f$seen_bytes, f$missing_bytes; + print canonical_file_name(f), f$seen_bytes, f$missing_bytes; if ( f?$conns ) for ( cid in f$conns ) print cid;