GH-1450: Improve printing/logging of large double/interval/time values

The modp_dtoa/modp_dtoa2 functions aren't capable of handling double
values larger than INT_MAX and fallback on using sprintf() in that
situation.  Previously, the format string to that sprintf() was "%e",
defaulting to a precision of 6, which is already too few digits to
represent a number known to be larger than INT_MAX.  Now, an sprintf()
is still performed for values larger than INT_MAX and still uses a
scientific notation format, but in a way that uses as many decimal
digits as needed to preserve information.

src/modp_numtoa.c

@@ -7,6 +7,7 @@
 #include <stdio.h>
 #include <math.h>
 #include <limits.h>
+#include <float.h>
 
 // other interesting references on num to string convesion
 // http://www.jb.man.ac.uk/~slowe/cpp/itoa.html
@@ -30,6 +31,68 @@ static void strreverse(char* begin, char* end)
         aux = *end, *end-- = *begin, *begin++ = aux;
 }
 
+// Expects 'str' to have been made using "%e" scientific notation format string
+static void sn_strip_trailing_zeros(char* str)
+	{
+	char* frac = 0;
+
+	for ( ; ; )
+		{
+		if ( *str == '.' )
+			{
+			frac = str + 1;
+			break;
+			}
+
+		if ( *str == 0 )
+			break;
+
+		++str;
+		}
+
+	if ( ! frac )
+		return;
+
+	char* exp = 0;
+	char* trailing_zeros = 0;
+
+	for ( ; ; )
+		{
+		if ( *frac == 0 )
+			break;
+
+		if ( *frac == 'e' )
+			{
+			exp = frac;
+			break;
+			}
+
+		if ( *frac == '0' )
+			{
+			if ( ! trailing_zeros )
+				trailing_zeros = frac;
+			}
+		else
+			trailing_zeros = 0;
+
+		++frac;
+		}
+
+	if ( trailing_zeros && exp )
+		{
+		for ( ; ; )
+			{
+			*trailing_zeros = *exp;
+
+			if ( *exp == 0 )
+				break;
+
+			++trailing_zeros;
+			++exp;
+			}
+		}
+	}
+
 void modp_itoa10(int32_t value, char* str)
 {
     char* wstr=str;
@@ -108,7 +171,8 @@ void modp_dtoa(double value, char* str, int prec)
       which can be 100s of characters overflowing your buffers == bad
     */
     if (value >= thres_max) {
-        sprintf(str, "%e", neg ? -value : value);
+        sprintf(str, "%.*e", DBL_DECIMAL_DIG - 1, neg ? -value : value);
+        sn_strip_trailing_zeros(str);
         return;
     }
 
@@ -207,7 +271,8 @@ void modp_dtoa2(double value, char* str, int prec)
       which can be 100s of characters overflowing your buffers == bad
     */
     if (value >= thres_max) {
-        sprintf(str, "%e", neg ? -value : value);
+        sprintf(str, "%.*e", DBL_DECIMAL_DIG - 1, neg ? -value : value);
+        sn_strip_trailing_zeros(str);
         return;
     }
 

src/modp_numtoa.h

@@ -70,7 +70,8 @@ void modp_ulitoa10(uint64_t value, char* buf);
  * trailing zeros
  *
  * If the input value is greater than 1<<31, then the output format
- * will be switched exponential format.
+ * will be switched exponential format and include as many precision digits
+ * as needed to preserve information.
  *
  * \param[in] value
  * \param[out] buf  The allocated output buffer.  Should be 32 chars or more.
@@ -87,7 +88,8 @@ void modp_dtoa(double value, char* buf, int precision);
  * of format oddly does not exists with printf.
  *
  * If the input value is greater than 1<<31, then the output format
- * will be switched exponential format.
+ * will be switched exponential format and include as many precision digits
+ * as needed to preserve information.
  *
  * \param[in] value
  * \param[out] buf  The allocated output buffer.  Should be 32 chars or more.

testing/btest/Baseline/bifs.bytestring_to_double/out

@@ -1,6 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-3.140000e+15
+3.14e+15
--3.140000e+15
+-3.14e+15
 4.000000e-308
 0.000000e+00
 -0.000000e+00

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log

@@ -7,12 +7,13 @@
 #open XXXX-XX-XX-XX-XX-XX
 #fields	data
 #types	time
-XXXXXXXXXX.XXXXXX
+1234567890.000000
-XXXXXXXXXX.XXXXXX
+1234567890.000000
-XXXXXXXXXX.XXXXXX
+1234567890.010000
-XXXXXXXXXX.XXXXXX
+1234567890.001000
-XXXXXXXXXX.XXXXXX
+1234567890.000100
-XXXXXXXXXX.XXXXXX
+1234567890.000010
-XXXXXXXXXX.XXXXXX
+1234567890.000001
-XXXXXXXXXX.XXXXXX
+1234567890.000000
+2.385642157e+09
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.protocols.smb.smb2/.stdout

@@ -2,11 +2,11 @@
 smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=<share_root>, disposition=1, create_options=32]
 smb2_create_response 10.0.0.11 -> 10.0.0.12:445 [file_id=[persistent=69, volatile=18446744069414584321], size=8192, times=[modified=XXXXXXXXXX.XXXXXX, accessed=XXXXXXXXXX.XXXXXX, created=XXXXXXXXXX.XXXXXX, changed=XXXXXXXXXX.XXXXXX], attrs=[read_only=F, hidden=F, system=F, directory=T, archive=F, normal=F, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F], create_action=1]
 smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=srvsvc, disposition=1, create_options=4194368]
-smb2_create_response 10.0.0.11 -> 10.0.0.12:445 [file_id=[persistent=73, volatile=18446744069414584325], size=0, times=[modified=-1.164447e+10, accessed=-1.164447e+10, created=-1.164447e+10, changed=-1.164447e+10], attrs=[read_only=F, hidden=F, system=F, directory=F, archive=F, normal=T, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F], create_action=1]
+smb2_create_response 10.0.0.11 -> 10.0.0.12:445 [file_id=[persistent=73, volatile=18446744069414584325], size=0, times=[modified=-1.16444736e+10, accessed=-1.16444736e+10, created=-1.16444736e+10, changed=-1.16444736e+10], attrs=[read_only=F, hidden=F, system=F, directory=F, archive=F, normal=T, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F], create_action=1]
 smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=<share_root>, disposition=2, create_options=2097185]
 smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=WP_SMBPlugin.pdf, disposition=2, create_options=68]
 smb2_create_response 10.0.0.11 -> 10.0.0.12:445 [file_id=[persistent=77, volatile=18446744069414584329], size=0, times=[modified=XXXXXXXXXX.XXXXXX, accessed=XXXXXXXXXX.XXXXXX, created=XXXXXXXXXX.XXXXXX, changed=XXXXXXXXXX.XXXXXX], attrs=[read_only=F, hidden=F, system=F, directory=F, archive=T, normal=F, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F], create_action=2]
-smb2_file_sattr 10.0.0.11 -> 10.0.0.12:445 [persistent=77, volatile=18446744069414584329] MACTimes:[modified=XXXXXXXXXX.XXXXXX, accessed=-1.164447e+10, created=-1.164447e+10, changed=-1.164447e+10] FileAttrs:[read_only=F, hidden=F, system=F, directory=F, archive=F, normal=F, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F]
+smb2_file_sattr 10.0.0.11 -> 10.0.0.12:445 [persistent=77, volatile=18446744069414584329] MACTimes:[modified=XXXXXXXXXX.XXXXXX, accessed=-1.16444736e+10, created=-1.16444736e+10, changed=-1.16444736e+10] FileAttrs:[read_only=F, hidden=F, system=F, directory=F, archive=F, normal=F, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F]
 smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=<share_root>, disposition=1, create_options=32]
 smb2_create_response 10.0.0.11 -> 10.0.0.12:445 [file_id=[persistent=81, volatile=18446744069414584333], size=8192, times=[modified=XXXXXXXXXX.XXXXXX, accessed=XXXXXXXXXX.XXXXXX, created=XXXXXXXXXX.XXXXXX, changed=XXXXXXXXXX.XXXXXX], attrs=[read_only=F, hidden=F, system=F, directory=T, archive=F, normal=F, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F], create_action=1]
 smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=<share_root>, disposition=1, create_options=32]

testing/btest/Baseline/scripts.base.protocols.smb.smb3-negotiate-context/.stdout

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-[dialect_revision=767, security_mode=1, server_guid=[persistent=7021797314968118638, volatile=25959], system_time=XXXXXXXXXX.XXXXXX, server_start_time=-1.164447e+10, negotiate_context_count=0, negotiate_context_values=[]]
+[dialect_revision=767, security_mode=1, server_guid=[persistent=7021797314968118638, volatile=25959], system_time=XXXXXXXXXX.XXXXXX, server_start_time=-1.16444736e+10, negotiate_context_count=0, negotiate_context_values=[]]
-[dialect_revision=785, security_mode=1, server_guid=[persistent=7021797314968118638, volatile=25959], system_time=XXXXXXXXXX.XXXXXX, server_start_time=-1.164447e+10, negotiate_context_count=2, negotiate_context_values=[[context_type=1, data_length=38, preauth_info=[hash_alg_count=1, salt_length=32, hash_alg=[1], salt=\x0d&\xc9\xca\xf2\xd8\xfc\x87\xa7\x10\x9c\x04W\x82p\x09T8Rl\xed\xe0\x84\x10\xca4\xaa\x87B\xb9Z\x80], encryption_info=<uninitialized>, compression_info=<uninitialized>, netname=<uninitialized>], [context_type=2, data_length=4, preauth_info=<uninitialized>, encryption_info=[cipher_count=1, ciphers=[1]], compression_info=<uninitialized>, netname=<uninitialized>]]]
+[dialect_revision=785, security_mode=1, server_guid=[persistent=7021797314968118638, volatile=25959], system_time=XXXXXXXXXX.XXXXXX, server_start_time=-1.16444736e+10, negotiate_context_count=2, negotiate_context_values=[[context_type=1, data_length=38, preauth_info=[hash_alg_count=1, salt_length=32, hash_alg=[1], salt=\x0d&\xc9\xca\xf2\xd8\xfc\x87\xa7\x10\x9c\x04W\x82p\x09T8Rl\xed\xe0\x84\x10\xca4\xaa\x87B\xb9Z\x80], encryption_info=<uninitialized>, compression_info=<uninitialized>, netname=<uninitialized>], [context_type=2, data_length=4, preauth_info=<uninitialized>, encryption_info=[cipher_count=1, ciphers=[1]], compression_info=<uninitialized>, netname=<uninitialized>]]]

testing/btest/scripts/base/frameworks/logging/ascii-timestamps.zeek

@@ -1,6 +1,6 @@
 #
 # @TEST-EXEC: zeek -b %INPUT
-# @TEST-EXEC: btest-diff test.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-openclose-timestamps btest-diff test.log
 
 module Test;
 
@@ -23,5 +23,6 @@ event zeek_init()
 	Log::write(Test::LOG, [$data=double_to_time(1234567890.00001)]);
 	Log::write(Test::LOG, [$data=double_to_time(1234567890.000001)]);
 	Log::write(Test::LOG, [$data=double_to_time(1234567890.0000001)]);
+	Log::write(Test::LOG, [$data=double_to_time(2385642157)]);
 }