binpac: Extends BinPAC to support arbitrary line breakers via &linebreaker attribute

This feature is needed to run the FIX ASCII analyzer: https://github.com/reservoirlabs/fix-ascii

tools/binpac/lib/binpac_buffer.cc

@@ -25,6 +25,7 @@ FlowBuffer::FlowBuffer(LineBreakStyle linebreak_style)
 	orig_data_end_ = 0;
 
 	linebreak_style_ = linebreak_style;
+	linebreak_style_default = linebreak_style;
 	ResetLineState();
 
 	mode_ = UNKNOWN_MODE;
@@ -85,6 +86,8 @@ void FlowBuffer::ResetLineState()
 		case STRICT_CRLF:
 			state_ = STRICT_CRLF_0;
 			break;
+		case LINE_BREAKER:
+			break; // Nothing to reset
 		default:
 			BINPAC_ASSERT(0);
 			break;
@@ -115,6 +118,18 @@ void FlowBuffer::ExpandBuffer(int length)
 	buffer_ = new_buf;
 	}
 
+void FlowBuffer::SetLineBreaker(u_char *lbreaker) 
+	{
+	linebreaker_ = *lbreaker;
+	linebreak_style_default = linebreak_style_;
+	linebreak_style_ = LINE_BREAKER;
+	}
+
+void FlowBuffer::UnsetLineBreaker() 
+	{
+	linebreak_style_ = linebreak_style_default;
+	}
+
 void FlowBuffer::NewLine()
 	{
 	FlowBuffer::NewMessage();
@@ -261,6 +276,9 @@ void FlowBuffer::MarkOrCopyLine()
 		case STRICT_CRLF:
 			MarkOrCopyLine_STRICT_CRLF();
 			break;
+		case LINE_BREAKER:
+			MarkOrCopyLine_LINEBREAK();
+			break;
 		default:
 			BINPAC_ASSERT(0);
 			break;
@@ -400,6 +418,42 @@ found_end_of_line:
 #endif
 	}
 
+void FlowBuffer::MarkOrCopyLine_LINEBREAK()
+	{
+	if ( ! (orig_data_begin_ && orig_data_end_) )
+		return;
+
+	const_byteptr data;
+	for ( data = orig_data_begin_; data < orig_data_end_; ++data )
+		{
+		if ( *data == linebreaker_ )
+			goto found_end_of_line;
+		}
+
+	AppendToBuffer(orig_data_begin_, orig_data_end_ - orig_data_begin_);
+	return;
+
+found_end_of_line:
+	if ( buffer_n_ == 0 )
+		{
+		frame_length_ = data - orig_data_begin_;
+		}
+	else
+		{
+		AppendToBuffer(orig_data_begin_, data + 1 - orig_data_begin_);
+		// But eliminate the last CR or LF
+		--buffer_n_;
+		}
+	message_complete_ = true;
+
+#if DEBUG_FLOW_BUFFER
+	fprintf(stderr, "%.6f Line complete: [%s]\n",
+		network_time(),
+		string((const char *) begin(), (const char *) end()).c_str());
+#endif
+	}
+
+
 // Invariants:
 //
 // When buffer_n_ == 0:

tools/binpac/lib/binpac_buffer.h

@@ -12,6 +12,7 @@ public:
 		CR_OR_LF, 	// CR or LF or CRLF
 		STRICT_CRLF, 	// CR followed by LF
 		CR_LF_NUL,	// CR or LF or CR-LF or CR-NUL
+		LINE_BREAKER,   // User specified linebreaker 
 	};
 
 	FlowBuffer(LineBreakStyle linebreak_style = CR_OR_LF);
@@ -72,6 +73,8 @@ public:
 		return buffer_n_ > 0 || orig_data_end_ > orig_data_begin_;
 		}
 
+	void SetLineBreaker(u_char *lbreaker);
+	void UnsetLineBreaker();
 	void NewLine();
 	// A negative frame_length represents a frame till EOF
 	void NewFrame(int frame_length, bool chunked_);
@@ -119,6 +122,7 @@ protected:
 
 	void MarkOrCopyLine_CR_OR_LF();
 	void MarkOrCopyLine_STRICT_CRLF();
+	void MarkOrCopyLine_LINEBREAK();
 
 	int 	buffer_n_;		// number of bytes in the buffer
 	int 	buffer_length_;	// size of the buffer
@@ -129,6 +133,8 @@ protected:
 	const_byteptr orig_data_begin_, orig_data_end_;
 
 	LineBreakStyle linebreak_style_;
+	LineBreakStyle linebreak_style_default;
+	u_char  linebreaker_;
 
 	enum {
 		UNKNOWN_MODE,

tools/binpac/src/pac_type.cc

@@ -58,6 +58,7 @@ Type::Type(TypeType tot)
 	attr_length_expr_ = 0;
 	attr_letfields_ = 0;
 	attr_multiline_end_ = 0;
+	attr_linebreaker_ = 0;
 	attr_oneline_ = false;
 	attr_refcount_ = false;
 	attr_requires_ = new ExprList();
@@ -187,7 +188,9 @@ void Type::ProcessAttr(Attr* a)
 			break;
 
 		case ATTR_LINEBREAKER:
-			ASSERT(0);
+			if (strlen(a->expr()->orig()) != 6 )
+				throw Exception(this, "invalid line breaker length, must be a single ASCII character. (Ex: \"\\001\".)");
+			attr_linebreaker_ = a->expr();
 			break;
 
 		case ATTR_MULTILINE:
@@ -445,7 +448,12 @@ void Type::GenBufferConfiguration(Output *out_cc, Env *env)
 				env->RValue(buffering_state_id));
 			out_cc->inc_indent();
 			out_cc->println("{");
-
+			if(BufferableWithLineBreaker())
+				out_cc->println("%s->SetLineBreaker((u_char*)%s);", 
+				    env->LValue(flow_buffer_id), LineBreaker()->orig());
+			else 
+				out_cc->println("%s->UnsetLineBreaker();", 
+				    env->LValue(flow_buffer_id));
 			out_cc->println("%s->NewLine();",
 				env->LValue(flow_buffer_id));
 
@@ -999,6 +1007,17 @@ bool Type::Bufferable() const
 	return IsEmptyType() || BufferableByLength() || BufferableByLine();
 	}
 
+bool Type::BufferableWithLineBreaker() const
+	{
+	// If the input is an ASCII line with a given linebreaker;
+	return attr_linebreaker_ != 0;
+	}
+
+Expr* Type::LineBreaker() const
+	{
+	return attr_linebreaker_;
+	}
+
 Type::BufferMode Type::buffer_mode() const
 	{
 	if ( IsEmptyType() )

tools/binpac/src/pac_type.h

@@ -201,6 +201,8 @@ public:
 	bool Bufferable() const;
 	bool BufferableByLength() const;
 	bool BufferableByLine() const;
+	bool BufferableWithLineBreaker() const;
+	Expr* LineBreaker() const;
 
 	enum BufferMode {
 		NOT_BUFFERABLE,
@@ -288,6 +290,7 @@ protected:
 	Expr *attr_length_expr_;
 	FieldList *attr_letfields_;
 	Expr *attr_multiline_end_;
+	Expr *attr_linebreaker_;
 	bool attr_oneline_;
 	bool attr_refcount_;
 	ExprList *attr_requires_;