Optimize binary matching

2 files changed, 96 insertions, 55 deletions

diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index 02542cb1ab..9e037313bb 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -602,29 +602,72 @@ read_application_dist_data(DHandle, [], BufferSize, Rear) ->
     [Bin|Front] = lists:reverse(Rear),
     read_application_dist_data(DHandle, Front, BufferSize, [], Bin).
 %%
-read_application_dist_data(DHandle, Front0, BufferSize, Rear0, Bin) ->
-    case Bin of
+read_application_dist_data(DHandle, Front0, BufferSize, Rear0, Bin0) ->
+    case Bin0 of
+        %%
+        %% START Optimization
+        %% It is cheaper to match out several packets in one match operation than to loop for each
+        <<SizeA:32, DataA:SizeA/binary,
+          SizeB:32, DataB:SizeB/binary,
+          SizeC:32, DataC:SizeC/binary,
+          SizeD:32, DataD:SizeD/binary, Rest/binary>> ->
+            %% We have 4 complete packets in the first binary
+            erlang:dist_ctrl_put_data(DHandle, DataA),
+            erlang:dist_ctrl_put_data(DHandle, DataB),
+            erlang:dist_ctrl_put_data(DHandle, DataC),
+            erlang:dist_ctrl_put_data(DHandle, DataD),
+            read_application_dist_data(
+              DHandle, Front0, BufferSize - (4*4+SizeA+SizeB+SizeC+SizeD), Rear0, Rest);
+        <<SizeA:32, DataA:SizeA/binary,
+          SizeB:32, DataB:SizeB/binary,
+          SizeC:32, DataC:SizeC/binary, Rest/binary>> ->
+            %% We have 3 complete packets in the first binary
+            erlang:dist_ctrl_put_data(DHandle, DataA),
+            erlang:dist_ctrl_put_data(DHandle, DataB),
+            erlang:dist_ctrl_put_data(DHandle, DataC),
+            read_application_dist_data(
+              DHandle, Front0, BufferSize - (3*4+SizeA+SizeB+SizeC), Rear0, Rest);
+        <<SizeA:32, DataA:SizeA/binary,
+          SizeB:32, DataB:SizeB/binary, Rest/binary>> ->
+            %% We have 2 complete packets in the first binary
+            erlang:dist_ctrl_put_data(DHandle, DataA),
+            erlang:dist_ctrl_put_data(DHandle, DataB),
+            read_application_dist_data(
+              DHandle, Front0, BufferSize - (2*4+SizeA+SizeB), Rear0, Rest);
+        %% END Optimization
+        %%
+        %% Basic one packet code path
         <<Size:32, Data:Size/binary, Rest/binary>> ->
             %% We have a complete packet in the first binary
             erlang:dist_ctrl_put_data(DHandle, Data),
             read_application_dist_data(DHandle, Front0, BufferSize - (4+Size), Rear0, Rest);
         <<Size:32, FirstData/binary>> when 4+Size =< BufferSize ->
             %% We have a complete packet in the buffer
+            %% - fetch the missing content from the buffer front
             {Data,Front,Rear} = iovec_from_front(Size - byte_size(FirstData), Front0, Rear0, [FirstData]),
             erlang:dist_ctrl_put_data(DHandle, Data),
             read_application_dist_data(DHandle, Front, BufferSize - (4+Size), Rear);
-        <<_Size:32, _InsufficientData/binary>> ->
-            %% We can not have a complete packet in the buffer
-            {[Bin|Front0],BufferSize,Rear0};
-        <<IncompleteLengthField/binary>> when 4 < BufferSize ->
-            %% We might have a complete packet in the buffer
-            {LengthField,Front,Rear} =
-                iovec_from_front(4 - byte_size(IncompleteLengthField), Front0, Rear0, [IncompleteLengthField]),
-            LengthBin = iolist_to_binary(LengthField),
-            read_application_dist_data(DHandle, Front, BufferSize, Rear, LengthBin);
-        <<_IncompleteLengthField/binary>> ->
-            %% We can not have a complete packet in the buffer
-            {[Bin|Front0],BufferSize,Rear0}
+        <<Bin/binary>> ->
+            %% In OTP-21 the match context reuse optimization fails if we use Bin0 in recursion, so here we
+            %% match out the whole binary which will trick the optimization into keeping the match context
+            %% for the first binary contains complete packet code above
+            case Bin of
+                <<_Size:32, _InsufficientData/binary>> ->
+                    %% We have a length field in the first binary but there is not enough data
+                    %% in the buffer to form a complete packet - await more data
+                    {[Bin|Front0],BufferSize,Rear0};
+                <<IncompleteLengthField/binary>> when 4 < BufferSize ->
+                    %% We do not have a length field in the first binary but the buffer
+                    %% contains enough data to maybe form a packet
+                    %% - fetch a tiny binary from the buffer front to complete the length field
+                    {LengthField,Front,Rear} =
+                        iovec_from_front(4 - byte_size(IncompleteLengthField), Front0, Rear0, [IncompleteLengthField]),
+                    LengthBin = iolist_to_binary(LengthField),
+                    read_application_dist_data(DHandle, Front, BufferSize, Rear, LengthBin);
+                <<IncompleteLengthField/binary>> ->
+                    %% We do not have enough data in the buffer to even form a length field - await more data
+                    {[IncompleteLengthField|Front0],BufferSize,Rear0}
+            end
     end.
 
 iovec_from_front(Size, [], Rear, Acc) ->
diff --git a/lib/ssl/src/tls_record.erl b/lib/ssl/src/tls_record.erl
index 5cf238a685..b456197398 100644
--- a/lib/ssl/src/tls_record.erl
+++ b/lib/ssl/src/tls_record.erl
@@ -400,6 +400,7 @@ parse_tls_records(Versions, Q, undefined) ->
 parse_tls_records(Versions, Q, #ssl_tls{type = Type, version = Version, fragment = Length}) ->
     decode_tls_records(Versions, Q, [], Type, Version, Length).
 
+%% Generic code path
 decode_tls_records(Versions, {_,Size,_} = Q0, Acc, undefined, _Version, _Length) ->
     if
         5 =< Size ->
@@ -488,39 +489,38 @@ validate_tls_record_length(Versions, {_,Size0,_} = Q0, Acc, Type, Version, Lengt
     end.
 
 
-binary_from_front(BinSize, {Front,Size,Rear}) ->
-    binary_from_front(BinSize, Front, Size, Rear, []).
+binary_from_front(SplitSize, {Front,Size,Rear}) ->
+    binary_from_front(SplitSize, Front, Size, Rear, []).
 %%
-binary_from_front(BinSize, [], Size, [_] = Rear, Acc) ->
+binary_from_front(SplitSize, [], Size, [_] = Rear, Acc) ->
     %% Optimize a simple case
-    binary_from_front(BinSize, Rear, Size, [], Acc);
-binary_from_front(BinSize, [], Size, Rear, Acc) ->
-    binary_from_front(BinSize, lists:reverse(Rear), Size, [], Acc);
-binary_from_front(BinSize, [Bin|Front], Size, Rear, []) ->
+    binary_from_front(SplitSize, Rear, Size, [], Acc);
+binary_from_front(SplitSize, [], Size, Rear, Acc) ->
+    binary_from_front(SplitSize, lists:reverse(Rear), Size, [], Acc);
+binary_from_front(SplitSize, [Bin|Front], Size, Rear, []) ->
     %% Optimize a frequent case
-    case Bin of
-        <<RetBin:BinSize/binary, Rest/binary>> -> % More than enough - split here
-            {RetBin, {case Rest of
-                          <<>> ->
-                              Front;
-                          <<_/binary>> ->
-                              [Rest|Front]
-                      end,Size - BinSize,Rear}};
-        <<_/binary>> -> % Not enough
-            binary_from_front(BinSize - byte_size(Bin), Front, Size, Rear, [Bin])
+    BinSize = byte_size(Bin),
+    if
+        SplitSize < BinSize ->
+            {RetBin, Rest} = erlang:split_binary(Bin, SplitSize),
+            {RetBin, {[Rest|Front],Size - SplitSize,Rear}};
+        BinSize < SplitSize ->
+            binary_from_front(SplitSize - BinSize, Front, Size, Rear, [Bin]);
+        true -> % Perfect fit
+            {Bin, {Front,Size - SplitSize,Rear}}
     end;
-binary_from_front(BinSize, [Bin|Front], Size, Rear, Acc) ->
-    case Bin of
-        <<Last:BinSize/binary, Rest/binary>> -> % More than enough - split here
+binary_from_front(SplitSize, [Bin|Front], Size, Rear, Acc) ->
+    BinSize = byte_size(Bin),
+    if
+        SplitSize < BinSize ->
+            {Last, Rest} = erlang:split_binary(Bin, SplitSize),
             RetBin = iolist_to_binary(lists:reverse(Acc, [Last])),
-            {RetBin,{case Rest of
-                         <<>> ->
-                             Front;
-                         <<_/binary>> ->
-                             [Rest|Front]
-                     end,Size - byte_size(RetBin),Rear}};
-        <<_/binary>> -> % Not enough
-            binary_from_front(BinSize - byte_size(Bin), Front, Size, Rear, [Bin|Acc])
+            {RetBin, {[Rest|Front],Size - byte_size(RetBin),Rear}};
+        BinSize < SplitSize ->
+            binary_from_front(SplitSize - BinSize, Front, Size, Rear, [Bin|Acc]);
+        true -> % Perfect fit
+            RetBin = iolist_to_binary(lists:reverse(Acc, [Bin])),
+            {RetBin, {Front,Size - byte_size(RetBin),Rear}}
     end.
 
 %%--------------------------------------------------------------------
@@ -596,20 +596,18 @@ split_iovec(Data) ->
     {Part,Rest} = split_iovec(Data, ?MAX_PLAIN_TEXT_LENGTH, []),
     [Part|split_iovec(Rest)].
 %%
-split_iovec([Bin|Data], Size, Acc) ->
-    case Bin of
-        <<Last:Size/binary, Rest/binary>> ->
-            {lists:reverse(Acc, [Last]),
-             case Rest of
-                 <<>> ->
-                     Data;
-                 <<_/binary>> ->
-                     [Rest|Data]
-             end};
-        <<_/binary>> ->
-            split_iovec(Data, Size - byte_size(Bin), [Bin|Acc])
+split_iovec([Bin|Data], SplitSize, Acc) ->
+    BinSize = byte_size(Bin),
+    if
+        SplitSize < BinSize ->
+            {Last, Rest} = erlang:split_binary(Bin, SplitSize),
+            {lists:reverse(Acc, [Last]), [Rest|Data]};
+        BinSize < SplitSize ->
+            split_iovec(Data, SplitSize - BinSize, [Bin|Acc]);
+        true -> % Perfect match
+            {lists:reverse(Acc, [Bin]), Data}
     end;
-split_iovec([], _Size, Acc) ->
+split_iovec([], _SplitSize, Acc) ->
     {lists:reverse(Acc),[]}.
 
 %%--------------------------------------------------------------------