aboutsummaryrefslogtreecommitdiffstats
path: root/lib/stdlib
diff options
context:
space:
mode:
Diffstat (limited to 'lib/stdlib')
-rw-r--r--lib/stdlib/doc/src/beam_lib.xml39
-rw-r--r--lib/stdlib/src/array.erl21
-rw-r--r--lib/stdlib/src/beam_lib.erl52
-rw-r--r--lib/stdlib/src/string.erl8
-rw-r--r--lib/stdlib/test/beam_lib_SUITE.erl68
-rw-r--r--lib/stdlib/test/ets_SUITE.erl539
-rw-r--r--lib/stdlib/test/ets_SUITE_data/visualize_throughput.html10
-rw-r--r--lib/stdlib/test/stdlib.spec3
-rw-r--r--lib/stdlib/test/stdlib_bench.spec1
-rw-r--r--lib/stdlib/test/string_SUITE.erl51
-rw-r--r--[-rwxr-xr-x]lib/stdlib/uc_spec/gen_unicode_mod.escript194
11 files changed, 721 insertions, 265 deletions
diff --git a/lib/stdlib/doc/src/beam_lib.xml b/lib/stdlib/doc/src/beam_lib.xml
index 8bb4cf9101..bb44ca3201 100644
--- a/lib/stdlib/doc/src/beam_lib.xml
+++ b/lib/stdlib/doc/src/beam_lib.xml
@@ -470,6 +470,18 @@ CryptoKeyFun(clear) -> term()</code>
</func>
<func>
+ <name name="strip" arity="2" since=""/>
+ <fsummary>Remove chunks not needed by the loader from a BEAM file.
+ </fsummary>
+ <desc>
+ <p>Removes all chunks from a BEAM
+ file except those needed by the loader or passed in. In particular,
+ the debug information (chunk <c>debug_info</c> and <c>abstract_code</c>)
+ is removed.</p>
+ </desc>
+ </func>
+
+ <func>
<name name="strip_files" arity="1" since=""/>
<fsummary>Removes chunks not needed by the loader from BEAM files.
</fsummary>
@@ -483,6 +495,19 @@ CryptoKeyFun(clear) -> term()</code>
</func>
<func>
+ <name name="strip_files" arity="2" since=""/>
+ <fsummary>Removes chunks not needed by the loader from BEAM files.
+ </fsummary>
+ <desc>
+ <p>Removes all chunks except
+ those needed by the loader or passed in from BEAM files. In particular,
+ the debug information (chunk <c>debug_info</c> and <c>abstract_code</c>)
+ is removed. The returned list contains one element for each
+ specified filename, in the same order as in <c>Files</c>.</p>
+ </desc>
+ </func>
+
+ <func>
<name name="strip_release" arity="1" since=""/>
<fsummary>Remove chunks not needed by the loader from all BEAM files of
a release.</fsummary>
@@ -497,6 +522,20 @@ CryptoKeyFun(clear) -> term()</code>
</func>
<func>
+ <name name="strip_release" arity="2" since=""/>
+ <fsummary>Remove chunks not needed by the loader from all BEAM files of
+ a release.</fsummary>
+ <desc>
+ <p>Removes all chunks
+ except those needed by the loader or passed in from the BEAM files of a
+ release. <c><anno>Dir</anno></c> is to be the installation root
+ directory. For example, the current OTP release can be
+ stripped with the call
+ <c>beam_lib:strip_release(code:root_dir())</c>.</p>
+ </desc>
+ </func>
+
+ <func>
<name name="version" arity="1" since=""/>
<fsummary>Read the module version of the BEAM file.</fsummary>
<desc>
diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl
index 939b1fb488..1504326c61 100644
--- a/lib/stdlib/src/array.erl
+++ b/lib/stdlib/src/array.erl
@@ -126,11 +126,12 @@
%% per write than base 10, but the speedup is only 21%.)
-define(DEFAULT, undefined).
--define(LEAFSIZE, 10). % the "base"
--define(NODESIZE, ?LEAFSIZE). % (no reason to have a different size)
+-define(LEAFSIZE, 10). % the "base" (assumed to be > 1)
+-define(NODESIZE, ?LEAFSIZE). % must not be LEAFSIZE-1; keep same as leaf
-define(NODEPATTERN(S), {_,_,_,_,_,_,_,_,_,_,S}). % NODESIZE+1 elements!
--define(NEW_NODE(S), % beware of argument duplication!
- setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(S)),(S))).
+-define(NEW_NODE(E,S), % general case (currently unused)
+ setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(E)),(S))).
+-define(NEW_NODE(S), erlang:make_tuple((?NODESIZE+1),(S))). % when E = S
-define(NEW_LEAF(D), erlang:make_tuple(?LEAFSIZE,(D))).
-define(NODELEAFS, ?NODESIZE*?LEAFSIZE).
@@ -605,7 +606,7 @@ grow(I, E, M) ->
grow_1(I, E, M).
grow_1(I, E, M) when I >= M ->
- grow(I, setelement(1, ?NEW_NODE(M), E), ?extend(M));
+ grow_1(I, setelement(1, ?NEW_NODE(M), E), ?extend(M));
grow_1(_I, E, M) ->
{E, M}.
@@ -1631,12 +1632,11 @@ foldl_test_() ->
?_assert(foldl(Sum, 0, from_list(lists:seq(0,10))) =:= 55),
?_assert(foldl(Reverse, [], from_list(lists:seq(0,1000)))
=:= lists:reverse(lists:seq(0,1000))),
- ?_assert({999,[N0*100+1+2,N0*2+1+1,0]} =:=
- foldl(Vals, {0,[]},
+ ?_assertEqual({N0*100+1-2,[N0*100+1+2,N0*2+1+1,0]},
+ foldl(Vals, {0,[]},
set(N0*100+1,2,
set(N0*2+1,1,
set(0,0,new())))))
-
].
-endif.
@@ -1786,12 +1786,11 @@ foldr_test_() ->
?_assert(foldr(Sum, 0, from_list(lists:seq(0,10))) =:= 55),
?_assert(foldr(List, [], from_list(lists:seq(0,1000)))
=:= lists:seq(0,1000)),
- ?_assert({999,[0,N0*2+1+1,N0*100+1+2]} =:=
- foldr(Vals, {0,[]},
+ ?_assertEqual({N0*100+1-2,[0,N0*2+1+1,N0*100+1+2]},
+ foldr(Vals, {0,[]},
set(N0*100+1,2,
set(N0*2+1,1,
set(0,0,new())))))
-
].
-endif.
diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl
index 3386cfcbe6..aa992f17ab 100644
--- a/lib/stdlib/src/beam_lib.erl
+++ b/lib/stdlib/src/beam_lib.erl
@@ -32,8 +32,12 @@
all_chunks/1,
diff_dirs/2,
strip/1,
+ strip/2,
strip_files/1,
+ strip_files/2,
strip_release/1,
+ strip_release/2,
+ significant_chunks/0,
build_module/1,
version/1,
md5/1,
@@ -188,7 +192,16 @@ diff_dirs(Dir1, Dir2) ->
Beam2 :: beam().
strip(FileName) ->
- try strip_file(FileName)
+ strip(FileName, []).
+
+-spec strip(Beam1, AdditionalChunks) ->
+ {'ok', {module(), Beam2}} | {'error', 'beam_lib', info_rsn()} when
+ Beam1 :: beam(),
+ AdditionalChunks :: [chunkid()],
+ Beam2 :: beam().
+
+strip(FileName, AdditionalChunks) ->
+ try strip_file(FileName, AdditionalChunks)
catch Error -> Error end.
-spec strip_files(Files) ->
@@ -196,8 +209,17 @@ strip(FileName) ->
Files :: [beam()],
Beam :: beam().
-strip_files(Files) when is_list(Files) ->
- try strip_fils(Files)
+strip_files(Files) ->
+ strip_files(Files, []).
+
+-spec strip_files(Files, AdditionalChunks) ->
+ {'ok', [{module(), Beam}]} | {'error', 'beam_lib', info_rsn()} when
+ Files :: [beam()],
+ AdditionalChunks :: [chunkid()],
+ Beam :: beam().
+
+strip_files(Files, AdditionalChunks) when is_list(Files) ->
+ try strip_fils(Files, AdditionalChunks)
catch Error -> Error end.
-spec strip_release(Dir) ->
@@ -207,7 +229,17 @@ strip_files(Files) when is_list(Files) ->
Reason :: {'not_a_directory', term()} | info_rsn().
strip_release(Root) ->
- catch strip_rel(Root).
+ strip_release(Root, []).
+
+-spec strip_release(Dir, AdditionalChunks) ->
+ {'ok', [{module(), file:filename()}]}
+ | {'error', 'beam_lib', Reason} when
+ Dir :: atom() | file:filename(),
+ AdditionalChunks :: [chunkid()],
+ Reason :: {'not_a_directory', term()} | info_rsn().
+
+strip_release(Root, AdditionalChunks) ->
+ catch strip_rel(Root, AdditionalChunks).
-spec version(Beam) ->
{'ok', {module(), [Version :: term()]}} |
@@ -401,17 +433,17 @@ cmp_lists([{Id, C1} | R1], [{Id, C2} | R2]) ->
cmp_lists(_, _) ->
error(different_chunks).
-strip_rel(Root) ->
+strip_rel(Root, AdditionalChunks) ->
ok = assert_directory(Root),
- strip_fils(filelib:wildcard(filename:join(Root, "lib/*/ebin/*.beam"))).
+ strip_fils(filelib:wildcard(filename:join(Root, "lib/*/ebin/*.beam")), AdditionalChunks).
%% -> {ok, [{Mod, BinaryOrFileName}]} | throw(Error)
-strip_fils(Files) ->
- {ok, [begin {ok, Reply} = strip_file(F), Reply end || F <- Files]}.
+strip_fils(Files, AdditionalChunks) ->
+ {ok, [begin {ok, Reply} = strip_file(F, AdditionalChunks), Reply end || F <- Files]}.
%% -> {ok, {Mod, FileName}} | {ok, {Mod, binary()}} | throw(Error)
-strip_file(File) ->
- {ok, {Mod, Chunks}} = read_significant_chunks(File, significant_chunks()),
+strip_file(File, AdditionalChunks) ->
+ {ok, {Mod, Chunks}} = read_significant_chunks(File, AdditionalChunks ++ significant_chunks()),
{ok, Stripped0} = build_module(Chunks),
Stripped = compress(Stripped0),
case File of
diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl
index 2939e78d9d..1f8bdc5432 100644
--- a/lib/stdlib/src/string.erl
+++ b/lib/stdlib/src/string.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1996-2018. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2019. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -1247,18 +1247,20 @@ split_1(Bin, [_C|_]=Needle, Start, Where, Curr0, Acc) ->
end
end.
-lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps, Ts) when is_integer(CP) ->
+lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps0, Ts) when is_integer(CP) ->
case lists:member(CP, CPs) of
true ->
[GC|Cs2] = unicode_util:gc(Cs0),
case lists:member(GC, GCs) of
true ->
- lexemes_m(Cs2, Seps, Ts);
+ lexemes_m(Cs2, Seps0, Ts);
false ->
+ Seps = search_compile(Seps0),
{Lexeme,Rest} = lexeme_pick(Cs0, Seps, []),
lexemes_m(Rest, Seps, [Lexeme|Ts])
end;
false ->
+ Seps = search_compile(Seps0),
{Lexeme,Rest} = lexeme_pick(Cs0, Seps, []),
lexemes_m(Rest, Seps, [Lexeme|Ts])
end;
diff --git a/lib/stdlib/test/beam_lib_SUITE.erl b/lib/stdlib/test/beam_lib_SUITE.erl
index 6418dc7eb6..4b2694320e 100644
--- a/lib/stdlib/test/beam_lib_SUITE.erl
+++ b/lib/stdlib/test/beam_lib_SUITE.erl
@@ -35,7 +35,7 @@
-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
init_per_group/2,end_per_group/2,
- normal/1, error/1, cmp/1, cmp_literals/1, strip/1, otp_6711/1,
+ normal/1, error/1, cmp/1, cmp_literals/1, strip/1, strip_add_chunks/1, otp_6711/1,
building/1, md5/1, encrypted_abstr/1, encrypted_abstr_file/1]).
-export([init_per_testcase/2, end_per_testcase/2]).
@@ -45,7 +45,7 @@ suite() ->
{timetrap,{minutes,2}}].
all() ->
- [error, normal, cmp, cmp_literals, strip, otp_6711,
+ [error, normal, cmp, cmp_literals, strip, strip_add_chunks, otp_6711,
building, md5, encrypted_abstr, encrypted_abstr_file].
groups() ->
@@ -401,6 +401,69 @@ strip(Conf) when is_list(Conf) ->
Source5D1, BeamFile5D1]),
ok.
+strip_add_chunks(Conf) when is_list(Conf) ->
+ PrivDir = ?privdir,
+ {SourceD1, BeamFileD1} = make_beam(PrivDir, simple, member),
+ {Source2D1, BeamFile2D1} = make_beam(PrivDir, simple2, concat),
+ {Source3D1, BeamFile3D1} = make_beam(PrivDir, make_fun, make_fun),
+ {Source4D1, BeamFile4D1} = make_beam(PrivDir, constant, constant),
+ {Source5D1, BeamFile5D1} = make_beam(PrivDir, lines, lines),
+
+ NoOfTables = erlang:system_info(ets_count),
+ P0 = pps(),
+
+ %% strip binary
+ verify(not_a_beam_file, beam_lib:strip(<<>>)),
+ {ok, B1} = file:read_file(BeamFileD1),
+ {ok, {simple, NB1}} = beam_lib:strip(B1),
+
+ BId1 = chunk_ids(B1),
+ NBId1 = chunk_ids(NB1),
+ true = length(BId1) > length(NBId1),
+ compare_chunks(B1, NB1, NBId1),
+
+ %% Keep all the extra chunks
+ ExtraChunks = ["Abst" , "Dbgi" , "Attr" , "CInf" , "LocT" , "Atom" ],
+ {ok, {simple, AB1}} = beam_lib:strip(B1, ExtraChunks),
+ ABId1 = chunk_ids(AB1),
+ true = length(BId1) == length(ABId1),
+ compare_chunks(B1, AB1, ABId1),
+
+ %% strip file - Keep extra chunks
+ verify(file_error, beam_lib:strip(foo)),
+ {ok, {simple, _}} = beam_lib:strip(BeamFileD1, ExtraChunks),
+ compare_chunks(B1, BeamFileD1, ABId1),
+
+ %% strip_files
+ {ok, B2} = file:read_file(BeamFile2D1),
+ {ok, [{simple,_},{simple2,_}]} = beam_lib:strip_files([B1, B2], ExtraChunks),
+ {ok, [{simple,_},{simple2,_},{make_fun,_},{constant,_}]} =
+ beam_lib:strip_files([BeamFileD1, BeamFile2D1, BeamFile3D1, BeamFile4D1], ExtraChunks),
+
+ %% check that each module can be loaded.
+ {module, simple} = code:load_abs(filename:rootname(BeamFileD1)),
+ {module, simple2} = code:load_abs(filename:rootname(BeamFile2D1)),
+ {module, make_fun} = code:load_abs(filename:rootname(BeamFile3D1)),
+ {module, constant} = code:load_abs(filename:rootname(BeamFile4D1)),
+
+ %% check that line number information is still present after stripping
+ {module, lines} = code:load_abs(filename:rootname(BeamFile5D1)),
+ {'EXIT',{badarith,[{lines,t,1,Info}|_]}} = (catch lines:t(atom)),
+ false = code:purge(lines),
+ true = code:delete(lines),
+ {ok, {lines,BeamFile5D1}} = beam_lib:strip(BeamFile5D1),
+ {module, lines} = code:load_abs(filename:rootname(BeamFile5D1)),
+ {'EXIT',{badarith,[{lines,t,1,Info}|_]}} = (catch lines:t(atom)),
+
+ true = (P0 == pps()),
+ NoOfTables = erlang:system_info(ets_count),
+
+ delete_files([SourceD1, BeamFileD1,
+ Source2D1, BeamFile2D1,
+ Source3D1, BeamFile3D1,
+ Source4D1, BeamFile4D1,
+ Source5D1, BeamFile5D1]),
+ ok.
otp_6711(Conf) when is_list(Conf) ->
{'EXIT',{function_clause,_}} = (catch {a, beam_lib:info(3)}),
@@ -729,6 +792,7 @@ make_beam(Dir, Module, F) ->
FileBase = filename:join(Dir, atom_to_list(Module)),
Source = FileBase ++ ".erl",
BeamFile = FileBase ++ ".beam",
+ file:delete(BeamFile),
simple_file(Source, Module, F),
{ok, _} = compile:file(Source, [{outdir,Dir}, debug_info, report]),
{Source, BeamFile}.
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl
index 8561491d50..87ca9bd32c 100644
--- a/lib/stdlib/test/ets_SUITE.erl
+++ b/lib/stdlib/test/ets_SUITE.erl
@@ -70,7 +70,10 @@
-export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1,
smp_ordered_iteration/1,
smp_select_replace/1, otp_8166/1, otp_8732/1, delete_unfix_race/1]).
--export([throughput_benchmark/0, test_throughput_benchmark/1]).
+-export([throughput_benchmark/0,
+ throughput_benchmark/1,
+ test_throughput_benchmark/1,
+ long_throughput_benchmark/1]).
-export([exit_large_table_owner/1,
exit_many_large_table_owner/1,
exit_many_tables_owner/1,
@@ -93,6 +96,7 @@
-include_lib("stdlib/include/ms_transform.hrl"). % ets:fun2ms
-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
-define(m(A,B), assert_eq(A,B)).
-define(heap_binary_size, 64).
@@ -151,7 +155,8 @@ all() ->
take,
whereis_table,
delete_unfix_race,
- test_throughput_benchmark].
+ test_throughput_benchmark,
+ {group, benchmark}].
groups() ->
[{new, [],
@@ -179,7 +184,9 @@ groups() ->
{meta_smp, [],
[meta_lookup_unnamed_read, meta_lookup_unnamed_write,
meta_lookup_named_read, meta_lookup_named_write,
- meta_newdel_unnamed, meta_newdel_named]}].
+ meta_newdel_unnamed, meta_newdel_named]},
+ {benchmark, [],
+ [long_throughput_benchmark]}].
init_per_suite(Config) ->
erts_debug:set_internal_state(available_internal_state, true),
@@ -192,9 +199,61 @@ end_per_suite(_Config) ->
catch erts_debug:set_internal_state(available_internal_state, false),
ok.
+init_per_group(benchmark, Config) ->
+ P = self(),
+ %% Spawn owner of ETS table that is alive until end_per_group is run
+ EtsProcess =
+ spawn(
+ fun()->
+ Tab = ets:new(ets_benchmark_result_summary_tab, [public]),
+ P ! {the_table, Tab},
+ receive
+ kill -> ok
+ end
+ end),
+ Tab = receive {the_table, T} -> T end,
+ CounterNames = [nr_of_benchmarks,
+ total_throughput,
+ nr_of_set_benchmarks,
+ total_throughput_set,
+ nr_of_ordered_set_benchmarks,
+ total_throughput_ordered_set],
+ lists:foreach(fun(CtrName) ->
+ ets:insert(Tab, {CtrName, 0.0})
+ end,
+ CounterNames),
+ [{ets_benchmark_result_summary_tab, Tab},
+ {ets_benchmark_result_summary_tab_process, EtsProcess} | Config];
init_per_group(_GroupName, Config) ->
Config.
+end_per_group(benchmark, Config) ->
+ T = proplists:get_value(ets_benchmark_result_summary_tab, Config),
+ EtsProcess = proplists:get_value(ets_benchmark_result_summary_tab_process, Config),
+ Report =
+ fun(NOfBenchmarksCtr, TotThroughoutCtr, Name) ->
+ Average =
+ ets:lookup_element(T, TotThroughoutCtr, 2) /
+ ets:lookup_element(T, NOfBenchmarksCtr, 2),
+ io:format("~p ~p~n", [Name, Average]),
+ ct_event:notify(
+ #event{name = benchmark_data,
+ data = [{suite,"ets_bench"},
+ {name, Name},
+ {value, Average}]})
+ end,
+ Report(nr_of_benchmarks,
+ total_throughput,
+ "Average Throughput"),
+ Report(nr_of_set_benchmarks,
+ total_throughput_set,
+ "Average Throughput Set"),
+ Report(nr_of_ordered_set_benchmarks,
+ total_throughput_ordered_set,
+ "Average Throughput Ordered Set"),
+ ets:delete(T),
+ EtsProcess ! kill,
+ Config;
end_per_group(_GroupName, Config) ->
Config.
@@ -6530,8 +6589,8 @@ whereis_table(Config) when is_list(Config) ->
ok.
-%% The following work functions are used by
-%% throughput_benchmark/4. They are declared on the top level beacuse
+%% The following help functions are used by
+%% throughput_benchmark. They are declared on the top level beacuse
%% declaring them as function local funs cause a scalability issue.
get_op([{_,O}], _RandNum) ->
O;
@@ -6566,10 +6625,131 @@ prefill_table_loop(T, RS0, N, ObjFun) ->
ets:insert(T, ObjFun(Key)),
prefill_table_loop(T, RS1, N-1, ObjFun).
-throughput_benchmark() ->
- throughput_benchmark(false, not_set, not_set).
+-record(ets_throughput_bench_config,
+ {benchmark_duration_ms = 3000,
+ recover_time_ms = 1000,
+ thread_counts = not_set,
+ key_ranges = [1000000],
+ scenarios =
+ [
+ [
+ {0.5, insert},
+ {0.5, delete}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.8, lookup}
+ ],
+ [
+ {0.01, insert},
+ {0.01, delete},
+ {0.98, lookup}
+ ],
+ [
+ {1.0, lookup}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.4, lookup},
+ {0.4, nextseq10}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.4, lookup},
+ {0.4, nextseq100}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.4, lookup},
+ {0.4, nextseq1000}
+ ],
+ [
+ {1.0, nextseq1000}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.79, lookup},
+ {0.01, selectAll}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.7999, lookup},
+ {0.0001, selectAll}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.799999, lookup},
+ {0.000001, selectAll}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.79, lookup},
+ {0.01, partial_select1000}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.7999, lookup},
+ {0.0001, partial_select1000}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.799999, lookup},
+ {0.000001, partial_select1000}
+ ]
+ ],
+ table_types =
+ [
+ [ordered_set, public],
+ [ordered_set, public, {write_concurrency, true}],
+ [ordered_set, public, {read_concurrency, true}],
+ [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}],
+ [set, public],
+ [set, public, {write_concurrency, true}],
+ [set, public, {read_concurrency, true}],
+ [set, public, {write_concurrency, true}, {read_concurrency, true}]
+ ],
+ etsmem_fun = fun() -> ok end,
+ verify_etsmem_fun = fun(_) -> true end,
+ notify_res_fun = fun(_Name, _Throughput) -> ok end,
+ print_result_paths_fun =
+ fun(ResultPath, _LatestResultPath) ->
+ Comment =
+ io_lib:format("<a href=\"file:///~s\">Result visualization</a>",[ResultPath]),
+ {comment, Comment}
+ end
+ }).
+
+stdout_notify_res(ResultPath, LatestResultPath) ->
+ io:format("Result Location: /~s~n", [ResultPath]),
+ io:format("Latest Result Location: ~s~n", [LatestResultPath]).
-throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) ->
+throughput_benchmark() ->
+ throughput_benchmark(
+ #ets_throughput_bench_config{
+ print_result_paths_fun = fun stdout_notify_res/2}).
+
+throughput_benchmark(
+ #ets_throughput_bench_config{
+ benchmark_duration_ms = BenchmarkDurationMs,
+ recover_time_ms = RecoverTimeMs,
+ thread_counts = ThreadCountsOpt,
+ key_ranges = KeyRanges,
+ scenarios = Scenarios,
+ table_types = TableTypes,
+ etsmem_fun = ETSMemFun,
+ verify_etsmem_fun = VerifyETSMemFun,
+ notify_res_fun = NotifyResFun,
+ print_result_paths_fun = PrintResultPathsFun}) ->
NrOfSchedulers = erlang:system_info(schedulers),
%% Definitions of operations that are supported by the benchmark
NextSeqOp =
@@ -6634,7 +6814,7 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) ->
fun(T,KeyRange) -> NextSeqOp(T,KeyRange,1000) end,
selectAll =>
fun(T,_KeyRange) ->
- case -1 =:= ets:select_count(T, ets:fun2ms(fun(X) -> true end)) of
+ case -1 =:= ets:select_count(T, ets:fun2ms(fun(_X) -> true end)) of
true -> io:format("Will never be printed");
false -> ok
end
@@ -6683,11 +6863,28 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) ->
false -> ok
end
end,
+ DataHolder =
+ fun DataHolderFun(Data)->
+ receive
+ {get_data, Pid} -> Pid ! {ets_bench_data, Data};
+ D -> DataHolderFun([Data,D])
+ end
+ end,
+ DataHolderPid = spawn_link(fun()-> DataHolder([]) end),
+ PrintData =
+ fun (Str, List) ->
+ io:format(Str, List),
+ DataHolderPid ! io_lib:format(Str, List)
+ end,
+ GetData =
+ fun () ->
+ DataHolderPid ! {get_data, self()},
+ receive {ets_bench_data, Data} -> Data end
+ end,
%% Function that runs a benchmark instance and returns the number
%% of operations that were performed
RunBenchmark =
- fun(NrOfProcs, TableConfig, Scenario,
- Range, Duration, RecoverTime) ->
+ fun({NrOfProcs, TableConfig, Scenario, Range, Duration}) ->
ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0),
Table = ets:new(t, TableConfig),
Nobj = Range div 2,
@@ -6695,16 +6892,15 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) ->
Nobj = ets:info(Table, size),
SafeFixTableIfRequired(Table, Scenario, true),
ParentPid = self(),
+ Worker =
+ fun() ->
+ receive start -> ok end,
+ WorksDone =
+ do_work(0, Table, ProbHelpTab, Range, Operations),
+ ParentPid ! WorksDone
+ end,
ChildPids =
- lists:map(
- fun(_N) ->
- spawn(fun() ->
- receive start -> ok end,
- WorksDone =
- do_work(0, Table, ProbHelpTab, Range, Operations),
- ParentPid ! WorksDone
- end)
- end, lists:seq(1, NrOfProcs)),
+ lists:map(fun(_N) ->spawn_link(Worker)end, lists:seq(1, NrOfProcs)),
lists:foreach(fun(Pid) -> Pid ! start end, ChildPids),
timer:sleep(Duration),
lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids),
@@ -6716,185 +6912,194 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) ->
end, 0, ChildPids),
SafeFixTableIfRequired(Table, Scenario, false),
ets:delete(Table),
- timer:sleep(RecoverTime),
TotalWorksDone
end,
- %%
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
- %%%% Benchmark Configuration %%%%%%%%%%%%%%%%%%%%%%%%
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
- %%
- %% Change the following variables to configure the benchmark runs
- ThreadCounts =
- case TestMode of
- true -> [1, NrOfSchedulers];
- false -> CalculateThreadCounts([1])
- end,
- KeyRanges = % Sizes of the key ranges
- case TestMode of
- true -> [50000];
- false -> [1000000]
+ RunBenchmarkInSepProcess =
+ fun(ParameterTuple) ->
+ P = self(),
+ spawn_link(fun()-> P ! {bench_result, RunBenchmark(ParameterTuple)} end),
+ Result = receive {bench_result, Res} -> Res end,
+ timer:sleep(RecoverTimeMs),
+ Result
end,
- Duration =
- case BenchmarkRunMs of % Duration of a benchmark run in milliseconds
- not_set -> 30000;
- _ -> BenchmarkRunMs
+ RunBenchmarkAndReport =
+ fun(ThreadCount,
+ TableType,
+ Scenario,
+ KeyRange,
+ Duration) ->
+ Result = RunBenchmarkInSepProcess({ThreadCount,
+ TableType,
+ Scenario,
+ KeyRange,
+ Duration}),
+ Throughput = Result/(Duration/1000.0),
+ PrintData("; ~f",[Throughput]),
+ Name = io_lib:format("Scenario: ~w, Key Range Size: ~w, "
+ "# of Processes: ~w, Table Type: ~w",
+ [Scenario, KeyRange, ThreadCount, TableType]),
+ NotifyResFun(Name, Throughput)
end,
- TimeMsToSleepAfterEachBenchmarkRun =
- case RecoverTimeMs of
- not_set -> 1000;
- _ -> RecoverTimeMs
+ ThreadCounts =
+ case ThreadCountsOpt of
+ not_set ->
+ CalculateThreadCounts([1]);
+ _ -> ThreadCountsOpt
end,
- TableTypes = % The table types that will be benchmarked
- [
- [ordered_set, public],
- [ordered_set, public, {write_concurrency, true}],
- [ordered_set, public, {read_concurrency, true}],
- [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}],
- [set, public],
- [set, public, {write_concurrency, true}],
- [set, public, {read_concurrency, true}],
- [set, public, {write_concurrency, true}, {read_concurrency, true}]
- ],
- Scenarios = % Benchmark scenarios (the fractions should add up to approximately 1.0)
- [
- [
- {0.5, insert},
- {0.5, delete}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.8, lookup}
- ],
- [
- {0.01, insert},
- {0.01, delete},
- {0.98, lookup}
- ],
- [
- {1.0, lookup}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.4, lookup},
- {0.4, nextseq10}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.4, lookup},
- {0.4, nextseq100}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.4, lookup},
- {0.4, nextseq1000}
- ],
- [
- {1.0, nextseq1000}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.79, lookup},
- {0.01, selectAll}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.7999, lookup},
- {0.0001, selectAll}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.799999, lookup},
- {0.000001, selectAll}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.79, lookup},
- {0.01, partial_select1000}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.7999, lookup},
- {0.0001, partial_select1000}
- ],
- [
- {0.1, insert},
- {0.1, delete},
- {0.799999, lookup},
- {0.000001, partial_select1000}
- ]
- ],
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
- %%%% End of Benchmark Configuration %%%%%%%%%%%%%%%%
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
- %% Prepare for memory check
- EtsMem = case TestMode of
- true -> etsmem();
- false -> ok
- end,
%% Run the benchmark
- io:format("# Each instance of the benchmark runs for ~w seconds:~n", [Duration/1000]),
- io:format("# The result of a benchmark instance is presented as a number representing~n"),
- io:format("# the number of operations performed per second:~n~n~n"),
- io:format("# To plot graphs for the results below:~n"),
- io:format("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n"),
- io:format("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n"),
- io:format("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n"),
- io:format("# step 1 and press the Render button~n~n"),
- io:format("#BENCHMARK STARTED$~n"),
+ PrintData("# Each instance of the benchmark runs for ~w seconds:~n", [BenchmarkDurationMs/1000]),
+ PrintData("# The result of a benchmark instance is presented as a number representing~n",[]),
+ PrintData("# the number of operations performed per second:~n~n~n",[]),
+ PrintData("# To plot graphs for the results below:~n",[]),
+ PrintData("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n",[]),
+ PrintData("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n",[]),
+ PrintData("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n",[]),
+ PrintData("# step 1 and press the Render button~n~n",[]),
+ PrintData("#BENCHMARK STARTED$~n",[]),
+ EtsMem = ETSMemFun(),
%% The following loop runs all benchmark scenarios and prints the results (i.e, operations/second)
lists:foreach(
fun(KeyRange) ->
lists:foreach(
fun(Scenario) ->
- io:format("Scenario: ~s | Key Range Size: ~w$~n",
- [RenderScenario(Scenario, ""),
- KeyRange]),
+ PrintData("Scenario: ~s | Key Range Size: ~w$~n",
+ [RenderScenario(Scenario, ""), KeyRange]),
lists:foreach(
fun(ThreadCount) ->
- io:format("; ~w",[ThreadCount])
+ PrintData("; ~w",[ThreadCount])
end,
ThreadCounts),
- io:format("$~n",[]),
+ PrintData("$~n",[]),
lists:foreach(
fun(TableType) ->
- io:format("~w ",[TableType]),
+ PrintData("~w ",[TableType]),
lists:foreach(
fun(ThreadCount) ->
- Result = RunBenchmark(ThreadCount,
+ RunBenchmarkAndReport(ThreadCount,
TableType,
Scenario,
KeyRange,
- Duration,
- TimeMsToSleepAfterEachBenchmarkRun),
- io:format("; ~f",[Result/(Duration/1000.0)])
+ BenchmarkDurationMs)
end,
ThreadCounts),
- io:format("$~n",[])
+ PrintData("$~n",[])
end,
TableTypes)
end,
Scenarios)
end,
KeyRanges),
- io:format("~n#BENCHMARK ENDED$~n~n"),
- case TestMode of
- true -> verify_etsmem(EtsMem);
- false -> ok
- end.
+ PrintData("~n#BENCHMARK ENDED$~n~n",[]),
+ VerifyETSMemFun(EtsMem),
+ DataDir = filename:join(filename:dirname(code:which(?MODULE)), "ets_SUITE_data"),
+ TemplatePath = filename:join(DataDir, "visualize_throughput.html"),
+ {ok, Template} = file:read_file(TemplatePath),
+ OutputData = string:replace(Template, "#bench_data_placeholder", GetData()),
+ OutputPath1 = filename:join(DataDir, "ets_bench_result.html"),
+ {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:now_to_datetime(erlang:timestamp()),
+ StrTime = lists:flatten(io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w",[Year,Month,Day,Hour,Minute,Second])),
+ OutputPath2 = filename:join(DataDir, io_lib:format("ets_bench_result_~s.html", [StrTime])),
+ file:write_file(OutputPath1, OutputData),
+ file:write_file(OutputPath2, OutputData),
+ PrintResultPathsFun(OutputPath2, OutputPath1).
test_throughput_benchmark(Config) when is_list(Config) ->
- throughput_benchmark(true, 100, 0).
-
+ throughput_benchmark(
+ #ets_throughput_bench_config{
+ benchmark_duration_ms = 100,
+ recover_time_ms = 0,
+ thread_counts = [1, erlang:system_info(schedulers)],
+ key_ranges = [50000],
+ etsmem_fun = fun etsmem/0,
+ verify_etsmem_fun = fun verify_etsmem/1}).
+
+long_throughput_benchmark(Config) when is_list(Config) ->
+ N = erlang:system_info(schedulers),
+ throughput_benchmark(
+ #ets_throughput_bench_config{
+ benchmark_duration_ms = 3000,
+ recover_time_ms = 1000,
+ thread_counts = [1, N div 2, N],
+ key_ranges = [1000000],
+ scenarios =
+ [
+ [
+ {0.5, insert},
+ {0.5, delete}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.8, lookup}
+ ],
+ [
+ {0.01, insert},
+ {0.01, delete},
+ {0.98, lookup}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.4, lookup},
+ {0.4, nextseq100}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.79, lookup},
+ {0.01, selectAll}
+ ],
+ [
+ {0.1, insert},
+ {0.1, delete},
+ {0.79, lookup},
+ {0.01, partial_select1000}
+ ]
+ ],
+ table_types =
+ [
+ [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}],
+ [set, public, {write_concurrency, true}, {read_concurrency, true}]
+ ],
+ etsmem_fun = fun etsmem/0,
+ verify_etsmem_fun = fun verify_etsmem/1,
+ notify_res_fun =
+ fun(Name, Throughput) ->
+ SummaryTable =
+ proplists:get_value(ets_benchmark_result_summary_tab, Config),
+ AddToSummaryCounter =
+ case SummaryTable of
+ undefined ->
+ fun(_, _) ->
+ ok
+ end;
+ Tab ->
+ fun(CounterName, ToAdd) ->
+ OldVal = ets:lookup_element(Tab, CounterName, 2),
+ NewVal = OldVal + ToAdd,
+ ets:insert(Tab, {CounterName, NewVal})
+ end
+ end,
+ Record =
+ fun(NoOfBenchsCtr, TotThrputCtr) ->
+ AddToSummaryCounter(NoOfBenchsCtr, 1),
+ AddToSummaryCounter(TotThrputCtr, Throughput)
+ end,
+ Record(nr_of_benchmarks, total_throughput),
+ case string:find(Name, "ordered_set") of
+ nomatch ->
+ Record(nr_of_set_benchmarks, total_throughput_set);
+ _ ->
+ Record(nr_of_ordered_set_benchmarks,
+ total_throughput_ordered_set)
+ end,
+ ct_event:notify(
+ #event{name = benchmark_data,
+ data = [{suite,"ets_bench"},
+ {name, Name},
+ {value,Throughput}]})
+ end
+ }).
add_lists(L1,L2) ->
add_lists(L1,L2,[]).
diff --git a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html
index a2c61aa938..27d6849c60 100644
--- a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html
+++ b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html
@@ -42,7 +42,7 @@
</p>
Paste the generated data in the field below and press the Render button:
<br>
- <textarea id="dataField" rows="4" cols="50"></textarea>
+ <textarea id="dataField" rows="4" cols="50">#bench_data_placeholder</textarea>
<br>
<input type="checkbox" id="barPlot"> Bar Plot
<br>
@@ -56,13 +56,13 @@
<br>
<input type="checkbox" class="showCheck" value="[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]</code>
<br>
- <input type="checkbox" class="showCheck" value="[set,public]"> Show <code>[set,public]</code>
+ <input type="checkbox" class="showCheck" value="[set,public]" checked> Show <code>[set,public]</code>
<br>
- <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true}]</code>
+ <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true}]</code>
<br>
- <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]"> Show <code>[set,public,{read_concurrency,true}]</code>
+ <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]" checked> Show <code>[set,public,{read_concurrency,true}]</code>
<br>
- <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code>
+ <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code>
<br>
<button id="renderButton" type="button">Render</button>
diff --git a/lib/stdlib/test/stdlib.spec b/lib/stdlib/test/stdlib.spec
index 4de7c1a0eb..bf64eae2c7 100644
--- a/lib/stdlib/test/stdlib.spec
+++ b/lib/stdlib/test/stdlib.spec
@@ -2,3 +2,6 @@
{skip_groups,"../stdlib_test",stdlib_bench_SUITE,
[binary,base64,gen_server,gen_statem,unicode],
"Benchmark only"}.
+{skip_groups,"../stdlib_test",ets_SUITE,
+ [benchmark],
+ "Benchmark only"}.
diff --git a/lib/stdlib/test/stdlib_bench.spec b/lib/stdlib/test/stdlib_bench.spec
index 7a0da811a0..6d665f22b6 100644
--- a/lib/stdlib/test/stdlib_bench.spec
+++ b/lib/stdlib/test/stdlib_bench.spec
@@ -8,3 +8,4 @@
{skip_groups,"../stdlib_test",stdlib_bench_SUITE,
[gen_server_comparison,gen_statem_comparison],
"Not a benchmark"}.
+{groups,"../stdlib_test",ets_SUITE,[benchmark]}.
diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl
index 251e09121c..6afe9e7a76 100644
--- a/lib/stdlib/test/string_SUITE.erl
+++ b/lib/stdlib/test/string_SUITE.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2004-2018. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2019. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -52,7 +52,7 @@
suite() ->
[{ct_hooks,[ts_install_cth]},
- {timetrap,{minutes,1}}].
+ {timetrap,{minutes,2}}].
all() ->
[{group, chardata}, {group, list_string}].
@@ -737,10 +737,10 @@ meas(Config) ->
case ct:get_timetrap_info() of
{_,{_,Scale}} when Scale > 1 ->
{skip,{will_not_run_in_debug,Scale}};
- _ -> % No scaling, run at most 1.5 min
+ _ -> % No scaling, run at most 2 mins
Tester = spawn(Exec),
receive {test_done, Tester} -> ok
- after 90000 ->
+ after 120000 ->
io:format("Timelimit reached stopping~n",[]),
exit(Tester, die)
end,
@@ -754,19 +754,22 @@ do_measure(DataDir) ->
io:format("~p~n",[byte_size(Bin)]),
Do = fun(Name, Func, Mode) ->
{N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 20),
- io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n",
+ io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n",
[Name, Mode, Mean/1000, Stddev/1000, N])
end,
Do2 = fun(Name, Func, Mode) ->
{N, Mean, Stddev, _} = time_func(Func, binary, <<>>, 20),
- io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n",
+ io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n",
[Name, Mode, Mean/1000, Stddev/1000, N])
end,
+ %% lefty_list means a list balanced to the left, like
+ %% [[[30],31],32]. Only some functions check such lists.
+ Modes = [list, lefty_list, binary, {many_lists,1}, {many_lists, 4}],
io:format("----------------------~n"),
Do(old_tokens, fun(Str) -> string:tokens(Str, [$\n,$\r]) end, list),
Tokens = {lexemes, fun(Str) -> string:lexemes(Str, [$\n,$\r]) end},
- [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- [list, binary]],
+ [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- Modes],
S0 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....",
S0B = <<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....">>,
@@ -824,17 +827,17 @@ do_measure(DataDir) ->
io:format("--~n",[]),
NthTokens = {nth_lexemes, fun(Str) -> string:nth_lexeme(Str, 18000, [$\n,$\r]) end},
- [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- [list, binary]],
+ [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- Modes],
Do2(take_t, repeat(fun() -> string:take(S0, [$.,$y], false, trailing) end), list),
Do2(take_t, repeat(fun() -> string:take(S0B, [$.,$y], false, trailing) end), binary),
Do2(take_tc, repeat(fun() -> string:take(S0, [$x], true, trailing) end), list),
Do2(take_tc, repeat(fun() -> string:take(S0B, [$x], true, trailing) end), binary),
Length = {length, fun(Str) -> string:length(Str) end},
- [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- [list, binary]],
+ [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- Modes],
Reverse = {reverse, fun(Str) -> string:reverse(Str) end},
- [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- [list, binary]],
+ [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- Modes],
ok.
@@ -1064,7 +1067,33 @@ time_func(N,Sum,SumSq, _, _, Res, _) ->
{N, Mean, Stdev, Res}.
mode(binary, Bin) -> Bin;
-mode(list, Bin) -> unicode:characters_to_list(Bin).
+mode(list, Bin) -> unicode:characters_to_list(Bin);
+mode(lefty_list, Bin) ->
+ L = unicode:characters_to_list(Bin),
+ to_left(L);
+mode({many_lists, N}, Bin) ->
+ group(unicode:characters_to_list(Bin), N).
+
+group([], _N) ->
+ [];
+group(L, N) ->
+ try lists:split(N, L) of
+ {L1, L2} ->
+ [L1 | group(L2, N)]
+ catch
+ _:_ ->
+ [L]
+ end.
+
+to_left([]) ->
+ [];
+to_left([H|L]) ->
+ to_left([H], L).
+
+to_left(V, []) ->
+ V;
+to_left(V, [H|L]) ->
+ to_left([V,H], L).
%%
%% Old string lists Test cases starts here.
diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript
index 70eec1a6f2..8636c69a0d 100755..100644
--- a/lib/stdlib/uc_spec/gen_unicode_mod.escript
+++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript
@@ -4,7 +4,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2017. All Rights Reserved.
+%% Copyright Ericsson AB 2017-2019. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -460,17 +460,73 @@ gen_cp(Fd) ->
" maybe_improper_list() | {error, unicode:chardata()}.\n"),
io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"),
io:put_chars(Fd, "cp([List]) -> cp(List);\n"),
- io:put_chars(Fd, "cp([List|R]) ->\n"),
- io:put_chars(Fd, " case cp(List) of\n"),
- io:put_chars(Fd, " [] -> cp(R);\n"),
- io:put_chars(Fd, " [CP] -> [CP|R];\n"),
- io:put_chars(Fd, " [C|R0] -> [C|[R0|R]];\n"),
- io:put_chars(Fd, " {error,Error} -> {error,[Error|R]}\n"),
- io:put_chars(Fd, " end;\n"),
+ io:put_chars(Fd, "cp([List|R]) -> cpl(List, R);\n"),
io:put_chars(Fd, "cp([]) -> [];\n"),
io:put_chars(Fd, "cp(<<C/utf8, R/binary>>) -> [C|R];\n"),
io:put_chars(Fd, "cp(<<>>) -> [];\n"),
- io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n\n"),
+ io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cpl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"),
+ io:put_chars(Fd, "cpl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"),
+ io:put_chars(Fd, "cpl([List], R) -> cpl(List, R);\n"),
+ io:put_chars(Fd, "cpl([List|T], R) -> cpl(List, [T|R]);\n"),
+ io:put_chars(Fd, "cpl([], R) -> cp(R);\n"),
+ io:put_chars(Fd, "cpl(<<C/utf8, T/binary>>, R) -> [C,T|R];\n"),
+ io:put_chars(Fd, "cpl(<<>>, R) -> cp(R);\n"),
+ io:put_chars(Fd, "cpl(<<B/binary>>, R) -> {error,[B|R]}.\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "%%%\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cpl_cont([C|T], R) when is_integer(C) -> [C|cpl_cont2(T, R)];\n"),
+ io:put_chars(Fd, "cpl_cont([L], R) -> cpl_cont(L, R);\n"),
+ io:put_chars(Fd, "cpl_cont([L|T], R) -> cpl_cont(L, [T|R]);\n"),
+ io:put_chars(Fd, "cpl_cont([], R) -> cpl_1_cont(R);\n"),
+ io:put_chars(Fd, "cpl_cont(T, R) -> [T|R].\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cpl_cont2([C|T], R) when is_integer(C) -> [C|cpl_cont3(T, R)];\n"),
+ io:put_chars(Fd, "cpl_cont2([L], R) -> cpl_cont2(L, R);\n"),
+ io:put_chars(Fd, "cpl_cont2([L|T], R) -> cpl_cont2(L, [T|R]);\n"),
+ io:put_chars(Fd, "cpl_cont2([], R) -> cpl_1_cont2(R);\n"),
+ io:put_chars(Fd, "cpl_cont2(T, R) -> [T|R].\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cpl_cont3([C], R) when is_integer(C) -> [C|R];\n"),
+ io:put_chars(Fd, "cpl_cont3([C|T], R) when is_integer(C) -> [C,T|R];\n"),
+ io:put_chars(Fd, "cpl_cont3([L], R) -> cpl_cont3(L, R);\n"),
+ io:put_chars(Fd, "cpl_cont3([L|T], R) -> cpl_cont3(L, [T|R]);\n"),
+ io:put_chars(Fd, "cpl_cont3([], R) -> cpl_1_cont3(R);\n"),
+ io:put_chars(Fd, "cpl_cont3(T, R) -> [T|R].\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "%%%\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cpl_1_cont([C|T]) when is_integer(C) -> [C|cpl_1_cont2(T)];\n"),
+ io:put_chars(Fd, "cpl_1_cont([L]) -> cpl_1_cont(L);\n"),
+ io:put_chars(Fd, "cpl_1_cont([L|T]) -> cpl_cont(L, T);\n"),
+ io:put_chars(Fd, "cpl_1_cont(T) -> T.\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cpl_1_cont2([C|T]) when is_integer(C) -> [C|cpl_1_cont3(T)];\n"),
+ io:put_chars(Fd, "cpl_1_cont2([L]) -> cpl_1_cont2(L);\n"),
+ io:put_chars(Fd, "cpl_1_cont2([L|T]) -> cpl_cont2(L, T);\n"),
+ io:put_chars(Fd, "cpl_1_cont2(T) -> T.\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cpl_1_cont3([C|_]=T) when is_integer(C) -> T;\n"),
+ io:put_chars(Fd, "cpl_1_cont3([L]) -> cpl_1_cont3(L);\n"),
+ io:put_chars(Fd, "cpl_1_cont3([L|T]) -> cpl_cont3(L, T);\n"),
+ io:put_chars(Fd, "cpl_1_cont3(T) -> T.\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "%%%\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cp_no_bin([C|_]=L) when is_integer(C) -> L;\n"),
+ io:put_chars(Fd, "cp_no_bin([List]) -> cp_no_bin(List);\n"),
+ io:put_chars(Fd, "cp_no_bin([List|R]) -> cp_no_binl(List, R);\n"),
+ io:put_chars(Fd, "cp_no_bin([]) -> [];\n"),
+ io:put_chars(Fd, "cp_no_bin(_) -> binary_found.\n"),
+ io:put_chars(Fd, "\n"),
+ io:put_chars(Fd, "cp_no_binl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"),
+ io:put_chars(Fd, "cp_no_binl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"),
+ io:put_chars(Fd, "cp_no_binl([List], R) -> cp_no_binl(List, R);\n"),
+ io:put_chars(Fd, "cp_no_binl([List|T], R) -> cp_no_binl(List, [T|R]);\n"),
+ io:put_chars(Fd, "cp_no_binl([], R) -> cp_no_bin(R);\n"),
+ io:put_chars(Fd, "cp_no_binl(_, _) -> binary_found.\n\n"),
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -481,11 +537,26 @@ gen_gc(Fd, GBP) ->
"-spec gc(String::unicode:chardata()) ->"
" maybe_improper_list() | {error, unicode:chardata()}.\n"),
io:put_chars(Fd,
- "gc([CP1, CP2|_]=T)\n"
- " when CP1 < 256, CP2 < 256, CP1 =/= $\r -> %% Ascii Fast path\n"
- " T;\n"
+ "gc([]=R) -> R;\n"
+ "gc([CP]=R) when is_integer(CP) -> R;\n"
+ "gc([$\\r=CP|R0]) ->\n"
+ " case cp(R0) of % Don't break CRLF\n"
+ " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n"
+ " T -> [CP|T]\n"
+ " end;\n"
+ "gc([CP1|T1]=T) when CP1 < 256 ->\n"
+ " case T1 of\n"
+ " [CP2|_] when CP2 < 256 -> T; %% Ascii Fast path\n"
+ " _ -> %% Keep the tail binary.\n"
+ " case cp_no_bin(T1) of\n"
+ " [CP2|_]=T3 when CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n"
+ " binary_found -> gc_1(T);\n"
+ " T4 -> gc_1([CP1|T4])\n"
+ " end\n"
+ " end;\n"
+ "gc(<<>>) -> [];\n"
"gc(<<CP1/utf8, Rest/binary>>) ->\n"
- " if CP1 < 256, CP1 =/= $\r ->\n"
+ " if CP1 < 256, CP1 =/= $\\r ->\n"
" case Rest of\n"
" <<CP2/utf8, _/binary>> when CP2 < 256 -> %% Ascii Fast path\n"
" [CP1|Rest];\n"
@@ -493,13 +564,12 @@ gen_gc(Fd, GBP) ->
" end;\n"
" true -> gc_1([CP1|Rest])\n"
" end;\n"
+ "gc([CP|_]=T) when is_integer(CP) -> gc_1(T);\n"
"gc(Str) ->\n"
- " gc_1(cp(Str)).\n\n"
- "gc_1([$\\r|R0] = R) ->\n"
- " case cp(R0) of % Don't break CRLF\n"
- " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n"
- " _ -> R\n"
- " end;\n"
+ " case cp(Str) of\n"
+ " {error,_}=Error -> Error;\n"
+ " CPs -> gc(CPs)\n"
+ " end.\n"
),
GenExtP = fun(Range) -> io:format(Fd, "gc_1~s gc_ext_pict(R1,[CP]);\n", [gen_clause(Range)]) end,
@@ -507,7 +577,12 @@ gen_gc(Fd, GBP) ->
%% Pick codepoints below 256 (some data knowledge here)
{ExtendedPictographicLow,ExtendedPictographicHigh} =
lists:splitwith(fun({Start,undefined}) -> Start < 256 end,ExtendedPictographic0),
-
+ io:put_chars(Fd,
+ "\ngc_1([$\\r|R0] = R) ->\n"
+ " case cp(R0) of % Don't break CRLF\n"
+ " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n"
+ " _ -> R\n"
+ " end;\n"),
io:put_chars(Fd, "\n%% Handle control\n"),
GenControl = fun(Range) -> io:format(Fd, "gc_1~s R0;\n", [gen_clause(Range)]) end,
CRs0 = merge_ranges(maps:get(cr, GBP) ++ maps:get(lf, GBP) ++ maps:get(control, GBP), false),
@@ -516,7 +591,14 @@ gen_gc(Fd, GBP) ->
%%GenControl(R1),GenControl(R2),GenControl(R3),
io:put_chars(Fd, "\n%% Optimize Latin-1\n"),
[GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicLow)],
- io:format(Fd, "gc_1([CP|R]) when CP < 256 -> gc_extend(R,CP);\n\n", []),
+
+ io:format(Fd,
+ "gc_1([CP|R]=R0) when CP < 256 ->\n"
+ " case R of\n"
+ " [CP2|_] when CP2 < 256 -> R0;\n"
+ " _ -> gc_extend(cp(R), R, CP)\n"
+ " end;\n",
+ []),
io:put_chars(Fd, "\n%% Continue control\n"),
[GenControl(CP) || CP <- Crs],
%% One clause per CP
@@ -540,7 +622,7 @@ gen_gc(Fd, GBP) ->
io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, []);\n"),
io:put_chars(Fd, "\n%% Handle Regional\n"),
- GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,[CP]);\n", [gen_clause(Range)]) end,
+ GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end,
[GenRegional(CP) || CP <- merge_ranges(maps:get(regional_indicator,GBP))],
%% io:put_chars(Fd, "%% Handle E_Base\n"),
%% GenEBase = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end,
@@ -552,9 +634,7 @@ gen_gc(Fd, GBP) ->
io:put_chars(Fd, "%% Handle extended_pictographic\n"),
[GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicHigh)],
io:put_chars(Fd, "\n%% default clauses\n"),
- io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(R, CP);\n"),
- io:put_chars(Fd, "gc_1([]) -> [];\n"),
- io:put_chars(Fd, "gc_1({error,_}=Error) -> Error.\n\n"),
+ io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(cp(R), R, CP).\n\n"),
io:put_chars(Fd, "%% Handle Prepend\n"),
io:put_chars(Fd,
@@ -581,31 +661,24 @@ gen_gc(Fd, GBP) ->
"%% To simplify binary handling in libraries the tail should be kept binary\n"
"%% and not a lookahead CP\n"
),
- io:put_chars(Fd, "gc_extend(T, Acc) ->\n"
- " gc_extend(cp(T), T, Acc).\n\n"),
io:put_chars(Fd,
- "gc_extend([CP|T], T0, Acc0) ->\n"
+ "gc_extend([CP|T], T0, CP0) ->\n"
" case is_extend(CP) of\n"
- " false ->\n"
- " case Acc0 of\n"
- " [Acc] -> [Acc|T0];\n"
- " [_|_]=Acc -> [lists:reverse(Acc)|T0];\n"
- " Acc -> [Acc|T0]\n"
- " end;\n"
- " _TrueOrZWJ ->\n"
- " case Acc0 of\n"
- " [_|_] -> gc_extend(T, [CP|Acc0]);\n"
- " Acc -> gc_extend(T, [CP,Acc])\n"
- " end\n"
+ " false -> [CP0|T0]; % losing work done on T\n"
+ " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP,CP0])\n"
" end;\n"
- "gc_extend([], _, Acc0) ->\n"
- " case Acc0 of\n"
- " [_]=Acc -> Acc;\n"
- " [_|_]=Acc -> [lists:reverse(Acc)];\n"
- " Acc -> [Acc]\n"
+ "gc_extend([], _, CP) -> [CP];\n"
+ "gc_extend({error,R}, _, CP) -> [CP|R].\n\n"),
+ io:put_chars(Fd,
+ "gc_extend2([CP|T], T0, Acc) ->\n"
+ " case is_extend(CP) of\n"
+ " false -> [lists:reverse(Acc)|T0]; % losing work done on T\n"
+ " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP|Acc])\n"
" end;\n"
- "gc_extend({error,R}, T, Acc0) ->\n"
- " gc_extend([], T, Acc0) ++ [R].\n\n"
+ "gc_extend2([], _, Acc) ->\n"
+ " [lists:reverse(Acc)];\n"
+ "gc_extend2({error,R}, _, Acc) ->\n"
+ " [lists:reverse(Acc)] ++ [R].\n\n"
),
[ZWJ] = maps:get(zwj, GBP),
GenExtend = fun(R) when R =:= ZWJ -> io:format(Fd, "is_extend~s zwj;\n", [gen_single_clause(ZWJ)]);
@@ -660,10 +733,10 @@ gen_gc(Fd, GBP) ->
%% --------------------
io:put_chars(Fd, "%% Handle Regional\n"),
[{RLess,RLarge}] = merge_ranges(maps:get(regional_indicator,GBP)),
- io:put_chars(Fd,"gc_regional(R0, Acc) ->\n"
+ io:put_chars(Fd,"gc_regional(R0, CP0) ->\n"
" case cp(R0) of\n"),
- io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend(R1,[CP|Acc]);~n",[RLess, RLarge]),
- io:put_chars(Fd," R1 -> gc_extend(R1, R0, Acc)\n"
+ io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend2(cp(R1),R1,[CP,CP0]);~n",[RLess, RLarge]),
+ io:put_chars(Fd," R1 -> gc_extend(R1, R0, CP0)\n"
" end.\n\n"),
%% Special hangul
@@ -685,16 +758,23 @@ gen_gc(Fd, GBP) ->
GenHangulV_2 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n",
[$\s,gen_case_clause(Range)]) end,
[GenHangulV_2(CP) || CP <- merge_ranges(maps:get(t,GBP))],
- io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"),
-
+ io:put_chars(Fd,
+ " R1 ->\n"
+ " case Acc of\n"
+ " [CP] -> gc_extend(R1, R0, CP);\n"
+ " _ -> gc_extend2(R1, R0, Acc)\n"
+ " end\n end.\n\n"),
io:put_chars(Fd, "%% Handle Hangul T\n"),
io:put_chars(Fd, "gc_h_T(R0, Acc) ->\n case cp(R0) of\n"),
GenHangulT_1 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n",
[$\s,gen_case_clause(Range)]) end,
[GenHangulT_1(CP) || CP <- merge_ranges(maps:get(t,GBP))],
- io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"),
-
- io:put_chars(Fd, "gc_h_lv_lvt({error,_}=Error, Acc) -> gc_extend(Error, [], Acc);\n"),
+ io:put_chars(Fd,
+ " R1 ->\n"
+ " case Acc of\n"
+ " [CP] -> gc_extend(R1, R0, CP);\n"
+ " _ -> gc_extend2(R1, R0, Acc)\n"
+ " end\n end.\n\n"),
io:put_chars(Fd, "%% Handle Hangul LV\n"),
GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n",
[gen_clause2(Range)]) end,
@@ -703,8 +783,10 @@ gen_gc(Fd, GBP) ->
GenHangulLVT = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_T(R1,[CP|Acc]);\n",
[gen_clause2(Range)]) end,
[GenHangulLVT(CP) || CP <- merge_ranges(maps:get(lvt,GBP))],
- io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(R, CP);\n"), %% From gc_1/1
- io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend(R, Acc).\n\n"),
+ io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(cp(R), R, CP);\n"), %% From gc_1/1
+ io:put_chars(Fd, "%% Also handles error tuples\n"),
+ io:put_chars(Fd, "gc_h_lv_lvt(R, [CP]) -> gc_extend(R, R, CP);\n"),
+ io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend2(R, R, Acc).\n\n"),
ok.
gen_compose_pairs(Fd, ExclData, Data) ->
generated by cgit v1.2.3 (git 2.25.1) at 2025-05-09 04:07:36 +0000