From 881240485b96cfa7d4a75587f6f14c4e6c83eeeb Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 13 Dec 2017 17:09:24 +0100 Subject: stdlib: Make ets_SUITE memory check try again as memory stats do not guarantee consistency. A typical ETS test case ends by a lot of deallocating that may now trigger homecoming carrier migration, that in turn can cause quite large inconsistencies in memory stats when same carrier is accounted for twice or not at all. And that's my theory why I now sometimes see transient discrepancies between before and after memory stats. --- lib/stdlib/test/ets_SUITE.erl | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 00e02a06cc..a58497523a 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -5716,25 +5716,39 @@ etsmem() -> end}, {Mem,AllTabs}. -verify_etsmem({MemInfo,AllTabs}) -> + +verify_etsmem(MI) -> wait_for_test_procs(), + verify_etsmem(MI, 1). + +verify_etsmem({MemInfo,AllTabs}, Try) -> case etsmem() of {MemInfo,_} -> io:format("Ets mem info: ~p", [MemInfo]), - case MemInfo of - {ErlMem,EtsAlloc} when ErlMem == notsup; EtsAlloc == undefined -> + case {MemInfo, Try} of + {{ErlMem,EtsAlloc},_} when ErlMem == notsup; EtsAlloc == undefined -> %% Use 'erl +Mea max' to do more complete memory leak testing. {comment,"Incomplete or no mem leak testing"}; - _ -> - ok + {_, 1} -> + ok; + _ -> + {comment, "Transient memory discrepancy"} end; {MemInfo2, AllTabs2} -> io:format("Expected: ~p", [MemInfo]), io:format("Actual: ~p", [MemInfo2]), io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]), io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), - ets_test_spawn_logger ! {failed_memcheck, get('__ETS_TEST_CASE__')}, - {comment, "Failed memory check"} + case Try < 2 of + true -> + io:format("\nThis discrepancy could be caused by an " + "inconsistent memory \"snapshot\"" + "\nTry again...\n", []), + verify_etsmem({MemInfo, AllTabs}, Try+1); + false -> + ets_test_spawn_logger ! {failed_memcheck, get('__ETS_TEST_CASE__')}, + {comment, "Failed memory check"} + end end. -- cgit v1.2.3 From 55e929c4ed5cd854038c18697123ea94948ebf35 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 13 Dec 2017 11:22:45 +0100 Subject: kernel: Add os:cmd/2 with max_size option --- lib/kernel/doc/src/os.xml | 43 ++++++++++++++++++++++++++++++++++++++----- lib/kernel/src/os.erl | 37 ++++++++++++++++++++++++++----------- lib/kernel/test/os_SUITE.erl | 18 ++++++++++++++++-- 3 files changed, 80 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/kernel/doc/src/os.xml b/lib/kernel/doc/src/os.xml index 0e9add4161..7ce2f54542 100644 --- a/lib/kernel/doc/src/os.xml +++ b/lib/kernel/doc/src/os.xml @@ -38,17 +38,35 @@ most platforms.

+ + + + + + + +

Options for os:cmd/2

+ + max_size + +

The maximum size of the data returned by the os:cmd call. + See the os:cmd/2 + documentation for more details.

+ + + + + + + Execute a command in a shell of the target OS.

Executes Command in a command shell of the - target OS, - captures the standard output of the command, and returns this - result as a string. This function is a replacement of - the previous function unix:cmd/1; they are equivalent on a - Unix platform.

+ target OS, captures the standard output of the command, + and returns this result as a string.

Examples:

LsOut = os:cmd("ls"), % on unix platform @@ -57,6 +75,21 @@ DirOut = os:cmd("dir"), % on Win32 platform called from another program (for example, os:cmd/1) can differ, compared with the standard output of the command when called directly from an OS command shell.

+

os:cmd/2 was added in kernel-5.5 (OTP-20.2.1). It makes it + possible to pass an options map as the second argument in order to + control the behaviour of os:cmd. The possible options are: +

+ + max_size + +

The maximum size of the data returned by the os:cmd call. + This option is a safety feature that should be used when the command + executed can return a very large, possibly infinite, result.

+ +> os:cmd("cat /dev/zero", #{ max_size => 20 }). +[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + + diff --git a/lib/kernel/src/os.erl b/lib/kernel/src/os.erl index 0250783632..ae188ae932 100644 --- a/lib/kernel/src/os.erl +++ b/lib/kernel/src/os.erl @@ -21,7 +21,7 @@ %% Provides a common operating system interface. --export([type/0, version/0, cmd/1, find_executable/1, find_executable/2]). +-export([type/0, version/0, cmd/1, cmd/2, find_executable/1, find_executable/2]). -include("file.hrl"). @@ -32,6 +32,11 @@ putenv/2, set_signal/2, system_time/0, system_time/1, timestamp/0, unsetenv/1]). +-type os_command() :: atom() | io_lib:chars(). +-type os_command_opts() :: #{ max_size => non_neg_integer() | infinity }. + +-export_type([os_command/0, os_command_opts/0]). + -spec getenv() -> [string()]. getenv() -> erlang:nif_error(undef). @@ -232,15 +237,21 @@ extensions() -> %% Executes the given command in the default shell for the operating system. -spec cmd(Command) -> string() when - Command :: atom() | io_lib:chars(). + Command :: os_command(). cmd(Cmd) -> + cmd(Cmd, #{ }). + +-spec cmd(Command, Options) -> string() when + Command :: os_command(), + Options :: os_command_opts(). +cmd(Cmd, Opts) -> validate(Cmd), {SpawnCmd, SpawnOpts, SpawnInput, Eot} = mk_cmd(os:type(), Cmd), Port = open_port({spawn, SpawnCmd}, [binary, stderr_to_stdout, stream, in, hide | SpawnOpts]), MonRef = erlang:monitor(port, Port), true = port_command(Port, SpawnInput), - Bytes = get_data(Port, MonRef, Eot, []), + Bytes = get_data(Port, MonRef, Eot, [], 0, maps:get(max_size, Opts, infinity)), demonitor(MonRef, [flush]), String = unicode:characters_to_list(Bytes), if %% Convert to unicode list if possible otherwise return bytes @@ -291,12 +302,13 @@ validate1([List|Rest]) when is_list(List) -> validate1([]) -> ok. -get_data(Port, MonRef, Eot, Sofar) -> +get_data(Port, MonRef, Eot, Sofar, Size, Max) -> receive {Port, {data, Bytes}} -> - case eot(Bytes, Eot) of + case eot(Bytes, Eot, Size, Max) of more -> - get_data(Port, MonRef, Eot, [Sofar,Bytes]); + get_data(Port, MonRef, Eot, [Sofar, Bytes], + Size + byte_size(Bytes), Max); Last -> catch port_close(Port), flush_until_down(Port, MonRef), @@ -307,13 +319,16 @@ get_data(Port, MonRef, Eot, Sofar) -> iolist_to_binary(Sofar) end. -eot(_Bs, <<>>) -> +eot(_Bs, <<>>, _Size, _Max) -> more; -eot(Bs, Eot) -> +eot(Bs, Eot, Size, Max) -> case binary:match(Bs, Eot) of - nomatch -> more; - {Pos, _} -> - binary:part(Bs,{0, Pos}) + nomatch when Size + byte_size(Bs) < Max -> + more; + {Pos, _} when Size + Pos < Max -> + binary:part(Bs,{0, Pos}); + _ -> + binary:part(Bs,{0, Max - Size}) end. %% When port_close returns we know that all the diff --git a/lib/kernel/test/os_SUITE.erl b/lib/kernel/test/os_SUITE.erl index 53a9e168ef..079b9790f6 100644 --- a/lib/kernel/test/os_SUITE.erl +++ b/lib/kernel/test/os_SUITE.erl @@ -25,7 +25,8 @@ -export([space_in_cwd/1, quoting/1, cmd_unicode/1, space_in_name/1, bad_command/1, find_executable/1, unix_comment_in_command/1, deep_list_command/1, large_output_command/1, background_command/0, background_command/1, - message_leak/1, close_stdin/0, close_stdin/1, perf_counter_api/1]). + message_leak/1, close_stdin/0, close_stdin/1, max_size_command/1, + perf_counter_api/1]). -include_lib("common_test/include/ct.hrl"). @@ -37,7 +38,7 @@ all() -> [space_in_cwd, quoting, cmd_unicode, space_in_name, bad_command, find_executable, unix_comment_in_command, deep_list_command, large_output_command, background_command, message_leak, - close_stdin, perf_counter_api]. + close_stdin, max_size_command, perf_counter_api]. groups() -> []. @@ -312,6 +313,19 @@ close_stdin(Config) -> "-1" = os:cmd(Fds). +max_size_command(_Config) -> + + Res20 = os:cmd("cat /dev/zero", #{ max_size => 20 }), + 20 = length(Res20), + + Res0 = os:cmd("cat /dev/zero", #{ max_size => 0 }), + 0 = length(Res0), + + Res32768 = os:cmd("cat /dev/zero", #{ max_size => 32768 }), + 32768 = length(Res32768), + + ResHello = os:cmd("echo hello", #{ max_size => 20 }), + 6 = length(ResHello). %% Test that the os:perf_counter api works as expected perf_counter_api(_Config) -> -- cgit v1.2.3 From defd43985282606e841e2bcb29ad7414080d5a80 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 26 Jan 2018 18:42:32 +0100 Subject: erts: Add age order first fit allocator strategies ageffcaoff: Age First Fit Carrier, Address Order First Fit (within carrier) ageffcbf : Age First Fit Carrier, Best Fit (within carrier) ageffcaobf: Age First Fit Carrier, Address Order Best Fit (within carrier) Prefer old carriers, the older the better. --- lib/runtime_tools/src/erts_alloc_config.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/runtime_tools/src/erts_alloc_config.erl b/lib/runtime_tools/src/erts_alloc_config.erl index 514530332c..4b028681a0 100644 --- a/lib/runtime_tools/src/erts_alloc_config.erl +++ b/lib/runtime_tools/src/erts_alloc_config.erl @@ -265,7 +265,13 @@ strategy_str(aoff) -> strategy_str(aoffcbf) -> "Address order first fit carrier best fit"; strategy_str(aoffcaobf) -> - "Address order first fit carrier adress order best fit". + "Address order first fit carrier adress order best fit"; +strategy_str(ageffcaoff) -> + "Age order first fit carrier address order first fit"; +strategy_str(ageffcbf) -> + "Age order first fit carrier best fit"; +strategy_str(ageffcaobf) -> + "Age order first fit carrier adress order best fit". default_acul(A, S) -> case carrier_migration_support(S) of -- cgit v1.2.3 From 3162499ddc48d4ce877945775f084725ceab57c5 Mon Sep 17 00:00:00 2001 From: Erlang/OTP Date: Mon, 12 Feb 2018 13:38:24 +0100 Subject: Update version numbers --- lib/kernel/vsn.mk | 2 +- lib/runtime_tools/vsn.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 106bda01ca..91261e1d55 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 5.4.1 +KERNEL_VSN = 5.4.2 diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index 872bd5db1d..0dc6a48570 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.12.3 +RUNTIME_TOOLS_VSN = 1.12.4 -- cgit v1.2.3 From e11c649522be8849cc21e364734be62c7783f090 Mon Sep 17 00:00:00 2001 From: Erlang/OTP Date: Mon, 12 Feb 2018 13:38:43 +0100 Subject: Update release notes --- lib/kernel/doc/src/notes.xml | 20 ++++++++++++++++++++ lib/runtime_tools/doc/src/notes.xml | 17 +++++++++++++++++ 2 files changed, 37 insertions(+) (limited to 'lib') diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index d7f224c38e..65fe9b9c07 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,26 @@

This document describes the changes made to the Kernel application.

+
Kernel 5.4.2 + +
Fixed Bugs and Malfunctions + + +

+ Add os:cmd/2 that takes an options map as the + second argument.

+

+ Add max_size as an option to os:cmd/2 that + control the maximum size of the result that + os:cmd/2 will return.

+

+ Own Id: OTP-14823

+ + +
+ +
+
Kernel 5.4.1
Fixed Bugs and Malfunctions diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 93e3e26fda..74300ba3fc 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,23 @@

This document describes the changes made to the Runtime_Tools application.

+
Runtime_Tools 1.12.4 + +
Improvements and New Features + + +

+ New family of erts_alloc strategies: Age Order + First Fit. Similar to "address order", but instead the + oldest possible carrier is always chosen for allocation.

+

+ Own Id: OTP-14917 Aux Id: ERIERL-88

+ + +
+ +
+
Runtime_Tools 1.12.3
Fixed Bugs and Malfunctions -- cgit v1.2.3