diff options
33 files changed, 1074 insertions, 132 deletions
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index b83134c79c..6d4a895ef6 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3606,14 +3606,8 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix)); ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof(DbFixation), 0); } - else { - ASSERT(fix->counter == 0); - } db_unlock(tb, LCK_WRITE_REC); } - else { - ASSERT(fix->counter == 0); - } erts_bin_release(fix->tabs.btid); erts_free(ERTS_ALC_T_DB_FIXATION, fix); @@ -3855,11 +3849,8 @@ static void free_fixations_op(DbFixation* fix, void* vctx) { struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx; erts_aint_t diff; -#ifdef DEBUG - DbTable* dbg_tb = btid2tab(fix->tabs.btid); -#endif - ASSERT(!dbg_tb || dbg_tb == ctx->tb); + ASSERT(!btid2tab(fix->tabs.btid)); ASSERT(fix->counter > 0); ASSERT(ctx->tb->common.status & DB_DELETE); diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 19055c6110..7ce104a84c 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -220,6 +220,9 @@ typedef struct db_fixation { Process* p; } procs; + /* Number of fixations on table from procs.p + * Protected by table write lock or read lock + fixlock + */ Uint counter; } DbFixation; diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 1ab1e47254..4d7a86398a 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1474,10 +1474,10 @@ erts_port_task_schedule(Eterm id, } #endif - if (!pp) - goto fail; - if (type != ERTS_PORT_TASK_PROC_SIG) { + if (!pp) + goto fail; + ptp = port_task_alloc(); ptp->type = type; @@ -1515,6 +1515,9 @@ erts_port_task_schedule(Eterm id, ptp->u.alive.td.psig.callback = va_arg(argp, ErtsProc2PortSigCallback); ptp->u.alive.flags |= va_arg(argp, int); va_end(argp); + if (!pp) + goto fail; + if (!(ptp->u.alive.flags & ERTS_PT_FLG_NOSUSPEND)) set_tmp_handle(ptp, pthp); else { diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 63c838a91d..b72bac00c1 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1597,6 +1597,12 @@ erts_proclist_create(Process *p) return proclist_create(p); } +ErtsProcList * +erts_proclist_copy(ErtsProcList *plp) +{ + return proclist_copy(plp); +} + void erts_proclist_destroy(ErtsProcList *plp) { @@ -12512,6 +12518,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg.first = NULL; p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; + p->msg.saved_last = &p->msg.first; p->msg.len = 0; #ifdef ERTS_SMP p->msg_inq.first = NULL; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 9d7ba27c50..639818c20c 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1612,6 +1612,7 @@ Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); +ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 1560844521..c0a3838d42 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1923,7 +1923,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } result_bin = erts_bin_nrml_alloc(size); - result_bin->orig_bytes[0] = VERSION_MAGIC; + result_bin->orig_bytes[0] = (byte)VERSION_MAGIC; /* Next state immediately, no need to export context */ context->state = TTBEncode; context->s.ec.flags = flags; @@ -1981,7 +1981,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->s.cc.result_bin = result_bin; result_bin = erts_bin_nrml_alloc(real_size); - result_bin->orig_bytes[0] = VERSION_MAGIC; + result_bin->orig_bytes[0] = (byte) VERSION_MAGIC; context->s.cc.destination_bin = result_bin; context->s.cc.dest_len = 0; diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 50d8a35217..c1fa660cf9 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -789,6 +789,8 @@ signalterm_to_signum(Eterm signal) } } +#ifdef ERTS_SMP + static ERTS_INLINE Eterm signum_to_signalterm(int signum) { @@ -810,6 +812,8 @@ signum_to_signalterm(int signum) } } +#endif + #ifndef ERTS_SMP static ERTS_INLINE Uint signum_to_signalstate(int signum) diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl index a5f605bfff..28df36d293 100644 --- a/erts/emulator/test/iovec_SUITE.erl +++ b/erts/emulator/test/iovec_SUITE.erl @@ -20,7 +20,7 @@ -module(iovec_SUITE). --export([all/0, suite/0]). +-export([all/0, suite/0, init_per_suite/1, end_per_suite/1]). -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1, mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1, @@ -37,6 +37,13 @@ all() -> illegal_lists, improper_lists, cons_bomb, iolist_to_iovec_idempotence, iolist_to_iovec_correctness]. +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + application:stop(os_mon), + Config. + integer_lists(Config) when is_list(Config) -> Variations = gen_variations([I || I <- lists:seq(1, 255)]), diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 6ded7ff1c9..a9f20f9928 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -152,7 +152,11 @@ spawn_with_binaries(Config) when is_list(Config) -> TwoMeg = lists:duplicate(1024, L), Fun = fun() -> spawn(?MODULE, binary_owner, [list_to_binary(TwoMeg)]), receive after 1 -> ok end end, - test_server:do_times(150, Fun), + Iter = case test_server:is_valgrind() of + true -> 10; + false -> 150 + end, + test_server:do_times(Iter, Fun), ok. binary_owner(Bin) when is_binary(Bin) -> diff --git a/erts/emulator/valgrind/suppress.patched.3.6.0 b/erts/emulator/valgrind/suppress.patched.3.6.0 index fcde4a0123..29f2d3d62d 100644 --- a/erts/emulator/valgrind/suppress.patched.3.6.0 +++ b/erts/emulator/valgrind/suppress.patched.3.6.0 @@ -374,3 +374,10 @@ fun:erts_debug_set_internal_state_2 fun:process_main } +{ +Thread specific dlerror buffer. Either bug in libc or valgrind. +Memcheck:Leak +... +fun:_dlerror_run +... +} diff --git a/erts/emulator/valgrind/suppress.standard b/erts/emulator/valgrind/suppress.standard index bb07c92fc1..99a3ee4048 100644 --- a/erts/emulator/valgrind/suppress.standard +++ b/erts/emulator/valgrind/suppress.standard @@ -342,3 +342,11 @@ fun:erts_debug_set_internal_state_2 fun:process_main } +{ +Thread specific dlerror buffer. Either bug in libc or valgrind. +Memcheck:Leak +... +fun:_dlerror_run +... +} + diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index c61cecca4c..b98a704e28 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -266,7 +266,7 @@ replace_path(PathA, PathB) -> true = code:add_patha(PathB). join(Rule, Opts) -> - string:join([atom_to_list(Rule)|lists:map(fun atom_to_list/1, Opts)], "_"). + lists:join("_", [atom_to_list(Rule)|lists:map(fun atom_to_list/1, Opts)]). %%------------------------------------------------------------------------------ %% Test cases diff --git a/lib/asn1/test/testUniqueObjectSets.erl b/lib/asn1/test/testUniqueObjectSets.erl index 476d190651..cabdb44a0c 100644 --- a/lib/asn1/test/testUniqueObjectSets.erl +++ b/lib/asn1/test/testUniqueObjectSets.erl @@ -60,7 +60,7 @@ main(CaseDir, Rule, Opts) -> Objs = [gen_obj(I) || {I,_,_} <- D1], DupObjs = [gen_dup_obj(I, T) || {I,T,_} <- D1], DupObjRefs0 = [gen_dup_obj_refs(I) || {I,_,_} <- D1], - DupObjRefs = string:join(DupObjRefs0, " |\n"), + DupObjRefs = lists:join(" |\n", DupObjRefs0), Asn1Spec = 'UniqueObjectSets', A = ["UniqueObjectSets DEFINITIONS AUTOMATIC TAGS ::=\n", "BEGIN\n\n", diff --git a/lib/asn1/test/test_modified_x420.erl b/lib/asn1/test/test_modified_x420.erl index 6cd9e0e33b..15f7c70978 100644 --- a/lib/asn1/test/test_modified_x420.erl +++ b/lib/asn1/test/test_modified_x420.erl @@ -38,7 +38,7 @@ read_pem(File) -> extract_base64(Binary) -> - extract_base64_lines(string:tokens(binary_to_list(Binary), "\n")). + extract_base64_lines(string:lexemes(binary_to_list(Binary), "\n")). extract_base64_lines(["-----BEGIN"++_ | Lines]) -> take_base64_lines(Lines, _Acc = []); diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index ee3a5e4bba..dc6b7a536c 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -21,7 +21,7 @@ -define(DEFAULT_TIMETRAP_SECS, 60). %%% TEST_SERVER_CTRL INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --export([run_test_case_apply/1,init_target_info/0]). +-export([run_test_case_apply/1,init_target_info/0,init_valgrind/0]). -export([cover_compile/1,cover_analyse/2]). %%% TEST_SERVER_SUP INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -49,6 +49,10 @@ -export([break/1,break/2,break/3,continue/0,continue/1]). +%%% DEBUGGER INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-export([valgrind_new_leaks/0, valgrind_format/2, + is_valgrind/0]). + %%% PRIVATE EXPORTED %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([]). @@ -69,6 +73,10 @@ init_target_info() -> username=test_server_sup:get_username(), cookie=atom_to_list(erlang:get_cookie())}. +init_valgrind() -> + valgrind_new_leaks(). + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% cover_compile(#cover{app=App,incl=Include,excl=Exclude,cross=Cross}) -> %% {ok,#cover{mods=AnalyseModules}} | {error,Reason} @@ -358,11 +366,12 @@ stick_all_sticky(Node,Sticky) -> %% compensate timetraps for runtime delays introduced by e.g. tools like %% cover. -run_test_case_apply({Mod,Func,Args,Name,RunInit,TimetrapData}) -> +run_test_case_apply({CaseNum,Mod,Func,Args,Name,RunInit,TimetrapData}) -> case is_valgrind() of false -> ok; true -> + valgrind_format("Test case #~w ~w:~w/1", [CaseNum, Mod, Func]), os:putenv("VALGRIND_LOGFILE_INFIX",atom_to_list(Mod)++"."++ atom_to_list(Func)++"-") end, @@ -370,6 +379,7 @@ run_test_case_apply({Mod,Func,Args,Name,RunInit,TimetrapData}) -> Result = run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData), ProcAft = erlang:system_info(process_count), + valgrind_new_leaks(), DetFail = get(test_server_detected_fail), {Result,DetFail,ProcBef,ProcAft}. @@ -2735,11 +2745,36 @@ is_commercial() -> %% %% Returns true if valgrind is running, else false is_valgrind() -> - case os:getenv("TS_RUN_VALGRIND") of - false -> false; - _ -> true + case catch erlang:system_info({valgrind, running}) of + {'EXIT', _} -> false; + Res -> Res end. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% DEBUGGER INTERFACE %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% valgrind_new_leaks() -> ok +%% +%% Checks for new memory leaks if Valgrind is active. +valgrind_new_leaks() -> + catch erlang:system_info({valgrind, memory}), + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% valgrind_format(Format, Args) -> ok +%% Format = string() +%% Args = lists() +%% +%% Outputs the formatted string to Valgrind's logfile,if Valgrind is active. +valgrind_format(Format, Args) -> + (catch erlang:system_info({valgrind, io_lib:format(Format, Args)})), + ok. + + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% Apply given function and reply to caller or proxy. diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index 9412c43187..71978c7267 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -2163,6 +2163,7 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) -> %% Runs the specified tests, then displays/logs the summary. run_test_cases(TestSpec, Config, TimetrapData) -> + test_server:init_valgrind(), case lists:member(no_src, get(test_server_logopts)) of true -> ok; @@ -3796,7 +3797,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, %% run the test case {Result,DetectedFail,ProcsBefore,ProcsAfter} = - run_test_case_apply(Mod, Func, [UpdatedArgs], GrName, + run_test_case_apply(Num, Mod, Func, [UpdatedArgs], GrName, RunInit, TimetrapData), {Time,RetVal,Loc,Opts,Comment} = case Result of @@ -4366,7 +4367,7 @@ do_format_exception(Reason={Error,Stack}) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% run_test_case_apply(Mod, Func, Args, Name, RunInit, +%% run_test_case_apply(CaseNum, Mod, Func, Args, Name, RunInit, %% TimetrapData) -> %% {{Time,RetVal,Loc,Opts,Comment},DetectedFail,ProcessesBefore,ProcessesAfter} | %% {{died,Reason,unknown,Comment},DetectedFail,ProcessesBefore,ProcessesAfter} @@ -4380,9 +4381,9 @@ do_format_exception(Reason={Error,Stack}) -> %% ProcessesBefore = ProcessesAfter = integer() %% -run_test_case_apply(Mod, Func, Args, Name, RunInit, +run_test_case_apply(CaseNum, Mod, Func, Args, Name, RunInit, TimetrapData) -> - test_server:run_test_case_apply({Mod,Func,Args,Name,RunInit, + test_server:run_test_case_apply({CaseNum,Mod,Func,Args,Name,RunInit, TimetrapData}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/common_test/test_server/ts_lib.erl b/lib/common_test/test_server/ts_lib.erl index a7be740c5c..ea039a2c2b 100644 --- a/lib/common_test/test_server/ts_lib.erl +++ b/lib/common_test/test_server/ts_lib.erl @@ -120,7 +120,8 @@ specs(Dir) -> [] end end, Specs), - sort_tests(MainSpecs). + + sort_tests(filter_tests(MainSpecs)). test_categories(Dir, App) -> Specs = filelib:wildcard(filename:join([filename:dirname(Dir), @@ -141,10 +142,29 @@ suites(Dir, App) -> "*_SUITE.erl"]), Suites=filelib:wildcard(Glob), [filename_to_atom(Name) || Name <- Suites]. - + filename_to_atom(Name) -> list_to_atom(filename:rootname(filename:basename(Name))). +%% Filter out tests of applications that are not accessible + +filter_tests(Tests) -> + lists:filter( + fun(Special) when Special == epmd; + Special == emulator; + Special == system -> + true; + (Test) -> + case application:load(filename_to_atom(Test)) of + {error, {already_loaded, _}} -> + true; + {error,_NoSuchApplication} -> + false; + _ -> + true + end + end, Tests). + %% Sorts a list of either log files directories or spec files. sort_tests(Tests) -> diff --git a/lib/common_test/test_server/ts_run.erl b/lib/common_test/test_server/ts_run.erl index 82ae44ec06..e22fa8d196 100644 --- a/lib/common_test/test_server/ts_run.erl +++ b/lib/common_test/test_server/ts_run.erl @@ -96,6 +96,9 @@ ct_run_test(Dir, CommonTestArgs) -> case ct:run_test(CommonTestArgs) of {_,_,_} -> ok; + {error,{make_failed, _Modules} = Error} -> + io:format("ERROR: ~P\n", [Error,20]), + erlang:halt(123, [{flush,false}]); {error,Error} -> io:format("ERROR: ~P\n", [Error,20]); Other -> @@ -288,6 +291,10 @@ tricky_print_data(Port, Timeout) -> receive {Port, {exit_status, 0}} -> ok; + {Port, {exit_status, 123 = N}} -> + io:format(user, "Test run exited with status ~p," + "aborting rest of test~n", [N]), + erlang:halt(123, [{flush,false}]); {Port, {exit_status, N}} -> io:format(user, "Test run exited with status ~p~n", [N]) after 1 -> diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 0fd99bbc04..95c8b5ebce 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -772,12 +772,16 @@ tab_is_disj(K1, T1, T2) -> end. merge_tables(T1, T2) -> - ets:safe_fixtable(T1, true), tab_merge(ets:first(T1), T1, T2). tab_merge('$end_of_table', T1, T2) -> - true = ets:delete(T1), - T2; + case ets:first(T1) of % no safe_fixtable()... + '$end_of_table' -> + true = ets:delete(T1), + T2; + Key -> + tab_merge(Key, T1, T2) + end; tab_merge(K1, T1, T2) -> Vs = ets:lookup(T1, K1), NextK1 = ets:next(T1, K1), diff --git a/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc index 37389ce5ae..914e0e509c 100644 --- a/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc @@ -327,7 +327,7 @@ <section> <title>Initial Database Content</title> <p>After the insertion of the employee named <c>klacke</c>, - the databse has the following records:</p> + the database has the following records:</p> <marker id="table2_1"></marker> <table> <row> diff --git a/lib/public_key/src/pubkey_ssh.erl b/lib/public_key/src/pubkey_ssh.erl index 9bda76d670..75c1880655 100644 --- a/lib/public_key/src/pubkey_ssh.erl +++ b/lib/public_key/src/pubkey_ssh.erl @@ -79,7 +79,9 @@ dh_gex_group(Min, N, Max, undefined) -> dh_gex_group(Min, N, Max, Groups) -> case select_by_keylen(Min-10, N, Max+10, Groups) of {ok,{Sz,GPs}} -> - {ok, {Sz,lists:nth(crypto:rand_uniform(1, 1+length(GPs)), GPs)}}; + Rnd = rand:uniform( length(GPs) ), + %% 1 =< Rnd =< length(GPs) + {ok, {Sz, lists:nth(Rnd,GPs)}}; Other -> Other end. diff --git a/lib/public_key/test/erl_make_certs.erl b/lib/public_key/test/erl_make_certs.erl index e4118bab0d..e772ea1734 100644 --- a/lib/public_key/test/erl_make_certs.erl +++ b/lib/public_key/test/erl_make_certs.erl @@ -178,8 +178,9 @@ make_tbs(SubjectKey, Opts) -> _ -> subject(proplists:get_value(subject, Opts),false) end, - - {#'OTPTBSCertificate'{serialNumber = trunc(random:uniform()*100000000)*10000 + 1, + Rnd = rand:uniform( 1000000000000 ), + %% 1 =< Rnd < 1000000000001 + {#'OTPTBSCertificate'{serialNumber = Rnd, signature = SignAlgo, issuer = Issuer, validity = validity(Opts), @@ -466,7 +467,8 @@ odd_rand(Size) -> odd_rand(Min, Max). odd_rand(Min,Max) -> - Rand = crypto:rand_uniform(Min,Max), + %% Odd random number N such that Min =< N =< Max + Rand = (Min-1) + rand:uniform(Max-Min), % Min =< Rand < Max case Rand rem 2 of 0 -> Rand + 1; diff --git a/lib/ssh/doc/src/Makefile b/lib/ssh/doc/src/Makefile index a759854da4..adbda5a030 100644 --- a/lib/ssh/doc/src/Makefile +++ b/lib/ssh/doc/src/Makefile @@ -53,7 +53,8 @@ XML_PART_FILES = part_notes.xml \ XML_CHAPTER_FILES = notes.xml \ introduction.xml \ ssh_protocol.xml \ - using_ssh.xml + using_ssh.xml \ + configure_algos.xml BOOK_FILES = book.xml diff --git a/lib/ssh/doc/src/configure_algos.xml b/lib/ssh/doc/src/configure_algos.xml new file mode 100644 index 0000000000..dd60324851 --- /dev/null +++ b/lib/ssh/doc/src/configure_algos.xml @@ -0,0 +1,428 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2017</year> + <year>2017</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>Configuring algorithms in SSH</title> + <prepared></prepared> + <docno></docno> + <approved></approved> + <date></date> + <rev></rev> + <file>configure_algos.xml</file> + </header> + + <section> + <marker id="introduction"/> + <title>Introduction</title> + <p>To fully understand how to configure the algorithms, it is essential to have a basic understanding of the SSH protocol + and how OTP SSH app handles the corresponding items</p> + + <p>The first subsection will give a short background of the SSH protocol while later sections describes + the implementation and provides some examples</p> + + <section> + <title>Basics of the ssh protocol's algorithms handling</title> + + <p>SSH uses different sets of algorithms in different phases of a session. Which + algorithms to use is negotiated by the client and the server at the beginning of a session. + See <url href="https://tools.ietf.org/html/rfc4253">RFC 4253</url>, + "The Secure Shell (SSH) Transport Layer Protocol" for details. + </p> + + <p>The negotiation is simple: both peers sends their list of supported alghorithms to the other part. + The first algorithm on the client's list that also in on the server's list is selected. So it is the + client's orderering of the list that gives the priority for the algorithms.</p> + + <p>There are five lists exchanged in the connection setup. Three of them are also divided in two + directions, to and from the server.</p> + + <p>The lists are (named as in the SSH application's options):</p> + <taglist> + <tag><c>kex</c></tag> + <item> + <p>Key exchange.</p> + <p>An algorithm is selected for computing a secret encryption key. Among examples are: + the old nowadays week <c>'diffie-hellman-group-exchange-sha1'</c> and the very strong and modern + <c>'ecdh-sha2-nistp512'</c>.</p> + </item> + + <tag><c>public_key</c></tag> + <item> + <p>Server host key</p> + <p>The asymetric encryption algorithm used in the server's private-public host key pair. + Examples include the well-known RSA <c>'ssh-rsa'</c> and elliptic curve <c>'ecdsa-sha2-nistp521'</c>. + </p> + </item> + + <tag><c>cipher</c></tag> + <item> + <p>Symetric cipher algorithm used for the payload encryption. This algorithm will use the key calculated + in the kex phase (together with other info) to genereate the actual key used. Examples are + tripple-DES <c>'3des-cbc'</c> and one of many AES variants <c>'aes192-ctr'</c>. + </p> + <p>This list is actually two - one for each direction server-to-client and client-to-server. Therefore it + is possible but rare to have different algorithms in the two directions in one connection.</p> + </item> + + <tag><c>mac</c></tag> + <item> + <p>Message authentication code</p> + <p>"Check sum" of each message sent between the peers. Examples are SHA <c>'hmac-sha1'</c> and + SHA2 <c>'hmac-sha2-512'</c>.</p> + <p>This list is also divided into two for the both directions</p> + </item> + + <tag><c>compression</c></tag> + <item> + <p>If and how to compress the message. Examples are <c>none</c>, that is, no compression and + <c>zlib</c>.</p> + <p>This list is also divided into two for the both directions</p> + </item> + + </taglist> + </section> + + <section> + <title>The SSH app's mechanism</title> + <p>The set of algorithms that the SSH app uses by default depends on the algoritms supported by the:</p> + <list> + <item><p><seealso marker="crypto:crypto">crypto</seealso> app,</p> + </item> + <item><p>The cryptolib OTP is linked with, usally the one the OS uses, probably OpenSSL,</p> + </item> + <item><p>and finaly what the SSH app implements</p> + </item> + </list> + <p>Due to this, it impossible to list in documentation what algorithms that are available in a certain installation.</p> + <p>There is an important command to list the actual algorithms and their ordering: + <seealso marker="ssh#default_algorithms-0">ssh:default_algorithms/0</seealso>.</p> + <code type="erl"> +0> ssh:default_algorithms(). +[{kex,['ecdh-sha2-nistp384','ecdh-sha2-nistp521', + 'ecdh-sha2-nistp256','diffie-hellman-group-exchange-sha256', + 'diffie-hellman-group16-sha512', + 'diffie-hellman-group18-sha512', + 'diffie-hellman-group14-sha256', + 'diffie-hellman-group14-sha1', + 'diffie-hellman-group-exchange-sha1']}, + {public_key,['ecdsa-sha2-nistp384','ecdsa-sha2-nistp521', + 'ecdsa-sha2-nistp256','ssh-rsa','rsa-sha2-256', + 'rsa-sha2-512','ssh-dss']}, + {cipher,[{client2server,['[email protected]', + 'aes256-ctr','aes192-ctr','[email protected]', + 'aes128-ctr','aes128-cbc','3des-cbc']}, + {server2client,['[email protected]','aes256-ctr', + 'aes192-ctr','[email protected]','aes128-ctr', + 'aes128-cbc','3des-cbc']}]}, + {mac,[{client2server,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}, + {server2client,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}]}, + {compression,[{client2server,[none,'[email protected]',zlib]}, + {server2client,[none,'[email protected]',zlib]}]}] + + </code> + <p>To change the algorithm list, there are two options which can be used in + <seealso marker="ssh#connect-3">ssh:connect/2,3,4</seealso> + and + <seealso marker="ssh#daemon-2">ssh:daemon/2,3</seealso>. The options could of course + be used in all other functions that initiates connections.</p> + + <p>The options are <c>preferred_algorithms</c> and <c>modify_algorithms</c>. The first one + replaces the default set, while the latter modifies the default set.</p> + </section> + </section> + + <section> + <title>Replacing the default set: preferred_algorithms</title> + <p>See the <seealso marker="ssh#option_preferred_algorithms">Reference Manual</seealso> for details</p> + + <p>Here follows a series of examples ranging from simple to more complex.</p> + + <p>To forsee the effect of an option there is an experimental function <c>ssh:chk_algos_opts(Opts)</c>. + It mangles the options <c>preferred_algorithms</c> + and <c>modify_algorithms</c> in the same way as <c>ssh:dameon</c>, <c>ssh:connect</c> and their friends does.</p> + + <section> + <title>Example 1</title> + <p>Replace the kex algorithms list with the single algorithm <c>'diffie-hellman-group14-sha256'</c>:</p> + <code> +1> ssh:chk_algos_opts( + [{preferred_algorithms, + [{kex, ['diffie-hellman-group14-sha256']} + ] + } + ]). +[{kex,['diffie-hellman-group14-sha256']}, + {public_key,['ecdsa-sha2-nistp384','ecdsa-sha2-nistp521', + 'ecdsa-sha2-nistp256','ssh-rsa','rsa-sha2-256', + 'rsa-sha2-512','ssh-dss']}, + {cipher,[{client2server,['[email protected]', + 'aes256-ctr','aes192-ctr','[email protected]', + 'aes128-ctr','aes128-cbc','3des-cbc']}, + {server2client,['[email protected]','aes256-ctr', + 'aes192-ctr','[email protected]','aes128-ctr', + 'aes128-cbc','3des-cbc']}]}, + {mac,[{client2server,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}, + {server2client,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}]}, + {compression,[{client2server,[none,'[email protected]',zlib]}, + {server2client,[none,'[email protected]',zlib]}]}] + </code> + <p>Note that the unmentioned lists (<c>public_key</c>, <c>cipher</c>, <c>mac</c> and <c>compression</c>) + are un-changed.</p> + </section> + + <section> + <title>Example 2</title> + <p>In the lists that are divided in two for the two directions (c.f <c>cipher</c>) it is possible + to change both directions at once:</p> + <code> +2> ssh:chk_algos_opts( + [{preferred_algorithms, + [{cipher,['aes128-ctr']} + ] + } + ]). +[{kex,['ecdh-sha2-nistp384','ecdh-sha2-nistp521', + 'ecdh-sha2-nistp256','diffie-hellman-group-exchange-sha256', + 'diffie-hellman-group16-sha512', + 'diffie-hellman-group18-sha512', + 'diffie-hellman-group14-sha256', + 'diffie-hellman-group14-sha1', + 'diffie-hellman-group-exchange-sha1']}, + {public_key,['ecdsa-sha2-nistp384','ecdsa-sha2-nistp521', + 'ecdsa-sha2-nistp256','ssh-rsa','rsa-sha2-256', + 'rsa-sha2-512','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['aes128-ctr']}]}, + {mac,[{client2server,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}, + {server2client,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}]}, + {compression,[{client2server,[none,'[email protected]',zlib]}, + {server2client,[none,'[email protected]',zlib]}]}] + </code> + <p>Note that both lists in <c>cipher</c> has been changed to the provided value (<c>'aes128-ctr'</c>).</p> + </section> + + <section> + <title>Example 3</title> + <p>In the lists that are divided in two for the two directions (c.f <c>cipher</c>) it is possible + to change only one of the directions:</p> + <code> +3> ssh:chk_algos_opts( + [{preferred_algorithms, + [{cipher,[{client2server,['aes128-ctr']}]} + ] + } + ]). +[{kex,['ecdh-sha2-nistp384','ecdh-sha2-nistp521', + 'ecdh-sha2-nistp256','diffie-hellman-group-exchange-sha256', + 'diffie-hellman-group16-sha512', + 'diffie-hellman-group18-sha512', + 'diffie-hellman-group14-sha256', + 'diffie-hellman-group14-sha1', + 'diffie-hellman-group-exchange-sha1']}, + {public_key,['ecdsa-sha2-nistp384','ecdsa-sha2-nistp521', + 'ecdsa-sha2-nistp256','ssh-rsa','rsa-sha2-256', + 'rsa-sha2-512','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['[email protected]','aes256-ctr', + 'aes192-ctr','[email protected]','aes128-ctr', + 'aes128-cbc','3des-cbc']}]}, + {mac,[{client2server,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}, + {server2client,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}]}, + {compression,[{client2server,[none,'[email protected]',zlib]}, + {server2client,[none,'[email protected]',zlib]}]}] + </code> + </section> + + <section> + <title>Example 4</title> + <p>It is of course possible to change more than one list:</p> + <code> +4> ssh:chk_algos_opts( + [{preferred_algorithms, + [{cipher,['aes128-ctr']}, + {mac,['hmac-sha2-256']}, + {kex,['ecdh-sha2-nistp384']}, + {public_key,['ssh-rsa']}, + {compression,[{server2client,[none]}, + {client2server,[zlib]}]} + ] + } + ]). +[{kex,['ecdh-sha2-nistp384']}, + {public_key,['ssh-rsa']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['aes128-ctr']}]}, + {mac,[{client2server,['hmac-sha2-256']}, + {server2client,['hmac-sha2-256']}]}, + {compression,[{client2server,[zlib]}, + {server2client,[none]}]}] + + </code> + <p>Note that the ordering of the tuples in the lists didn't matter.</p> + </section> + </section> + + <section> + <title>Modifying the default set: modify_algorithms</title> + <p>A situation where it might be useful to add an algorithm is when one need to use a supported but disabled one. + An example is the <c>'diffie-hellman-group1-sha1'</c> which nowadays is very unsecure and therefore disabled. It is + however still supported and might be used.</p> + + <p>The option <c>preferred_algorithms</c> may be complicated to use for adding or removing single algorithms. + First one has to list them with <c>ssh:default_algorithms()</c> and then do changes in the lists.</p> + + <p>To facilitate addition or removal of algorithms the option <c>modify_algorithms</c> is available. + See the <seealso marker="ssh#option_modify_algorithms">Reference Manual</seealso> for details.</p> + + <p>The option takes a list with instructions to append, prepend or remove algorithms:</p> + <code type="erl"> +{modify_algorithms, [{append, ...}, + {prepend, ...}, + {rm, ...} + ]} + </code> + <p>Each of the <c>...</c> can be a <c>algs_list()</c> as the argument to the <c>preferred_algorithms</c> option.</p> + <section> + <title>Example 5</title> + <p>As an example let's add the Diffie-Hellman Group1 first in the kex list. It is supported according to + <seealso marker="SSH_app#supported_algos">Supported algoritms</seealso>.</p> + <code type="erl"> +5> ssh:chk_algos_opts( + [{modify_algorithms, + [{prepend, + [{kex,['diffie-hellman-group1-sha1']}] + } + ] + } + ]). +[{kex,['diffie-hellman-group1-sha1','ecdh-sha2-nistp384', + 'ecdh-sha2-nistp521','ecdh-sha2-nistp256', + 'diffie-hellman-group-exchange-sha256', + 'diffie-hellman-group16-sha512', + 'diffie-hellman-group18-sha512', + 'diffie-hellman-group14-sha256', + 'diffie-hellman-group14-sha1', + 'diffie-hellman-group-exchange-sha1']}, + {public_key,['ecdsa-sha2-nistp384','ecdsa-sha2-nistp521', + 'ecdsa-sha2-nistp256','ssh-rsa','rsa-sha2-256', + 'rsa-sha2-512','ssh-dss']}, + {cipher,[{client2server,['[email protected]', + 'aes256-ctr','aes192-ctr','[email protected]', + 'aes128-ctr','aes128-cbc','3des-cbc']}, + {server2client,['[email protected]','aes256-ctr', + 'aes192-ctr','[email protected]','aes128-ctr', + 'aes128-cbc','3des-cbc']}]}, + {mac,[{client2server,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}, + {server2client,['hmac-sha2-256','hmac-sha2-512', + 'hmac-sha1']}]}, + {compression,[{client2server,[none,'[email protected]',zlib]}, + {server2client,[none,'[email protected]',zlib]}]}] + + </code> + <p>And the result shows that the Diffie-Hellman Group1 is added at the head of the kex list</p> + </section> + + <section> + <title>Example 6</title> + <p>In this example, we in put the 'diffie-hellman-group1-sha1' first and also move the + <c>'ecdh-sha2-nistp521'</c> to the end in the kex list, that is, <c>append</c> it.</p> + <code type="erl"> +6> ssh:chk_algos_opts( + [{modify_algorithms, + [{prepend, + [{kex, ['diffie-hellman-group1-sha1']} + ]}, + {append, + [{kex, ['ecdh-sha2-nistp521']} + ]} + ] + } + ]). +[{kex,['diffie-hellman-group1-sha1','ecdh-sha2-nistp384', + 'ecdh-sha2-nistp256','diffie-hellman-group-exchange-sha256', + 'diffie-hellman-group16-sha512', + 'diffie-hellman-group18-sha512', + 'diffie-hellman-group14-sha256', + 'diffie-hellman-group14-sha1', + 'diffie-hellman-group-exchange-sha1','ecdh-sha2-nistp521']}, + {public_key,['ecdsa-sha2-nistp384','ecdsa-sha2-nistp521', + ..... +] + </code> + <p>Note that the appended algorithm is removed from its original place and then appended to the same list.</p> + </section> + + <section> + <title>Example 7</title> + <p>In this example, we use both options (<c>preferred_algorithms</c> and <c>modify_algorithms</c>) and + also try to prepend an unsupported algorithm. Any unsupported algorithm is quietly removed.</p> + <code type="erl"> +7> ssh:chk_algos_opts( + [{preferred_algorithms, + [{cipher,['aes128-ctr']}, + {mac,['hmac-sha2-256']}, + {kex,['ecdh-sha2-nistp384']}, + {public_key,['ssh-rsa']}, + {compression,[{server2client,[none]}, + {client2server,[zlib]}]} + ] + }, + {modify_algorithms, + [{prepend, + [{kex, ['some unsupported algorithm']} + ]}, + {append, + [{kex, ['diffie-hellman-group1-sha1']} + ]} + ] + } + ]). +[{kex,['ecdh-sha2-nistp384','diffie-hellman-group1-sha1']}, + {public_key,['ssh-rsa']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['aes128-ctr']}]}, + {mac,[{client2server,['hmac-sha2-256']}, + {server2client,['hmac-sha2-256']}]}, + {compression,[{client2server,[zlib]}, + {server2client,[none]}]}] + + </code> + <p>It is of course questionable why anyone would like to use the both these options together, + but it is possible if an unforeseen need should arise.</p> + </section> + + + + </section> + +</chapter> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index ea7e975ef5..d9516fff12 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -108,6 +108,9 @@ <tag><c>double_algs() =</c></tag> <item><p><c>[{client2serverlist,simple_algs()},{server2client,simple_algs()}] | simple_algs()</c></p></item> + + <tag><c>modify_algs_list() =</c></tag> + <item><p><c>list( {append,algs_list()} | {prepend,algs_list()} | {rm,algs_list()} )</c></p></item> </taglist> </section> @@ -254,7 +257,8 @@ </p> </item> - <tag><c><![CDATA[{preferred_algorithms, algs_list()}]]></c></tag> + <tag><marker id="option_preferred_algorithms"></marker> + <c><![CDATA[{preferred_algorithms, algs_list()}]]></c></tag> <item> <p>List of algorithms to use in the algorithm negotiation. The default <c>algs_list()</c> can be obtained from <seealso marker="#default_algorithms/0">default_algorithms/0</seealso>. @@ -275,6 +279,8 @@ for cipher but specifies the same algorithms for mac and compression in both directions. The kex (key exchange) is implicit but public_key is set explicitly.</p> + <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> + <warning> <p>Changing the values can make a connection less secure. Do not change unless you know exactly what you are doing. If you do not understand the values then you @@ -282,6 +288,62 @@ </warning> </item> + <tag><marker id="option_modify_algorithms"></marker> + <c><![CDATA[{modify_algorithms, modify_algs_list()}]]></c></tag> + <item> + <p>Modifies the list of algorithms to use in the algorithm negotiation. The modifications are + applied after the option <c>preferred_algorithms</c> (if existing) is applied.</p> + <p>The algoritm for modifications works like this:</p> + <list> + <item> + <p>Input is the <c>modify_algs_list()</c> and a set of algorithms <c>A</c> + obtained from the <c>preferred_algorithms</c> option if existing, or else from the + <seealso marker="ssh#default_algorithms-0">ssh:default_algorithms/0</seealso>. + </p> + </item> + <item> + <p>The head of the <c>modify_algs_list()</c> modifies <c>A</c> giving the result <c>A'</c>.</p> + <p>The possible modifications are:</p> + <list> + <item> + <p>Append or prepend supported but not enabled algorithm(s) to the list of + algorithms. If the wanted algorithms already are in <c>A</c> they will first + be removed and then appended or prepended, + </p> + </item> + <item> + <p>Remove (rm) one or more algorithms from <c>A</c>. + </p> + </item> + </list> + </item> + <item> + <p>Repeat the modification step with the tail of <c>modify_algs_list()</c> and the resulting + <c>A'</c>. + </p> + </item> + </list> + <p>If an unsupported algorithm is in the <c>modify_algs_list()</c>, it will be silently ignored</p> + <p>If there are more than one modify_algorithms options, the result is undefined.</p> + <p>Here is an example of this option:</p> + <code> +{modify_algorithms, + [{prepend, [{kex, ['diffie-hellman-group1-sha1']}], + {rm, [{compression, [none]}]} + ] +} +</code> + <p>The example specifies that:</p> + <list> + <item><p>the old key exchange algorithm 'diffie-hellman-group1-sha1' should be + the main alternative. It will be the main alternative since it is prepened to the list</p> + </item> + <item><p>The compression algorithm none (= no compression) is removed so compression is enforced</p> + </item> + </list> + <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> + </item> + <tag><c><![CDATA[{dh_gex_limits,{Min=integer(),I=integer(),Max=integer()}}]]></c></tag> <item> <p>Sets the three diffie-hellman-group-exchange parameters that guides the connected server in choosing a group. @@ -555,6 +617,8 @@ for cipher but specifies the same algorithms for mac and compression in both directions. The kex (key exchange) is implicit but public_key is set explicitly.</p> + <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> + <warning> <p>Changing the values can make a connection less secure. Do not change unless you know exactly what you are doing. If you do not understand the values then you @@ -562,6 +626,41 @@ </warning> </item> + <tag><marker id="option_modify_algorithms"></marker> + <c><![CDATA[{modify_algorithms, modify_algs_list()}]]></c></tag> + <item> + <p>Modifies the list of algorithms to use in the algorithm negotiation. The modifications are + applied after the option <c>preferred_algorithms</c> is applied (if existing)</p> + <p>The possible modifications are to:</p> + <list> + <item><p>Append or prepend supported but not enabled algorithm(s) to the list of + algorithms.</p><p>If the wanted algorithms already are in the list of algorithms, they will first + be removed and then appended or prepended. + </p> + </item> + <item><p>Remove (rm) one or more algorithms from the list of algorithms.</p></item> + </list> + <p>If an unsupported algorithm is in the list, it will be silently ignored</p> + + <p>Here is an example of this option:</p> + <code> +{modify_algorithms, + [{prepend, [{kex, ['diffie-hellman-group1-sha1']}], + {rm, [{compression, [none]}]} + ] +} +</code> + <p>The example specifies that:</p> + <list> + <item><p>the old key exchange algorithm 'diffie-hellman-group1-sha1' should be + the main alternative. It will be the main alternative since it is prepened to the list</p> + </item> + <item><p>The compression algorithm none (= no compression) is removed so compression is enforced</p> + </item> + </list> + <p>For background and more examples see the <seealso marker="configure_algos#introduction">User's Guide</seealso>.</p> + </item> + <tag><c><![CDATA[{dh_gex_groups, [{Size=integer(),G=integer(),P=integer()}] | {file,filename()} {ssh_moduli_file,filename()} }]]></c></tag> <item> <p>Defines the groups the server may choose among when diffie-hellman-group-exchange is negotiated. diff --git a/lib/ssh/doc/src/ssh_app.xml b/lib/ssh/doc/src/ssh_app.xml index 33ec7aaee0..1cbbdfcf38 100644 --- a/lib/ssh/doc/src/ssh_app.xml +++ b/lib/ssh/doc/src/ssh_app.xml @@ -97,7 +97,7 @@ <p>The <c>known_hosts</c> file contains a list of approved servers and their public keys. Once a server is listed, it can be verified without user interaction. - </p> + </p> </section> <section> <title>Authorized Keys</title> @@ -135,7 +135,7 @@ </p> <p>Supported algorithms are:</p> - + <marker id="supported_algos"></marker> <taglist> <tag>Key exchange algorithms</tag> <item> diff --git a/lib/ssh/doc/src/usersguide.xml b/lib/ssh/doc/src/usersguide.xml index 70051ba771..d902df6848 100644 --- a/lib/ssh/doc/src/usersguide.xml +++ b/lib/ssh/doc/src/usersguide.xml @@ -36,4 +36,5 @@ </description> <xi:include href="introduction.xml"/> <xi:include href="using_ssh.xml"/> + <xi:include href="configure_algos.xml"/> </part> diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index 5ebab43c30..1a5d48baca 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -35,6 +35,7 @@ daemon/1, daemon/2, daemon/3, daemon_info/1, default_algorithms/0, + chk_algos_opts/1, stop_listener/1, stop_listener/2, stop_listener/3, stop_daemon/1, stop_daemon/2, stop_daemon/3, shell/1, shell/2, shell/3 @@ -381,6 +382,27 @@ default_algorithms() -> ssh_transport:default_algorithms(). %%-------------------------------------------------------------------- +-spec chk_algos_opts(list(any())) -> algs_list() . +%%-------------------------------------------------------------------- +chk_algos_opts(Opts) -> + case lists:foldl( + fun({preferred_algorithms,_}, Acc) -> Acc; + ({modify_algorithms,_}, Acc) -> Acc; + (KV, Acc) -> [KV|Acc] + end, [], Opts) + of + [] -> + case ssh_options:handle_options(client, Opts) of + M when is_map(M) -> + maps:get(preferred_algorithms, M); + Others -> + Others + end; + OtherOps -> + {error, {non_algo_opts_found,OtherOps}} + end. + +%%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- %% The handle_daemon_args/2 function basically only sets the ip-option in Opts diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index b41ad8b33b..6939094401 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -170,9 +170,10 @@ handle_options(Role, PropList0, Opts0) when is_map(Opts0), OptionDefinitions), %% Enter the user's values into the map; unknown keys are %% treated as socket options - lists:foldl(fun(KV, Vals) -> - save(KV, OptionDefinitions, Vals) - end, InitialMap, PropList1) + final_preferred_algorithms( + lists:foldl(fun(KV, Vals) -> + save(KV, OptionDefinitions, Vals) + end, InitialMap, PropList1)) catch error:{eoptions, KV, undefined} -> {error, {eoptions,KV}}; @@ -509,6 +510,15 @@ default(common) -> class => user_options }, + %% NOTE: This option is supposed to be used only in this very module (?MODULE). There is + %% a final stage in handle_options that "merges" the preferred_algorithms option and this one. + %% The preferred_algorithms is the one to use in the rest of the ssh application! + {modify_algorithms, def} => + #{default => undefined, % signals error if unsupported algo in preferred_algorithms :( + chk => fun check_modify_algorithms/1, + class => user_options + }, + {id_string, def} => #{default => undefined, % FIXME: see ssh_transport:ssh_vsn/0 chk => fun(random) -> @@ -820,83 +830,190 @@ valid_hash(L, Ss) when is_list(L) -> lists:all(fun(S) -> valid_hash(S,Ss) end, L valid_hash(X, _) -> error_in_check(X, "Expect atom or list in fingerprint spec"). %%%---------------------------------------------------------------- -check_preferred_algorithms(Algs) -> - [error_in_check(K,"Bad preferred_algorithms key") - || {K,_} <- Algs, - not lists:keymember(K,1,ssh:default_algorithms())], +check_modify_algorithms(M) when is_list(M) -> + [error_in_check(Op_KVs, "Bad modify_algorithms") + || Op_KVs <- M, + not is_tuple(Op_KVs) + orelse (size(Op_KVs) =/= 2) + orelse (not lists:member(element(1,Op_KVs), [append,prepend,rm]))], + {true, [{Op,normalize_mod_algs(KVs,false)} || {Op,KVs} <- M]}; +check_modify_algorithms(_) -> + error_in_check(modify_algorithms, "Bad option value. List expected."). + + + + +normalize_mod_algs(KVs, UseDefaultAlgs) -> + normalize_mod_algs(ssh_transport:algo_classes(), KVs, [], UseDefaultAlgs). + +normalize_mod_algs([K|Ks], KVs0, Acc, UseDefaultAlgs) -> + %% Pick the expected keys in order and check if they are in the user's list + {Vs1, KVs} = + case lists:keytake(K, 1, KVs0) of + {value, {K,Vs0}, KVs1} -> + {Vs0, KVs1}; + false -> + {[], KVs0} + end, + Vs = normalize_mod_alg_list(K, Vs1, UseDefaultAlgs), + normalize_mod_algs(Ks, KVs, [{K,Vs} | Acc], UseDefaultAlgs); +normalize_mod_algs([], [], Acc, _) -> + %% No values left in the key-value list after removing the expected entries + %% (thats good) + lists:reverse(Acc); +normalize_mod_algs([], [{K,_}|_], _, _) -> + %% Some values left in the key-value list after removing the expected entries + %% (thats bad) + case ssh_transport:algo_class(K) of + true -> error_in_check(K, "Duplicate key"); + false -> error_in_check(K, "Unknown key") + end; +normalize_mod_algs([], [X|_], _, _) -> + error_in_check(X, "Bad list element"). - try alg_duplicates(Algs, [], []) - of - [] -> - {true, - [case proplists:get_value(Key, Algs) of - undefined -> - {Key,DefAlgs}; - Vals -> - handle_pref_alg(Key,Vals,SupAlgs) - end - || {{Key,DefAlgs}, {Key,SupAlgs}} <- lists:zip(ssh:default_algorithms(), - ssh_transport:supported_algorithms()) - ] - }; - - Dups -> - error_in_check(Dups, "Duplicates") - catch - _:_ -> - false - end. -alg_duplicates([{K,V}|KVs], Ks, Dups0) -> - Dups = - case lists:member(K,Ks) of - true -> [K|Dups0]; - false -> Dups0 - end, - case V--lists:usort(V) of - [] -> alg_duplicates(KVs, [K|Ks], Dups); - Ds -> alg_duplicates(KVs, [K|Ks], Dups++Ds) + +%%% Handle the algorithms list +normalize_mod_alg_list(K, Vs, UseDefaultAlgs) -> + normalize_mod_alg_list(K, + ssh_transport:algo_two_spec_class(K), + Vs, + def_alg(K,UseDefaultAlgs)). + + +normalize_mod_alg_list(_K, _, [], Default) -> + Default; + +normalize_mod_alg_list(K, true, [{client2server,L1}], [_,{server2client,L2}]) -> + [nml1(K,{client2server,L1}), + {server2client,L2}]; + +normalize_mod_alg_list(K, true, [{server2client,L2}], [{client2server,L1},_]) -> + [{client2server,L1}, + nml1(K,{server2client,L2})]; + +normalize_mod_alg_list(K, true, [{server2client,L2},{client2server,L1}], _) -> + [nml1(K,{client2server,L1}), + nml1(K,{server2client,L2})]; + +normalize_mod_alg_list(K, true, [{client2server,L1},{server2client,L2}], _) -> + [nml1(K,{client2server,L1}), + nml1(K,{server2client,L2})]; + +normalize_mod_alg_list(K, true, L0, _) -> + L = nml(K,L0), % Throws errors + [{client2server,L}, + {server2client,L}]; + +normalize_mod_alg_list(K, false, L, _) -> + nml(K,L). + + +nml1(K, {T,V}) when T==client2server ; T==server2client -> + {T, nml({K,T}, V)}. + +nml(K, L) -> + [error_in_check(K, "Bad value for this key") % This is a throw + || V <- L, + not is_atom(V) + ], + case L -- lists:usort(L) of + [] -> ok; + Dups -> error_in_check({K,Dups}, "Duplicates") % This is a throw + end, + L. + + +def_alg(K, false) -> + case ssh_transport:algo_two_spec_class(K) of + false -> []; + true -> [{client2server,[]}, {server2client,[]}] end; -alg_duplicates([], _Ks, Dups) -> - Dups. - -handle_pref_alg(Key, - Vs=[{client2server,C2Ss=[_|_]},{server2client,S2Cs=[_|_]}], - [{client2server,Sup_C2Ss},{server2client,Sup_S2Cs}] - ) -> - chk_alg_vs(Key, C2Ss, Sup_C2Ss), - chk_alg_vs(Key, S2Cs, Sup_S2Cs), - {Key, Vs}; - -handle_pref_alg(Key, - Vs=[{server2client,[_|_]},{client2server,[_|_]}], - Sup=[{client2server,_},{server2client,_}] - ) -> - handle_pref_alg(Key, lists:reverse(Vs), Sup); - -handle_pref_alg(Key, - Vs=[V|_], - Sup=[{client2server,_},{server2client,_}] - ) when is_atom(V) -> - handle_pref_alg(Key, [{client2server,Vs},{server2client,Vs}], Sup); - -handle_pref_alg(Key, - Vs=[V|_], - Sup=[S|_] - ) when is_atom(V), is_atom(S) -> - chk_alg_vs(Key, Vs, Sup), - {Key, Vs}; - -handle_pref_alg(Key, Vs, _) -> - error_in_check({Key,Vs}, "Badly formed list"). - -chk_alg_vs(OptKey, Values, SupportedValues) -> - case (Values -- SupportedValues) of - [] -> Values; - [none] -> [none]; % for testing only - Bad -> error_in_check({OptKey,Bad}, "Unsupported value(s) found") +def_alg(K, true) -> + ssh_transport:default_algorithms(K). + + + +check_preferred_algorithms(Algs) when is_list(Algs) -> + check_input_ok(Algs), + {true, normalize_mod_algs(Algs, true)}; + +check_preferred_algorithms(_) -> + error_in_check(modify_algorithms, "Bad option value. List expected."). + + +check_input_ok(Algs) -> + [error_in_check(KVs, "Bad preferred_algorithms") + || KVs <- Algs, + not is_tuple(KVs) + orelse (size(KVs) =/= 2)]. + +%%%---------------------------------------------------------------- +final_preferred_algorithms(Options) -> + Result = + case ?GET_OPT(modify_algorithms, Options) of + undefined -> + rm_non_supported(true, + ?GET_OPT(preferred_algorithms, Options)); + ModAlgs -> + rm_non_supported(false, + eval_ops(?GET_OPT(preferred_algorithms, Options), + ModAlgs)) + end, + error_if_empty(Result), % Throws errors if any value list is empty + ?PUT_OPT({preferred_algorithms,Result}, Options). + +eval_ops(PrefAlgs, ModAlgs) -> + lists:foldl(fun eval_op/2, PrefAlgs, ModAlgs). + +eval_op({Op,AlgKVs}, PrefAlgs) -> + eval_op(Op, AlgKVs, PrefAlgs, []). + +eval_op(Op, [{C,L1}|T1], [{C,L2}|T2], Acc) -> + eval_op(Op, T1, T2, [{C,eval_op(Op,L1,L2,[])} | Acc]); + +eval_op(_, [], [], Acc) -> lists:reverse(Acc); +eval_op(rm, Opt, Pref, []) when is_list(Opt), is_list(Pref) -> Pref -- Opt; +eval_op(append, Opt, Pref, []) when is_list(Opt), is_list(Pref) -> (Pref--Opt) ++ Opt; +eval_op(prepend, Opt, Pref, []) when is_list(Opt), is_list(Pref) -> Opt ++ (Pref--Opt). + + +rm_non_supported(UnsupIsErrorFlg, KVs) -> + [{K,rmns(K,Vs, UnsupIsErrorFlg)} || {K,Vs} <- KVs]. + +rmns(K, Vs, UnsupIsErrorFlg) -> + case ssh_transport:algo_two_spec_class(K) of + false -> + rm_unsup(Vs, ssh_transport:supported_algorithms(K), UnsupIsErrorFlg, K); + true -> + [{C, rm_unsup(Vsx, Sup, UnsupIsErrorFlg, {K,C})} + || {{C,Vsx},{C,Sup}} <- lists:zip(Vs,ssh_transport:supported_algorithms(K)) + ] end. +rm_unsup(A, B, Flg, ErrInf) -> + case A--B of + Unsup=[_|_] when Flg==true -> error({eoptions, + {preferred_algorithms,{ErrInf,Unsup}}, + "Unsupported value(s) found" + }); + Unsup -> A -- Unsup + end. + + +error_if_empty([{K,[]}|_]) -> + error({eoptions, K, "Empty resulting algorithm list"}); +error_if_empty([{K,[{client2server,[]}, {server2client,[]}]}]) -> + error({eoptions, K, "Empty resulting algorithm list"}); +error_if_empty([{K,[{client2server,[]}|_]} | _]) -> + error({eoptions, {K,client2server}, "Empty resulting algorithm list"}); +error_if_empty([{K,[_,{server2client,[]}|_]} | _]) -> + error({eoptions, {K,server2client}, "Empty resulting algorithm list"}); +error_if_empty([_|T]) -> + error_if_empty(T); +error_if_empty([]) -> + ok. + %%%---------------------------------------------------------------- forbidden_option(K,V) -> Txt = io_lib:format("The option '~s' is used internally. The " diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 412f5de9de..c48c0800e4 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -34,6 +34,8 @@ -export([next_seqnum/1, supported_algorithms/0, supported_algorithms/1, default_algorithms/0, default_algorithms/1, + algo_classes/0, algo_class/1, + algo_two_spec_classes/0, algo_two_spec_class/1, handle_packet_part/4, handle_hello_version/1, key_exchange_init_msg/1, @@ -81,6 +83,23 @@ default_algorithms() -> [{K,default_algorithms(K)} || K <- algo_classes()]. algo_classes() -> [kex, public_key, cipher, mac, compression]. +algo_class(kex) -> true; +algo_class(public_key) -> true; +algo_class(cipher) -> true; +algo_class(mac) -> true; +algo_class(compression) -> true; +algo_class(_) -> false. + + +algo_two_spec_classes() -> [cipher, mac, compression]. + +algo_two_spec_class(cipher) -> true; +algo_two_spec_class(mac) -> true; +algo_two_spec_class(compression) -> true; +algo_two_spec_class(_) -> false. + + + default_algorithms(kex) -> supported_algorithms(kex, [ 'diffie-hellman-group1-sha1' % Gone in OpenSSH 7.3.p1 diff --git a/lib/ssh/test/ssh_protocol_SUITE.erl b/lib/ssh/test/ssh_protocol_SUITE.erl index 0837fe7eaf..7da921adb2 100644 --- a/lib/ssh/test/ssh_protocol_SUITE.erl +++ b/lib/ssh/test/ssh_protocol_SUITE.erl @@ -34,8 +34,8 @@ -define(NEWLINE, <<"\r\n">>). -define(REKEY_DATA_TMO, 65000). -%%-define(DEFAULT_KEX, 'diffie-hellman-group1-sha1'). -define(DEFAULT_KEX, 'diffie-hellman-group14-sha256'). +-define(EXTRA_KEX, 'diffie-hellman-group1-sha1'). -define(CIPHERS, ['aes256-ctr','aes192-ctr','aes128-ctr','aes128-cbc','3des-cbc']). -define(DEFAULT_CIPHERS, [{client2server,?CIPHERS}, {server2client,?CIPHERS}]). @@ -60,7 +60,8 @@ all() -> {group,authentication}, {group,packet_size_error}, {group,field_size_error}, - {group,ext_info} + {group,ext_info}, + {group,preferred_algorithms} ]. groups() -> @@ -96,7 +97,13 @@ groups() -> no_ext_info_s2, ext_info_s, ext_info_c - ]} + ]}, + {preferred_algorithms, [], [preferred_algorithms, + modify_append, + modify_prepend, + modify_rm, + modify_combo + ]} ]. @@ -701,8 +708,6 @@ ext_info_s(Config) -> %%%-------------------------------------------------------------------- %%% The client sends the extension ext_info_c(Config) -> - {User,_Pwd} = server_user_password(Config), - %% Create a listening socket as server socket: {ok,InitialState} = ssh_trpt_test_lib:exec(listen), HostPort = ssh_trpt_test_lib:server_host_port(InitialState), @@ -757,10 +762,135 @@ ext_info_c(Config) -> {result, Pid, Error} -> ct:fail("Error: ~p",[Error]) end. + +%%%---------------------------------------------------------------- +%%% +preferred_algorithms(Config) -> + Ciphers = filter_supported(cipher, ?CIPHERS), + {error,{eoptions,{{preferred_algorithms,{kex,[some_unknown_algo]}}, + "Unsupported value(s) found"}}} = + chk_pref_algs(Config, + [?DEFAULT_KEX], + Ciphers, + [{preferred_algorithms, [{kex,[some_unknown_algo,?DEFAULT_KEX]}, + {cipher,Ciphers} + ]} + ]). + +%%%---------------------------------------------------------------- +%%% +modify_append(Config) -> + Ciphers = filter_supported(cipher, ?CIPHERS), + {ok,_} = + chk_pref_algs(Config, + [?DEFAULT_KEX, ?EXTRA_KEX], + Ciphers, + [{preferred_algorithms, [{kex,[?DEFAULT_KEX]}, + {cipher,Ciphers} + ]}, + {modify_algorithms, [{append,[{kex,[some_unknown_algo,?EXTRA_KEX]}]}]} + ]). + +%%%---------------------------------------------------------------- +%%% +modify_prepend(Config) -> + Ciphers = filter_supported(cipher, ?CIPHERS), + {ok,_} = + chk_pref_algs(Config, + [?EXTRA_KEX, ?DEFAULT_KEX], + Ciphers, + [{preferred_algorithms, [{kex,[?DEFAULT_KEX]}, + {cipher,Ciphers} + ]}, + {modify_algorithms, [{prepend,[{kex,[some_unknown_algo,?EXTRA_KEX]}]}]} + ]). + +%%%---------------------------------------------------------------- +%%% +modify_rm(Config) -> + Ciphers = filter_supported(cipher, ?CIPHERS), + {ok,_} = + chk_pref_algs(Config, + [?DEFAULT_KEX], + tl(Ciphers), + [{preferred_algorithms, [{kex,[?DEFAULT_KEX,?EXTRA_KEX]}, + {cipher,Ciphers} + ]}, + {modify_algorithms, [{rm,[{kex,[some_unknown_algo,?EXTRA_KEX]}, + {cipher,[hd(Ciphers)]} + ]} + ]} + ]). + + +%%%---------------------------------------------------------------- +%%% +modify_combo(Config) -> + Ciphers = filter_supported(cipher, ?CIPHERS), + LastC = lists:last(Ciphers), + {ok,_} = + chk_pref_algs(Config, + [?DEFAULT_KEX], + [LastC] ++ (tl(Ciphers)--[LastC]) ++ [hd(Ciphers)], + [{preferred_algorithms, [{kex,[?DEFAULT_KEX,?EXTRA_KEX]}, + {cipher,Ciphers} + ]}, + {modify_algorithms, [{rm,[{kex,[some_unknown_algo,?EXTRA_KEX]} + ]}, + {prepend,[{cipher,[{server2client,[LastC]}]} + ]}, + {append,[{cipher,[a,hd(Ciphers),b]} + ]} + ]} + ]). + %%%================================================================ %%%==== Internal functions ======================================== %%%================================================================ +chk_pref_algs(Config, + ExpectedKex, + ExpectedCiphers, + ServerPrefOpts) -> + %% Start the dameon + case ssh_test_lib:daemon( + [{send_ext_info,false}, + {recv_ext_info,false}, + {system_dir, system_dir(Config)} + | ServerPrefOpts]) + of + {_,Host,Port} -> + %% Check the Kex part + ssh_trpt_test_lib:exec( + [{set_options, [print_ops, {print_messages,detail}]}, + {connect, Host, Port, + [{silently_accept_hosts, true}, + {user_dir, user_dir(Config)}, + {user_interaction, false} + ]}, + {send, hello}, + receive_hello, + {match, + #ssh_msg_kexinit{ + kex_algorithms = to_lists(ExpectedKex), + encryption_algorithms_server_to_client = to_lists(ExpectedCiphers), + _ = '_'}, + receive_msg} + ]); + Error -> + Error + end. + + +filter_supported(K, Algs) -> Algs -- (Algs--supported(K)). + +supported(K) -> proplists:get_value( + server2client, + ssh_transport:supported_algorithms(cipher)). + +to_lists(L) -> lists:map(fun erlang:atom_to_list/1, L). + + %%%---- init_suite and end_suite --------------------------------------- start_apps(Config) -> catch ssh:stop(), diff --git a/lib/stdlib/test/id_transform_SUITE.erl b/lib/stdlib/test/id_transform_SUITE.erl index 3d4ae1a189..186df41d3f 100644 --- a/lib/stdlib/test/id_transform_SUITE.erl +++ b/lib/stdlib/test/id_transform_SUITE.erl @@ -61,8 +61,13 @@ id_transform(Config) when is_list(Config) -> "erl_id_trans.erl"]), {ok,erl_id_trans,Bin} = compile:file(File,[binary]), {module,erl_id_trans} = code:load_binary(erl_id_trans, File, Bin), - ct:timetrap({hours,1}), - run_in_test_suite(). + case test_server:is_valgrind() of + false -> + ct:timetrap({hours,1}), + run_in_test_suite(); + true -> + {skip,"Valgrind (too slow)"} + end. run_in_test_suite() -> SuperDir = filename:dirname(filename:dirname(code:which(?MODULE))), diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 9a3985541b..3eff13e1d3 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -2084,12 +2084,6 @@ This function is aware of imported functions." (when funcname (erlang-man-find-function (current-buffer) funcname)))))) -(defun erlang-default-function-or-module () - (let ((id (erlang-get-identifier-at-point))) - (if (eq (erlang-id-kind id) 'qualified-function) - (format "%s:%s" (erlang-id-module id) (erlang-id-name id)) - (erlang-id-name id)))) - ;; Should the defadvice be at the top level, the package `advice' would ;; be required. Now it is only required when this functionality @@ -3793,6 +3787,21 @@ of arguments could be found, otherwise nil." (nth 3 (erlang-id-to-list id))) +(defun erlang-default-function-or-module () + (erlang-with-id (kind module name) (erlang-get-identifier-at-point) + (let ((x (cond ((eq kind 'module) + (format "%s:" name)) + ((eq kind 'record) + (format "-record(%s" name)) + ((eq kind 'macro) + (format "-define(%s" name)) + (t + name)))) + (if module + (format "%s:%s" module x) + x)))) + + ;; TODO: Escape single quotes inside the string without ;; replace-regexp-in-string. (defun erlang-add-quotes-if-needed (str) @@ -5000,9 +5009,10 @@ considered first when it is time to jump to the definition.") (and (fboundp 'xref-make) (fboundp 'xref-make-file-location) (let* ((first-time t) + (cbuf (current-buffer)) xrefs matching-files) (save-excursion - (while (visit-tags-table-buffer (not first-time)) + (while (erlang-visit-tags-table-buffer (not first-time) cbuf) (setq first-time nil) (let ((files (tags-table-files))) (while files @@ -5018,6 +5028,10 @@ considered first when it is time to jump to the definition.") (setq files (cdr files)))))) (nreverse xrefs)))) +(defun erlang-visit-tags-table-buffer (cont cbuf) + (if (< emacs-major-version 26) + (visit-tags-table-buffer cont) + (visit-tags-table-buffer cont cbuf))) (defun erlang-xref-find-definitions-module-tag (module tag |