diff options
47 files changed, 964 insertions, 497 deletions
diff --git a/OTP_VERSION b/OTP_VERSION index 07256210c7..ab1e4f6a31 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -20.0-rc1 +20.0-rc2 diff --git a/erts/doc/src/erl_tracer.xml b/erts/doc/src/erl_tracer.xml index 63feebb0b5..fd3c17f337 100644 --- a/erts/doc/src/erl_tracer.xml +++ b/erts/doc/src/erl_tracer.xml @@ -653,7 +653,7 @@ ok <0.37.0> 3> erlang:trace(new, true, [send,{tracer, erl_msg_tracer, Tracer}]). 0 -{<0.39.0>,<0.27.0>} +{trace,<0.39.0>,<0.27.0>} 4> {ok, D} = file:open("/tmp/tmp.data",[write]). {trace,#Port<0.486>,<0.40.0>} {trace,<0.40.0>,<0.21.0>} @@ -758,18 +758,21 @@ static ERL_NIF_TERM enabled(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) /* * argv[0]: TraceTag, should only be 'send' - * argv[1]: TracerState, process to send {argv[2], argv[4]} to + * argv[1]: TracerState, process to send {Tracee, Recipient} to * argv[2]: Tracee - * argv[3]: Recipient - * argv[4]: Options, ignored + * argv[3]: Message + * argv[4]: Options, map containing Recipient */ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifPid to_pid; + ERL_NIF_TERM recipient, msg; if (enif_get_local_pid(env, argv[1], &to_pid)) { - ERL_NIF_TERM msg = enif_make_tuple3(env, enif_make_atom(env, "trace"), argv[2], argv[4]); + if (enif_get_map_value(env, argv[4], enif_make_atom(env, "extra"), &recipient)) { + msg = enif_make_tuple3(env, enif_make_atom(env, "trace"), argv[2], recipient); enif_send(env, &to_pid, NULL, msg); + } } return enif_make_atom(env, "ok"); diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 08f6732036..8ae87ac280 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -32,6 +32,26 @@ <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 8.3.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Add option to make SIGTERM trigger the OS default + behaviour instead of doing a gracefull shutdown. To + activate this bahviour the environment variable + ERL_ZZ_SIGTERM_KILL should be set to "true". This option + only works in OTP 19 as OTP 20 will have a different way + to deal with SIGTERM.</p> + <p> + Own Id: OTP-14418 Aux Id: ERIERL-15 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 8.3.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 254e59f2c7..61c1e14741 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -149,6 +149,8 @@ endif LIBS += $(TYPE_LIBS) +ORIG_LIBS:= $(LIBS) + comma:=, space:= space+= @@ -965,7 +967,7 @@ $(OBJDIR)/%.o: hipe/%.c $(V_CC) $(subst O2,O3, $(CFLAGS)) $(INCLUDES) -c $< -o $@ $(BINDIR)/hipe_mkliterals$(TF_MARKER): $(OBJDIR)/hipe_mkliterals.o - $(ld_verbose)$(CC) $(LDFLAGS) -o $@ $< $(TYPE_LIBS) + $(ld_verbose)$(CC) $(LDFLAGS) -o $@ $< $(ORIG_LIBS) $(OBJDIR)/hipe_mkliterals.o: $(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h $(DTRACE_HEADERS) \ $(TTF_DIR)/OPCODES-GENERATED $(TTF_DIR)/TABLES-GENERATED diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 7134d2da56..40dd4129d2 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2047,8 +2047,6 @@ ebif_bang_2(BIF_ALIST_2) #define SEND_YIELD_CONTINUE (-8) -Sint do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext*); - static Sint remote_send(Process *p, DistEntry *dep, Eterm to, Eterm full_to, Eterm msg, ErtsSendContext* ctx) @@ -2102,8 +2100,8 @@ static Sint remote_send(Process *p, DistEntry *dep, return res; } -Sint -do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) +static Sint +do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) { Eterm portid; Port *pt; diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 4889fac923..6fddba4b34 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -821,7 +821,7 @@ static void clear_literal_range(void* start, Uint size) #if HAVE_ERTS_MSEG -void* +static void* erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; @@ -832,7 +832,7 @@ erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) return res; } -void* +static void* erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) { @@ -845,7 +845,7 @@ erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg, return res; } -void +static void erts_alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt); @@ -996,7 +996,7 @@ erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) #endif /* HAVE_ERTS_MSEG */ -void* +static void* erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) { void *res; @@ -1013,7 +1013,7 @@ erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) return res; } -void* +static void* erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign) { void *res; @@ -1035,7 +1035,7 @@ erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, return res; } -void +static void erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) { #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f570703f25..7a96bd0319 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -195,10 +195,6 @@ extern UWord erts_literal_vspace_map[]; # define ERTS_VSPACE_WORD_BITS (sizeof(UWord)*8) #endif -void* erts_alcu_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); -void* erts_alcu_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); -void erts_alcu_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); - #if HAVE_ERTS_MSEG # if defined(ARCH_32) void* erts_alcu_literal_32_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); @@ -218,9 +214,6 @@ void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif #endif /* HAVE_ERTS_MSEG */ -void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); -void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); -void erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #ifdef ARCH_32 void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign); void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index eea55b3239..71c64997c1 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1636,7 +1636,7 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) return LSB[0] | (LSB[1]<<8) | (LSB[2]<<16) | (LSB[3]<<24); } -void +static void erts_align_utf8_bytes(ErlBinMatchBuffer* mb, byte* buf) { Uint bits = mb->size - mb->offset; diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 632855255e..5da2b28a89 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -202,7 +202,6 @@ void erts_new_bs_put_string(ERL_BITS_PROTO_2(byte* iptr, Uint num_bytes)); Uint erts_bits_bufs_size(void); Uint32 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb); -void erts_align_utf8_bytes(ErlBinMatchBuffer* mb, byte* buf); Eterm erts_bs_get_utf8(ErlBinMatchBuffer* mb); Eterm erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags); Eterm erts_bs_append(Process* p, Eterm* reg, Uint live, Eterm build_size_term, diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 4347f9f2b7..d0fd763798 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -2243,45 +2243,6 @@ add_cpu_groups(int groups, return cgm; } -static void -remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) -{ - erts_cpu_groups_map_t *prev_cgm, *cgm; - erts_cpu_groups_callback_list_t *prev_cgcl, *cgcl; - - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); - - no_cpu_groups_callbacks--; - - prev_cgm = NULL; - for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { - prev_cgcl = NULL; - for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { - if (cgcl->callback == callback && cgcl->arg == arg) { - if (prev_cgcl) - prev_cgcl->next = cgcl->next; - else - cgm->callback_list = cgcl->next; - erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgcl); - if (!cgm->callback_list) { - if (prev_cgm) - prev_cgm->next = cgm->next; - else - cpu_groups_maps = cgm->next; - if (cgm->array) - erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm->array); - erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm); - } - return; - } - prev_cgcl = cgcl; - } - prev_cgm = cgm; - } - - erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); -} - static int cpu_groups_lookup(erts_cpu_groups_map_t *map, ErtsSchedulerData *esdp) @@ -2321,21 +2282,3 @@ update_cpu_groups_maps(void) for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) make_cpu_groups_map(cgm, 0); } - -void -erts_add_cpu_groups(int groups, - erts_cpu_groups_callback_t callback, - void *arg) -{ - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); - add_cpu_groups(groups, callback, arg); - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); -} - -void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, - void *arg) -{ - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); - remove_cpu_groups(callback, arg); - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); -} diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 08a0f0e83b..0addfaa3c7 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -676,7 +676,7 @@ int db_create_hash(Process *p, DbTable *tbl) sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, - "db_hash_slot", make_small(i)); + "db_hash_slot", tb->common.the_name); } /* This important property is needed to guarantee the two buckets * involved in a grow/shrink operation it protected by the same lock: diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 0acc2432a7..f8341f788a 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -969,17 +969,21 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, fdrec.sfv_len = SENDFILE_CHUNK_SIZE; else fdrec.sfv_len = *nbytes; + retval = sendfilev(out_fd, &fdrec, 1, &len); - /* Sometimes sendfilev can return -1 and still send data. - When that happens we just pretend that no error happend. */ - if (retval != -1 || errno == EAGAIN || errno == EINTR || - len != 0) { + if (retval == -1 && errno == EINVAL) { + /* On some solaris versions (I've seen it on SunOS 5.10), + using a sfv_len larger then a filesize will result in + a -1 && errno == EINVAL return. We translate this so + a successful send of the data.*/ + retval = len; + } + + if (retval != -1 || errno == EAGAIN || errno == EINTR) { *offset += len; *nbytes -= len; written += len; - if (errno != EAGAIN && errno != EINTR && len != 0) - retval = len; } } while (len == SENDFILE_CHUNK_SIZE); #elif defined(__DARWIN__) diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl index 35dd147550..2c2cb9c32d 100644 --- a/erts/emulator/test/gc_SUITE.erl +++ b/erts/emulator/test/gc_SUITE.erl @@ -203,89 +203,90 @@ long_receive() -> end. minor_major_gc_option_self(_Config) -> - Endless = fun Endless() -> - receive - {gc, Type} -> erlang:garbage_collect(self(), [{type, Type}]) - after 100 -> ok end, - Endless() - end, - - %% Try as major, a test process will self-trigger GC - P1 = spawn(Endless), - erlang:garbage_collect(P1, []), - erlang:trace(P1, true, [garbage_collection]), - P1 ! {gc, major}, - expect_trace_messages(P1, [gc_major_start, gc_major_end]), - erlang:trace(P1, false, [garbage_collection]), - erlang:exit(P1, kill), - - %% Try as minor, a test process will self-trigger GC - P2 = spawn(Endless), - erlang:garbage_collect(P2, []), - erlang:trace(P2, true, [garbage_collection]), - P2 ! {gc, minor}, - expect_trace_messages(P2, [gc_minor_start, gc_minor_end]), - erlang:trace(P2, false, [garbage_collection]), - erlang:exit(P2, kill). + %% Try as major, the test process will self-trigger GC + check_gc_tracing_around( + fun(Pid, Ref) -> + Pid ! {gc, Ref, major} + end, [gc_major_start, gc_major_end]), + + %% Try as minor, the test process will self-trigger GC + check_gc_tracing_around( + fun(Pid, Ref) -> + Pid ! {gc, Ref, minor} + end, [gc_minor_start, gc_minor_end]). minor_major_gc_option_async(_Config) -> - Endless = fun Endless() -> - receive after 100 -> ok end, - Endless() - end, - - %% Try with default option, must be major gc - P1 = spawn(Endless), - erlang:garbage_collect(P1, []), - erlang:trace(P1, true, [garbage_collection]), - erlang:garbage_collect(P1, []), - expect_trace_messages(P1, [gc_major_start, gc_major_end]), - erlang:trace(P1, false, [garbage_collection]), - erlang:exit(P1, kill), + %% Try with default option, must be major GC + check_gc_tracing_around( + fun(Pid, _Ref) -> + erlang:garbage_collect(Pid, []) + end, [gc_major_start, gc_major_end]), %% Try with the 'major' type - P2 = spawn(Endless), - erlang:garbage_collect(P2, []), - erlang:trace(P2, true, [garbage_collection]), - erlang:garbage_collect(P2, [{type, major}]), - expect_trace_messages(P2, [gc_major_start, gc_major_end]), - erlang:trace(P2, false, [garbage_collection]), - erlang:exit(P2, kill), + check_gc_tracing_around( + fun(Pid, _Ref) -> + erlang:garbage_collect(Pid, [{type, major}]) + end, [gc_major_start, gc_major_end]), %% Try with 'minor' option, once - P3 = spawn(Endless), - erlang:garbage_collect(P3, []), - erlang:trace(P3, true, [garbage_collection]), - erlang:garbage_collect(P3, [{type, minor}]), - expect_trace_messages(P3, [gc_minor_start, gc_minor_end]), - erlang:trace(P3, false, [garbage_collection]), - erlang:exit(P3, kill), + check_gc_tracing_around( + fun(Pid, _Ref) -> + erlang:garbage_collect(Pid, [{type, minor}]) + end, [gc_minor_start, gc_minor_end]), %% Try with 'minor' option, once, async - P4 = spawn(Endless), + check_gc_tracing_around( + fun(Pid, Ref) -> + ?assertEqual(async, + erlang:garbage_collect(Pid, [{type, minor}, {async, Ref}])), + + receive + {garbage_collect, Ref, true} -> + ok + after 10000 -> + ct:fail("Did not receive a completion notification on async GC") + end + end, [gc_minor_start, gc_minor_end]). + +%% Traces garbage collection around the given operation, and fails the test if +%% it results in any unexpected messages or if the expected trace tags are not +%% received. +check_gc_tracing_around(Fun, ExpectedTraceTags) -> Ref = erlang:make_ref(), - erlang:garbage_collect(P4, []), - erlang:trace(P4, true, [garbage_collection]), - ?assertEqual(async, - erlang:garbage_collect(P4, [{type, minor}, {async, Ref}])), - expect_trace_messages(P4, [gc_minor_start, gc_minor_end]), - erlang:trace(P4, false, [garbage_collection]), - receive {garbage_collect, Ref, true} -> ok; - Other4 -> ct:pal("Unexpected message: ~p~n" - ++ "while waiting for async gc result", [Other4]) - after 2000 -> ?assert(false) - end, - erlang:exit(P4, kill). - -%% Given a list of atoms, trace tags - receives messages and checks if they are -%% trace events, and if the tag matches. Else will crash failing the test. -expect_trace_messages(_Pid, []) -> ok; + Pid = spawn( + fun Endless() -> + receive + {gc, Ref, Type} -> + erlang:garbage_collect(self(), [{type, Type}]) + after 100 -> + ok + end, + Endless() + end), + erlang:garbage_collect(Pid, []), + erlang:trace(Pid, true, [garbage_collection]), + Fun(Pid, Ref), + expect_trace_messages(Pid, ExpectedTraceTags), + erlang:trace(Pid, false, [garbage_collection]), + erlang:exit(Pid, kill), + check_no_unexpected_messages(). + +%% Ensures that trace messages with the provided tags have all been received +%% within a reasonable timeframe. +expect_trace_messages(_Pid, []) -> + ok; expect_trace_messages(Pid, [Tag | TraceTags]) -> receive - {trace, Pid, Tag, _Data} -> ok; - AnythingElse -> - ct:pal("Unexpected message: ~p~nWhile expected {trace, _, ~p, _}", - [AnythingElse, Tag]), - ?assert(false) - end, - expect_trace_messages(Pid, TraceTags). + {trace, Pid, Tag, _Data} -> + expect_trace_messages(Pid, TraceTags) + after 4000 -> + ct:fail("Didn't receive tag ~p within 4000ms", [Tag]) + end. + +check_no_unexpected_messages() -> + receive + Anything -> + ct:fail("Unexpected message: ~p", [Anything]) + after 0 -> + ok + end. diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 4ae7edd8b2..688ec339aa 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -609,7 +609,7 @@ struct digest_type_t { }md; }; -struct digest_type_t digest_types[] = +static struct digest_type_t digest_types[] = { {{"md4"}, {&EVP_md4}}, {{"md5"}, {&EVP_md5}}, @@ -666,7 +666,7 @@ struct cipher_type_t { #define COND_NO_DES_PTR(Ptr) (Ptr) #endif -struct cipher_type_t cipher_types[] = +static struct cipher_type_t cipher_types[] = { {{"rc2_cbc"}, #ifndef OPENSSL_NO_RC2 diff --git a/lib/eldap/test/eldap_basic_SUITE.erl b/lib/eldap/test/eldap_basic_SUITE.erl index ac3447cfe6..4bfb0dd291 100644 --- a/lib/eldap/test/eldap_basic_SUITE.erl +++ b/lib/eldap/test/eldap_basic_SUITE.erl @@ -119,7 +119,10 @@ init_per_suite(Config) -> {ldaps_server, LDAPS_server} | Config]. end_per_suite(_Config) -> - ssl:stop(). + try ssl:stop() + catch + _:_ -> ok + end. init_per_group(return_values, Config) -> diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 89c17a8679..4b2bcc7242 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1175,17 +1175,20 @@ handle_empty_queue(Session, ProfileName, TimeOut, State) -> %% If a pipline | keep_alive session has been idle for some time is not %% closed by the server, the client may want to close it. NewState = activate_queue_timeout(TimeOut, State), - update_session(ProfileName, Session, #session.queue_length, 0), - %% Note mfa will be initialized when a new request - %% arrives. - {noreply, - NewState#state{request = undefined, - mfa = undefined, - status_line = undefined, - headers = undefined, - body = undefined - } - }. + case update_session(ProfileName, Session, #session.queue_length, 0) of + {stop, Reason} -> + {stop, {shutdown, Reason}, State}; + _ -> + %% Note mfa will be initialized when a new request + %% arrives. + {noreply, + NewState#state{request = undefined, + mfa = undefined, + status_line = undefined, + headers = undefined, + body = undefined + }} + end. receive_response(Request, Session, Data, State) -> NewState = init_wait_for_response_state(Request, State), @@ -1677,7 +1680,7 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> Session2 = erlang:setelement(Pos, Session, Value), insert_session(Session2, ProfileName); error:badarg -> - exit(normal); %% Manager has been shutdown + {stop, normal}; T:E -> %% Unexpected this must be an error! Stacktrace = erlang:get_stacktrace(), @@ -1697,14 +1700,14 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> Session, (catch httpc_manager:lookup_session(SessionId, ProfileName)), T, E]), - exit({failed_updating_session, - [{profile, ProfileName}, - {session_id, SessionId}, - {pos, Pos}, - {value, Value}, - {etype, T}, - {error, E}, - {stacktrace, Stacktrace}]}) + {stop, {failed_updating_session, + [{profile, ProfileName}, + {session_id, SessionId}, + {pos, Pos}, + {value, Value}, + {etype, T}, + {error, E}, + {stacktrace, Stacktrace}]}} end. diff --git a/lib/kernel/test/os_SUITE.erl b/lib/kernel/test/os_SUITE.erl index 3cbb75a633..1233e362f2 100644 --- a/lib/kernel/test/os_SUITE.erl +++ b/lib/kernel/test/os_SUITE.erl @@ -319,31 +319,38 @@ perf_counter_api(_Config) -> true = is_integer(os:perf_counter()), true = os:perf_counter() > 0, - T1 = os:perf_counter(), + Conv = fun(T1, T2) -> + erlang:convert_time_unit(T2 - T1, perf_counter, nanosecond) + end, + + do_perf_counter_test([], Conv, 120000000, 80000000), + do_perf_counter_test([1000], fun(T1, T2) -> T2 - T1 end, 120, 80). + +do_perf_counter_test(CntArgs, Conv, Upper, Lower) -> + %% We run the test multiple times to try to get a somewhat + %% stable value... what does this test? That the + %% calculate_perf_counter_unit in sys_time.c works somewhat ok. + do_perf_counter_test(CntArgs, Conv, Upper, Lower, 10). + +do_perf_counter_test(CntArgs, _Conv, Upper, Lower, 0) -> + ct:fail("perf_counter_test ~p ~p ~p",[CntArgs, Upper, Lower]); +do_perf_counter_test(CntArgs, Conv, Upper, Lower, Iters) -> + + T1 = apply(os, perf_counter, CntArgs), timer:sleep(100), - T2 = os:perf_counter(), - TsDiff = erlang:convert_time_unit(T2 - T1, perf_counter, nanosecond), - ct:pal("T1: ~p~n" + T2 = apply(os, perf_counter, CntArgs), + TsDiff = Conv(T1, T2), + ct:log("T1: ~p~n" "T2: ~p~n" "TsDiff: ~p~n", [T1,T2,TsDiff]), - %% We allow a 15% diff - true = TsDiff < 115000000, - true = TsDiff > 85000000, - - T1Ms = os:perf_counter(1000), - timer:sleep(100), - T2Ms = os:perf_counter(1000), - MsDiff = T2Ms - T1Ms, - ct:pal("T1Ms: ~p~n" - "T2Ms: ~p~n" - "MsDiff: ~p~n", - [T1Ms,T2Ms,MsDiff]), - - %% We allow a 15% diff - true = MsDiff < 115, - true = MsDiff > 85. + if + TsDiff < Upper, TsDiff > Lower -> + ok; + true -> + do_perf_counter_test(CntArgs, Conv, Upper, Lower, Iters-1) + end. %% Util functions diff --git a/lib/kernel/test/sendfile_SUITE.erl b/lib/kernel/test/sendfile_SUITE.erl index 2673c38494..e839959623 100644 --- a/lib/kernel/test/sendfile_SUITE.erl +++ b/lib/kernel/test/sendfile_SUITE.erl @@ -100,13 +100,13 @@ init_per_testcase(TC,Config) when TC == t_sendfile_recvduring; %% Check if sendfile is supported on this platform case catch sendfile_send(Send) of ok -> - Config; + init_per_testcase(t_sendfile, Config); Error -> ct:log("Error: ~p",[Error]), {skip,"Not supported"} end; init_per_testcase(_Tc,Config) -> - Config. + Config ++ [{sendfile_opts,[{use_threads,false}]}]. t_sendfile_small(Config) when is_list(Config) -> diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 17b47c059e..93a7e21f4e 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -1888,7 +1888,7 @@ info_format(Tab, Size, Mem, Media) -> StrT = mnesia_lib:pad_name(atom_to_list(Tab), 15, []), StrS = mnesia_lib:pad_name(integer_to_list(Size), 8, []), StrM = mnesia_lib:pad_name(integer_to_list(Mem), 8, []), - io:format("~s: with ~s records occupying ~s ~s~n", + io:format("~ts: with ~s records occupying ~s ~s~n", [StrT, StrS, StrM, Media]). %% Handle early arrived messages diff --git a/lib/observer/src/observer_lib.erl b/lib/observer/src/observer_lib.erl index 7352af936c..c7ee294719 100644 --- a/lib/observer/src/observer_lib.erl +++ b/lib/observer/src/observer_lib.erl @@ -636,12 +636,12 @@ user_term_multiline(Parent, Title, Default) -> parse_string(Str) -> try - Tokens = case erl_scan:string(Str) of + Tokens = case erl_scan:string(Str, 1, [text]) of {ok, Ts, _} -> Ts; {error, {_SLine, SMod, SError}, _} -> throw(io_lib:format("~s", [SMod:format_error(SError)])) end, - case erl_parse:parse_term(Tokens) of + case lib:extended_parse_term(Tokens) of {error, {_PLine, PMod, PError}} -> throw(io_lib:format("~s", [PMod:format_error(PError)])); Res -> Res diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 6cf659f830..ac64a7bf14 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -28,7 +28,8 @@ -include("ssh_auth.hrl"). -include("ssh_transport.hrl"). --export([publickey_msg/1, password_msg/1, keyboard_interactive_msg/1, +-export([get_public_key/2, + publickey_msg/1, password_msg/1, keyboard_interactive_msg/1, service_request_msg/1, init_userauth_request_msg/1, userauth_request_msg/1, handle_userauth_request/3, handle_userauth_info_request/2, handle_userauth_info_response/2 @@ -136,41 +137,49 @@ keyboard_interactive_msg([#ssh{user = User, Ssh) end. -publickey_msg([SigAlg, #ssh{user = User, - session_id = SessionId, - service = Service, - opts = Opts} = Ssh]) -> - Hash = ssh_transport:sha(SigAlg), + +get_public_key(SigAlg, #ssh{opts = Opts}) -> KeyAlg = key_alg(SigAlg), {KeyCb,KeyCbOpts} = ?GET_OPT(key_cb, Opts), UserOpts = ?GET_OPT(user_options, Opts), case KeyCb:user_key(KeyAlg, [{key_cb_private,KeyCbOpts}|UserOpts]) of - {ok, PrivKey} -> - SigAlgStr = atom_to_list(SigAlg), + {ok, PrivKey} -> try Key = ssh_transport:extract_public_key(PrivKey), public_key:ssh_encode(Key, ssh2_pubkey) of - PubKeyBlob -> - SigData = build_sig_data(SessionId, User, Service, - PubKeyBlob, SigAlgStr), - Sig = ssh_transport:sign(SigData, Hash, PrivKey), - SigBlob = list_to_binary([?string(SigAlgStr), - ?binary(Sig)]), - ssh_transport:ssh_packet( - #ssh_msg_userauth_request{user = User, - service = Service, - method = "publickey", - data = [?TRUE, - ?string(SigAlgStr), - ?binary(PubKeyBlob), - ?binary(SigBlob)]}, - Ssh) + PubKeyBlob -> {ok,{PrivKey,PubKeyBlob}} catch _:_ -> - {not_ok, Ssh} + not_ok end; - _Error -> + _Error -> + not_ok + end. + + +publickey_msg([SigAlg, #ssh{user = User, + session_id = SessionId, + service = Service} = Ssh]) -> + case get_public_key(SigAlg, Ssh) of + {ok, {PrivKey,PubKeyBlob}} -> + SigAlgStr = atom_to_list(SigAlg), + SigData = build_sig_data(SessionId, User, Service, + PubKeyBlob, SigAlgStr), + Hash = ssh_transport:sha(SigAlg), + Sig = ssh_transport:sign(SigData, Hash, PrivKey), + SigBlob = list_to_binary([?string(SigAlgStr), + ?binary(Sig)]), + ssh_transport:ssh_packet( + #ssh_msg_userauth_request{user = User, + service = Service, + method = "publickey", + data = [?TRUE, + ?string(SigAlgStr), + ?binary(PubKeyBlob), + ?binary(SigBlob)]}, + Ssh); + _ -> {not_ok, Ssh} end. diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index f1ce337947..8d3ddb09a4 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -453,16 +453,20 @@ init_ssh_record(Role, _Socket, PeerAddr, Opts) -> PeerName0 when is_list(PeerName0) -> PeerName0 end, - S0#ssh{c_vsn = Vsn, - c_version = Version, - io_cb = case ?GET_OPT(user_interaction, Opts) of - true -> ssh_io; - false -> ssh_no_io - end, - userauth_pubkeys = ?GET_OPT(pref_public_key_algs, Opts), - userauth_quiet_mode = ?GET_OPT(quiet_mode, Opts), - peer = {PeerName, PeerAddr} - }; + S1 = + S0#ssh{c_vsn = Vsn, + c_version = Version, + io_cb = case ?GET_OPT(user_interaction, Opts) of + true -> ssh_io; + false -> ssh_no_io + end, + userauth_quiet_mode = ?GET_OPT(quiet_mode, Opts), + peer = {PeerName, PeerAddr} + }, + S1#ssh{userauth_pubkeys = [K || K <- ?GET_OPT(pref_public_key_algs, Opts), + is_usable_user_pubkey(K, S1) + ] + }; server -> S0#ssh{s_vsn = Vsn, @@ -1700,29 +1704,49 @@ handle_ssh_msg_ext_info(#ssh_msg_ext_info{data=Data}, D0) -> lists:foldl(fun ext_info/2, D0, Data). -ext_info({"server-sig-algs",SigAlgs}, D0 = #data{ssh_params=#ssh{role=client, - userauth_pubkeys=ClientSigAlgs}=Ssh0}) -> - %% Make strings to eliminate risk of beeing bombed with odd strings that fills the atom table: - SupportedAlgs = lists:map(fun erlang:atom_to_list/1, ssh_transport:supported_algorithms(public_key)), - ServerSigAlgs = [list_to_atom(SigAlg) || SigAlg <- string:tokens(SigAlgs,","), - %% length of SigAlg is implicitly checked by the comparison - %% in member/2: - lists:member(SigAlg, SupportedAlgs) - ], - CommonAlgs = [Alg || Alg <- ServerSigAlgs, - lists:member(Alg, ClientSigAlgs)], - SelectedAlgs = - case CommonAlgs of - [] -> ClientSigAlgs; % server-sig-algs value is just an advice - _ -> CommonAlgs - end, - D0#data{ssh_params = Ssh0#ssh{userauth_pubkeys = SelectedAlgs} }; +ext_info({"server-sig-algs",SigAlgsStr}, + D0 = #data{ssh_params=#ssh{role=client, + userauth_pubkeys=ClientSigAlgs}=Ssh0}) -> + %% ClientSigAlgs are the pub_key algortithms that: + %% 1) is usable, that is, the user has such a public key and + %% 2) is either the default list or set by the caller + %% with the client option 'pref_public_key_algs' + %% + %% The list is already checked for duplicates. + + SigAlgs = [A || Astr <- string:tokens(SigAlgsStr, ","), + A <- try [list_to_existing_atom(Astr)] + %% list_to_existing_atom will fail for unknown algorithms + catch _:_ -> [] + end], + + CommonAlgs = [A || A <- SigAlgs, + lists:member(A, ClientSigAlgs)], + + %% Re-arrange the client supported public-key algorithms so that the server + %% preferred ones are tried first. + %% Trying algorithms not mentioned by the server is ok, since the server can't know + %% if the client supports 'server-sig-algs' or not. + + D0#data{ + ssh_params = + Ssh0#ssh{ + userauth_pubkeys = + CommonAlgs ++ (ClientSigAlgs -- CommonAlgs) + }}; ext_info(_, D0) -> %% Not implemented D0. %%%---------------------------------------------------------------- +is_usable_user_pubkey(A, Ssh) -> + case ssh_auth:get_public_key(A, Ssh) of + {ok,_} -> true; + _ -> false + end. + +%%%---------------------------------------------------------------- handle_request(ChannelPid, ChannelId, Type, Data, WantReply, From, D) -> case ssh_channel:cache_lookup(cache(D), ChannelId) of #channel{remote_id = Id} = Channel -> diff --git a/lib/ssh/src/ssh_dbg.erl b/lib/ssh/src/ssh_dbg.erl index 003b3856e6..3f742ad9b6 100644 --- a/lib/ssh/src/ssh_dbg.erl +++ b/lib/ssh/src/ssh_dbg.erl @@ -22,10 +22,8 @@ -module(ssh_dbg). --export([messages/0, messages/1, messages/2, - ct_messages/0, - auth/0, auth/1, auth/2, - ct_auth/0, +-export([messages/0, messages/1, messages/2, messages/3, + auth/0, auth/1, auth/2, auth/3, stop/0 ]). @@ -37,50 +35,52 @@ -include("ssh_connect.hrl"). -include("ssh_auth.hrl"). --record(data, { - writer, - acc = []}). %%%================================================================ -messages() -> - messages(fun(String,_D) -> io:format(String) end). +messages() -> start(msg). +messages(F) -> start(msg,F). +messages(F,X) -> start(msg,F,X). +messages(F,M,I) -> start(msg,F,M,I). -ct_messages() -> - messages(fun(String,_D) -> ct:log(String,[]) end). +auth() -> start(auth). +auth(F) -> start(auth,F). +auth(F,X) -> start(auth,F,X). +auth(F,M,I) -> start(auth,F,M,I). -messages(Write) when is_function(Write,2) -> - messages(Write, fun(X) -> X end). - -messages(Write, MangleArg) when is_function(Write,2), - is_function(MangleArg,1) -> - cond_start(msg, Write, MangleArg), - dbg_ssh_messages(), - dbg_ssh_auth(). +stop() -> dbg:stop(). +%%%---------------------------------------------------------------- +start(Type) -> start(Type, fun io:format/2). -auth() -> - auth(fun(String,_D) -> io:format(String) end). +start(Type, F) when is_function(F,2) -> start(Type, fmt_fun(F)); +start(Type, F) when is_function(F,3) -> start(Type, F, id_fun()). -ct_auth() -> - auth(fun(String,_D) -> ct:log(String,[]) end). +start(Type, WriteFun, MangleArgFun) when is_function(WriteFun, 3), + is_function(MangleArgFun, 1) -> + start(Type, WriteFun, MangleArgFun, []); +start(Type, WriteFun, InitValue) -> + start(Type, WriteFun, id_fun(), InitValue). -auth(Write) when is_function(Write,2) -> - auth(Write, fun(X) -> X end). +start(Type, WriteFun, MangleArgFun, InitValue) when is_function(WriteFun, 3), + is_function(MangleArgFun, 1) -> + cond_start(Type, WriteFun, MangleArgFun, InitValue), + dbg_ssh(Type). -auth(Write, MangleArg) when is_function(Write,2), - is_function(MangleArg,1) -> - cond_start(auth, Write, MangleArg), - dbg_ssh_auth(). +%%%---------------------------------------------------------------- +fmt_fun(F) -> fun(Fmt,Args,Data) -> F(Fmt,Args), Data end. +id_fun() -> fun(X) -> X end. -dbg_ssh_messages() -> +%%%---------------------------------------------------------------- +dbg_ssh(msg) -> + dbg_ssh(auth), dbg:tp(ssh_message,encode,1, x), dbg:tp(ssh_message,decode,1, x), dbg:tpl(ssh_transport,select_algorithm,4, x), dbg:tp(ssh_transport,hello_version_msg,1, x), dbg:tp(ssh_transport,handle_hello_version,1, x), - dbg:tpl(ssh_connection_handler,ext_info,2, x). + dbg:tpl(ssh_connection_handler,ext_info,2, x); -dbg_ssh_auth() -> +dbg_ssh(auth) -> dbg:tp(ssh_transport,hello_version_msg,1, x), dbg:tp(ssh_transport,handle_hello_version,1, x), dbg:tp(ssh_message,encode,1, x), @@ -89,15 +89,11 @@ dbg_ssh_auth() -> lists:foreach(fun(F) -> dbg:tp(ssh_auth, F, x) end, [publickey_msg, password_msg, keyboard_interactive_msg]). -%%%---------------------------------------------------------------- -stop() -> - dbg:stop(). - %%%================================================================ -cond_start(Type, Write, MangleArg) -> +cond_start(Type, WriteFun, MangleArgFun, Init) -> try dbg:start(), - setup_tracer(Type, Write, MangleArg), + setup_tracer(Type, WriteFun, MangleArgFun, Init), dbg:p(new,[c,timestamp]) catch _:_ -> ok @@ -136,12 +132,13 @@ msg_formater(_, {trace_ts,Pid,call,{ssh_transport,handle_hello_version,[Hello]}, msg_formater(_, {trace_ts,_Pid,return_from,{ssh_transport,handle_hello_version,1},_,_TS}, D) -> D; -msg_formater(_, {trace_ts,Pid,call,{ssh_connection_handler,ext_info,[{"server-sig-algs",_SigAlgs},State]},TS}, D) -> +msg_formater(_, {trace_ts,Pid,call,{ssh_connection_handler,ext_info,[{"server-sig-algs",SigAlgs},State]},TS}, D) -> try lists:keyfind(ssh, 1, tuple_to_list(State)) of false -> D; #ssh{userauth_pubkeys = PKs} -> - fmt("~n~s ~p Client got suggestion to use user public key sig-algs~n ~p~n", [ts(TS),Pid,PKs], D) + fmt("~n~s ~p Client got suggestion to use user public key sig-algs~n ~p~n and can use~n ~p~n", + [ts(TS),Pid,string:tokens(SigAlgs,","),PKs], D) catch _:_ -> D @@ -206,21 +203,25 @@ msg_formater(msg, {trace_ts,Pid,'receive',ErlangMsg,TS}, D) -> msg_formater(_, _, D) -> D. +%%%---------------------------------------------------------------- +-record(data, {writer, + acc}). -fmt(Fmt, Args, D=#data{writer=Write,acc=Acc}) -> - D#data{acc = Write(io_lib:format(Fmt, Args), Acc)}. +fmt(Fmt, Args, D=#data{writer=Write, acc=Acc}) -> + D#data{acc = Write(Fmt,Args,Acc)}. ts({_,_,Usec}=Now) -> {_Date,{HH,MM,SS}} = calendar:now_to_local_time(Now), io_lib:format("~.2.0w:~.2.0w:~.2.0w.~.6.0w",[HH,MM,SS,Usec]); ts(_) -> "-". -%%%---------------------------------------------------------------- -setup_tracer(Type, Write, MangleArg) -> + +setup_tracer(Type, WriteFun, MangleArgFun, Init) -> Handler = fun(Arg, D) -> - msg_formater(Type, MangleArg(Arg), D) + msg_formater(Type, MangleArgFun(Arg), D) end, - InitialData = #data{writer = Write}, + InitialData = #data{writer = WriteFun, + acc = Init}, {ok,_} = dbg:tracer(process, {Handler, InitialData}), ok. diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index aebb5a7062..7eeed70739 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -674,7 +674,11 @@ check_pref_public_key_algs(V) -> PKs = ssh_transport:supported_algorithms(public_key), CHK = fun(A, Ack) -> case lists:member(A, PKs) of - true -> [A|Ack]; + true -> + case lists:member(A,Ack) of + false -> [A|Ack]; + true -> Ack % Remove duplicates + end; false -> error_in_check(A, "Not supported public key") end end, diff --git a/lib/ssh/test/ssh_algorithms_SUITE.erl b/lib/ssh/test/ssh_algorithms_SUITE.erl index 0f69910e40..98964a2c8a 100644 --- a/lib/ssh/test/ssh_algorithms_SUITE.erl +++ b/lib/ssh/test/ssh_algorithms_SUITE.erl @@ -131,9 +131,14 @@ init_per_group(public_key=Tag, Alg, Config) -> ct:log("Init tests for public_key ~p",[Alg]), PrefAlgs = {preferred_algorithms,[{Tag,[Alg]}]}, %% Daemon started later in init_per_testcase - [{pref_algs,PrefAlgs}, - {tag_alg,{Tag,Alg}} - | Config]; + try + setup_pubkey(Alg, + [{pref_algs,PrefAlgs}, + {tag_alg,{Tag,Alg}} + | Config]) + catch + _:_ -> {skip, io_lib:format("Unsupported: ~p",[Alg])} + end; init_per_group(Tag, Alg, Config) -> PA = @@ -167,18 +172,24 @@ init_per_testcase(TC, Config) -> init_per_testcase(TC, proplists:get_value(tag_alg,Config), Config). -init_per_testcase(_, {public_key,Alg}, Config) -> - Opts = pubkey_opts(Config), +init_per_testcase(TC, {public_key,Alg}, Config) -> + ExtraOpts = case TC of + simple_connect -> + [{user_dir, proplists:get_value(priv_dir,Config)}]; + _ -> + [] + end, + Opts = pubkey_opts(Config) ++ ExtraOpts, case {ssh_file:user_key(Alg,Opts), ssh_file:host_key(Alg,Opts)} of {{ok,_}, {ok,_}} -> - ssh_dbg:ct_auth(), - start_pubkey_daemon([proplists:get_value(pref_algs,Config)], + start_pubkey_daemon([proplists:get_value(pref_algs,Config) + | ExtraOpts], [{extra_daemon,true}|Config]); - {{ok,_}, _} -> - {skip, "No host key"}; + {{ok,_}, {error,Err}} -> + {skip, io_lib:format("No host key: ~p",[Err])}; - {_, {ok,_}} -> - {skip, "No user key"}; + {{error,Err}, {ok,_}} -> + {skip, io_lib:format("No user key: ~p",[Err])}; _ -> {skip, "Neither host nor user key"} @@ -193,7 +204,6 @@ init_per_testcase(_, _, Config) -> end_per_testcase(_TC, Config) -> - catch ssh_dbg:stop(), case proplists:get_value(extra_daemon, Config, false) of true -> case proplists:get_value(srvr_pid,Config) of @@ -223,6 +233,19 @@ simple_exec(Config) -> ssh_test_lib:std_simple_exec(Host, Port, Config). %%-------------------------------------------------------------------- +%% A simple exec call +simple_connect(Config) -> + {Host,Port} = proplists:get_value(srvr_addr, Config), + Opts = + case proplists:get_value(tag_alg, Config) of + {public_key,Alg} -> [{pref_public_key_algs,[Alg]}]; + _ -> [] + end, + ConnectionRef = ssh_test_lib:std_connect(Config, Host, Port, Opts), + ct:log("~p:~p connected! ~p",[?MODULE,?LINE,ConnectionRef]), + ssh:close(ConnectionRef). + +%%-------------------------------------------------------------------- %% Testing if no group matches simple_exec_groups_no_match_too_small(Config) -> try_exec_simple_group({400,500,600}, Config). @@ -304,9 +327,15 @@ sshc_simple_exec_os_cmd(Config) -> %%-------------------------------------------------------------------- %% Connect to the ssh server of the OS sshd_simple_exec(Config) -> + ClientPubKeyOpts = + case proplists:get_value(tag_alg,Config) of + {public_key,Alg} -> [{pref_public_key_algs,[Alg]}]; + _ -> [] + end, ConnectionRef = ssh_test_lib:connect(22, [{silently_accept_hosts, true}, proplists:get_value(pref_algs,Config), - {user_interaction, false}]), + {user_interaction, false} + | ClientPubKeyOpts]), {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), success = ssh_connection:exec(ConnectionRef, ChannelId0, "echo testing", infinity), @@ -363,8 +392,8 @@ split(Alg) -> ssh_test_lib:to_atoms(string:tokens(atom_to_list(Alg), " + ")). specific_test_cases(Tag, Alg, SshcAlgos, SshdAlgos, TypeSSH) -> case Tag of - public_key -> []; - _ -> [simple_exec, simple_sftp] + public_key -> [simple_connect]; + _ -> [simple_connect, simple_exec, simple_sftp] end ++ case supports(Tag, Alg, SshcAlgos) of true when TypeSSH == openSSH -> @@ -439,10 +468,26 @@ setup_pubkey(Config) -> Keys = [ssh_test_lib:setup_dsa(DataDir, UserDir), ssh_test_lib:setup_rsa(DataDir, UserDir), - ssh_test_lib:setup_ecdsa("256", DataDir, UserDir)], + ssh_test_lib:setup_ecdsa("256", DataDir, UserDir) + ], ssh_test_lib:write_auth_keys(Keys, UserDir), % 'authorized_keys' shall contain ALL pub keys Config. +setup_pubkey(Alg, Config) -> + DataDir = proplists:get_value(data_dir, Config), + UserDir = proplists:get_value(priv_dir, Config), + ct:log("Setup keys for ~p",[Alg]), + case Alg of + 'ssh-dss' -> ssh_test_lib:setup_dsa(DataDir, UserDir); + 'ssh-rsa' -> ssh_test_lib:setup_rsa(DataDir, UserDir); + 'rsa-sha2-256' -> ssh_test_lib:setup_rsa(DataDir, UserDir); + 'rsa-sha2-512' -> ssh_test_lib:setup_rsa(DataDir, UserDir); + 'ecdsa-sha2-nistp256' -> ssh_test_lib:setup_ecdsa("256", DataDir, UserDir); + 'ecdsa-sha2-nistp384' -> ssh_test_lib:setup_ecdsa("384", DataDir, UserDir); + 'ecdsa-sha2-nistp521' -> ssh_test_lib:setup_ecdsa("521", DataDir, UserDir) + end, + Config. + simple_exec_group(I, Config) when is_integer(I) -> simple_exec_group({I,I,I}, Config); diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa384 b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa384 new file mode 100644 index 0000000000..4c39e916e9 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa384 @@ -0,0 +1,6 @@ +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDAughXu55DNyhxe6x+MNjv4oZKWUDh7bhi4CqjvxhCp9KMpsybltcq+ +lsuKTarzTdKgBwYFK4EEACKhZANiAASu1vvDL0SQoXGtzlltaPHPyDfEVMG/sKLA +pqv8vfRN5Wcs7+yaRKw92nYEKGXfZLbhVX8ArFPMtXPWHcRHCntvL1Acn2kJQ8Gc +7iL4NAr8JhTIUBv4YMhHDa9Pv/CH2zk= +-----END EC PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa384.pub b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa384.pub new file mode 100644 index 0000000000..caa9604c84 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa384.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBK7W+8MvRJChca3OWW1o8c/IN8RUwb+wosCmq/y99E3lZyzv7JpErD3adgQoZd9ktuFVfwCsU8y1c9YdxEcKe28vUByfaQlDwZzuIvg0CvwmFMhQG/hgyEcNr0+/8IfbOQ== uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa521 b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa521 new file mode 100644 index 0000000000..1e16fcbd57 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa521 @@ -0,0 +1,7 @@ +-----BEGIN EC PRIVATE KEY----- +MIHbAgEBBEEWXGoVLiNwQVUwAGZWxOu6uxtU8ntxyZNlcWU4Z8pze9kq3eK7a9XH +l/wxL75Vk1QdOiR/rE3s/L/zOuChp44o1aAHBgUrgQQAI6GBiQOBhgAEAfCrtwjO +kQYKr4/F3uanS7Eby1+SYDdRl1ABuDFhNC3CivVBFt4CnRneV+Mf0viDAxD+HEpd +/GaE2CdsFoVpglN5AVG+fEePY2PiCLHmjc4/pBuR+tWhErzcWAd0KLBCBuc4OAvl +aLLYV1NAJI6COnnfGTCVvYYE5nKMG4LLX0zaWtWl +-----END EC PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa521.pub b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa521.pub new file mode 100644 index 0000000000..069683eba7 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/id_ecdsa521.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHwq7cIzpEGCq+Pxd7mp0uxG8tfkmA3UZdQAbgxYTQtwor1QRbeAp0Z3lfjH9L4gwMQ/hxKXfxmhNgnbBaFaYJTeQFRvnxHj2Nj4gix5o3OP6QbkfrVoRK83FgHdCiwQgbnODgL5Wiy2FdTQCSOgjp53xkwlb2GBOZyjBuCy19M2lrVpQ== uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key384 b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key384 new file mode 100644 index 0000000000..5835bcd74c --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key384 @@ -0,0 +1,6 @@ +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDB+l0+SMLYgQ3ZRzg2Pn5u+1ZwKbEnJzXsTKTJM9QSJbKkbA7uCnjdS +CvEW+66CoHqgBwYFK4EEACKhZANiAAT6awCCIrcCr9H4wq0bJ/rQou3tpLHyyf33 +c8D6FPn48/hNqinpx7b0le/0D+Rrhdl9edIplAf6oki7yoFFGl4yuzWtv7rag9jB +vv6w1508ChOmyQ094rFt/xj4KVBhEHI= +-----END EC PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key384.pub b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key384.pub new file mode 100644 index 0000000000..714fc4eb89 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key384.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBPprAIIitwKv0fjCrRsn+tCi7e2ksfLJ/fdzwPoU+fjz+E2qKenHtvSV7/QP5GuF2X150imUB/qiSLvKgUUaXjK7Na2/utqD2MG+/rDXnTwKE6bJDT3isW3/GPgpUGEQcg== uabhnil@elxadlj3q32 diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key521 b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key521 new file mode 100644 index 0000000000..81aa8df39f --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key521 @@ -0,0 +1,7 @@ +-----BEGIN EC PRIVATE KEY----- +MIHbAgEBBEHHxgYEfDclsu5bW+pZfg+bkaqWpgEpXtuzLVm++FFPjhAPhMkurSRj +WQ+CuI2TxgYkBbYFNjn9JqgdMF7FzaiojKAHBgUrgQQAI6GBiQOBhgAEAFTM8TKG +xexxmfAGuyl/Tpk4wytB/OyuVfkF+Q3H1v17HLcpMacA5xUFr80+D5XnjxGttBsS ++X0uexR7QbPbhhPqADgQzFqvTsB1mUNAZnJBD6QNCZkfWwRRwFYQWSmisb43H6G3 +iUTKqiCXMXO8drKLA+Wi+L7VyfoI1CvatBBlDHbV +-----END EC PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key521.pub b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key521.pub new file mode 100644 index 0000000000..17b9a1d834 --- /dev/null +++ b/lib/ssh/test/ssh_algorithms_SUITE_data/ssh_host_ecdsa_key521.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBABUzPEyhsXscZnwBrspf06ZOMMrQfzsrlX5BfkNx9b9exy3KTGnAOcVBa/NPg+V548RrbQbEvl9LnsUe0Gz24YT6gA4EMxar07AdZlDQGZyQQ+kDQmZH1sEUcBWEFkporG+Nx+ht4lEyqoglzFzvHayiwPlovi+1cn6CNQr2rQQZQx21Q== uabhnil@elxadlj3q32 diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index 2de947d8b4..f338471829 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -115,7 +115,7 @@ send_handshake_flight(#state{socket = Socket, {Encoded, ConnectionStates} = encode_handshake_flight(lists:reverse(Flight), Version, 1400, Epoch, ConnectionStates0), send(Transport, Socket, Encoded), - start_flight(State0#state{connection_states = ConnectionStates}); + {State0#state{connection_states = ConnectionStates}, []}; send_handshake_flight(#state{socket = Socket, transport_cb = Transport, @@ -129,7 +129,7 @@ send_handshake_flight(#state{socket = Socket, {EncChangeCipher, ConnectionStates} = encode_change_cipher(ChangeCipher, Version, Epoch, ConnectionStates1), send(Transport, Socket, [HsBefore, EncChangeCipher]), - start_flight(State0#state{connection_states = ConnectionStates}); + {State0#state{connection_states = ConnectionStates}, []}; send_handshake_flight(#state{socket = Socket, transport_cb = Transport, @@ -145,7 +145,7 @@ send_handshake_flight(#state{socket = Socket, {HsAfter, ConnectionStates} = encode_handshake_flight(lists:reverse(Flight1), Version, 1400, Epoch, ConnectionStates2), send(Transport, Socket, [HsBefore, EncChangeCipher, HsAfter]), - start_flight(State0#state{connection_states = ConnectionStates}); + {State0#state{connection_states = ConnectionStates}, []}; send_handshake_flight(#state{socket = Socket, transport_cb = Transport, @@ -159,7 +159,7 @@ send_handshake_flight(#state{socket = Socket, {HsAfter, ConnectionStates} = encode_handshake_flight(lists:reverse(Flight1), Version, 1400, Epoch, ConnectionStates1), send(Transport, Socket, [EncChangeCipher, HsAfter]), - start_flight(State0#state{connection_states = ConnectionStates}). + {State0#state{connection_states = ConnectionStates}, []}. queue_change_cipher(ChangeCipher, #state{flight_buffer = Flight, connection_states = ConnectionStates0} = State) -> @@ -235,12 +235,14 @@ init([Role, Host, Port, Socket, Options, User, CbInfo]) -> end. callback_mode() -> - state_functions. + [state_functions, state_enter]. %%-------------------------------------------------------------------- %% State functions %%-------------------------------------------------------------------- +init(enter, _, State) -> + {keep_state, State}; init({call, From}, {start, Timeout}, #state{host = Host, port = Port, role = client, ssl_options = SslOpts, @@ -282,6 +284,8 @@ init({call, _} = Type, Event, #state{role = server} = State) -> init(Type, Event, State) -> ssl_connection:init(Type, Event, State, ?MODULE). +error(enter, _, State) -> + {keep_state, State}; error({call, From}, {start, _Timeout}, {Error, State}) -> {stop_and_reply, normal, {reply, From, {error, Error}}, State}; error({call, From}, Msg, State) -> @@ -295,6 +299,11 @@ error(_, _, _) -> #state{}) -> gen_statem:state_function_result(). %%-------------------------------------------------------------------- +hello(enter, _, #state{role = server} = State) -> + {keep_state, State}; +hello(enter, _, #state{role = client} = State0) -> + {State, Actions} = handle_flight_timer(State0), + {keep_state, State, Actions}; hello(internal, #client_hello{cookie = <<>>, client_version = Version} = Hello, #state{role = server, transport_cb = Transport, @@ -374,6 +383,9 @@ hello(state_timeout, Event, State) -> hello(Type, Event, State) -> ssl_connection:hello(Type, Event, State, ?MODULE). +abbreviated(enter, _, State0) -> + {State, Actions} = handle_flight_timer(State0), + {keep_state, State, Actions}; abbreviated(info, Event, State) -> handle_info(Event, abbreviated, State); abbreviated(internal = Type, @@ -391,6 +403,9 @@ abbreviated(state_timeout, Event, State) -> abbreviated(Type, Event, State) -> ssl_connection:abbreviated(Type, Event, State, ?MODULE). +certify(enter, _, State0) -> + {State, Actions} = handle_flight_timer(State0), + {keep_state, State, Actions}; certify(info, Event, State) -> handle_info(Event, certify, State); certify(internal = Type, #server_hello_done{} = Event, State) -> @@ -400,6 +415,9 @@ certify(state_timeout, Event, State) -> certify(Type, Event, State) -> ssl_connection:certify(Type, Event, State, ?MODULE). +cipher(enter, _, State0) -> + {State, Actions} = handle_flight_timer(State0), + {keep_state, State, Actions}; cipher(info, Event, State) -> handle_info(Event, cipher, State); cipher(internal = Type, #change_cipher_spec{type = <<1>>} = Event, @@ -417,6 +435,8 @@ cipher(state_timeout, Event, State) -> cipher(Type, Event, State) -> ssl_connection:cipher(Type, Event, State, ?MODULE). +connection(enter, _, State) -> + {keep_state, State}; connection(info, Event, State) -> handle_info(Event, connection, State); connection(internal, #hello_request{}, #state{host = Host, port = Port, @@ -449,6 +469,9 @@ connection(internal, #client_hello{}, #state{role = server, allow_renegotiate = connection(Type, Event, State) -> ssl_connection:connection(Type, Event, State, ?MODULE). +%%TODO does this make sense for DTLS ? +downgrade(enter, _, State) -> + {keep_state, State}; downgrade(Type, Event, State) -> ssl_connection:downgrade(Type, Event, State, ?MODULE). @@ -750,14 +773,16 @@ next_event(connection = StateName, no_record, {#ssl_tls{epoch = Epoch, type = ?HANDSHAKE, version = _Version}, State1} = _Record when Epoch == CurrentEpoch-1 -> - {State, MoreActions} = send_handshake_flight(State1, CurrentEpoch), - {next_state, StateName, State, Actions ++ MoreActions}; + {State2, MoreActions} = send_handshake_flight(State1, CurrentEpoch), + {NextRecord, State} = next_record(State2), + next_event(StateName, NextRecord, State, Actions ++ MoreActions); %% From FLIGHT perspective CHANGE_CIPHER_SPEC is treated as a handshake {#ssl_tls{epoch = Epoch, type = ?CHANGE_CIPHER_SPEC, version = _Version}, State1} = _Record when Epoch == CurrentEpoch-1 -> - {State, MoreActions} = send_handshake_flight(State1, CurrentEpoch), - {next_state, StateName, State, Actions ++ MoreActions}; + {State2, MoreActions} = send_handshake_flight(State1, CurrentEpoch), + {NextRecord, State} = next_record(State2), + next_event(StateName, NextRecord, State, Actions ++ MoreActions); {#ssl_tls{epoch = _Epoch, version = _Version}, State1} -> %% TODO maybe buffer later epoch @@ -774,14 +799,16 @@ next_event(connection = StateName, Record, #ssl_tls{epoch = Epoch, type = ?HANDSHAKE, version = _Version} when Epoch == CurrentEpoch-1 -> - {State, MoreActions} = send_handshake_flight(State0, CurrentEpoch), - {next_state, StateName, State, Actions ++ MoreActions}; + {State1, MoreActions} = send_handshake_flight(State0, CurrentEpoch), + {NextRecord, State} = next_record(State1), + next_event(StateName, NextRecord, State, Actions ++ MoreActions); %% From FLIGHT perspective CHANGE_CIPHER_SPEC is treated as a handshake #ssl_tls{epoch = Epoch, type = ?CHANGE_CIPHER_SPEC, version = _Version} when Epoch == CurrentEpoch-1 -> - {State, MoreActions} = send_handshake_flight(State0, CurrentEpoch), - {next_state, StateName, State, Actions ++ MoreActions}; + {State1, MoreActions} = send_handshake_flight(State0, CurrentEpoch), + {NextRecord, State} = next_record(State1), + next_event(StateName, NextRecord, State, Actions ++ MoreActions); _ -> next_event(StateName, no_record, State0, Actions) end; @@ -841,13 +868,13 @@ next_flight(Flight) -> change_cipher_spec => undefined, handshakes_after_change_cipher_spec => []}. -start_flight(#state{transport_cb = gen_udp, - flight_state = {retransmit, Timeout}} = State) -> +handle_flight_timer(#state{transport_cb = gen_udp, + flight_state = {retransmit, Timeout}} = State) -> start_retransmision_timer(Timeout, State); -start_flight(#state{transport_cb = gen_udp, +handle_flight_timer(#state{transport_cb = gen_udp, flight_state = connection} = State) -> {State, []}; -start_flight(State) -> +handle_flight_timer(State) -> %% No retransmision needed i.e DTLS over SCTP {State#state{flight_state = reliable}, []}. diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 96c3ab86e9..352874c77d 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -600,8 +600,12 @@ next_record(#state{protocol_buffers = next_record(#state{protocol_buffers = #protocol_buffers{tls_packets = [], tls_cipher_texts = []}, socket = Socket, transport_cb = Transport} = State) -> - tls_socket:setopts(Transport, Socket, [{active,once}]), - {no_record, State}; + case tls_socket:setopts(Transport, Socket, [{active,once}]) of + ok -> + {no_record, State}; + _ -> + {socket_closed, State} + end; next_record(State) -> {no_record, State}. @@ -626,10 +630,15 @@ passive_receive(State0 = #state{user_data_buffer = Buffer}, StateName) -> next_event(StateName, Record, State) -> next_event(StateName, Record, State, []). +next_event(StateName, socket_closed, State, _) -> + ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), + {stop, {shutdown, transport_closed}, State}; next_event(connection = StateName, no_record, State0, Actions) -> case next_record_if_active(State0) of {no_record, State} -> ssl_connection:hibernate_after(StateName, State, Actions); + {socket_closed, State} -> + next_event(StateName, socket_closed, State, Actions); {#ssl_tls{} = Record, State} -> {next_state, StateName, State, [{next_event, internal, {protocol_record, Record}} | Actions]}; {#alert{} = Alert, State} -> diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 58870a3419..d13b1b3f2a 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -240,6 +240,7 @@ error_handling_tests()-> error_handling_tests_tls()-> [controller_dies, tls_client_closes_socket, + tls_closed_in_active_once, tls_tcp_error_propagation_in_active_mode, tls_tcp_connect, tls_tcp_connect_big, @@ -430,6 +431,7 @@ init_per_testcase(prf, Config) -> init_per_testcase(TestCase, Config) when TestCase == tls_ssl_accept_timeout; TestCase == tls_client_closes_socket; + TestCase == tls_closed_in_active_once; TestCase == tls_downgrade -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:timetrap({seconds, 15}), @@ -961,6 +963,48 @@ tls_client_closes_socket(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, {error,closed}). %%-------------------------------------------------------------------- +tls_closed_in_active_once() -> + [{doc, "Test that ssl_closed is delivered in active once with non-empty buffer, check ERL-420."}]. + +tls_closed_in_active_once(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + {_ClientNode, _ServerNode, Hostname} = ssl_test_lib:run_where(Config), + TcpOpts = [binary, {reuseaddr, true}], + Port = ssl_test_lib:inet_port(node()), + Server = fun() -> + {ok, Listen} = gen_tcp:listen(Port, TcpOpts), + {ok, TcpServerSocket} = gen_tcp:accept(Listen), + {ok, ServerSocket} = ssl:ssl_accept(TcpServerSocket, ServerOpts), + lists:foreach( + fun(_) -> + ssl:send(ServerSocket, "some random message\r\n") + end, lists:seq(1, 20)), + %% Close TCP instead of SSL socket to trigger the bug: + gen_tcp:close(TcpServerSocket), + gen_tcp:close(Listen) + end, + spawn_link(Server), + {ok, Socket} = ssl:connect(Hostname, Port, [{active, false} | ClientOpts]), + Result = tls_closed_in_active_once_loop(Socket), + ssl:close(Socket), + case Result of + ok -> ok; + _ -> ct:fail(Result) + end. + +tls_closed_in_active_once_loop(Socket) -> + ssl:setopts(Socket, [{active, once}]), + receive + {ssl, Socket, _} -> + tls_closed_in_active_once_loop(Socket); + {ssl_closed, Socket} -> + ok + after 5000 -> + no_ssl_closed_received + end. + +%%-------------------------------------------------------------------- connect_dist() -> [{doc,"Test a simple connect as is used by distribution"}]. diff --git a/lib/xmerl/doc/src/notes.xml b/lib/xmerl/doc/src/notes.xml index ef4831e6ec..56856d026e 100644 --- a/lib/xmerl/doc/src/notes.xml +++ b/lib/xmerl/doc/src/notes.xml @@ -32,6 +32,27 @@ <p>This document describes the changes made to the Xmerl application.</p> +<section><title>Xmerl 1.3.14</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> A couple of bugs are fixed in the sax parser + (xmerl_sax_parser). </p> <list> <item>The continuation + function was not called correctly when the XML directive + was fragmented. </item> <item>When the event callback + modules (xmerl_sax_old_dom and xmerl_sax_simple) got an + endDocument event at certain conditions the parser + crashed.</item> <item>Replaced internal ets table with + map to avoid table leakage.</item> </list> + <p> + Own Id: OTP-14430</p> + </item> + </list> + </section> + +</section> + <section><title>Xmerl 1.3.13</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/xmerl/src/xmerl_sax_old_dom.erl b/lib/xmerl/src/xmerl_sax_old_dom.erl index fefcf03fce..411121370f 100644 --- a/lib/xmerl/src/xmerl_sax_old_dom.erl +++ b/lib/xmerl/src/xmerl_sax_old_dom.erl @@ -127,9 +127,10 @@ build_dom(endDocument, State#xmerl_sax_old_dom_state{dom=[Decl, Current#xmlElement{ content=lists:reverse(C) }]}; - _ -> - %%?dbg("~p\n", [D]), - ?error("we're not at end the document when endDocument event is encountered.") + _ -> + %% endDocument is also sent by the parser when a fault occur to tell + %% the event receiver that no more input will be sent + State end; %% Element diff --git a/lib/xmerl/src/xmerl_sax_parser.erl b/lib/xmerl/src/xmerl_sax_parser.erl index 1aef6c58c4..e383c4c349 100644 --- a/lib/xmerl/src/xmerl_sax_parser.erl +++ b/lib/xmerl/src/xmerl_sax_parser.erl @@ -64,7 +64,7 @@ %% Description: Parse file containing an XML document. %%---------------------------------------------------------------------- file(Name,Options) -> - case file:open(Name, [raw, read,binary]) of + case file:open(Name, [raw, read_ahead, read,binary]) of {error, Reason} -> {error,{Name, file:format_error(Reason)}}; {ok, FD} -> @@ -120,21 +120,22 @@ stream(Xml, Options, InputType) when is_binary(Xml), is_list(Options) -> normal -> parse end, - case detect_charset(Xml, State) of - {error, Reason} -> {fatal_error, - { - State#xmerl_sax_parser_state.current_location, - State#xmerl_sax_parser_state.entity, - 1 - }, - Reason, - [], - State#xmerl_sax_parser_state.event_state}; - {Xml1, State1} -> - parse_binary(Xml1, - State1#xmerl_sax_parser_state{input_type = InputType}, - ParseFunction) - end + try + {Xml1, State1} = detect_charset(Xml, State), + parse_binary(Xml1, + State1#xmerl_sax_parser_state{input_type = InputType}, + ParseFunction) + catch + throw:{fatal_error, {State2, Reason}} -> + {fatal_error, + { + State2#xmerl_sax_parser_state.current_location, + State2#xmerl_sax_parser_state.entity, + 1 + }, + Reason, [], + State2#xmerl_sax_parser_state.event_state} + end end. %%---------------------------------------------------------------------- @@ -156,8 +157,8 @@ parse_binary(Xml, #xmerl_sax_parser_state{encoding={utf16,big}}=State, F) -> xmerl_sax_parser_utf16be:F(Xml, State); parse_binary(Xml, #xmerl_sax_parser_state{encoding=latin1}=State, F) -> xmerl_sax_parser_latin1:F(Xml, State); -parse_binary(_, #xmerl_sax_parser_state{encoding=Enc}, _) -> - {error, lists:flatten(io_lib:format("Charcter set ~p not supported", [Enc]))}. +parse_binary(_, #xmerl_sax_parser_state{encoding=Enc}, State) -> + ?fatal_error(State, lists:flatten(io_lib:format("Charcter set ~p not supported", [Enc]))). %%---------------------------------------------------------------------- %% Function: initial_state/0 @@ -211,8 +212,7 @@ parse_options([{entity, Entity} |Options], State) -> parse_options([skip_external_dtd |Options], State) -> parse_options(Options, State#xmerl_sax_parser_state{skip_external_dtd = true}); parse_options([O |_], _State) -> - {error, - lists:flatten(io_lib:format("Option: ~p not supported", [O]))}. + {error, lists:flatten(io_lib:format("Option: ~p not supported", [O]))}. check_encoding_option(E) when E==utf8; E=={utf16,little}; E=={utf16,big}; @@ -230,16 +230,10 @@ check_encoding_option(E) -> %% Output: {utf8|utf16le|utf16be|iso8859, Xml, State} %% Description: Detects which character set is used in a binary stream. %%---------------------------------------------------------------------- -detect_charset(<<>>, #xmerl_sax_parser_state{continuation_fun = undefined} = _) -> - {error, "Can't detect character encoding due to no indata"}; -detect_charset(<<>>, #xmerl_sax_parser_state{continuation_fun = CFun, - continuation_state = CState} = State) -> - case CFun(CState) of - {<<>>, _} -> - {error, "Can't detect character encoding due to lack of indata"}; - {NewBytes, NewContState} -> - detect_charset(NewBytes, State#xmerl_sax_parser_state{continuation_state = NewContState}) - end; +detect_charset(<<>>, #xmerl_sax_parser_state{continuation_fun = undefined} = State) -> + ?fatal_error(State, "Can't detect character encoding due to lack of indata"); +detect_charset(<<>>, State) -> + cf(<<>>, State, fun detect_charset/2); detect_charset(Bytes, State) -> case unicode:bom_to_encoding(Bytes) of {latin1, 0} -> @@ -249,25 +243,47 @@ detect_charset(Bytes, State) -> {RealBytes, State#xmerl_sax_parser_state{encoding=Enc}} end. +detect_charset_1(<<16#00>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); +detect_charset_1(<<16#00, 16#3C>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); +detect_charset_1(<<16#00, 16#3C, 16#00>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); detect_charset_1(<<16#00, 16#3C, 16#00, 16#3F, _/binary>> = Xml, State) -> {Xml, State#xmerl_sax_parser_state{encoding={utf16, big}}}; +detect_charset_1(<<16#3C>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); +detect_charset_1(<<16#3C, 16#00>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); +detect_charset_1(<<16#3C, 16#00, 16#3F>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); detect_charset_1(<<16#3C, 16#00, 16#3F, 16#00, _/binary>> = Xml, State) -> {Xml, State#xmerl_sax_parser_state{encoding={utf16, little}}}; -detect_charset_1(<<16#3C, 16#3F, 16#78, 16#6D, 16#6C, Xml2/binary>> = Xml, State) -> - case parse_xml_directive(Xml2) of +detect_charset_1(<<16#3C>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); +detect_charset_1(<<16#3C, 16#3F>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); +detect_charset_1(<<16#3C, 16#3F, 16#78>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); +detect_charset_1(<<16#3C, 16#3F, 16#78, 16#6D>> = Xml, State) -> + cf(Xml, State, fun detect_charset_1/2); +detect_charset_1(<<16#3C, 16#3F, 16#78, 16#6D, 16#6C, Xml2/binary>>, State) -> + {Xml3, State1} = read_until_end_of_xml_directive(Xml2, State), + case parse_xml_directive(Xml3) of {error, Reason} -> - {error, Reason}; + ?fatal_error(State, Reason); AttrList -> case lists:keysearch("encoding", 1, AttrList) of {value, {_, E}} -> case convert_encoding(E) of {error, Reason} -> - {error, Reason}; + ?fatal_error(State, Reason); Enc -> - {Xml, State#xmerl_sax_parser_state{encoding=Enc}} + {<<16#3C, 16#3F, 16#78, 16#6D, 16#6C, Xml3/binary>>, + State1#xmerl_sax_parser_state{encoding=Enc}} end; _ -> - {Xml, State} + {<<16#3C, 16#3F, 16#78, 16#6D, 16#6C, Xml3/binary>>, State1} end end; detect_charset_1(Xml, State) -> @@ -377,7 +393,7 @@ parse_value_1(<<C, Rest/binary>>, Stop, Acc) -> parse_value_1(Rest, Stop, [C |Acc]). %%====================================================================== -%%Default functions +%% Default functions %%====================================================================== %%---------------------------------------------------------------------- %% Function: default_event_cb(Event, LineNo, State) -> Result @@ -393,7 +409,7 @@ default_event_cb(_Event, _LineNo, State) -> %%---------------------------------------------------------------------- %% Function: default_continuation_cb(IoDevice) -> Result %% IoDevice = iodevice() -%% Output: Result = {[char()], State} +%% Output: Result = {binary(), IoDevice} %% Description: Default continuation callback reading blocks. %%---------------------------------------------------------------------- default_continuation_cb(IoDevice) -> @@ -403,3 +419,82 @@ default_continuation_cb(IoDevice) -> {ok, FileBin} -> {FileBin, IoDevice} end. + +%%---------------------------------------------------------------------- +%% Function: read_until_end_of_xml_directive(Rest, State) -> Result +%% Rest = binary() +%% Output: Result = {binary(), State} +%% Description: Reads a utf8 or latin1 until it finds '?>' +%%---------------------------------------------------------------------- +read_until_end_of_xml_directive(Rest, State) -> + case binary:match(Rest, <<"?>">>) of + nomatch -> + case cf(Rest, State) of + {<<>>, _} -> + ?fatal_error(State, "Can't detect character encoding due to lack of indata"); + {NewBytes, NewState} -> + read_until_end_of_xml_directive(NewBytes, NewState) + end; + _ -> + {Rest, State} + end. + + +%%---------------------------------------------------------------------- +%% Function : cf(Rest, State) -> Result +%% Parameters: Rest = binary() +%% State = #xmerl_sax_parser_state{} +%% NextCall = fun() +%% Result : {Rest, State} +%% Description: Function that uses provided fun to read another chunk from +%% input stream and calls the fun in NextCall. +%%---------------------------------------------------------------------- +cf(_Rest, #xmerl_sax_parser_state{continuation_fun = undefined} = State) -> + ?fatal_error(State, "Continuation function undefined"); +cf(Rest, #xmerl_sax_parser_state{continuation_fun = CFun, continuation_state = CState} = State) -> + Result = + try + CFun(CState) + catch + throw:ErrorTerm -> + ?fatal_error(State, ErrorTerm); + exit:Reason -> + ?fatal_error(State, {'EXIT', Reason}) + end, + case Result of + {<<>>, _} -> + ?fatal_error(State, "Can't detect character encoding due to lack of indata"); + {NewBytes, NewContState} -> + {<<Rest/binary, NewBytes/binary>>, + State#xmerl_sax_parser_state{continuation_state = NewContState}} + end. + +%%---------------------------------------------------------------------- +%% Function : cf(Rest, State, NextCall) -> Result +%% Parameters: Rest = binary() +%% State = #xmerl_sax_parser_state{} +%% NextCall = fun() +%% Result : {Rest, State} +%% Description: Function that uses provided fun to read another chunk from +%% input stream and calls the fun in NextCall. +%%---------------------------------------------------------------------- +cf(_Rest, #xmerl_sax_parser_state{continuation_fun = undefined} = State, _) -> + ?fatal_error(State, "Continuation function undefined"); +cf(Rest, #xmerl_sax_parser_state{continuation_fun = CFun, continuation_state = CState} = State, + NextCall) -> + Result = + try + CFun(CState) + catch + throw:ErrorTerm -> + ?fatal_error(State, ErrorTerm); + exit:Reason -> + ?fatal_error(State, {'EXIT', Reason}) + end, + case Result of + {<<>>, _} -> + ?fatal_error(State, "Can't detect character encoding due to lack of indata"); + {NewBytes, NewContState} -> + NextCall(<<Rest/binary, NewBytes/binary>>, + State#xmerl_sax_parser_state{continuation_state = NewContState}) + end. diff --git a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc index f3470b2809..1dca9608cb 100644 --- a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc @@ -64,38 +64,42 @@ %% Description: Parsing XML from input stream. %%---------------------------------------------------------------------- parse(Xml, State) -> - RefTable = ets:new(xmerl_sax_entity_refs, [private]), - - State1 = event_callback(startDocument, State), - - case catch parse_document(Xml, State1#xmerl_sax_parser_state{ref_table=RefTable}) of - {ok, Rest, State2} -> - State3 = event_callback(endDocument, State2), - ets:delete(RefTable), - case check_if_rest_ok(State3#xmerl_sax_parser_state.input_type, Rest) of - true -> - {ok, State3#xmerl_sax_parser_state.event_state, Rest}; - false -> - format_error(fatal_error, State3, "Input found after legal document") - end; - {fatal_error, {State2, Reason}} -> - State3 = event_callback(endDocument, State2), - ets:delete(RefTable), - format_error(fatal_error, State3, Reason); - {event_receiver_error, State2, {Tag, Reason}} -> - State3 = event_callback(endDocument, State2), - ets:delete(RefTable), - format_error(Tag, State3, Reason); - {endDocument, Rest, State2} -> - State3 = event_callback(endDocument, State2), - ets:delete(RefTable), - {ok, State3#xmerl_sax_parser_state.event_state, Rest}; - Other -> - _State2 = event_callback(endDocument, State1), - ets:delete(RefTable), - {fatal_error, Other} + RefTable = maps:new(), + + try + State1 = event_callback(startDocument, State), + Result = parse_document(Xml, State1#xmerl_sax_parser_state{ref_table=RefTable}), + handle_end_document(Result) + catch + throw:Exception -> + handle_end_document(Exception); + _:OtherError -> + handle_end_document({other, OtherError, State}) end. + % case catch parse_document(Xml, State1#xmerl_sax_parser_state{ref_table=RefTable}) of + % {ok, Rest, State2} -> + % State3 = event_callback(endDocument, State2), + % case check_if_rest_ok(State3#xmerl_sax_parser_state.input_type, Rest) of + % true -> + % {ok, State3#xmerl_sax_parser_state.event_state, Rest}; + % false -> + % format_error(fatal_error, State3, "Input found after legal document") + % end; + % {fatal_error, {State2, Reason}} -> + % State3 = event_callback(endDocument, State2), + % format_error(fatal_error, State3, Reason); + % {event_receiver_error, State2, {Tag, Reason}} -> + % State3 = event_callback(endDocument, State2), + % format_error(Tag, State3, Reason); + % {endDocument, Rest, State2} -> + % State3 = event_callback(endDocument, State2), + % {ok, State3#xmerl_sax_parser_state.event_state, Rest}; + % Other -> + % _State2 = event_callback(endDocument, State1), + % {fatal_error, Other} + % end. + %%---------------------------------------------------------------------- %% Function: parse_dtd(Xml, State) -> Result %% Input: Xml = string() | binary() @@ -105,38 +109,120 @@ parse(Xml, State) -> %% Description: Parsing XML DTD from input stream. %%---------------------------------------------------------------------- parse_dtd(Xml, State) -> - RefTable = ets:new(xmerl_sax_entity_refs, [private]), - - State1 = event_callback(startDocument, State), - - case catch parse_external_entity_1(Xml, State1#xmerl_sax_parser_state{ref_table=RefTable}) of - {fatal_error, {State2, Reason}} -> - State3 = event_callback(endDocument, State2), - ets:delete(RefTable), - format_error(fatal_error, State3, Reason); - {event_receiver_error, State2, {Tag, Reason}} -> - State3 = event_callback(endDocument, State2), - format_error(Tag, State3, Reason); - {Rest, State2} when is_record(State2, xmerl_sax_parser_state) -> - State3 = event_callback(endDocument, State2), - ets:delete(RefTable), - {ok, State3#xmerl_sax_parser_state.event_state, Rest}; - {endDocument, Rest, State2} when is_record(State2, xmerl_sax_parser_state) -> - State3 = event_callback(endDocument, State2), - ets:delete(RefTable), - {ok, State3#xmerl_sax_parser_state.event_state, Rest}; - Other -> - _State2 = event_callback(endDocument, State1), - ets:delete(RefTable), - {fatal_error, Other} + RefTable = maps:new(), + + try + State1 = event_callback(startDocument, State), + Result = parse_external_entity_1(Xml, State1#xmerl_sax_parser_state{ref_table=RefTable}), + handle_end_document(Result) + catch + throw:Exception -> + handle_end_document(Exception); + _:OtherError -> + handle_end_document({other, OtherError, State}) end. + % case catch parse_external_entity_1(Xml, State1#xmerl_sax_parser_state{ref_table=RefTable}) of + % {fatal_error, {State2, Reason}} -> + % State3 = event_callback(endDocument, State2), + % format_error(fatal_error, State3, Reason); + % {event_receiver_error, State2, {Tag, Reason}} -> + % State3 = event_callback(endDocument, State2), + % format_error(Tag, State3, Reason); + % {Rest, State2} when is_record(State2, xmerl_sax_parser_state) -> + % State3 = event_callback(endDocument, State2), + % {ok, State3#xmerl_sax_parser_state.event_state, Rest}; + % {endDocument, Rest, State2} when is_record(State2, xmerl_sax_parser_state) -> + % State3 = event_callback(endDocument, State2), + % {ok, State3#xmerl_sax_parser_state.event_state, Rest}; + % Other -> + % _State2 = event_callback(endDocument, State1), + % {fatal_error, Other} + % end. + + %%====================================================================== %% Internal functions %%====================================================================== %%---------------------------------------------------------------------- +%% Function: handle_end_document(ParserResult) -> Result +%% Input: ParseResult = term() +%% Output: Result = {ok, Rest, EventState} | +%% EventState = term() +%% Description: Ends the parsing and formats output +%%---------------------------------------------------------------------- +handle_end_document({ok, Rest, State}) -> + %%ok case from parse + try + State1 = event_callback(endDocument, State), + case check_if_rest_ok(State1#xmerl_sax_parser_state.input_type, Rest) of + true -> + {ok, State1#xmerl_sax_parser_state.event_state, Rest}; + false -> + format_error(fatal_error, State1, "Input found after legal document") + end + catch + throw:{event_receiver_error, State2, {Tag, Reason}} -> + format_error(Tag, State2, Reason); + _:Other -> + {fatal_error, Other} + end; +handle_end_document({endDocument, Rest, State}) -> + %% ok case from parse and parse_dtd + try + State1 = event_callback(endDocument, State), + {ok, State1#xmerl_sax_parser_state.event_state, Rest} + catch + throw:{event_receiver_error, State2, {Tag, Reason}} -> + format_error(Tag, State2, Reason); + _:Other -> + {fatal_error, Other} + end; +handle_end_document({fatal_error, {State, Reason}}) -> + try + State1 = event_callback(endDocument, State), + format_error(fatal_error, State1, Reason) + catch + throw:{event_receiver_error, State2, {Tag, Reason}} -> + format_error(Tag, State2, Reason); + _:Other -> + {fatal_error, Other} + end; +handle_end_document({event_receiver_error, State, {Tag, Reason}}) -> + try + State1 = event_callback(endDocument, State), + format_error(Tag, State1, Reason) + catch + throw:{event_receiver_error, State2, {Tag, Reason}} -> + format_error(Tag, State2, Reason); + _:Other -> + {fatal_error, Other} + end; +handle_end_document({Rest, State}) when is_record(State, xmerl_sax_parser_state) -> + %%ok case from parse_dtd + try + State1 = event_callback(endDocument, State), + {ok, State1#xmerl_sax_parser_state.event_state, Rest} + catch + throw:{event_receiver_error, State2, {Tag, Reason}} -> + format_error(Tag, State2, Reason); + _:Other -> + {fatal_error, Other} + end; +handle_end_document({other, Error, State}) -> + try + _State1 = event_callback(endDocument, State), + {fatal_error, Error} + catch + throw:{event_receiver_error, State2, {Tag, Reason}} -> + format_error(Tag, State2, Reason); + _:Other -> + {fatal_error, Other} + end. + +%%---------------------------------------------------------------------- %% Function: parse_document(Rest, State) -> Result %% Input: Rest = string() | binary() %% State = #xmerl_sax_parser_state{} @@ -439,6 +525,7 @@ check_if_rest_ok(stream, _) -> check_if_rest_ok(_, _) -> false. + %%---------------------------------------------------------------------- %% Function: parse_pi_1(Rest, State) -> Result %% Input: Rest = string() | binary() @@ -883,11 +970,11 @@ send_end_prefix_mapping_event([{Prefix, _Uri} |Ns], State) -> parse_eq(?STRING_EMPTY, State) -> cf(?STRING_EMPTY, State, fun parse_eq/2); parse_eq(?STRING_REST("=", Rest), State) -> - {Rest, State}; + {Rest, State}; parse_eq(?STRING_UNBOUND_REST(C, _) = Bytes, State) when ?is_whitespace(C) -> - {_WS, Rest, State1} = - whitespace(Bytes, State, []), - parse_eq(Rest, State1); + {_WS, Rest, State1} = + whitespace(Bytes, State, []), + parse_eq(Rest, State1); parse_eq(Bytes, State) -> unicode_incomplete_check([Bytes, State, fun parse_eq/2], "expecting = or whitespace"). @@ -905,11 +992,11 @@ parse_eq(Bytes, State) -> parse_att_value(?STRING_EMPTY, State) -> cf(?STRING_EMPTY, State, fun parse_att_value/2); parse_att_value(?STRING_UNBOUND_REST(C, Rest), State) when C == $'; C == $" -> - parse_att_value(Rest, State, C, []); + parse_att_value(Rest, State, C, []); parse_att_value(?STRING_UNBOUND_REST(C, _) = Bytes, State) when ?is_whitespace(C) -> - {_WS, Rest, State1} = - whitespace(Bytes, State, []), - parse_att_value(Rest, State1); + {_WS, Rest, State1} = + whitespace(Bytes, State, []), + parse_att_value(Rest, State1); parse_att_value(Bytes, State) -> unicode_incomplete_check([Bytes, State, fun parse_att_value/2], "\', \" or whitespace expected"). @@ -1360,17 +1447,20 @@ parse_pe_reference_1(Bytes, State, Name) -> %%---------------------------------------------------------------------- -%% Function: insert_reference(Reference, State) -> Result -%% Parameters: Reference = string() +%% Function: insert_reference(Name, Ref, State) -> Result +%% Parameters: Name = string() +%% Ref = {Type, Value} +%% Type = atom() +%% Value = term() %% State = #xmerl_sax_parser_state{} %% Result : %%---------------------------------------------------------------------- -insert_reference({Name, Type, Value}, Table) -> - case ets:lookup(Table, Name) of - [{Name, _, _}] -> - ok; +insert_reference(Name, Value, #xmerl_sax_parser_state{ref_table = Map} = State) -> + case maps:find(Name, Map) of + error -> + State#xmerl_sax_parser_state{ref_table = maps:put(Name, Value, Map)}; _ -> - ets:insert(Table, {Name, Type, Value}) + State end. @@ -1391,8 +1481,8 @@ look_up_reference("apos", _, _) -> look_up_reference("quot", _, _) -> {internal_general, "quot", "\""}; look_up_reference(Name, HaveToExist, State) -> - case ets:lookup(State#xmerl_sax_parser_state.ref_table, Name) of - [{Name, Type, Value}] -> + case maps:find(Name, State#xmerl_sax_parser_state.ref_table) of + {ok, {Type, Value}} -> {Type, Name, Value}; _ -> case HaveToExist of @@ -1474,7 +1564,7 @@ parse_system_litteral(?STRING_EMPTY, State, Stop, Acc) -> parse_system_litteral(?STRING_UNBOUND_REST(Stop, Rest), State, Stop, Acc) -> {lists:reverse(Acc), Rest, State}; parse_system_litteral(?STRING_UNBOUND_REST(C, Rest), State, Stop, Acc) -> - parse_system_litteral(Rest, State, Stop, [C |Acc]); + parse_system_litteral(Rest, State, Stop, [C |Acc]); parse_system_litteral(Bytes, State, Stop, Acc) -> unicode_incomplete_check([Bytes, State, Stop, Acc, fun parse_system_litteral/4], undefined). @@ -1646,9 +1736,11 @@ parse_external_entity(State, _PubId, SysId) -> end_tags = []}, - EventState = handle_external_entity(ExtRef, State1), + {EventState, RefTable} = handle_external_entity(ExtRef, State1), - NewState = event_callback({endEntity, SysId}, SaveState#xmerl_sax_parser_state{event_state=EventState}), + NewState = event_callback({endEntity, SysId}, + SaveState#xmerl_sax_parser_state{event_state=EventState, + ref_table=RefTable}), NewState#xmerl_sax_parser_state{file_type=normal}. @@ -1675,7 +1767,8 @@ handle_external_entity({file, FileToOpen}, State) -> entity=filename:basename(FileToOpen), input_type=file}), ok = file:close(FD), - EntityState#xmerl_sax_parser_state.event_state + {EntityState#xmerl_sax_parser_state.event_state, + EntityState#xmerl_sax_parser_state.ref_table} end; handle_external_entity({http, Url}, State) -> @@ -1695,7 +1788,9 @@ handle_external_entity({http, Url}, State) -> input_type=file}), ok = file:close(FD), ok = file:delete(TmpFile), - EntityState#xmerl_sax_parser_state.event_state + {EntityState#xmerl_sax_parser_state.event_state, + EntityState#xmerl_sax_parser_state.ref_table} + end catch throw:{error, Error} -> @@ -1716,7 +1811,7 @@ handle_external_entity({Tag, _Url}, State) -> parse_external_entity_1(?STRING_EMPTY, #xmerl_sax_parser_state{file_type=Type} = State) -> case catch cf(?STRING_EMPTY, State, fun parse_external_entity_1/2) of {Rest, State1} when is_record(State1, xmerl_sax_parser_state) -> - {Rest, State}; + {Rest, State1}; {fatal_error, {State1, "No more bytes"}} when Type == dtd; Type == entity -> {?STRING_EMPTY, State1}; Other -> @@ -2442,24 +2537,24 @@ parse_entity_def(?STRING_EMPTY, State, Name) -> cf(?STRING_EMPTY, State, Name, fun parse_entity_def/3); parse_entity_def(?STRING_UNBOUND_REST(C, Rest), State, Name) when C == $'; C == $" -> {Value, Rest1, State1} = parse_entity_value(Rest, State, C, []), - insert_reference({Name, internal_general, Value}, State1#xmerl_sax_parser_state.ref_table), - State2 = event_callback({internalEntityDecl, Name, Value}, State1), - {_WS, Rest2, State3} = whitespace(Rest1, State2, []), - parse_def_end(Rest2, State3); + State2 = insert_reference(Name, {internal_general, Value}, State1), + State3 = event_callback({internalEntityDecl, Name, Value}, State2), + {_WS, Rest2, State4} = whitespace(Rest1, State3, []), + parse_def_end(Rest2, State4); parse_entity_def(?STRING_UNBOUND_REST(C, _) = Rest, State, Name) when C == $S; C == $P -> {PubId, SysId, Rest1, State1} = parse_external_id(Rest, State, false), {Ndata, Rest2, State2} = parse_ndata(Rest1, State1), case Ndata of undefined -> - insert_reference({Name, external_general, {PubId, SysId}}, - State2#xmerl_sax_parser_state.ref_table), - State3 = event_callback({externalEntityDecl, Name, PubId, SysId}, State2), - {Rest2, State3}; + State3 = insert_reference(Name, {external_general, {PubId, SysId}}, + State2), + State4 = event_callback({externalEntityDecl, Name, PubId, SysId}, State3), + {Rest2, State4}; _ -> - insert_reference({Name, unparsed, {PubId, SysId, Ndata}}, - State2#xmerl_sax_parser_state.ref_table), - State3 = event_callback({unparsedEntityDecl, Name, PubId, SysId, Ndata}, State2), - {Rest2, State3} + State3 = insert_reference(Name, {unparsed, {PubId, SysId, Ndata}}, + State2), + State4 = event_callback({unparsedEntityDecl, Name, PubId, SysId, Ndata}, State3), + {Rest2, State4} end; parse_entity_def(Bytes, State, Name) -> unicode_incomplete_check([Bytes, State, Name, fun parse_entity_def/3], @@ -2636,19 +2731,19 @@ parse_pe_def(?STRING_EMPTY, State, Name) -> parse_pe_def(?STRING_UNBOUND_REST(C, Rest), State, Name) when C == $'; C == $" -> {Value, Rest1, State1} = parse_entity_value(Rest, State, C, []), Name1 = "%" ++ Name, - insert_reference({Name1, internal_parameter, Value}, - State1#xmerl_sax_parser_state.ref_table), - State2 = event_callback({internalEntityDecl, Name1, Value}, State1), - {_WS, Rest2, State3} = whitespace(Rest1, State2, []), - parse_def_end(Rest2, State3); + State2 = insert_reference(Name1, {internal_parameter, Value}, + State1), + State3 = event_callback({internalEntityDecl, Name1, Value}, State2), + {_WS, Rest2, State4} = whitespace(Rest1, State3, []), + parse_def_end(Rest2, State4); parse_pe_def(?STRING_UNBOUND_REST(C, _) = Bytes, State, Name) when C == $S; C == $P -> {PubId, SysId, Rest1, State1} = parse_external_id(Bytes, State, false), Name1 = "%" ++ Name, - insert_reference({Name1, external_parameter, {PubId, SysId}}, - State1#xmerl_sax_parser_state.ref_table), - State2 = event_callback({externalEntityDecl, Name1, PubId, SysId}, State1), - {_WS, Rest2, State3} = whitespace(Rest1, State2, []), - parse_def_end(Rest2, State3); + State2 = insert_reference(Name1, {external_parameter, {PubId, SysId}}, + State1), + State3 = event_callback({externalEntityDecl, Name1, PubId, SysId}, State2), + {_WS, Rest2, State4} = whitespace(Rest1, State3, []), + parse_def_end(Rest2, State4); parse_pe_def(Bytes, State, Name) -> unicode_incomplete_check([Bytes, State, Name, fun parse_pe_def/3], "\", \', SYSTEM or PUBLIC expected"). diff --git a/lib/xmerl/src/xmerl_sax_simple_dom.erl b/lib/xmerl/src/xmerl_sax_simple_dom.erl index 7eb3afd499..d842bd982b 100644 --- a/lib/xmerl/src/xmerl_sax_simple_dom.erl +++ b/lib/xmerl/src/xmerl_sax_simple_dom.erl @@ -129,8 +129,9 @@ build_dom(endDocument, State#xmerl_sax_simple_dom_state{dom=[Decl, {Tag, Attributes, lists:reverse(Content)}]}; _ -> - ?dbg("~p\n", [D]), - ?error("we're not at end the document when endDocument event is encountered.") + %% endDocument is also sent by the parser when a fault occur to tell + %% the event receiver that no more input will be sent + State end; %% Element diff --git a/lib/xmerl/test/Makefile b/lib/xmerl/test/Makefile index e5c89f84b9..3204f081ba 100644 --- a/lib/xmerl/test/Makefile +++ b/lib/xmerl/test/Makefile @@ -126,5 +126,6 @@ release_tests_spec: opt @tar cfh - xmerl_xsd_MS2002-01-16_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) @tar cfh - xmerl_xsd_NIST2002-01-16_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) @tar cfh - xmerl_xsd_Sun2002-01-16_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) + @tar cfh - xmerl_sax_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) @tar cfh - xmerl_sax_stream_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) chmod -R u+w "$(RELSYSDIR)" diff --git a/lib/xmerl/test/xmerl_sax_SUITE.erl b/lib/xmerl/test/xmerl_sax_SUITE.erl index eb9cefe0df..68b9bcc4a2 100644 --- a/lib/xmerl/test/xmerl_sax_SUITE.erl +++ b/lib/xmerl/test/xmerl_sax_SUITE.erl @@ -33,7 +33,6 @@ %%====================================================================== %% External functions %%====================================================================== - %%---------------------------------------------------------------------- %% Initializations %%---------------------------------------------------------------------- @@ -42,12 +41,15 @@ all() -> [{group, bugs}]. groups() -> - [{bugs, [], [ticket_8213, ticket_8214, ticket_11551]}]. + [{bugs, [], [ticket_8213, ticket_8214, ticket_11551, + fragmented_xml_directive, + old_dom_event_fun_endDocument_bug, + event_fun_endDocument_error_test, + event_fun_startDocument_error_test]}]. -%%---------------------------------------------------------------------- +%%====================================================================== %% Tests -%%---------------------------------------------------------------------- - +%%====================================================================== %%---------------------------------------------------------------------- %% Test Case %% ID: ticket_8213 @@ -56,7 +58,6 @@ ticket_8213(_Config) -> {ok,ok,[]} = xmerl_sax_parser:stream("<elem/>", [{event_fun, fun (_E,_,_) -> ok end}]), ok. - %%---------------------------------------------------------------------- %% Test Case %% ID: ticket_8214 @@ -99,3 +100,66 @@ ticket_11551(_Config) -> <a>hej</a>">>, {ok, undefined, <<"\n\n<?xml", _/binary>>} = xmerl_sax_parser:stream(Stream3, []), ok. + +%%---------------------------------------------------------------------- +%% Test Case +%% ID: fragmented_xml_directive +%% Test of fragmented xml directive by reading one byte per continuation ca +fragmented_xml_directive(Config) -> + DataDir = proplists:get_value(data_dir, Config), + Name = filename:join(DataDir, "test_data_1.xml"), + {ok, Fd} = file:open(Name, [raw, read,binary]), + Cf = fun cf_fragmented_xml_directive/1, + {ok, undefined, _} = xmerl_sax_parser:stream(<<>>, + [{continuation_fun, Cf}, + {continuation_state, Fd}]), + ok. + +%%---------------------------------------------------------------------- +%% Test Case +%% ID: old_dom_event_fun_endDocument_bug +%% The old_dom backend previous generateded an uncatched exception +%% instead of the correct fatal_error from the parser. +old_dom_event_fun_endDocument_bug(_Config) -> + %% Stream contains bad characters, + {fatal_error, _, _, _, _} = + xmerl_sax_parser:stream([60,63,120,109,108,32,118,101,114,115,105,111,110,61,39,49,46,48,39,32,101,110,99,111,100,105,110,103,61,39,117,116,102,45,56,39,63,62,60, + 99,111,109,109,97,110,100,62,60,104,101,97,100,101,114,62,60,116,114,97,110,115,97,99,116,105,111,110,73,100,62,49,60,47,116,114,97,110, + 115,97,99,116,105,111,110,73,100,62,60,47,104,101,97,100,101,114,62,60,98,111,100,121,62,95,226,130,172,59,60,60,47,98,111,100,121,62,60, + 47,99,111,109,109,97,110,100,62,60,47,120,49,95,49,62], + [{event_fun,fun xmerl_sax_old_dom:event/3}, + {event_state,xmerl_sax_old_dom:initial_state()}]), + ok. + +%%---------------------------------------------------------------------- +%% Test Case +%% ID: event_fun_endDocument_error_test +event_fun_endDocument_error_test(_Config) -> + Stream = <<"<?xml version=\"1.0\" encoding=\"utf-8\"?><a>hej</a>">>, + Ef = fun(endDocument, _ , _) -> throw({event_error, "endDocument error"}); + (_, _, S) -> S + end, + {event_error, _, _, _, _} = xmerl_sax_parser:stream(Stream, [{event_fun, Ef}]), + ok. + +%%---------------------------------------------------------------------- +%% Test Case +%% ID: event_fun_startDocument_error_test +event_fun_startDocument_error_test(_Config) -> + Stream = <<"<?xml version=\"1.0\" encoding=\"utf-8\"?><a>hej</a>">>, + Ef = fun(startDocument, _ , _) -> throw({event_error, "endDocument error"}); + (_, _, S) -> S + end, + {event_error, _, _, _, _} = xmerl_sax_parser:stream(Stream, [{event_fun, Ef}]), + ok. + +%%====================================================================== +%% Internal functions +%%====================================================================== +cf_fragmented_xml_directive(IoDevice) -> + case file:read(IoDevice, 1) of + eof -> + {<<>>, IoDevice}; + {ok, FileBin} -> + {FileBin, IoDevice} + end. diff --git a/lib/xmerl/test/xmerl_sax_SUITE_data/test_data_1.xml b/lib/xmerl/test/xmerl_sax_SUITE_data/test_data_1.xml new file mode 100644 index 0000000000..efbaee6b81 --- /dev/null +++ b/lib/xmerl/test/xmerl_sax_SUITE_data/test_data_1.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="utf-8"?> +<a> +Hej +</a> diff --git a/lib/xmerl/test/xmerl_sax_stream_SUITE.erl b/lib/xmerl/test/xmerl_sax_stream_SUITE.erl index a306eb66a2..7315f67374 100644 --- a/lib/xmerl/test/xmerl_sax_stream_SUITE.erl +++ b/lib/xmerl/test/xmerl_sax_stream_SUITE.erl @@ -40,7 +40,8 @@ all() -> [ one_document, two_documents, - one_document_and_junk + one_document_and_junk, + end_of_stream ]. %%---------------------------------------------------------------------- @@ -62,6 +63,9 @@ end_per_testcase(_Func, _Config) -> %%---------------------------------------------------------------------- %% Tests %%---------------------------------------------------------------------- + +%%---------------------------------------------------------------------- +%% Send One doc over stream one_document(Config) -> Port = 11111, @@ -107,6 +111,8 @@ one_document(Config) -> end, ok. +%%---------------------------------------------------------------------- +%% Send Two doc over stream two_documents(Config) -> Port = 11111, @@ -152,6 +158,8 @@ two_documents(Config) -> end, ok. +%%---------------------------------------------------------------------- +%% Send one doc and then junk on stream one_document_and_junk(Config) -> Port = 11111, @@ -196,6 +204,13 @@ one_document_and_junk(Config) -> ct:fail("Timeout") end, ok. + +%%---------------------------------------------------------------------- +%% Test of continuation when end of stream +end_of_stream(Config) -> + Stream = <<"<?xml version=\"1.0\" encoding=\"utf-8\"?><a>hej</a>">>, + {ok, undefined, <<>>} = xmerl_sax_parser:stream(Stream, []), + ok. %%---------------------------------------------------------------------- %% Utility functions diff --git a/lib/xmerl/vsn.mk b/lib/xmerl/vsn.mk index 1515a4e37d..4e741d59a8 100644 --- a/lib/xmerl/vsn.mk +++ b/lib/xmerl/vsn.mk @@ -1 +1 @@ -XMERL_VSN = 1.3.13 +XMERL_VSN = 1.3.14 diff --git a/otp_versions.table b/otp_versions.table index 62eaa62fed..e86ace6e02 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,4 @@ +OTP-19.3.5 : erts-8.3.4 xmerl-1.3.14 # asn1-4.0.4 common_test-1.14 compiler-7.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2 ssl-8.1.3 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 : OTP-19.3.4 : inets-6.3.9 ssl-8.1.3 # asn1-4.0.4 common_test-1.14 compiler-7.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 erts-8.3.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.13 : OTP-19.3.3 : dialyzer-3.1.1 erts-8.3.3 inets-6.3.8 # asn1-4.0.4 common_test-1.14 compiler-7.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2 ssl-8.1.2 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.13 : OTP-19.3.2 : erts-8.3.2 # asn1-4.0.4 common_test-1.14 compiler-7.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.7 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2 ssl-8.1.2 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.13 : |