diff options
81 files changed, 1301 insertions, 463 deletions
diff --git a/HOWTO/INSTALL.md b/HOWTO/INSTALL.md index bbde5bc08c..368947b36c 100644 --- a/HOWTO/INSTALL.md +++ b/HOWTO/INSTALL.md @@ -217,7 +217,7 @@ Step 4: Run the following commands to configure the build: $ ./configure [ options ] -If you are building it from git you will need to run `autoconf` to generate configure file. +If you are building it from git you will need to run `./otp_build autoconf` to generate configure file. By default, Erlang/OTP will be installed in `/usr/local/{bin,lib/erlang}`. To instead install in `<BaseDir>/{bin,lib/erlang}`, use the `--prefix=<BaseDir>` option. diff --git a/bootstrap/lib/compiler/ebin/cerl.beam b/bootstrap/lib/compiler/ebin/cerl.beam Binary files differindex 686281467d..d5e97e4f68 100644 --- a/bootstrap/lib/compiler/ebin/cerl.beam +++ b/bootstrap/lib/compiler/ebin/cerl.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_clauses.beam b/bootstrap/lib/compiler/ebin/cerl_clauses.beam Binary files differindex c55d8be59d..2357df79f4 100644 --- a/bootstrap/lib/compiler/ebin/cerl_clauses.beam +++ b/bootstrap/lib/compiler/ebin/cerl_clauses.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_inline.beam b/bootstrap/lib/compiler/ebin/cerl_inline.beam Binary files differindex 5450bd23ab..2f7f220ebd 100644 --- a/bootstrap/lib/compiler/ebin/cerl_inline.beam +++ b/bootstrap/lib/compiler/ebin/cerl_inline.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_trees.beam b/bootstrap/lib/compiler/ebin/cerl_trees.beam Binary files differindex faafbe20c8..1338631c23 100644 --- a/bootstrap/lib/compiler/ebin/cerl_trees.beam +++ b/bootstrap/lib/compiler/ebin/cerl_trees.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex 81261e8c5c..eb682b953c 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/core_parse.beam b/bootstrap/lib/compiler/ebin/core_parse.beam Binary files differindex 8c5816c52d..121ef65275 100644 --- a/bootstrap/lib/compiler/ebin/core_parse.beam +++ b/bootstrap/lib/compiler/ebin/core_parse.beam diff --git a/bootstrap/lib/compiler/ebin/core_pp.beam b/bootstrap/lib/compiler/ebin/core_pp.beam Binary files differindex 5de821db8b..fbfd68a93b 100644 --- a/bootstrap/lib/compiler/ebin/core_pp.beam +++ b/bootstrap/lib/compiler/ebin/core_pp.beam diff --git a/bootstrap/lib/compiler/ebin/erl_bifs.beam b/bootstrap/lib/compiler/ebin/erl_bifs.beam Binary files differindex e2dc90c887..27136581c1 100644 --- a/bootstrap/lib/compiler/ebin/erl_bifs.beam +++ b/bootstrap/lib/compiler/ebin/erl_bifs.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold.beam b/bootstrap/lib/compiler/ebin/sys_core_fold.beam Binary files differindex 41b702efcf..9bdab52015 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold.beam diff --git a/bootstrap/lib/compiler/ebin/v3_core.beam b/bootstrap/lib/compiler/ebin/v3_core.beam Binary files differindex add7d2be2f..b876bd9970 100644 --- a/bootstrap/lib/compiler/ebin/v3_core.beam +++ b/bootstrap/lib/compiler/ebin/v3_core.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel.beam b/bootstrap/lib/compiler/ebin/v3_kernel.beam Binary files differindex 1ac5932127..f5dcecb807 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam Binary files differindex bc67dfe0c6..1798a279b0 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam diff --git a/bootstrap/lib/kernel/ebin/dist_util.beam b/bootstrap/lib/kernel/ebin/dist_util.beam Binary files differindex 4cd98bd217..ea09695075 100644 --- a/bootstrap/lib/kernel/ebin/dist_util.beam +++ b/bootstrap/lib/kernel/ebin/dist_util.beam diff --git a/bootstrap/lib/kernel/include/dist.hrl b/bootstrap/lib/kernel/include/dist.hrl index e32c112e63..77556d1303 100644 --- a/bootstrap/lib/kernel/include/dist.hrl +++ b/bootstrap/lib/kernel/include/dist.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2013. All Rights Reserved. +%% Copyright Ericsson AB 1999-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -37,3 +37,4 @@ -define(DFLAG_DIST_HDR_ATOM_CACHE,16#2000). -define(DFLAG_SMALL_ATOM_TAGS, 16#4000). -define(DFLAG_UTF8_ATOMS, 16#10000). +-define(DFLAG_MAP_TAG, 16#20000). diff --git a/bootstrap/lib/stdlib/ebin/epp.beam b/bootstrap/lib/stdlib/ebin/epp.beam Binary files differindex 9bb4b6fb1f..ab22f9971b 100644 --- a/bootstrap/lib/stdlib/ebin/epp.beam +++ b/bootstrap/lib/stdlib/ebin/epp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_eval.beam b/bootstrap/lib/stdlib/ebin/erl_eval.beam Binary files differindex 63a3b3aa65..aab2f38b91 100644 --- a/bootstrap/lib/stdlib/ebin/erl_eval.beam +++ b/bootstrap/lib/stdlib/ebin/erl_eval.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_lint.beam b/bootstrap/lib/stdlib/ebin/erl_lint.beam Binary files differindex 6e9d06ea44..9660509270 100644 --- a/bootstrap/lib/stdlib/ebin/erl_lint.beam +++ b/bootstrap/lib/stdlib/ebin/erl_lint.beam diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 523130d01a..2a9a0a5c2f 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -414,13 +414,6 @@ else UNIX_ONLY_BUILDS = endif -ifeq ($(TARGET), win32) -TMPVAR := $(shell LANG=C $(PERL) utils/make_compiler_flags -o $(TTF_DIR)/erl_compile_flags.h -v CONFIG_H "N/A" -v CFLAGS "$(CFLAGS)" -v LDFLAGS "$(LDFLAGS)") -else -# We force this to be run every time this makefile is executed -TMPVAR := $(shell LANG=C $(PERL) utils/make_compiler_flags -o $(TTF_DIR)/erl_compile_flags.h -f CONFIG_H "$(ERL_TOP)/erts/$(TARGET)/config.h" -v CFLAGS "$(CFLAGS)" -v LDFLAGS "$(LDFLAGS)") -endif - .PHONY: all ifdef VOID_EMULATOR all: @@ -499,6 +492,15 @@ release_docs_spec: _create_dirs := $(shell mkdir -p $(CREATE_DIRS)) + +# has to be run after _create_dirs +ifeq ($(TARGET), win32) +TMPVAR := $(shell LANG=C $(PERL) utils/make_compiler_flags -o $(TTF_DIR)/erl_compile_flags.h -v CONFIG_H "N/A" -v CFLAGS "$(CFLAGS)" -v LDFLAGS "$(LDFLAGS)") +else +# We force this to be run every time this makefile is executed +TMPVAR := $(shell LANG=C $(PERL) utils/make_compiler_flags -o $(TTF_DIR)/erl_compile_flags.h -f CONFIG_H "$(ERL_TOP)/erts/$(TARGET)/config.h" -v CFLAGS "$(CFLAGS)" -v LDFLAGS "$(LDFLAGS)") +endif + GENERATE = HIPE_ASM = @@ -1087,7 +1089,9 @@ BEAM_SRC=$(wildcard beam/*.c) DRV_COMMON_SRC=$(wildcard drivers/common/*.c) DRV_OSTYPE_SRC=$(wildcard drivers/$(ERLANG_OSTYPE)/*.c) ALL_SYS_SRC=$(wildcard sys/$(ERLANG_OSTYPE)/*.c) $(wildcard sys/common/*.c) -TARGET_SRC=$(wildcard $(TARGET)/*.c) $(wildcard $(TTF_DIR)/*.c) +# We use $(shell ls) here instead of wildcard as $(wildcard ) resolved at +# loadtime of the makefile and at that time these files are not generated yet. +TARGET_SRC=$(shell ls $(TARGET)/*.c) $(shell ls $(TTF_DIR)/*.c) # I do not want the -MG flag on windows, it does not work properly for a # windows build. diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 25029ba90f..a62a83a928 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -485,7 +485,7 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret) *ret = am_EOT; return DB_ERROR_NONE; } - /* Walk down to the tree to the left */ + /* Walk down the tree to the left */ if ((stack = get_static_stack(tb)) != NULL) { stack->pos = stack->slot = 0; } @@ -531,7 +531,7 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret) *ret = am_EOT; return DB_ERROR_NONE; } - /* Walk down to the tree to the left */ + /* Walk down the tree to the right */ if ((stack = get_static_stack(tb)) != NULL) { stack->pos = stack->slot = 0; } diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 73630fda8e..68fcc177ae 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1484,7 +1484,8 @@ new_map j d I I update_map_assoc j s d I I update_map_exact j s d I I -is_map Fail cq => jump Fail +is_map Fail Literal=q => move Literal x | is_map Fail x +is_map Fail c => jump Fail %macro: is_map IsMap -fail_action is_map f r diff --git a/lib/common_test/src/ct_gen_conn.erl b/lib/common_test/src/ct_gen_conn.erl index 078d6b1a44..239f5b5f25 100644 --- a/lib/common_test/src/ct_gen_conn.erl +++ b/lib/common_test/src/ct_gen_conn.erl @@ -344,7 +344,7 @@ loop(Opts) -> link(NewPid), put(conn_pid,NewPid), loop(Opts#gen_opts{conn_pid=NewPid, - cb_state=NewState}); + cb_state=NewState}); Error -> ct_util:unregister_connection(self()), log("Reconnect failed. Giving up!", diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index b4d82a53cf..8c3ce03732 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -281,8 +281,16 @@ open(KeyOrName,ConnType,TargetMod,Extra) -> end, log(undefined,open,"Connecting to ~p(~p)", [KeyOrName,Addr1]), - ct_gen_conn:start(KeyOrName,full_addr(Addr1,ConnType), - {TargetMod,KeepAlive,Extra},?MODULE) + Reconnect = + case ct:get_config({telnet_settings,reconnection_attempts}) of + 0 -> false; + _ -> true + end, + ct_gen_conn:start(full_addr(Addr1,ConnType), + {TargetMod,KeepAlive,Extra}, + ?MODULE, [{name,KeyOrName}, + {reconnect,Reconnect}, + {old,true}]) end. %%%----------------------------------------------------------------- @@ -601,11 +609,9 @@ handle_msg({cmd,Cmd,Timeout},State) -> end_gen_log(), {Return,State#state{buffer=NewBuffer,prompt=Prompt}}; handle_msg({send,Cmd},State) -> - log(State,send,"Cmd: ~p",[Cmd]), - + log(State,send,"Sending: ~p",[Cmd]), debug_cont_gen_log("Throwing Buffer:",[]), debug_log_lines(State#state.buffer), - case {State#state.type,State#state.prompt} of {ts,_} -> silent_teln_expect(State#state.name, @@ -783,66 +789,61 @@ log(#state{name=Name,teln_pid=TelnPid,host=Host,port=Port}, true -> Name end, Silent = get(silent), - case ct_util:get_testdata({cth_conn_log,?MODULE}) of - Result when Result /= undefined, Result /= silent, Silent /= true -> - {PrintHeader,PreBR} = if Action==undefined -> - {false,""}; - true -> - {true,"\n"} - end, - error_logger:info_report(#conn_log{header=PrintHeader, - client=self(), - conn_pid=TelnPid, - address={Host,Port}, - name=Name1, - action=Action, - module=?MODULE}, - {PreBR++String,Args}); - Result when Result /= undefined -> - ok; - _ when Action == open; Action == close; Action == reconnect; - Action == info; Action == error -> - ct_gen_conn:log(heading(Action,Name1),String,Args); - _ when ForcePrint == false -> - case ct_util:is_silenced(telnet) of - true -> - ok; - false -> - ct_gen_conn:cont_log(String,Args) + + if Action == general_io -> + case ct_util:get_testdata({cth_conn_log,?MODULE}) of + HookMode when HookMode /= undefined, HookMode /= silent, + Silent /= true -> + error_logger:info_report(#conn_log{header=false, + client=self(), + conn_pid=TelnPid, + address={Host,Port}, + name=Name1, + action=Action, + module=?MODULE}, + {String,Args}); + _ -> %% hook inactive or silence requested + ok end; - _ when ForcePrint == true -> - case ct_util:is_silenced(telnet) of - true -> - %% call log/3 now instead of cont_log/2 since - %% start_gen_log/1 will not have been previously called + + true -> + if Action == open; Action == close; Action == reconnect; + Action == info; Action == error -> ct_gen_conn:log(heading(Action,Name1),String,Args); - false -> - ct_gen_conn:cont_log(String,Args) + + ForcePrint == false -> + case ct_util:is_silenced(telnet) of + true -> + ok; + false -> + ct_gen_conn:cont_log(String,Args) + end; + + ForcePrint == true -> + case ct_util:is_silenced(telnet) of + true -> + %% call log/3 now instead of cont_log/2 since + %% start_gen_log/1 will not have been previously + %% called + ct_gen_conn:log(heading(Action,Name1),String,Args); + false -> + ct_gen_conn:cont_log(String,Args) + end end end. start_gen_log(Heading) -> - case ct_util:get_testdata({cth_conn_log,?MODULE}) of - undefined -> - %% check if output is suppressed - case ct_util:is_silenced(telnet) of - true -> ok; - false -> ct_gen_conn:start_log(Heading) - end; - _ -> - ok + %% check if output is suppressed + case ct_util:is_silenced(telnet) of + true -> ok; + false -> ct_gen_conn:start_log(Heading) end. end_gen_log() -> - case ct_util:get_testdata({cth_conn_log,?MODULE}) of - undefined -> - %% check if output is suppressed - case ct_util:is_silenced(telnet) of - true -> ok; - false -> ct_gen_conn:end_log() - end; - _ -> - ok + %% check if output is suppressed + case ct_util:is_silenced(telnet) of + true -> ok; + false -> ct_gen_conn:end_log() end. %%% @hidden @@ -1027,19 +1028,25 @@ teln_expect1(Name,Pid,Data,Pattern,Acc,EO) -> NotFinished -> %% Get more data Fun = fun() -> get_data1(EO#eo.teln_pid) end, - case ct_gen_conn:do_within_time(Fun, EO#eo.timeout) of - {error,Reason} -> + case timer:tc(ct_gen_conn, do_within_time, [Fun, EO#eo.timeout]) of + {_,{error,Reason}} -> %% A timeout will occur when the telnet connection %% is idle for EO#eo.timeout milliseconds. {error,Reason}; - {ok,Data1} -> - case NotFinished of - {nomatch,Rest} -> - %% One expect - teln_expect1(Name,Pid,Rest++Data1,Pattern,[],EO); - {continue,Patterns1,Acc1,Rest} -> - %% Sequence - teln_expect1(Name,Pid,Rest++Data1,Patterns1,Acc1,EO) + {Elapsed,{ok,Data1}} -> + TVal = trunc(EO#eo.timeout - (Elapsed/1000)), + if TVal =< 0 -> + {error,timeout}; + true -> + EO1 = EO#eo{timeout = TVal}, + case NotFinished of + {nomatch,Rest} -> + %% One expect + teln_expect1(Name,Pid,Rest++Data1,Pattern,[],EO1); + {continue,Patterns1,Acc1,Rest} -> + %% Sequence + teln_expect1(Name,Pid,Rest++Data1,Patterns1,Acc1,EO1) + end end end end. diff --git a/lib/common_test/src/ct_telnet_client.erl b/lib/common_test/src/ct_telnet_client.erl index 2cbcba9c77..ce30dcb74b 100644 --- a/lib/common_test/src/ct_telnet_client.erl +++ b/lib/common_test/src/ct_telnet_client.erl @@ -32,7 +32,9 @@ -module(ct_telnet_client). --export([open/1, open/2, open/3, open/4, close/1]). +%% -define(debug, true). + +-export([open/2, open/3, open/4, open/5, close/1]). -export([send_data/2, get_data/1]). -define(TELNET_PORT, 23). @@ -64,20 +66,23 @@ -define(TERMINAL_TYPE, 24). -define(WINDOW_SIZE, 31). --record(state,{get_data, keep_alive=true}). +-record(state,{conn_name, get_data, keep_alive=true, log_pos=1}). -open(Server) -> - open(Server, ?TELNET_PORT, ?OPEN_TIMEOUT, true). +open(Server, ConnName) -> + open(Server, ?TELNET_PORT, ?OPEN_TIMEOUT, true, ConnName). -open(Server, Port) -> - open(Server, Port, ?OPEN_TIMEOUT, true). +open(Server, Port, ConnName) -> + open(Server, Port, ?OPEN_TIMEOUT, true, ConnName). -open(Server, Port, Timeout) -> - open(Server, Port, Timeout, true). +open(Server, Port, Timeout, ConnName) -> + open(Server, Port, Timeout, true, ConnName). -open(Server, Port, Timeout, KeepAlive) -> +open(Server, Port, Timeout, KeepAlive, ConnName) -> Self = self(), - Pid = spawn(fun() -> init(Self, Server, Port, Timeout, KeepAlive) end), + Pid = spawn(fun() -> + init(Self, Server, Port, Timeout, + KeepAlive, ConnName) + end), receive {open,Pid} -> {ok,Pid}; @@ -86,29 +91,34 @@ open(Server, Port, Timeout, KeepAlive) -> end. close(Pid) -> - Pid ! close. + Pid ! {close,self()}, + receive closed -> ok + after 5000 -> ok + end. send_data(Pid, Data) -> Pid ! {send_data, Data++"\n"}, ok. get_data(Pid) -> - Pid ! {get_data, self()}, + Pid ! {get_data,self()}, receive {data,Data} -> - {ok, Data} + {ok,Data} end. %%%----------------------------------------------------------------- %%% Internal functions -init(Parent, Server, Port, Timeout, KeepAlive) -> +init(Parent, Server, Port, Timeout, KeepAlive, ConnName) -> case gen_tcp:connect(Server, Port, [list,{packet,0}], Timeout) of {ok,Sock} -> - dbg("Connected to: ~p (port: ~w, keep_alive: ~w)\n", [Server,Port,KeepAlive]), - send([?IAC,?DO,?SUPPRESS_GO_AHEAD], Sock), + dbg("~p connected to: ~p (port: ~w, keep_alive: ~w)\n", + [ConnName,Server,Port,KeepAlive]), + send([?IAC,?DO,?SUPPRESS_GO_AHEAD], Sock, ConnName), Parent ! {open,self()}, - loop(#state{get_data=10, keep_alive=KeepAlive}, Sock, []), + loop(#state{conn_name=ConnName, get_data=10, keep_alive=KeepAlive}, + Sock, []), gen_tcp:close(Sock); Error -> Parent ! {Error,self()} @@ -118,6 +128,13 @@ loop(State, Sock, Acc) -> receive {tcp_closed,_} -> dbg("Connection closed\n", []), + Data = lists:reverse(lists:append(Acc)), + dbg("Printing queued messages: ~tp",[Data]), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + length(Data))]), receive {get_data,Pid} -> Pid ! closed @@ -125,11 +142,11 @@ loop(State, Sock, Acc) -> ok end; {tcp,_,Msg0} -> - dbg("tcp msg: ~p~n",[Msg0]), + dbg("tcp msg: ~tp~n",[Msg0]), Msg = check_msg(Sock,Msg0,[]), loop(State, Sock, [Msg | Acc]); {send_data,Data} -> - send(Data, Sock), + send(Data, Sock, State#state.conn_name), loop(State, Sock, Acc); {get_data,Pid} -> NewState = @@ -144,54 +161,100 @@ loop(State, Sock, Acc) -> end; _ -> Data = lists:reverse(lists:append(Acc)), - dbg("get_data ~p\n",[Data]), + Len = length(Data), + dbg("get_data ~tp\n",[Data]), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + Len)]), Pid ! {data,Data}, - State + State#state{log_pos = 1} end, loop(NewState, Sock, []); {get_data_delayed,Pid} -> NewState = case State of #state{keep_alive = true, get_data = 0} -> - if Acc == [] -> send([?IAC,?NOP], Sock); + if Acc == [] -> send([?IAC,?NOP], Sock, + State#state.conn_name); true -> ok end, State#state{get_data=10}; _ -> State end, - NewAcc = + {NewAcc,Pos} = case erlang:is_process_alive(Pid) of - true -> + true when Acc /= [] -> Data = lists:reverse(lists:append(Acc)), - dbg("get_data_delayed ~p\n",[Data]), + Len = length(Data), + dbg("get_data_delayed ~tp\n",[Data]), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + Len)]), Pid ! {data,Data}, - []; + {[],1}; + true when Acc == [] -> + dbg("get_data_delayed nodata\n",[]), + Pid ! {data,[]}, + {[],1}; false -> - Acc + {Acc,NewState#state.log_pos} end, - loop(NewState, Sock, NewAcc); - close -> + loop(NewState#state{log_pos=Pos}, Sock, NewAcc); + {close,Pid} -> dbg("Closing connection\n", []), + if Acc == [] -> + ok; + true -> + Data = lists:reverse(lists:append(Acc)), + dbg("Printing queued messages: ~tp",[Data]), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + length(Data))]) + end, gen_tcp:close(Sock), - ok + Pid ! closed after wait(State#state.keep_alive,?IDLE_TIMEOUT) -> - if - Acc == [] -> send([?IAC,?NOP], Sock); - true -> ok - end, - loop(State, Sock, Acc) + Data = lists:reverse(lists:append(Acc)), + case Data of + [] -> + send([?IAC,?NOP], Sock, State#state.conn_name), + loop(State, Sock, Acc); + _ when State#state.log_pos == length(Data)+1 -> + loop(State, Sock, Acc); + _ -> + dbg("Idle timeout, printing ~tp\n",[Data]), + Len = length(Data), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + Len)]), + loop(State#state{log_pos = Len+1}, Sock, Acc) + end end. wait(true, Time) -> Time; wait(false, _) -> infinity. -send(Data, Sock) -> +send(Data, Sock, ConnName) -> case Data of [?IAC|_] = Cmd -> cmd_dbg(Cmd); _ -> - dbg("Sending: ~p\n", [Data]) + dbg("Sending: ~tp\n", [Data]), + try io_lib:format("[~w] ~ts", [?MODULE,Data]) of + Str -> + ct_telnet:log(ConnName, general_io, Str, []) + catch + _:_ -> ok + end end, gen_tcp:send(Sock, Data), ok. diff --git a/lib/common_test/src/unix_telnet.erl b/lib/common_test/src/unix_telnet.erl index e049c3bf39..b05386a5ab 100644 --- a/lib/common_test/src/unix_telnet.erl +++ b/lib/common_test/src/unix_telnet.erl @@ -109,7 +109,7 @@ connect(ConnName,Ip,Port,Timeout,KeepAlive,Extra) -> connect1(Name,Ip,Port,Timeout,KeepAlive,Username,Password) -> start_gen_log("unix_telnet connect"), Result = - case ct_telnet_client:open(Ip,Port,Timeout,KeepAlive) of + case ct_telnet_client:open(Ip,Port,Timeout,KeepAlive,Name) of {ok,Pid} -> case ct_telnet:silent_teln_expect(Name,Pid,[], [prompt],?prx,[]) of @@ -143,13 +143,13 @@ connect1(Name,Ip,Port,Timeout,KeepAlive,Username,Password) -> {ok,[{prompt,_OtherPrompt1},{prompt,_OtherPrompt2}],_} -> {ok,Pid}; Error -> - log(Name,error, + log(Name,conn_error, "Did not get expected prompt from ~p:~p\n~p\n", [Ip,Port,Error]), {error,Error} end; Error -> - log(Name,error, + log(Name,conn_error, "Could not open telnet connection to ~p:~p\n~p\n", [Ip,Port,Error]), Error diff --git a/lib/common_test/test/ct_telnet_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE.erl index acce4eca14..f5cff76fd1 100644 --- a/lib/common_test/test/ct_telnet_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE.erl @@ -72,19 +72,32 @@ init_per_suite(Config) -> end_per_suite(Config) -> ct_test_support:end_per_suite(Config). -init_per_testcase(TestCase, Config) when TestCase=/=unix_telnet-> +init_per_testcase(TestCase, Config) when TestCase /= unix_telnet -> + ct:pal("Testcase ~p starting!", [TestCase]), TS = telnet_server:start([{port,?erl_telnet_server_port}, {users,[{?erl_telnet_server_user, ?erl_telnet_server_pwd}]}]), ct_test_support:init_per_testcase(TestCase, [{telnet_server,TS}|Config]); init_per_testcase(TestCase, Config) -> - ct_test_support:init_per_testcase(TestCase, Config). - + ct:pal("Testcase ~p starting. Checking connection to telnet server...", + [TestCase]), + ct:require(testconn, {unix,[telnet]}), + case {os:type(),ct_telnet:open(testconn)} of + {_,{ok,Handle}} -> + ok = ct_telnet:close(Handle), + ct:pal("Connection ok, starting tests!", []), + ct_test_support:init_per_testcase(TestCase, Config); + {{unix,_},{error,Reason}} -> + ct:fail("No connection to telnet server! Reason: ~tp", [Reason]); + {_,{error,Reason}} -> + {skip,{no_access_to_telnet_server,Reason}} + end. + +end_per_testcase(TestCase, Config) when TestCase /= unix_telnet -> + ct:pal("Stopping the telnet_server now!", []), + telnet_server:stop(?config(telnet_server,Config)), + ct_test_support:end_per_testcase(TestCase, Config); end_per_testcase(TestCase, Config) -> - case ?config(telnet_server,Config) of - undefined -> ok; - TS -> telnet_server:stop(TS) - end, ct_test_support:end_per_testcase(TestCase, Config). @@ -179,7 +192,12 @@ telnet_config(_, LogType) -> {port, ?erl_telnet_server_port}, {username,?erl_telnet_server_user}, {password,?erl_telnet_server_pwd}, - {keep_alive,true}]} | + {keep_alive,true}]}, + {telnet_settings, [{connect_timeout,10000}, + {command_timeout,10000}, + {reconnection_attempts,0}, + {reconnection_interval,0}, + {keep_alive,true}]} | if LogType == legacy -> [{ct_conn_log,[]}]; true -> diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl index 8d142e85a8..394d64c2ed 100644 --- a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl @@ -28,7 +28,9 @@ all() -> ignore_prompt, ignore_prompt_repeat, ignore_prompt_sequence, - ignore_prompt_timeout]. + ignore_prompt_timeout, + server_speaks, + server_disconnects]. groups() -> []. @@ -188,3 +190,37 @@ no_prompt_check_timeout(_) -> {timeout,1000}]), ok = ct_telnet:close(Handle), ok. + +%% The server says things. Manually check that it gets printed correctly +%% in the general IO log. +server_speaks(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the first message\r\n"), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the second message\r\n"), + %% let ct_telnet_client get an idle timeout + timer:sleep(15000), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the third message\r\n"), + {ok,_} = ct_telnet:expect(Handle, ["the"], [no_prompt_check]), + {error,timeout} = ct_telnet:expect(Handle, ["the"], [no_prompt_check, + {timeout,1000}]), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the fourth message\r\n"), + %% give the server time to respond + timer:sleep(2000), + %% closing the connection should print last message in log + ok = ct_telnet:close(Handle), + ok. + +%% Let the server close the connection. Make sure buffered data gets printed +%% to the general IO log. +server_disconnects(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "disconnect_after 1500"), + %% wait until the get_data operation (triggered by send/2) times out + %% before sending the msg + timer:sleep(500), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the message\r\n"), + %% when the server closes the connection, the last message should be + %% printed in the log + timer:sleep(3000), + _ = ct_telnet:close(Handle), + ok. diff --git a/lib/common_test/test/telnet_server.erl b/lib/common_test/test/telnet_server.erl index 1760100d8e..3b0c3cbcb5 100644 --- a/lib/common_test/test/telnet_server.erl +++ b/lib/common_test/test/telnet_server.erl @@ -51,32 +51,51 @@ stop(Pid) -> init(Opts) -> Port = proplists:get_value(port,Opts), Users = proplists:get_value(users,Opts,[]), - {ok, LSock} = gen_tcp:listen(Port, [list, {packet, 0}, - {active, true}]), + {ok, LSock} = listen(5, Port, [list, {packet, 0}, + {active, true}, + {reuseaddr,true}]), State = #state{listen=LSock,users=Users}, accept(State), - ok = gen_tcp:close(LSock). + ok = gen_tcp:close(LSock), + dbg("telnet_server closed the listen socket ~p\n", [LSock]), + timer:sleep(1000), + ok. + +listen(0, _Port, _Opts) -> + {error,eaddrinuse}; +listen(Retries, Port, Opts) -> + case gen_tcp:listen(Port, Opts) of + {error,eaddrinuse} -> + dbg("Listen port not released, trying again..."), + timer:sleep(5000), + listen(Retries-1, Port, Opts); + Ok = {ok,_LSock} -> + Ok; + Error -> + exit(Error) + end. accept(#state{listen=LSock}=State) -> Server = self(), Acceptor = spawn_link(fun() -> do_accept(LSock,Server) end), receive {Acceptor,Sock} when is_port(Sock) -> + dbg("Connected to client on socket ~p\n", [Sock]), case init_client(State#state{client=Sock}) of stopped -> - io:format("[telnet_server] telnet_server stopped\n"), + dbg("telnet_server stopped\n"), ok; R -> - io:format("[telnet_server] connection to client" - "closed with reason ~p~n",[R]), + dbg("Connection to client " + "closed with reason ~p~n",[R]), accept(State) end; {Acceptor,closed} -> - io:format("[telnet_server] listen socket closed unexpectedly, " - "terminating telnet_server\n"), + dbg("Listen socket closed unexpectedly, " + "terminating telnet_server\n"), ok; stop -> - io:format("[telnet_server] telnet_server stopped\n"), + dbg("telnet_server stopped\n"), ok end. @@ -97,19 +116,21 @@ init_client(#state{client=Sock}=State) -> dbg("Server sending: ~p~n",["login: "]), R = case gen_tcp:send(Sock,"login: ") of ok -> - loop(State); + loop(State, 1); Error -> Error end, _ = gen_tcp:close(Sock), R. -loop(State) -> +loop(State, N) -> receive {tcp,_,Data} -> try handle_data(Data,State) of {ok,State1} -> - loop(State1) + loop(State1, N); + closed -> + closed catch throw:Error -> Error @@ -118,6 +139,11 @@ loop(State) -> closed; {tcp_error,_,Error} -> {error,tcp,Error}; + disconnect -> + Sock = State#state.client, + dbg("Server closing connection on socket ~p~n", [Sock]), + ok = gen_tcp:close(Sock), + closed; stop -> stopped end. @@ -130,10 +156,16 @@ handle_data(Data,State) -> case get_line(Data,[]) of {Line,Rest} -> WholeLine = lists:flatten(lists:reverse(State#state.buffer,Line)), - {ok,State1} = do_handle_data(WholeLine,State), - case Rest of - [] -> {ok,State1}; - _ -> handle_data(Rest,State1) + case do_handle_data(WholeLine,State) of + {ok,State1} -> + case Rest of + [] -> {ok,State1}; + _ -> handle_data(Rest,State1) + end; + {close,State1} -> + dbg("Server closing connection~n",[]), + gen_tcp:close(State1#state.client), + closed end; false -> {ok,State#state{buffer=[Data|State#state.buffer]}} @@ -163,22 +195,29 @@ do_handle_data(Data,#state{authorized=false}=State) -> check_user(Data,State); do_handle_data(Data,#state{authorized={user,_}}=State) -> check_pwd(Data,State); -do_handle_data("echo "++ Data,State) -> +do_handle_data("echo " ++ Data,State) -> send(Data++"\r\n> ",State), {ok,State}; -do_handle_data("echo_no_prompt "++ Data,State) -> +do_handle_data("echo_no_prompt " ++ Data,State) -> send(Data,State), {ok,State}; -do_handle_data("echo_ml "++ Data,State) -> +do_handle_data("echo_ml " ++ Data,State) -> Lines = string:tokens(Data," "), ReturnData = string:join(Lines,"\n"), send(ReturnData++"\r\n> ",State), {ok,State}; -do_handle_data("echo_ml_no_prompt "++ Data,State) -> +do_handle_data("echo_ml_no_prompt " ++ Data,State) -> Lines = string:tokens(Data," "), ReturnData = string:join(Lines,"\n"), send(ReturnData,State), {ok,State}; +do_handle_data("disconnect_after " ++WaitStr,State) -> + Wait = list_to_integer(string:strip(WaitStr,right,$\n)), + dbg("Server will close connection in ~w ms...", [Wait]), + erlang:send_after(Wait,self(),disconnect), + {ok,State}; +do_handle_data("disconnect" ++_,State) -> + {close,State}; do_handle_data([],State) -> send("> ",State), {ok,State}; @@ -226,4 +265,4 @@ get_line([],_) -> dbg(_F) -> dbg(_F,[]). dbg(_F,_A) -> - io:format("[telnet_server] "++_F,_A). + io:format("[telnet_server] " ++ _F,_A). diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index 3c121f3b04..ecc4b2c9b1 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -124,9 +124,9 @@ %% keep map exports here for now map_es/1, - map_val/1, + map_arg/1, update_c_map/3, - ann_c_map/3, + ann_c_map/2, ann_c_map/3, map_pair_op/1,map_pair_key/1,map_pair_val/1, update_c_map_pair/4, ann_c_map_pair/4 @@ -135,6 +135,9 @@ -export_type([c_binary/0, c_call/0, c_clause/0, c_cons/0, c_fun/0, c_literal/0, c_module/0, c_tuple/0, c_values/0, c_var/0, cerl/0, var_name/0]). +%% HiPE does not understand Maps +%% (guard functions is_map/1 and map_size/1 in ann_c_map/3) +-compile(no_native). %% %% needed by the include file below -- do not move %% @@ -1575,20 +1578,70 @@ ann_make_list(_, [], Node) -> %% --------------------------------------------------------------------- %% maps --spec map_es(c_map()) -> [cerl()]. +-spec map_es(c_map()) -> [c_map_pair()]. map_es(#c_map{es = Es}) -> Es. --spec map_val(c_map()) -> cerl(). -map_val(#c_map{var = M}) -> +-spec map_arg(c_map()) -> c_map() | c_literal(). + +map_arg(#c_map{arg = M}) -> M. +-spec ann_c_map([term()], [cerl()]) -> c_map() | c_literal(). + +ann_c_map(As,Es) -> + ann_c_map(As, #c_literal{val=#{}}, Es). + +-spec ann_c_map([term()], c_map() | c_literal(), [c_map_pair()]) -> c_map() | c_literal(). + +ann_c_map(As,#c_literal{val=Mval}=M,Es) when is_map(Mval), map_size(Mval) =:= 0 -> + Pairs = [[Ck,Cv]||#c_map_pair{key=Ck,val=Cv}<-Es], + IsLit = lists:foldl(fun(Pair,Res) -> + Res andalso is_lit_list(Pair) + end, true, Pairs), + Fun = fun(Pair) -> [K,V] = lit_list_vals(Pair), {K,V} end, + case IsLit of + false -> + #c_map{arg=M, es=Es, anno=As }; + true -> + #c_literal{anno=As, val=maps:from_list(lists:map(Fun, Pairs))} + end; +ann_c_map(As,#c_literal{val=M},Es) when is_map(M) -> + fold_map_pairs(As,Es,M); ann_c_map(As,M,Es) -> - #c_map{var=M,es = Es, anno = As }. + #c_map{arg=M, es=Es, anno=As }. + +fold_map_pairs(As,[],M) -> #c_literal{anno=As,val=M}; +%% M#{ K => V} +fold_map_pairs(As,[#c_map_pair{op=#c_literal{val=assoc},key=Ck,val=Cv}=E|Es],M) -> + case is_lit_list([Ck,Cv]) of + true -> + [K,V] = lit_list_vals([Ck,Cv]), + fold_map_pairs(As,Es,maps:put(K,V,M)); + false -> + #c_map{arg=#c_literal{val=M,anno=As}, es=[E|Es], anno=As } + end; +%% M#{ K := V} +fold_map_pairs(As,[#c_map_pair{op=#c_literal{val=exact},key=Ck,val=Cv}=E|Es],M) -> + case is_lit_list([Ck,Cv]) of + true -> + [K,V] = lit_list_vals([Ck,Cv]), + case maps:is_key(K,M) of + true -> fold_map_pairs(As,Es,maps:put(K,V,M)); + false -> + #c_map{arg=#c_literal{val=M,anno=As}, es=[E|Es], anno=As } + end; + false -> + #c_map{arg=#c_literal{val=M,anno=As}, es=[E|Es], anno=As } + end; +fold_map_pairs(As,Es,M) -> + #c_map{arg=#c_literal{val=M,anno=As}, es=Es, anno=As }. + +%-spec update_c_map(c_map() | c_literal(), [c_map_pair()]) -> c_map() | c_literal(). update_c_map(Old,M,Es) -> - #c_map{var=M, es = Es, anno = get_ann(Old)}. + #c_map{arg=M, es = Es, anno = get_ann(Old)}. map_pair_key(#c_map_pair{key=K}) -> K. map_pair_val(#c_map_pair{val=V}) -> V. diff --git a/lib/compiler/src/cerl_clauses.erl b/lib/compiler/src/cerl_clauses.erl index 76d70dcabf..87bd47c08b 100644 --- a/lib/compiler/src/cerl_clauses.erl +++ b/lib/compiler/src/cerl_clauses.erl @@ -356,14 +356,19 @@ match(P, E, Bs) -> end; map -> %% The most we can do is to say "definitely no match" if a - %% binary pattern is matched against non-binary data. + %% map pattern is matched against non-map data. case E of any -> {false, Bs}; _ -> case type(E) of - literal -> - none; + literal -> + case is_map(concrete(E)) of + false -> + none; + true -> + {false, Bs} + end; cons -> none; tuple -> diff --git a/lib/compiler/src/cerl_inline.erl b/lib/compiler/src/cerl_inline.erl index fa1d34cc9b..75740e8b9d 100644 --- a/lib/compiler/src/cerl_inline.erl +++ b/lib/compiler/src/cerl_inline.erl @@ -64,7 +64,7 @@ seq_body/1, set_ann/2, try_arg/1, try_body/1, try_vars/1, try_evars/1, try_handler/1, tuple_es/1, tuple_arity/1, type/1, values_es/1, var_name/1, - map_val/1, map_es/1, update_c_map/3, + map_arg/1, map_es/1, update_c_map/3, update_c_map_pair/4, map_pair_op/1, map_pair_key/1, map_pair_val/1 ]). @@ -1343,7 +1343,7 @@ i_bitstr(E, Ren, Env, S) -> i_map(E, Ctx, Ren, Env, S) -> %% Visit the segments for value. - {M1, S1} = i(map_val(E), value, Ren, Env, S), + {M1, S1} = i(map_arg(E), value, Ren, Env, S), {Es, S2} = mapfoldl(fun (E, S) -> i_map_pair(E, Ctx, Ren, Env, S) end, S1, map_es(E)), @@ -1420,8 +1420,8 @@ i_pattern(E, Ren, Env, Ren0, Env0, S) -> S2 = count_size(weight(binary), S1), {update_c_binary(E, Es), S2}; map -> - %% map patterns should not have vals - M = map_val(E), + %% map patterns should not have args + M = map_arg(E), {Es, S1} = mapfoldl(fun (E, S) -> i_map_pair_pattern(E, Ren, Env, Ren0, Env0, S) diff --git a/lib/compiler/src/cerl_trees.erl b/lib/compiler/src/cerl_trees.erl index 2ebeab243f..e53bdd4efb 100644 --- a/lib/compiler/src/cerl_trees.erl +++ b/lib/compiler/src/cerl_trees.erl @@ -57,7 +57,7 @@ update_c_try/6, update_c_tuple/2, update_c_tuple_skel/2, update_c_values/2, values_es/1, var_name/1, - map_val/1, map_es/1, + map_arg/1, map_es/1, ann_c_map/3, update_c_map/3, map_pair_key/1,map_pair_val/1,map_pair_op/1, @@ -138,7 +138,7 @@ map_1(F, T) -> tuple -> update_c_tuple_skel(T, map_list(F, tuple_es(T))); map -> - update_c_map(T, map(F,map_val(T)), map_list(F, map_es(T))); + update_c_map(T, map(F, map_arg(T)), map_list(F, map_es(T))); map_pair -> update_c_map_pair(T, map(F, map_pair_op(T)), map(F, map_pair_key(T)), @@ -372,7 +372,7 @@ mapfold(F, S0, T) -> {Ts, S1} = mapfold_list(F, S0, tuple_es(T)), F(update_c_tuple_skel(T, Ts), S1); map -> - {M , S1} = mapfold(F, S0, map_val(T)), + {M , S1} = mapfold(F, S0, map_arg(T)), {Ts, S2} = mapfold_list(F, S1, map_es(T)), F(update_c_map(T, M, Ts), S2); map_pair -> @@ -724,7 +724,7 @@ label(T, N, Env) -> {As, N2} = label_ann(T, N1), {ann_c_tuple_skel(As, Ts), N2}; map -> - {M, N1} = label(map_val(T), N, Env), + {M, N1} = label(map_arg(T), N, Env), {Ts, N2} = label_list(map_es(T), N1, Env), {As, N3} = label_ann(T, N2), {ann_c_map(As, M, Ts), N3}; diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 9030dd998b..c7d91070f6 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -234,7 +234,9 @@ format_error({crash,Pass,Reason}) -> format_error({bad_return,Pass,Reason}) -> io_lib:format("internal error in ~p;\nbad return value: ~ts", [Pass,format_error_reason(Reason)]); format_error({module_name,Mod,Filename}) -> - io_lib:format("Module name '~s' does not match file name '~ts'", [Mod,Filename]). + io_lib:format("Module name '~s' does not match file name '~ts'", [Mod,Filename]); +format_error(reparsing_invalid_unicode) -> + "Non-UTF-8 character(s) detected, but no encoding declared. Encode the file in UTF-8 or add \"%% coding: latin-1\" at the beginning of the file. Retrying with latin-1 encoding.". format_error_reason({Reason, Stack}) when is_list(Stack) -> StackFun = fun @@ -792,20 +794,59 @@ no_native_compilation(BeamFile, #compile{options=Opts0}) -> _ -> false end. -parse_module(St) -> - Opts = St#compile.options, - Cwd = ".", - IncludePath = [Cwd, St#compile.dir|inc_paths(Opts)], - R = epp:parse_file(St#compile.ifile, IncludePath, pre_defs(Opts)), +parse_module(St0) -> + case do_parse_module(utf8, St0) of + {ok,_}=Ret -> + Ret; + {error,_}=Ret -> + Ret; + {invalid_unicode,File,Line} -> + case do_parse_module(latin1, St0) of + {ok,St} -> + Es = [{File,[{Line,?MODULE,reparsing_invalid_unicode}]}], + {ok,St#compile{warnings=Es++St#compile.warnings}}; + {error,St} -> + Es = [{File,[{Line,?MODULE,reparsing_invalid_unicode}]}], + {error,St#compile{errors=Es++St#compile.errors}} + end + end. + +do_parse_module(DefEncoding, #compile{ifile=File,options=Opts,dir=Dir}=St) -> + R = epp:parse_file(File, + [{includes,[".",Dir|inc_paths(Opts)]}, + {macros,pre_defs(Opts)}, + {default_encoding,DefEncoding}, + extra]), case R of - {ok,Forms} -> - Encoding = epp:read_encoding(St#compile.ifile), - {ok,St#compile{code=Forms,encoding=Encoding}}; + {ok,Forms,Extra} -> + Encoding = proplists:get_value(encoding, Extra), + case find_invalid_unicode(Forms, File) of + none -> + {ok,St#compile{code=Forms,encoding=Encoding}}; + {invalid_unicode,_,_}=Ret -> + case Encoding of + none -> + Ret; + _ -> + {ok,St#compile{code=Forms,encoding=Encoding}} + end + end; {error,E} -> Es = [{St#compile.ifile,[{none,?MODULE,{epp,E}}]}], {error,St#compile{errors=St#compile.errors ++ Es}} end. +find_invalid_unicode([H|T], File0) -> + case H of + {attribute,_,file,{File,_}} -> + find_invalid_unicode(T, File); + {error,{Line,file_io_server,invalid_unicode}} -> + {invalid_unicode,File0,Line}; + _Other -> + find_invalid_unicode(T, File0) + end; +find_invalid_unicode([], _) -> none. + parse_core(St) -> case file:read_file(St#compile.ifile) of {ok,Bin} -> diff --git a/lib/compiler/src/core_lib.erl b/lib/compiler/src/core_lib.erl index ed181e3baa..93ec3bbad5 100644 --- a/lib/compiler/src/core_lib.erl +++ b/lib/compiler/src/core_lib.erl @@ -105,7 +105,7 @@ vu_expr(V, #c_cons{hd=H,tl=T}) -> vu_expr(V, H) orelse vu_expr(V, T); vu_expr(V, #c_tuple{es=Es}) -> vu_expr_list(V, Es); -vu_expr(V, #c_map{var=M,es=Es}) -> +vu_expr(V, #c_map{arg=M,es=Es}) -> vu_expr(V, M) orelse vu_expr_list(V, Es); vu_expr(V, #c_map_pair{key=Key,val=Val}) -> vu_expr_list(V, [Key,Val]); diff --git a/lib/compiler/src/core_parse.hrl b/lib/compiler/src/core_parse.hrl index d54715ef59..20f3a46991 100644 --- a/lib/compiler/src/core_parse.hrl +++ b/lib/compiler/src/core_parse.hrl @@ -103,5 +103,5 @@ val}). -record(c_map, {anno=[], - var=#c_literal{val=[]} :: #c_var{} | #c_literal{}, + arg=#c_literal{val=#{}} :: #c_var{} | #c_literal{}, es :: [#c_map_pair{}]}). diff --git a/lib/compiler/src/core_parse.yrl b/lib/compiler/src/core_parse.yrl index b8db0f683a..a66ad4235f 100644 --- a/lib/compiler/src/core_parse.yrl +++ b/lib/compiler/src/core_parse.yrl @@ -21,7 +21,7 @@ %% Have explicit productions for annotated phrases named anno_XXX. %% This just does an XXX and adds the annotation. -Expect 1. +Expect 0. Nonterminals @@ -285,9 +285,9 @@ tuple -> '{' '}' : c_tuple([]). tuple -> '{' anno_expressions '}' : c_tuple('$2'). map_expr -> '~' '{' '}' '~' : #c_map{es=[]}. -map_expr -> '~' '{' map_pairs '}' '~' : #c_map{es='$3'}. -map_expr -> variable '~' '{' '}' '~' : #c_map{var='$1',es=[]}. -map_expr -> variable '~' '{' map_pairs '}' '~' : #c_map{var='$1',es='$4'}. +map_expr -> '~' '{' map_pairs '}' '~' : #c_map{es='$3'}. +map_expr -> '~' '{' map_pairs '|' variable '}' '~' : #c_map{arg='$5',es='$3'}. +map_expr -> '~' '{' map_pairs '|' map_expr '}' '~' : #c_map{arg='$5',es='$3'}. map_pairs -> map_pair : ['$1']. map_pairs -> map_pair ',' map_pairs : ['$1' | '$3']. diff --git a/lib/compiler/src/core_pp.erl b/lib/compiler/src/core_pp.erl index faa26ec6df..a76327457d 100644 --- a/lib/compiler/src/core_pp.erl +++ b/lib/compiler/src/core_pp.erl @@ -118,6 +118,12 @@ format_1(#c_literal{val=Tuple}, Ctxt) when is_tuple(Tuple) -> format_1(#c_literal{anno=A,val=Bitstring}, Ctxt) when is_bitstring(Bitstring) -> Segs = segs_from_bitstring(Bitstring), format_1(#c_binary{anno=A,segments=Segs}, Ctxt); +format_1(#c_literal{anno=A,val=M},Ctxt) when is_map(M) -> + Pairs = maps:to_list(M), + Cpairs = [#c_map_pair{op=#c_literal{val=assoc}, + key=#c_literal{val=V}, + val=#c_literal{val=K}} || {K,V} <- Pairs], + format_1(#c_map{anno=A,arg=#c_literal{val=#{}},es=Cpairs},Ctxt); format_1(#c_var{name={I,A}}, _) -> [core_atom(I),$/,integer_to_list(A)]; format_1(#c_var{name=V}, _) -> @@ -161,15 +167,15 @@ format_1(#c_tuple{es=Es}, Ctxt) -> format_hseq(Es, ",", add_indent(Ctxt, 1), fun format/2), $} ]; -format_1(#c_map{var=#c_var{}=Var,es=Es}, Ctxt) -> - [format_1(Var, Ctxt), - "~{", +format_1(#c_map{arg=#c_literal{val=M},es=Es}, Ctxt) when is_map(M),map_size(M)=:=0 -> + ["~{", format_hseq(Es, ",", add_indent(Ctxt, 1), fun format/2), "}~" ]; -format_1(#c_map{es=Es}, Ctxt) -> +format_1(#c_map{arg=Var,es=Es}, Ctxt) -> ["~{", format_hseq(Es, ",", add_indent(Ctxt, 1), fun format/2), + "|",format(Var, add_indent(Ctxt, 1)), "}~" ]; format_1(#c_map_pair{op=#c_literal{val=assoc},key=K,val=V}, Ctxt) -> diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl index 3ad3c8c690..6c75538194 100644 --- a/lib/compiler/src/erl_bifs.erl +++ b/lib/compiler/src/erl_bifs.erl @@ -91,6 +91,7 @@ is_pure(erlang, is_float, 1) -> true; is_pure(erlang, is_function, 1) -> true; is_pure(erlang, is_integer, 1) -> true; is_pure(erlang, is_list, 1) -> true; +is_pure(erlang, is_map, 1) -> true; is_pure(erlang, is_number, 1) -> true; is_pure(erlang, is_pid, 1) -> true; is_pure(erlang, is_port, 1) -> true; diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 058abd3357..52d6dfe184 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -72,7 +72,7 @@ -import(lists, [map/2,foldl/3,foldr/3,mapfoldl/3,all/2,any/2, reverse/1,reverse/2,member/2,nth/2,flatten/1,unzip/1]). --import(cerl, [ann_c_cons/3,ann_c_tuple/2]). +-import(cerl, [ann_c_cons/3,ann_c_map/3,ann_c_tuple/2]). -include("core_parse.hrl"). @@ -246,7 +246,7 @@ expr(#c_tuple{anno=Anno,es=Es0}=Tuple, Ctxt, Sub) -> value -> ann_c_tuple(Anno, Es) end; -expr(#c_map{var=V0,es=Es0}=Map, Ctxt, Sub) -> +expr(#c_map{anno=Anno,arg=V0,es=Es0}=Map, Ctxt, Sub) -> Es = pair_list(Es0, Ctxt, Sub), case Ctxt of effect -> @@ -254,7 +254,7 @@ expr(#c_map{var=V0,es=Es0}=Map, Ctxt, Sub) -> expr(make_effect_seq(Es, Sub), Ctxt, Sub); value -> V = expr(V0, Ctxt, Sub), - Map#c_map{var=V,es=Es} + ann_c_map(Anno,V,Es) end; expr(#c_binary{segments=Ss}=Bin0, Ctxt, Sub) -> %% Warn for useless building, but always build the binary @@ -1378,6 +1378,7 @@ eval_is_record(Call, _, _, _, _) -> Call. is_not_integer(#c_literal{val=Val}) when not is_integer(Val) -> true; is_not_integer(#c_tuple{}) -> true; is_not_integer(#c_cons{}) -> true; +is_not_integer(#c_map{}) -> true; is_not_integer(_) -> false. %% is_not_tuple(Core) -> true | false. @@ -1385,6 +1386,7 @@ is_not_integer(_) -> false. is_not_tuple(#c_literal{val=Val}) when not is_tuple(Val) -> true; is_not_tuple(#c_cons{}) -> true; +is_not_tuple(#c_map{}) -> true; is_not_tuple(_) -> false. %% eval_setelement(Call, Pos, Tuple, NewVal) -> Core. diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index 3d17557e01..04210ae243 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -77,7 +77,8 @@ splitwith/2,keyfind/3,sort/1,foreach/2,droplast/1,last/1]). -import(ordsets, [add_element/2,del_element/2,is_element/2, union/1,union/2,intersection/2,subtract/2]). --import(cerl, [ann_c_cons/3,ann_c_cons_skel/3,ann_c_tuple/2,c_tuple/1]). +-import(cerl, [ann_c_cons/3,ann_c_cons_skel/3,ann_c_tuple/2,c_tuple/1, + ann_c_map/2, ann_c_map/3]). -include("core_parse.hrl"). @@ -515,12 +516,20 @@ expr({map,L,Es0}, St0) -> % in map construction. {Es1,Eps,St1} = map_pair_list(Es0, St0), A = lineno_anno(L, St1), - {#c_map{anno=A,es=Es1},Eps,St1}; + {ann_c_map(A,Es1),Eps,St1}; expr({map,L,M0,Es0}, St0) -> - {M1,Mps,St1} = safe(M0, St0), - {Es1,Eps,St2} = map_pair_list(Es0, St1), - A = lineno_anno(L, St2), - {#c_map{anno=A,var=M1,es=Es1},Mps++Eps,St2}; + try expr_map(M0,Es0,lineno_anno(L, St0),St0) of + {_,_,_}=Res -> Res + catch + throw:bad_map -> + St = add_warning(L, bad_map, St0), + LineAnno = lineno_anno(L, St), + As = [#c_literal{anno=LineAnno,val=badarg}], + {#icall{anno=#a{anno=LineAnno}, %Must have an #a{} + module=#c_literal{anno=LineAnno,val=erlang}, + name=#c_literal{anno=LineAnno,val=error}, + args=As},[],St} + end; expr({bin,L,Es0}, St0) -> try expr_bin(Es0, lineno_anno(L, St0), St0) of {_,_,_}=Res -> Res @@ -730,6 +739,37 @@ make_bool_switch_guard(L, E, V, T, F) -> {clause,NegL,[V],[],[V]} ]}. +expr_map(M0,Es0,A,St0) -> + {M1,Mps,St1} = safe(M0, St0), + case is_valid_map_src(M1) of + true -> + case {M1,Es0} of + {#c_var{}, []} -> + %% transform M#{} to is_map(M) + {Vpat,St2} = new_var(St1), + {Fpat,St3} = new_var(St2), + Cs = [#iclause{ + anno=A, + pats=[Vpat], + guard=[#icall{anno=#a{anno=A}, + module=#c_literal{anno=A,val=erlang}, + name=#c_literal{anno=A,val=is_map}, + args=[Vpat]}], + body=[Vpat]}], + Fc = fail_clause([Fpat], A, #c_literal{val=badarg}), + {#icase{anno=#a{anno=A},args=[M1],clauses=Cs,fc=Fc},Mps,St3}; + {_,_} -> + {Es1,Eps,St2} = map_pair_list(Es0, St1), + {ann_c_map(A,M1,Es1),Mps++Eps,St2} + end; + false -> throw(bad_map) + end. + +is_valid_map_src(#c_literal{val = M}) when is_map(M) -> true; +is_valid_map_src(#c_map{}) -> true; +is_valid_map_src(#c_var{}) -> true; +is_valid_map_src(_) -> false. + map_pair_list(Es, St) -> foldr(fun ({map_field_assoc,L,K0,V0}, {Ces,Esp,St0}) -> @@ -2166,6 +2206,8 @@ lit_vars(Lit) -> lit_vars(Lit, []). lit_vars(#c_cons{hd=H,tl=T}, Vs) -> lit_vars(H, lit_vars(T, Vs)); lit_vars(#c_tuple{es=Es}, Vs) -> lit_list_vars(Es, Vs); +lit_vars(#c_map{arg=V,es=Es}, Vs) -> lit_vars(V, lit_list_vars(Es, Vs)); +lit_vars(#c_map_pair{key=K,val=V}, Vs) -> lit_vars(K, lit_vars(V, Vs)); lit_vars(#c_var{name=V}, Vs) -> add_element(V, Vs); lit_vars(_, Vs) -> Vs. %These are atomic @@ -2256,7 +2298,9 @@ is_simple_list(Es) -> lists:all(fun is_simple/1, Es). format_error(nomatch) -> "pattern cannot possibly match"; format_error(bad_binary) -> - "binary construction will fail because of a type mismatch". + "binary construction will fail because of a type mismatch"; +format_error(bad_map) -> + "map construction will fail because of a type mismatch". add_warning(Line, Term, #core{ws=Ws,file=[{file,File}]}=St) when Line >= 0 -> St#core{ws=[{File,[{location(Line),?MODULE,Term}]}|Ws]}; diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index d00dd56f30..d3b785aa14 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -272,9 +272,18 @@ expr(#c_cons{anno=A,hd=Ch,tl=Ct}, Sub, St0) -> expr(#c_tuple{anno=A,es=Ces}, Sub, St0) -> {Kes,Ep,St1} = atomic_list(Ces, Sub, St0), {#k_tuple{anno=A,es=Kes},Ep,St1}; -expr(#c_map{anno=A,var=Var0,es=Ces}, Sub, St0) -> - {Var,[],St1} = expr(Var0, Sub, St0), - map_split_pairs(A, Var, Ces, Sub, St1); +expr(#c_map{anno=A,arg=Var,es=Ces}, Sub, St0) -> + try expr_map(A,Var,Ces,Sub,St0) of + {_,_,_}=Res -> Res + catch + throw:bad_map -> + St1 = add_warning(get_line(A), bad_map, A, St0), + Erl = #c_literal{val=erlang}, + Name = #c_literal{val=error}, + Args = [#c_literal{val=badarg}], + Error = #c_call{anno=A,module=Erl,name=Name,args=Args}, + expr(Error, Sub, St1) + end; expr(#c_binary{anno=A,segments=Cv}, Sub, St0) -> try atomic_bin(Cv, Sub, St0) of {Kv,Ep,St1} -> @@ -496,6 +505,21 @@ translate_match_fail_1(Anno, As, Sub, #kern{ff=FF}) -> translate_fc(Args) -> [#c_literal{val=function_clause},make_list(Args)]. +expr_map(A,Var0,Ces,Sub,St0) -> + %% An extra pass of validation of Map src because of inlining + {Var,Mps,St1} = expr(Var0, Sub, St0), + case is_valid_map_src(Var) of + true -> + {Km,Eps,St2} = map_split_pairs(A, Var, Ces, Sub, St1), + {Km,Eps++Mps,St2}; + false -> throw(bad_map) + end. + +is_valid_map_src(#k_map{}) -> true; +is_valid_map_src(#k_literal{val=M}) when is_map(M) -> true; +is_valid_map_src(#k_var{}) -> true; +is_valid_map_src(_) -> false. + map_split_pairs(A, Var, Ces, Sub, St0) -> %% two steps %% 1. force variables @@ -1986,7 +2010,9 @@ format_error(nomatch_shadow) -> format_error(bad_call) -> "invalid module and/or function name; this call will always fail"; format_error(bad_segment_size) -> - "binary construction will fail because of a type mismatch". + "binary construction will fail because of a type mismatch"; +format_error(bad_map) -> + "map construction will fail because of a type mismatch". add_warning(none, Term, Anno, #kern{ws=Ws}=St) -> File = get_file(Anno), diff --git a/lib/compiler/src/v3_kernel_pp.erl b/lib/compiler/src/v3_kernel_pp.erl index b4e486f97c..b33eba50eb 100644 --- a/lib/compiler/src/v3_kernel_pp.erl +++ b/lib/compiler/src/v3_kernel_pp.erl @@ -104,20 +104,26 @@ format_1(#k_tuple{es=Es}, Ctxt) -> format_hseq(Es, ",", ctxt_bump_indent(Ctxt, 1), fun format/2), $} ]; -format_1(#k_map{var=#k_var{}=Var,es=Es}, Ctxt) -> - [$~,${, +format_1(#k_map{var=#k_literal{val=M},op=assoc,es=Es}, Ctxt) when is_map(M), map_size(M) =:= 0 -> + ["~{", format_hseq(Es, ",", ctxt_bump_indent(Ctxt, 1), fun format/2), - " | ",format_1(Var, Ctxt), - $},$~ + "}~" ]; -format_1(#k_map{op=assoc,es=Es}, Ctxt) -> +format_1(#k_map{var=#k_literal{val=M},op=exact,es=Es}, Ctxt) when is_map(M), map_size(M) =:= 0 -> + ["::{", + format_hseq(Es, ",", ctxt_bump_indent(Ctxt, 1), fun format/2), + "}::" + ]; +format_1(#k_map{var=Var,op=assoc,es=Es}, Ctxt) -> ["~{", format_hseq(Es, ",", ctxt_bump_indent(Ctxt, 1), fun format/2), + " | ",format_1(Var, Ctxt), "}~" ]; -format_1(#k_map{es=Es}, Ctxt) -> +format_1(#k_map{var=Var,op=exact,es=Es}, Ctxt) -> ["::{", format_hseq(Es, ",", ctxt_bump_indent(Ctxt, 1), fun format/2), + " | ",format_1(Var, Ctxt), "}::" ]; format_1(#k_map_pair{key=K,val=V}, Ctxt) -> diff --git a/lib/compiler/test/core_SUITE_data/map_core_test.core b/lib/compiler/test/core_SUITE_data/map_core_test.core index 7ece8a8bbd..2aa853d450 100644 --- a/lib/compiler/test/core_SUITE_data/map_core_test.core +++ b/lib/compiler/test/core_SUITE_data/map_core_test.core @@ -67,7 +67,7 @@ module 'map_core_test' ['map_core_test'/0, (Val, V) in let <_cor5> = %% Line 21 - M~{~<1337,_cor4>,~<'val',_cor2>}~ + ~{~<1337,_cor4>,~<'val',_cor2>|M}~ in %% Line 21 apply 'call'/2 (_cor5, Vs) @@ -92,4 +92,4 @@ module 'map_core_test' ['map_core_test'/0, fun (_cor0) -> call 'erlang':'get_module_info' ('map_core_test', _cor0) -end
\ No newline at end of file +end diff --git a/lib/compiler/test/error_SUITE.erl b/lib/compiler/test/error_SUITE.erl index 5cdf429a5f..bd877bb528 100644 --- a/lib/compiler/test/error_SUITE.erl +++ b/lib/compiler/test/error_SUITE.erl @@ -23,7 +23,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, head_mismatch_line/1,warnings_as_errors/1, bif_clashes/1, - transforms/1,forbidden_maps/1]). + transforms/1,forbidden_maps/1,bad_utf8/1]). %% Used by transforms/1 test case. -export([parse_transform/2]). @@ -36,7 +36,8 @@ all() -> groups() -> [{p,test_lib:parallel(), - [head_mismatch_line,warnings_as_errors,bif_clashes,transforms,forbidden_maps]}]. + [head_mismatch_line,warnings_as_errors,bif_clashes, + transforms,forbidden_maps,bad_utf8]}]. init_per_suite(Config) -> Config. @@ -254,6 +255,23 @@ forbidden_maps(Config) when is_list(Config) -> [] = run2(Config, Ts1), ok. +bad_utf8(Config) -> + Ts = [{bad_utf8, + %% If coding is specified explicitly as utf-8, there should be + %% a compilation error; we must not fallback to parsing the + %% file in latin-1 mode. + <<"%% coding: utf-8 + %% Bj",246,"rn + t() -> \"",246,"\". + ">>, + [], + {error,[{2,epp,cannot_parse}, + {2,file_io_server,invalid_unicode}], + []} + }], + [] = run2(Config, Ts), + ok. + run(Config, Tests) -> ?line File = test_filename(Config), @@ -318,6 +336,7 @@ run_test(Test0, File, Warnings, WriteBeam) -> ?line compile:file(File, [binary,report|Warnings]), %% Test result of compilation. + io:format("~p\n", [Opts]), ?line Res = case compile:file(File, Opts) of {ok,Mod,_,[{_File,Ws}]} -> %io:format("compile:file(~s,~p) ->~n~p~n", @@ -335,6 +354,11 @@ run_test(Test0, File, Warnings, WriteBeam) -> %io:format("compile:file(~s,~p) ->~n~p~n", % [File,Opts,_ZZ]), {error,Es,Ws}; + {error,[{XFile,Es1},{XFile,Es2}],Ws} = _ZZ + when is_list(XFile) -> + %io:format("compile:file(~s,~p) ->~n~p~n", + % [File,Opts,_ZZ]), + {error,Es1++Es2,Ws}; {error,Es,[{_File,Ws}]} = _ZZ-> %io:format("compile:file(~s,~p) ->~n~p~n", % [File,Opts,_ZZ]), diff --git a/lib/compiler/test/map_SUITE.erl b/lib/compiler/test/map_SUITE.erl index b7e27afef1..90eae6fb4f 100644 --- a/lib/compiler/test/map_SUITE.erl +++ b/lib/compiler/test/map_SUITE.erl @@ -43,7 +43,8 @@ %% errors in 17.0-rc1 t_update_values/1, - t_expand_map_update/1 + t_expand_map_update/1, + t_export/1 ]). suite() -> []. @@ -70,7 +71,8 @@ all() -> [ %% errors in 17.0-rc1 t_update_values, - t_expand_map_update + t_expand_map_update, + t_export ]. groups() -> []. @@ -285,6 +287,12 @@ t_expand_map_update(Config) when is_list(Config) -> #{<<"hello">> := <<"les gens">>} = M, ok. +t_export(Config) when is_list(Config) -> + Raclette = id(#{}), + case brie of brie -> Fromage = Raclette end, + Raclette = Fromage#{}, + ok. + check_val(#{val1:=V1, val2:=V2},V1,V2) -> ok. get_val(#{ "wazzup" := _, val := V}) -> V; diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl index de56a59e12..c3b02819f9 100644 --- a/lib/compiler/test/warnings_SUITE.erl +++ b/lib/compiler/test/warnings_SUITE.erl @@ -37,8 +37,9 @@ -export([pattern/1,pattern2/1,pattern3/1,pattern4/1, guard/1,bad_arith/1,bool_cases/1,bad_apply/1, - files/1,effect/1,bin_opt_info/1,bin_construction/1, comprehensions/1, - maps/1,redundant_boolean_clauses/1]). + files/1,effect/1,bin_opt_info/1,bin_construction/1, + comprehensions/1,maps/1,redundant_boolean_clauses/1, + latin1_fallback/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(2)). @@ -63,7 +64,7 @@ groups() -> [pattern,pattern2,pattern3,pattern4,guard, bad_arith,bool_cases,bad_apply,files,effect, bin_opt_info,bin_construction,comprehensions,maps, - redundant_boolean_clauses]}]. + redundant_boolean_clauses,latin1_fallback]}]. init_per_suite(Config) -> Config. @@ -591,6 +592,37 @@ redundant_boolean_clauses(Config) when is_list(Config) -> run(Config, Ts), ok. +latin1_fallback(Conf) when is_list(Conf) -> + DataDir = ?privdir, + IncFile = filename:join(DataDir, "include_me.hrl"), + file:write_file(IncFile, <<"%% ",246," in include file\n">>), + Ts1 = [{latin1_fallback1, + %% Test that the compiler fall backs to latin-1 with + %% a warning if a file has no encoding and does not + %% contain correct UTF-8 sequences. + <<"%% Bj",246,"rn + t(_) -> \"",246,"\"; + t(x) -> ok. + ">>, + [], + {warnings,[{1,compile,reparsing_invalid_unicode}, + {3,sys_core_fold,{nomatch_shadow,2}}]}}], + [] = run(Conf, Ts1), + + Ts2 = [{latin1_fallback2, + %% Test that the compiler fall backs to latin-1 with + %% a warning if a file has no encoding and does not + %% contain correct UTF-8 sequences. + <<" + + -include(\"include_me.hrl\"). + ">>, + [], + {warnings,[{1,compile,reparsing_invalid_unicode}]} + }], + [] = run(Conf, Ts2), + ok. + %%% %%% End of test cases. %%% diff --git a/lib/debugger/src/dbg_ieval.erl b/lib/debugger/src/dbg_ieval.erl index 1d36aae8ee..0653ce4c00 100644 --- a/lib/debugger/src/dbg_ieval.erl +++ b/lib/debugger/src/dbg_ieval.erl @@ -663,12 +663,16 @@ expr({map,Line,Fs0}, Bs0, Ieval) -> expr({map,Line,E0,Fs0}, Bs0, Ieval0) -> Ieval = Ieval0#ieval{line=Line,top=false}, {value,E,Bs1} = expr(E0, Bs0, Ieval), - {Fs,Bs2} = eval_map_fields(Fs0, Bs1, Ieval), - Value = lists:foldl(fun ({map_assoc,K,V}, Mi) -> maps:put(K,V,Mi); - ({map_exact,K,V}, Mi) -> maps:update(K,V,Mi) end, - E, Fs), - {value,Value,Bs2}; - + case E of + #{} -> + {Fs,Bs2} = eval_map_fields(Fs0, Bs1, Ieval), + Value = lists:foldl(fun ({map_assoc,K,V}, Mi) -> maps:put(K,V,Mi); + ({map_exact,K,V}, Mi) -> maps:update(K,V,Mi) + end, E, Fs), + {value,Value,Bs2}; + _ -> + exception(error, {badarg,E}, Bs1, Ieval) + end; %% A block of statements expr({block,Line,Es},Bs,Ieval) -> seq(Es, Bs, Ieval#ieval{line=Line}); diff --git a/lib/debugger/src/debugger.erl b/lib/debugger/src/debugger.erl index 8a2ac28df5..77fd0acb70 100644 --- a/lib/debugger/src/debugger.erl +++ b/lib/debugger/src/debugger.erl @@ -51,12 +51,6 @@ %% ------------------------------ %% Help window for creating new breakpoints. %% -%% dbg_wx_edit, dbg_wx_edit_win -%% -------------------------------------- -%% Help window for editing terms, used for setting backtrace size -%% (i.e. how many stack frames to display in the attach process window) -%% and changing variable values. -%% %% dbg_wx_interpret, dbg_wx_filedialog_win %% -------------------------------------- %% Help window for selecting modules to interpret. diff --git a/lib/debugger/test/int_eval_SUITE.erl b/lib/debugger/test/int_eval_SUITE.erl index 4ffcf7888e..ecbd68ab40 100644 --- a/lib/debugger/test/int_eval_SUITE.erl +++ b/lib/debugger/test/int_eval_SUITE.erl @@ -28,7 +28,7 @@ bifs_outside_erlang/1, spawning/1, applying/1, catch_and_throw/1, external_call/1, test_module_info/1, apply_interpreted_fun/1, apply_uninterpreted_fun/1, - interpreted_exit/1, otp_8310/1, stacktrace/1]). + interpreted_exit/1, otp_8310/1, stacktrace/1, maps/1]). %% Helpers. -export([applier/3]). @@ -44,7 +44,7 @@ all() -> [bifs_outside_erlang, spawning, applying, catch_and_throw, external_call, test_module_info, apply_interpreted_fun, apply_uninterpreted_fun, - interpreted_exit, otp_8310, stacktrace]. + interpreted_exit, otp_8310, stacktrace, maps]. groups() -> []. @@ -291,6 +291,11 @@ stacktrace(Config) when is_list(Config) -> end, ok. +maps(Config) when is_list(Config) -> + Fun = fun () -> ?IM:empty_map_update([camembert]) end, + {'EXIT',{{badarg,[camembert]},_}} = spawn_eval(Fun), + ok. + do_eval(Config, Mod) -> ?line DataDir = ?config(data_dir, Config), diff --git a/lib/debugger/test/int_eval_SUITE_data/my_int_eval_module.erl b/lib/debugger/test/int_eval_SUITE_data/my_int_eval_module.erl index ab485fd350..e047a33d8c 100644 --- a/lib/debugger/test/int_eval_SUITE_data/my_int_eval_module.erl +++ b/lib/debugger/test/int_eval_SUITE_data/my_int_eval_module.erl @@ -29,6 +29,7 @@ -export([more_catch/1,more_nocatch/1,exit_me/0]). -export([f/1, f_try/1, f_catch/1]). -export([otp_5837/1, otp_8310/0]). +-export([empty_map_update/1]). %% Internal exports. -export([echo/2,my_subtract/2,catch_a_ball/0,throw_a_ball/0]). @@ -241,3 +242,5 @@ otp_8310() -> true = begin (X3 = true) orelse X3, X3 end, false = begin (X4 = false) orelse X4, X4 end, ok. + +empty_map_update(Map) -> Map#{}. diff --git a/lib/debugger/test/map_SUITE.erl b/lib/debugger/test/map_SUITE.erl index 741ad2dc41..0076193725 100644 --- a/lib/debugger/test/map_SUITE.erl +++ b/lib/debugger/test/map_SUITE.erl @@ -226,8 +226,8 @@ t_update_map_expressions(Config) when is_list(Config) -> #{ "a" :=1, "b":=42, "c":=42 } = (maps:from_list([{"a",1},{"b",2}]))#{ "b" := 42, "c" => 42 }, %% Error cases, FIXME: should be 'badmap'? - {'EXIT',{badarg,_}} = (catch (id(<<>>))#{ a := 42, b => 2 }), - {'EXIT',{badarg,_}} = (catch (id([]))#{ a := 42, b => 2 }), + {'EXIT',{{badarg,<<>>},_}} = (catch (id(<<>>))#{ a := 42, b => 2 }), + {'EXIT',{{badarg,[]},_}} = (catch (id([]))#{ a := 42, b => 2 }), ok. @@ -244,7 +244,7 @@ t_update_assoc(Config) when is_list(Config) -> %% Errors cases. BadMap = id(badmap), - {'EXIT',{badarg,_}} = (catch BadMap#{nonexisting=>val}), + {'EXIT',{{badarg,BadMap},_}} = (catch BadMap#{nonexisting=>val}), ok. diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index 3e68d64d53..793efe4b50 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -504,7 +504,9 @@ hipe_compile(Files, #options{erlang_mode = ErlangMode} = Options) -> _ -> Mods = [lists, dict, digraph, digraph_utils, ets, gb_sets, gb_trees, ordsets, sets, sofs, - cerl, cerl_trees, erl_types, erl_bif_types, + %cerl, % uses maps instructions + %erl_types, % uses maps instructions + cerl_trees, erl_bif_types, dialyzer_analysis_callgraph, dialyzer, dialyzer_behaviours, dialyzer_codeserver, dialyzer_contracts, dialyzer_coordinator, dialyzer_dataflow, dialyzer_dep, @@ -533,7 +535,7 @@ hc(Mod) -> case code:is_module_native(Mod) of true -> ok; false -> - %% io:format(" ~s", [Mod]), + %% io:format(" ~w", [Mod]), {ok, Mod} = hipe:c(Mod), ok end. diff --git a/lib/hipe/cerl/Makefile b/lib/hipe/cerl/Makefile index 506e993ff4..d13dfb33c2 100644 --- a/lib/hipe/cerl/Makefile +++ b/lib/hipe/cerl/Makefile @@ -42,8 +42,8 @@ RELSYSDIR = $(RELEASE_PATH)/lib/hipe-$(VSN) # ---------------------------------------------------- # Target Specs # ---------------------------------------------------- -MODULES = cerl_cconv cerl_closurean cerl_hipeify \ - cerl_lib cerl_messagean cerl_pmatch cerl_prettypr cerl_to_icode \ +MODULES = cerl_cconv cerl_closurean cerl_hipeify cerl_lib \ + cerl_messagean cerl_pmatch cerl_prettypr cerl_to_icode \ cerl_typean erl_bif_types erl_types HRL_FILES= cerl_hipe_primops.hrl @@ -65,7 +65,7 @@ DOC_FILES= $(MODULES:%=$(DOCS)/%.html) include ../native.mk -ERL_COMPILE_FLAGS += +inline +warn_exported_vars +warn_unused_import +warn_missing_spec# +warn_untyped_record +ERL_COMPILE_FLAGS += -Werror +inline +warn_exported_vars +warn_unused_import +warn_missing_spec #+warn_untyped_record # ---------------------------------------------------- # Targets @@ -107,7 +107,6 @@ release_spec: opt release_docs_spec: -$(EBIN)/cerl_to_icode.beam: cerl_hipe_primops.hrl ../icode/hipe_icode_primops.hrl +$(EBIN)/cerl_cconv.beam: cerl_hipe_primops.hrl $(EBIN)/cerl_hipeify.beam: cerl_hipe_primops.hrl -$(EBIN)/cerl_lambdalift.beam: cerl_hipe_primops.hrl -$(EBIN)/erl_bif_types.beam: ../icode/hipe_icode_primops.hrl +$(EBIN)/cerl_to_icode.beam: cerl_hipe_primops.hrl ../icode/hipe_icode_primops.hrl diff --git a/lib/hipe/cerl/cerl_to_icode.erl b/lib/hipe/cerl/cerl_to_icode.erl index 1c1c10d9b0..2645056be1 100644 --- a/lib/hipe/cerl/cerl_to_icode.erl +++ b/lib/hipe/cerl/cerl_to_icode.erl @@ -29,9 +29,9 @@ -define(NO_UNUSED, true). --export([module/2]). +-export([module/1, module/2]). -ifndef(NO_UNUSED). --export([function/3, function/4, module/1]). +-export([function/3, function/4]). -endif. %% Added in an attempt to suppress message by Dialyzer, but I run into @@ -102,36 +102,32 @@ %% Record definitions --record(ctxt, {final = false :: boolean(), - effect = false, - fail = [], % [] or fail-to label - class = expr, % expr | guard - line = 0, % current line number - 'receive' % undefined | #receive{} - }). - -record('receive', {loop}). -record(cerl_to_icode__var, {name}). -record('fun', {label, vars}). +-record(ctxt, {final = false :: boolean(), + effect = false :: boolean(), + fail = [], % [] or fail-to label + class = expr :: 'expr' | 'guard', + line = 0 :: erl_scan:line(), % current line number + 'receive' :: 'undefined' | #'receive'{} + }). %% --------------------------------------------------------------------- %% Code - -%% @spec module(Module::cerl()) -> [icode()] +%% @spec module(Module::cerl()) -> [{mfa(), icode()}] %% @equiv module(Module, []) --ifndef(NO_UNUSED). +-spec module(cerl:c_module()) -> [{mfa(), hipe_icode:icode()}]. + module(E) -> module(E, []). --endif. -%% @clear - -%% @spec module(Module::cerl(), Options::[term()]) -> [icode()] +%% @spec module(Module::cerl(), Options::[term()]) -> [{mfa(), icode()}] %% -%% cerl() = cerl:cerl() +%% cerl() = cerl:c_module() %% icode() = hipe_icode:icode() %% %% @doc Transforms a Core Erlang module to linear HiPE Icode. The result @@ -149,7 +145,7 @@ module(E) -> %% @see function/4 %% @see cerl_hipeify:transform/1 -%% -spec module(cerl:c_module(), [term()]) -> [{mfa(), hipe_icode:icode()}]. +-spec module(cerl:c_module(), [term()]) -> [{mfa(), hipe_icode:icode()}]. module(E, Options) -> module_1(cerl_hipeify:transform(E, Options), Options). @@ -163,8 +159,8 @@ module_1(E, Options) -> throw(error) end, S0 = init(M), - S1 = s__set_pmatch(proplists:get_value(pmatch, Options), S0), - S2 = s__set_bitlevel_binaries(proplists:get_value( + S1 = s__set_pmatch(proplists:get_value(pmatch, Options), S0), + S2 = s__set_bitlevel_binaries(proplists:get_value( bitlevel_binaries, Options), S1), {Icode, _} = lists:mapfoldl(fun function_definition/2, S2, cerl:module_defs(E)), diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index 32390045e3..aa69b57fa2 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -218,6 +218,10 @@ %%-define(DO_ERL_TYPES_TEST, true). -compile({no_auto_import,[min/2,max/2]}). +%% HiPE does not understand Maps +%% (guard function is_map/1 in t_from_term/1) +-compile(no_native). + -ifdef(DO_ERL_TYPES_TEST). -export([test/0]). -else. @@ -2156,6 +2160,7 @@ t_from_term(T) when is_integer(T) -> t_integer(T); t_from_term(T) when is_pid(T) -> t_pid(); t_from_term(T) when is_port(T) -> t_port(); t_from_term(T) when is_reference(T) -> t_reference(); +t_from_term(T) when is_map(T) -> t_map(); t_from_term(T) when is_tuple(T) -> t_tuple([t_from_term(E) || E <- tuple_to_list(T)]). diff --git a/lib/hipe/icode/hipe_icode.erl b/lib/hipe/icode/hipe_icode.erl index 0e651a351c..7b3d087e2d 100644 --- a/lib/hipe/icode/hipe_icode.erl +++ b/lib/hipe/icode/hipe_icode.erl @@ -503,7 +503,6 @@ enter_args_update/2, enter_type/1, is_enter/1, - mk_return/1, %% mk_return(Vars) %% mk_fail/1, %% mk_fail(Args) class = exit @@ -606,6 +605,12 @@ -export([highest_var/1, highest_label/1]). +%% +%% Exported types +%% + +-export_type([icode/0]). + %%--------------------------------------------------------------------- %% %% Icode @@ -614,7 +619,7 @@ -spec mk_icode(mfa(), [icode_var()], boolean(), boolean(), [icode_instr()], {non_neg_integer(),non_neg_integer()}, - {icode_lbl(),icode_lbl()}) -> #icode{}. + {icode_lbl(),icode_lbl()}) -> icode(). mk_icode(Fun, Params, IsClosure, IsLeaf, Code, VarRange, LabelRange) -> #icode{'fun'=Fun, params=Params, code=Code, is_closure=IsClosure, @@ -1434,8 +1439,8 @@ subst1([_|Pairs], I) -> subst1(Pairs, I). %% %% @doc Returns the successors of an Icode instruction. %% In CFG form only branch instructions have successors, -%% but in linear form other instructions like e.g. moves and -%% others might be the last instruction of some basic block. +%% but in linear form other instructions like e.g. moves +%% might be the last instruction of some basic block. %% -spec successors(icode_instr()) -> [icode_lbl()]. diff --git a/lib/hipe/icode/hipe_icode.hrl b/lib/hipe/icode/hipe_icode.hrl index 060493e61e..25deac5152 100644 --- a/lib/hipe/icode/hipe_icode.hrl +++ b/lib/hipe/icode/hipe_icode.hrl @@ -178,5 +178,6 @@ var_range :: {non_neg_integer(), non_neg_integer()}, label_range :: {icode_lbl(), icode_lbl()}, info = [] :: icode_info()}). +-type icode() :: #icode{}. %%--------------------------------------------------------------------- diff --git a/lib/hipe/icode/hipe_icode_fp.erl b/lib/hipe/icode/hipe_icode_fp.erl index c0cd9bd2d1..38b3881a77 100644 --- a/lib/hipe/icode/hipe_icode_fp.erl +++ b/lib/hipe/icode/hipe_icode_fp.erl @@ -424,7 +424,7 @@ redirect_phis([I|Is] = Code, OldFrom, NewFrom, Acc) -> NewI = hipe_icode:phi_redirect_pred(I, OldFrom, NewFrom), redirect_phis(Is, OldFrom, NewFrom, [NewI|Acc]); _ -> - lists:reverse(Acc) ++ Code + lists:reverse(Acc, Code) end; redirect_phis([], _OldFrom, _NewFrom, Acc) -> lists:reverse(Acc). diff --git a/lib/hipe/icode/hipe_icode_mulret.erl b/lib/hipe/icode/hipe_icode_mulret.erl index 2402bad42c..99522f6430 100644 --- a/lib/hipe/icode/hipe_icode_mulret.erl +++ b/lib/hipe/icode/hipe_icode_mulret.erl @@ -1166,9 +1166,9 @@ printCallList([]) -> io:format("~n"). %% removeUnElems([#icode_call{'fun'={unsafe_element,_}, args=Var}|List], Var, Res) -> %% removeUnElems(List, Var, Res); %% removeUnElems([I=#icode_move{dst=Var}|List], [Var], Res) -> -%% lists:reverse(Res) ++ [I|List]; +%% lists:reverse(Res, [I|List]); %% removeUnElems([I=#icode_call{dstlist=Var}|List], Var, Res) -> -%% lists:reverse(Res) ++ [I|List]; +%% lists:reverse(Res, [I|List]); %% removeUnElems([I|List], Var, Res) -> %% removeUnElems(List, Var, [I|Res]); %% removeUnElems([], _, Res) -> lists:reverse(Res). @@ -1187,7 +1187,7 @@ printCallList([]) -> io:format("~n"). %% false -> %% case lists:member(Var, Defs) of %% true -> -%% lists:reverse(Res) ++ [I|List]; +%% lists:reverse(Res, [I|List]); %% false -> %% removeUnElems(List, Var, [I|Res]) %% end @@ -1195,7 +1195,7 @@ printCallList([]) -> io:format("~n"). %% false -> %% case lists:member(Var, Defs) of %% true -> -%% lists:reverse(Res) ++ [I|List]; +%% lists:reverse(Res, [I|List]); %% false -> %% removeUnElems(List, Var, [I|Res]) %% end @@ -1203,7 +1203,7 @@ printCallList([]) -> io:format("~n"). %% false -> %% case lists:member(Var, Defs) of %% true -> -%% lists:reverse(Res) ++ [I|List]; +%% lists:reverse(Res, [I|List]); %% false -> %% removeUnElems(List, Var, [I|Res]) %% end @@ -1248,16 +1248,16 @@ printCallList([]) -> io:format("~n"). %% modifyCode([I|Code], Var, Res) -> %% case scanInstr(I, Var) of %% {move, Arity, VarLst} -> -%% Code2 = [#icode_return{vars=VarLst}, I |lists:reverse(Res) ++ Code], +%% Code2 = [#icode_return{vars=VarLst}, I |lists:reverse(Res, Code)], %% {Arity, lists:reverse(Code2)}; %% {mktuple, Arity, VarLst} -> -%% Code2 = [#icode_return{vars=VarLst}|lists:reverse(Res) ++ Code], +%% Code2 = [#icode_return{vars=VarLst}|lists:reverse(Res, Code)], %% {Arity, lists:reverse(Code2)}; %% other -> %% modifyCode(Code, Var, [I|Res]) %% end; %% modifyCode([], Var, Res) -> -%% {1, lists:reverse(Res) ++ [#icode_return{vars=Var}]}. +%% {1, lists:reverse(Res, [#icode_return{vars=Var}]}. %% scanInstr(#icode_call{dstlist=Var, 'fun'=mktuple, args=Lst}, Var) -> %% {mktuple, length(Lst), Lst}; diff --git a/lib/hipe/regalloc/hipe_ls_regalloc.erl b/lib/hipe/regalloc/hipe_ls_regalloc.erl index 4276b8f968..7a00a0534a 100644 --- a/lib/hipe/regalloc/hipe_ls_regalloc.erl +++ b/lib/hipe/regalloc/hipe_ls_regalloc.erl @@ -722,7 +722,7 @@ is_free(R, Free) -> is_free(R, Free, []). is_free(R, [{R,_}|Rest], Acc) -> - {true,lists:reverse(Acc)++Rest}; + {true, lists:reverse(Acc, Rest)}; is_free(R, [X|Rs],Acc) -> is_free(R, Rs, [X|Acc]); is_free(_, [], _) -> @@ -733,7 +733,7 @@ exists_free_register(Start, Regs) -> exists_free_register(Start, [{Phys, Start0}|Rest], Acc) when Start > Start0 -> - {true, Phys, lists:reverse(Acc)++Rest}; + {true, Phys, lists:reverse(Acc, Rest)}; exists_free_register(Start, [Free|Rest], Acc) -> exists_free_register(Start, Rest, [Free|Acc]); exists_free_register(_, [], _) -> diff --git a/lib/hipe/regalloc/hipe_optimistic_regalloc.erl b/lib/hipe/regalloc/hipe_optimistic_regalloc.erl index 5bad31ade9..0278a896d2 100644 --- a/lib/hipe/regalloc/hipe_optimistic_regalloc.erl +++ b/lib/hipe/regalloc/hipe_optimistic_regalloc.erl @@ -958,9 +958,9 @@ splits_2({Cols, NonCols, OldSpillCost}, L, SpillCost) -> %% Merge two ordered sub-splits into one. spillCostOrderedMerge(Spl1, [], Spl) -> - lists:reverse(Spl) ++ Spl1; + lists:reverse(Spl, Spl1); spillCostOrderedMerge([], Spl2, Spl) -> - lists:reverse(Spl) ++ Spl2; + lists:reverse(Spl, Spl2); spillCostOrderedMerge(Spl1, Spl2, Spl) -> {_, _, SpillCost1} = hd(Spl1), {_, _, SpillCost2} = hd(Spl2), diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index a89a457a51..5598185ad3 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1829,7 +1829,7 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> [ProfileName, SessionId, Pos, Value, (catch httpc_manager:which_session_info(ProfileName)), Session, - (catch httpc_manager:lookup_session(ProfileName, SessionId)), + (catch httpc_manager:lookup_session(SessionId, ProfileName)), T, E]), exit({failed_updating_session, [{profile, ProfileName}, diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index b3ec9fd33d..6d4b1cb2db 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -1630,6 +1630,11 @@ <desc> <p>Sets the current working directory of the file server to <c><anno>Dir</anno></c>. Returns <c>ok</c> if successful.</p> + <p>The functions in the <c>file</c> module usually treat binaries + as raw filenames, i.e. they are passed as is even when the encoding + of the binary does not agree with <c>file:native_name_encoding()</c>. + This function however expects binaries to be encoded according to the + value returned by <c>file:native_name_encoding()</c>.</p> <p>Typical error reasons are:</p> <taglist> <tag><c>enoent</c></tag> @@ -1654,8 +1659,8 @@ <tag><c>no_translation</c></tag> <item> <p><c><anno>Dir</anno></c> is a <c>binary()</c> with - characters coded in ISO-latin-1 and the VM was started - with the parameter <c>+fnue</c>.</p> + characters coded in ISO-latin-1 and the VM is operating + with unicode file name encoding.</p> </item> </taglist> <warning> diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 23cf74f80f..20b703e084 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -199,7 +199,8 @@ get_cwd(Drive) -> check_and_call(get_cwd, [file_name(Drive)]). -spec set_cwd(Dir) -> ok | {error, Reason} when - Dir :: name(), + Dir :: name() | EncodedBinary, + EncodedBinary :: binary(), Reason :: posix() | badarg | no_translation. set_cwd(Dirname) -> diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 245cd3e280..74ca7ca699 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -1654,7 +1654,16 @@ dec_hello_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_EXT), ?UINT16(Len), dec_hello_extensions(<<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), ExtData:Len/binary, Rest/binary>>, Acc) -> <<?UINT16(_), EllipticCurveList/binary>> = ExtData, - EllipticCurves = [tls_v1:enum_to_oid(X) || <<X:16>> <= EllipticCurveList], + %% Ignore unknown curves + Pick = fun(Enum) -> + case tls_v1:enum_to_oid(Enum) of + undefined -> + false; + Oid -> + {true, Oid} + end + end, + EllipticCurves = lists:filtermap(Pick, [ECC || <<ECC:16>> <= EllipticCurveList]), dec_hello_extensions(Rest, Acc#hello_extensions{elliptic_curves = #elliptic_curves{elliptic_curve_list = EllipticCurves}}); diff --git a/lib/ssl/src/tls_v1.erl b/lib/ssl/src/tls_v1.erl index 7c7fdd64c3..7b1f53b969 100644 --- a/lib/ssl/src/tls_v1.erl +++ b/lib/ssl/src/tls_v1.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -439,7 +439,9 @@ enum_to_oid(24) -> ?secp384r1; enum_to_oid(25) -> ?secp521r1; enum_to_oid(26) -> ?brainpoolP256r1; enum_to_oid(27) -> ?brainpoolP384r1; -enum_to_oid(28) -> ?brainpoolP512r1. +enum_to_oid(28) -> ?brainpoolP512r1; +enum_to_oid(_) -> + undefined. sufficent_ec_support() -> CryptoSupport = crypto:supports(), diff --git a/lib/ssl/test/ssl_handshake_SUITE.erl b/lib/ssl/test/ssl_handshake_SUITE.erl index 7e8e8d2611..6d020c472b 100644 --- a/lib/ssl/test/ssl_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_handshake_SUITE.erl @@ -34,6 +34,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [decode_hello_handshake, decode_single_hello_extension_correctly, + decode_supported_elliptic_curves_hello_extension_correctly, decode_unknown_hello_extension_correctly, encode_single_hello_sni_extension_correctly]. @@ -67,6 +68,17 @@ decode_single_hello_extension_correctly(_Config) -> #renegotiation_info{renegotiated_connection = <<0>>} = Extensions#hello_extensions.renegotiation_info. +decode_supported_elliptic_curves_hello_extension_correctly(_Config) -> + % List of supported and unsupported curves (RFC4492:S5.1.1) + ClientEllipticCurves = [0, tls_v1:oid_to_enum(?sect233k1), 37, tls_v1:oid_to_enum(?sect193r2), 16#badc], + % Construct extension binary - modified version of ssl_handshake:encode_hello_extensions([#elliptic_curves{}], _) + EllipticCurveList = << <<X:16>> || X <- ClientEllipticCurves>>, + ListLen = byte_size(EllipticCurveList), + Len = ListLen + 2, + Extension = <<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), ?UINT16(ListLen), EllipticCurveList/binary>>, + % after decoding we should see only valid curves + #hello_extensions{elliptic_curves = DecodedCurves} = ssl_handshake:decode_hello_extensions(Extension), + #elliptic_curves{elliptic_curve_list = [?sect233k1, ?sect193r2]} = DecodedCurves. decode_unknown_hello_extension_correctly(_Config) -> FourByteUnknown = <<16#CA,16#FE, ?UINT16(4), 3, 0, 1, 2>>, diff --git a/lib/stdlib/doc/src/epp.xml b/lib/stdlib/doc/src/epp.xml index cf33530395..452341f7d2 100644 --- a/lib/stdlib/doc/src/epp.xml +++ b/lib/stdlib/doc/src/epp.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2013</year> + <year>1996</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -46,8 +46,10 @@ valid encodings are <c>Latin-1</c> and <c>UTF-8</c> where the case of the characters can be chosen freely. Examples:</p> <pre> -%% coding: utf-8 -%% For this file we have chosen encoding = Latin-1 +%% coding: utf-8</pre> + <pre> +%% For this file we have chosen encoding = Latin-1</pre> + <pre> %% -*- coding: latin-1 -*-</pre> </description> <datatypes> @@ -64,11 +66,29 @@ </datatypes> <funcs> <func> + <name name="open" arity="1"/> + <fsummary>Open a file for preprocessing</fsummary> + <desc> + <p>Opens a file for preprocessing.</p> + <p>If <c>extra</c> is given in + <c><anno>Options</anno></c>, the return value will be + <c>{ok, <anno>Epp</anno>, <anno>Extra</anno>}</c> instead + of <c>{ok, <anno>Epp</anno>}</c>.</p> + </desc> + </func> + <func> <name name="open" arity="2"/> + <fsummary>Open a file for preprocessing</fsummary> + <desc> + <p>Equivalent to <c>epp:open([{name, FileName}, {includes, IncludePath}])</c>.</p> + </desc> + </func> + <func> <name name="open" arity="3"/> <fsummary>Open a file for preprocessing</fsummary> <desc> - <p>Opens a file for preprocessing.</p> + <p>Equivalent to <c>epp:open([{name, FileName}, {includes, IncludePath}, + {macros, PredefMacros}])</c>.</p> </desc> </func> <func> @@ -89,12 +109,24 @@ </desc> </func> <func> - <name name="parse_file" arity="3"/> + <name name="parse_file" arity="2"/> <fsummary>Preprocess and parse an Erlang source file</fsummary> <desc> <p>Preprocesses and parses an Erlang source file. - Note that the tuple <c>{eof, <anno>Line</anno>}</c> returned at end-of-file is - included as a "form".</p> + Note that the tuple <c>{eof, <anno>Line</anno>}</c> returned + at end-of-file is included as a "form".</p> + <p>If <c>extra</c> is given in + <c><anno>Options</anno></c>, the return value will be + <c>{ok, [<anno>Form</anno>], <anno>Extra</anno>}</c> instead + of <c>{ok, [<anno>Form</anno>]}</c>.</p> + </desc> + </func> + <func> + <name name="parse_file" arity="3"/> + <fsummary>Preprocess and parse an Erlang source file</fsummary> + <desc> + <p>Equivalent to <c>epp:parse_file(FileName, [{includes, IncludePath}, + {macros, PredefMacros}])</c>.</p> </desc> </func> <func> @@ -111,7 +143,7 @@ <p>Returns a string representation of an encoding. The string is recognized by <c>read_encoding/1,2</c>, <c>read_encoding_from_binary/1,2</c>, and - <c>set_encoding/1</c> as a valid encoding.</p> + <c>set_encoding/1,2</c> as a valid encoding.</p> </desc> </func> <func> @@ -157,6 +189,22 @@ </desc> </func> <func> + <name name="set_encoding" arity="2"/> + <fsummary>Read and set the encoding of an IO device</fsummary> + <desc> + <p>Reads the <seealso marker="#encoding">encoding</seealso> from + an IO device and sets the encoding of the device + accordingly. The position of the IO device referenced by + <c><anno>File</anno></c> is not affected. If no valid + encoding can be read from the IO device the encoding of the + IO device is set to the + <seealso marker="#encoding">encoding</seealso> given by + <c><anno>Default</anno></c>.</p> + <p>Returns the read encoding, or <c>none</c> if no valid + encoding was found.</p> + </desc> + </func> + <func> <name name="format_error" arity="1"/> <fsummary>Format an error descriptor</fsummary> <desc> diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index 68e079b7e5..d212a55b47 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -20,12 +20,12 @@ %% An Erlang code preprocessor. --export([open/2,open/3,open/5,close/1,format_error/1]). +-export([open/1, open/2,open/3,open/5,close/1,format_error/1]). -export([scan_erl_form/1,parse_erl_form/1,macro_defs/1]). --export([parse_file/1, parse_file/3]). +-export([parse_file/1, parse_file/2, parse_file/3]). -export([default_encoding/0, encoding_to_string/1, read_encoding_from_binary/1, read_encoding_from_binary/2, - set_encoding/1, read_encoding/1, read_encoding/2]). + set_encoding/1, set_encoding/2, read_encoding/1, read_encoding/2]). -export([interpret_file_attribute/1]). -export([normalize_typed_record_fields/1,restore_typed_record_fields/1]). @@ -37,9 +37,11 @@ -type epp_handle() :: pid(). -type source_encoding() :: latin1 | utf8. +-define(DEFAULT_ENCODING, utf8). + %% Epp state record. -record(epp, {file, %Current file - location, %Current location + location=1, %Current location delta, %Offset from Location (-file) name="", %Current file name name2="", %-"-, modified by -file @@ -48,6 +50,7 @@ path=[], %Include-path macs = dict:new() :: dict:dict(),%Macros (don't care locations) uses = dict:new() :: dict:dict(),%Macro use structure + default_encoding = ?DEFAULT_ENCODING :: source_encoding(), pre_opened = false :: boolean() }). @@ -58,6 +61,7 @@ %%% distinction in the internal representation would simplify the code %%% a little. +%% open(Options) %% open(FileName, IncludePath) %% open(FileName, IncludePath, PreDefMacros) %% open(FileName, IoDevice, StartLocation, IncludePath, PreDefMacros) @@ -65,6 +69,7 @@ %% scan_erl_form(Epp) %% parse_erl_form(Epp) %% parse_file(Epp) +%% parse_file(FileName, Options) %% parse_file(FileName, IncludePath, PreDefMacros) %% macro_defs(Epp) @@ -87,14 +92,43 @@ open(Name, Path) -> ErrorDescriptor :: term(). open(Name, Path, Pdm) -> - Self = self(), - Epp = spawn(fun() -> server(Self, Name, Path, Pdm) end), - epp_request(Epp). + internal_open([{name, Name}, {includes, Path}, {macros, Pdm}], #epp{}). open(Name, File, StartLocation, Path, Pdm) -> - Self = self(), - Epp = spawn(fun() -> server(Self, Name, File, StartLocation,Path,Pdm) end), - epp_request(Epp). + internal_open([{name, Name}, {includes, Path}, {macros, Pdm}], + #epp{file=File, pre_opened=true, location=StartLocation}). + +-spec open(Options) -> + {'ok', Epp} | {'ok', Epp, Extra} | {'error', ErrorDescriptor} when + Options :: [{'default_encoding', DefEncoding :: source_encoding()} | + {'includes', IncludePath :: [DirectoryName :: file:name()]} | + {'macros', PredefMacros :: macros()} | + {'name',FileName :: file:name()} | + 'extra'], + Epp :: epp_handle(), + Extra :: [{'encoding', source_encoding() | 'none'}], + ErrorDescriptor :: term(). + +open(Options) -> + internal_open(Options, #epp{}). + +internal_open(Options, St) -> + case proplists:get_value(name, Options) of + undefined -> + erlang:error(badarg); + Name -> + Self = self(), + Epp = spawn(fun() -> server(Self, Name, Options, St) end), + case epp_request(Epp) of + {ok, Pid, Encoding} -> + case proplists:get_bool(extra, Options) of + true -> {ok, Pid, [{encoding, Encoding}]}; + false -> {ok, Pid} + end; + Other -> + Other + end + end. -spec close(Epp) -> 'ok' when Epp :: epp_handle(). @@ -170,9 +204,6 @@ format_error({'NYI',What}) -> io_lib:format("not yet implemented '~s'", [What]); format_error(E) -> file:format_error(E). -%% parse_file(FileName, IncludePath, [PreDefMacro]) -> -%% {ok,[Form]} | {error,OpenError} - -spec parse_file(FileName, IncludePath, PredefMacros) -> {'ok', [Form]} | {error, OpenError} when FileName :: file:name(), @@ -184,17 +215,40 @@ format_error(E) -> file:format_error(E). OpenError :: file:posix() | badarg | system_limit. parse_file(Ifile, Path, Predefs) -> - case open(Ifile, Path, Predefs) of + parse_file(Ifile, [{includes, Path}, {macros, Predefs}]). + +-spec parse_file(FileName, Options) -> + {'ok', [Form]} | {'ok', [Form], Extra} | {error, OpenError} when + FileName :: file:name(), + Options :: [{'includes', IncludePath :: [DirectoryName :: file:name()]} | + {'macros', PredefMacros :: macros()} | + {'default_encoding', DefEncoding :: source_encoding()} | + 'extra'], + Form :: erl_parse:abstract_form() | {'error', ErrorInfo} | {'eof',Line}, + Line :: erl_scan:line(), + ErrorInfo :: erl_scan:error_info() | erl_parse:error_info(), + Extra :: [{'encoding', source_encoding() | 'none'}], + OpenError :: file:posix() | badarg | system_limit. + +parse_file(Ifile, Options) -> + case internal_open([{name, Ifile} | Options], #epp{}) of {ok,Epp} -> Forms = parse_file(Epp), close(Epp), {ok,Forms}; + {ok,Epp,Extra} -> + Forms = parse_file(Epp), + close(Epp), + {ok,Forms,Extra}; {error,E} -> {error,E} end. -%% parse_file(Epp) -> -%% [Form] +-spec parse_file(Epp) -> [Form] when + Epp :: epp_handle(), + Form :: erl_parse:abstract_form() | {'error', ErrorInfo} | {'eof',Line}, + Line :: erl_scan:line(), + ErrorInfo :: erl_scan:error_info() | erl_parse:error_info(). parse_file(Epp) -> case parse_erl_form(Epp) of @@ -219,8 +273,6 @@ parse_file(Epp) -> [{eof,Location}] end. --define(DEFAULT_ENCODING, utf8). - -spec default_encoding() -> source_encoding(). default_encoding() -> @@ -258,9 +310,16 @@ read_encoding(Name, Options) -> File :: io:device(). % pid(); raw files don't work set_encoding(File) -> + set_encoding(File, ?DEFAULT_ENCODING). + +-spec set_encoding(File, Default) -> source_encoding() | none when + Default :: source_encoding(), + File :: io:device(). % pid(); raw files don't work + +set_encoding(File, Default) -> Encoding = read_encoding_from_file(File, true), Enc = case Encoding of - none -> default_encoding(); + none -> Default; Encoding -> Encoding end, ok = io:setopts(File, [{encoding, Enc}]), @@ -446,35 +505,37 @@ restore_typed_record_fields([{attribute,La,type,{{record,Record},Fields,[]}}| restore_typed_record_fields([Form|Forms]) -> [Form|restore_typed_record_fields(Forms)]. -%% server(StarterPid, FileName, Path, PreDefMacros) - -server(Pid, Name, Path, Pdm) -> +server(Pid, Name, Options, #epp{pre_opened=PreOpened}=St) -> process_flag(trap_exit, true), - case file:open(Name, [read]) of - {ok,File} -> - Location = 1, - init_server(Pid, Name, File, Location, Path, Pdm, false); - {error,E} -> - epp_reply(Pid, {error,E}) + case PreOpened of + false -> + case file:open(Name, [read]) of + {ok,File} -> + init_server(Pid, Name, Options, St#epp{file = File}); + {error,E} -> + epp_reply(Pid, {error,E}) + end; + true -> + init_server(Pid, Name, Options, St) end. -%% server(StarterPid, FileName, IoDevice, Location, Path, PreDefMacros) -server(Pid, Name, File, AtLocation, Path, Pdm) -> - process_flag(trap_exit, true), - init_server(Pid, Name, File, AtLocation, Path, Pdm, true). - -init_server(Pid, Name, File, AtLocation, Path, Pdm, Pre) -> +init_server(Pid, Name, Options, St0) -> + Pdm = proplists:get_value(macros, Options, []), Ms0 = predef_macros(Name), case user_predef(Pdm, Ms0) of {ok,Ms1} -> - _ = set_encoding(File), - epp_reply(Pid, {ok,self()}), + #epp{file = File, location = AtLocation} = St0, + DefEncoding = proplists:get_value(default_encoding, Options, + ?DEFAULT_ENCODING), + Encoding = set_encoding(File, DefEncoding), + epp_reply(Pid, {ok,self(),Encoding}), %% ensure directory of current source file is %% first in path - Path1 = [filename:dirname(Name) | Path], - St = #epp{file=File, location=AtLocation, delta=0, - name=Name, name2=Name, path=Path1, macs=Ms1, - pre_opened = Pre}, + Path = [filename:dirname(Name) | + proplists:get_value(includes, Options, [])], + St = St0#epp{delta=0, name=Name, name2=Name, + path=Path, macs=Ms1, + default_encoding=DefEncoding}, From = wait_request(St), enter_file_reply(From, Name, AtLocation, AtLocation), wait_req_scan(St); @@ -600,9 +661,11 @@ enter_file2(NewF, Pname, From, St0, AtLocation) -> %% the path) must be dropped, otherwise the path used within the current %% file will depend on the order of file inclusions in the parent files Path = [filename:dirname(Pname) | tl(St0#epp.path)], - _ = set_encoding(NewF), + DefEncoding = St0#epp.default_encoding, + _ = set_encoding(NewF, DefEncoding), #epp{file=NewF,location=Loc,name=Pname,name2=Pname,delta=0, - sstk=[St0|St0#epp.sstk],path=Path,macs=Ms}. + sstk=[St0|St0#epp.sstk],path=Path,macs=Ms, + default_encoding=DefEncoding}. enter_file_reply(From, Name, Location, AtLocation) -> Attr = loc_attr(AtLocation), diff --git a/lib/stdlib/src/erl_eval.erl b/lib/stdlib/src/erl_eval.erl index 3a4108e297..acde3ad5d6 100644 --- a/lib/stdlib/src/erl_eval.erl +++ b/lib/stdlib/src/erl_eval.erl @@ -18,6 +18,9 @@ %% -module(erl_eval). +%% Guard is_map/1 is not yet supported in HiPE. +-compile(no_native). + %% An evaluator for Erlang abstract syntax. -export([exprs/2,exprs/3,exprs/4,expr/2,expr/3,expr/4,expr/5, @@ -243,11 +246,18 @@ expr({record,_,_,Name,_}, _Bs, _Lf, _Ef, _RBs) -> %% map expr({map,_, Binding,Es}, Bs0, Lf, Ef, RBs) -> {value, Map0, Bs1} = expr(Binding, Bs0, Lf, Ef, RBs), - {Vs,Bs} = eval_map_fields(Es, Bs1, Lf, Ef), - ret_expr(lists:foldl(fun - ({map_assoc,K,V}, Mi) -> maps:put(K,V,Mi); - ({map_exact,K,V}, Mi) -> maps:update(K,V,Mi) - end, Map0, Vs), Bs, RBs); + case Map0 of + #{} -> + {Vs,Bs} = eval_map_fields(Es, Bs1, Lf, Ef), + Map1 = lists:foldl(fun ({map_assoc,K,V}, Mi) -> + maps:put(K, V, Mi); + ({map_exact,K,V}, Mi) -> + maps:update(K, V, Mi) + end, Map0, Vs), + ret_expr(Map1, Bs, RBs); + _ -> + erlang:raise(error, {badarg,Map0}, stacktrace()) + end; expr({map,_,Es}, Bs0, Lf, Ef, RBs) -> {Vs,Bs} = eval_map_fields(Es, Bs0, Lf, Ef), ret_expr(lists:foldl(fun @@ -1113,9 +1123,10 @@ match1({tuple,_,Elts}, Tuple, Bs, BBs) match_tuple(Elts, Tuple, 1, Bs, BBs); match1({tuple,_,_}, _, _Bs, _BBs) -> throw(nomatch); -match1({map,_,Fs}, Map, Bs, BBs) -> +match1({map,_,Fs}, #{}=Map, Bs, BBs) -> match_map(Fs, Map, Bs, BBs); - +match1({map,_,_}, _, _Bs, _BBs) -> + throw(nomatch); match1({bin, _, Fs}, <<_/bitstring>>=B, Bs0, BBs) -> eval_bits:match_bits(Fs, B, Bs0, BBs, match_fun(BBs), diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 269e4b34cf..4c0261a1ad 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -225,6 +225,8 @@ format_error({too_many_arguments,Arity}) -> "maximum allowed is ~w", [Arity,?MAX_ARGUMENTS]); %% --- patterns and guards --- format_error(illegal_pattern) -> "illegal pattern"; +format_error(illegal_map_key) -> + "illegal map key"; format_error({illegal_map_key_variable,K}) -> io_lib:format("illegal use of variable ~w in map",[K]); format_error(illegal_bin_pattern) -> @@ -1385,19 +1387,20 @@ pattern({cons,_Line,H,T}, Vt, Old, Bvt, St0) -> pattern({tuple,_Line,Ps}, Vt, Old, Bvt, St) -> pattern_list(Ps, Vt, Old, Bvt, St); pattern({map,_Line,Ps}, Vt, Old, Bvt, St) -> - foldl(fun ({map_field_assoc,L,_,_}, {Psvt,Bvt0,St0}) -> - {Psvt,Bvt0,add_error(L, illegal_pattern, St0)}; - ({map_field_exact,L,KP,VP}, {Psvt,Bvt0,St0}) -> - case expr(KP, [], St0) of - {[],_} -> - {Pvt,Bvt1,St1} = pattern(VP, Vt, Old, Bvt, St0), - {vtmerge_pat(Pvt, Psvt),vtmerge_pat(Bvt0, Bvt1), - St1}; - {[Var|_],_} -> - Error = {illegal_map_key_variable,element(1, Var)}, - {Psvt,Bvt0,add_error(L, Error, St0)} - end - end, {[],[],St}, Ps); + foldl(fun + ({map_field_assoc,L,_,_}, {Psvt,Bvt0,St0}) -> + {Psvt,Bvt0,add_error(L, illegal_pattern, St0)}; + ({map_field_exact,L,KP,VP}, {Psvt,Bvt0,St0}) -> + case is_valid_map_key(KP, St0) of + true -> + {Pvt,Bvt1,St1} = pattern(VP, Vt, Old, Bvt, St0), + {vtmerge_pat(Pvt, Psvt),vtmerge_pat(Bvt0, Bvt1), St1}; + false -> + {Psvt,Bvt0,add_error(L, illegal_map_key, St0)}; + {false,variable,Var} -> + {Psvt,Bvt0,add_error(L, {illegal_map_key_variable,Var}, St0)} + end + end, {[],[],St}, Ps); %%pattern({struct,_Line,_Tag,Ps}, Vt, Old, Bvt, St) -> %% pattern_list(Ps, Vt, Old, Bvt, St); pattern({record_index,Line,Name,Field}, _Vt, _Old, _Bvt, St) -> @@ -2237,9 +2240,10 @@ check_assoc_fields([], St) -> map_fields([{Tag,Line,K,V}|Fs], Vt, St, F) when Tag =:= map_field_assoc; Tag =:= map_field_exact -> St1 = case is_valid_map_key(K, St) of - true -> St; - {false,Var} -> add_error(Line, {illegal_map_key_variable,Var}, St) - end, + true -> St; + false -> add_error(Line, illegal_map_key, St); + {false,variable,Var} -> add_error(Line, {illegal_map_key_variable,Var}, St) + end, {Pvt,St2} = F([K,V], Vt, St1), {Vts,St3} = map_fields(Fs, Vt, St2, F), {vtupdate(Pvt, Vts),St3}; @@ -2298,11 +2302,64 @@ is_valid_call(Call) -> _ -> true end. +%% is_valid_map_key(K,St) -> true | false | {false, Var::atom()} +%% check for value expression without variables + is_valid_map_key(K,St) -> case expr(K,[],St) of - {[],_} -> true; + {[],_} -> + is_valid_map_key_value(K); {[Var|_],_} -> - {false,element(1,Var)} + {false,variable,element(1,Var)} + end. + +is_valid_map_key_value(K) -> + case K of + {char,_,_} -> true; + {integer,_,_} -> true; + {float,_,_} -> true; + {string,_,_} -> true; + {nil,_} -> true; + {atom,_,_} -> true; + {cons,_,H,T} -> + is_valid_map_key_value(H) andalso + is_valid_map_key_value(T); + {tuple,_,Es} -> + foldl(fun(E,B) -> + B andalso is_valid_map_key_value(E) + end,true,Es); + {map,_,Arg,Ps} -> + % only check for value expressions to be valid + % invalid map expressions are later checked in + % core and kernel + is_valid_map_key_value(Arg) andalso foldl(fun + ({Tag,_,Ke,Ve},B) when Tag =:= map_field_assoc; + Tag =:= map_field_exact -> + B andalso is_valid_map_key_value(Ke) + andalso is_valid_map_key_value(Ve) + end,true,Ps); + {map,_,Ps} -> + foldl(fun + ({Tag,_,Ke,Ve},B) when Tag =:= map_field_assoc; + Tag =:= map_field_exact -> + B andalso is_valid_map_key_value(Ke) + andalso is_valid_map_key_value(Ve) + end, true, Ps); + {record,_,_,Fs} -> + foldl(fun + ({record_field,_,Ke,Ve},B) -> + B andalso is_valid_map_key_value(Ke) + andalso is_valid_map_key_value(Ve) + end,true,Fs); + {bin,_,Es} -> + % only check for value expressions to be valid + % invalid binary expressions are later checked in + % core and kernel + foldl(fun + ({bin_element,_,E,_,_},B) -> + B andalso is_valid_map_key_value(E) + end,true,Es); + _ -> false end. %% record_def(Line, RecordName, [RecField], State) -> State. @@ -2727,6 +2784,7 @@ check_record_types([], _Name, _DefFields, SeenVars, St, _SeenFields) -> {SeenVars, St}. is_var_arity_type(tuple) -> true; +is_var_arity_type(map) -> true; is_var_arity_type(product) -> true; is_var_arity_type(union) -> true; is_var_arity_type(record) -> true; @@ -2759,7 +2817,6 @@ is_default_type({iodata, 0}) -> true; is_default_type({iolist, 0}) -> true; is_default_type({list, 0}) -> true; is_default_type({list, 1}) -> true; -is_default_type({map, 0}) -> true; is_default_type({maybe_improper_list, 0}) -> true; is_default_type({maybe_improper_list, 2}) -> true; is_default_type({mfa, 0}) -> true; @@ -2790,6 +2847,7 @@ is_default_type({timeout, 0}) -> true; is_default_type({var, 1}) -> true; is_default_type(_) -> false. +%% OTP 17.0 is_newly_introduced_builtin_type({Name, _}) when is_atom(Name) -> false. is_obsolete_builtin_type(TypePair) -> diff --git a/lib/stdlib/test/epp_SUITE.erl b/lib/stdlib/test/epp_SUITE.erl index 0b4726c07a..b17e8bd186 100644 --- a/lib/stdlib/test/epp_SUITE.erl +++ b/lib/stdlib/test/epp_SUITE.erl @@ -26,7 +26,7 @@ pmod/1, not_circular/1, skip_header/1, otp_6277/1, otp_7702/1, otp_8130/1, overload_mac/1, otp_8388/1, otp_8470/1, otp_8503/1, otp_8562/1, otp_8665/1, otp_8911/1, otp_10302/1, otp_10820/1, - otp_11728/1]). + otp_11728/1, encoding/1]). -export([epp_parse_erl_form/2]). @@ -68,7 +68,8 @@ all() -> {group, variable}, otp_4870, otp_4871, otp_5362, pmod, not_circular, skip_header, otp_6277, otp_7702, otp_8130, overload_mac, otp_8388, otp_8470, otp_8503, otp_8562, - otp_8665, otp_8911, otp_10302, otp_10820, otp_11728]. + otp_8665, otp_8911, otp_10302, otp_10820, otp_11728, + encoding]. groups() -> [{upcase_mac, [], [upcase_mac_1, upcase_mac_2]}, @@ -123,10 +124,22 @@ include_local(Config) when is_list(Config) -> %%% regular epp:parse_file, the test case will time out, and then epp %%% server will go on growing until we dump core. epp_parse_file(File, Inc, Predef) -> - {ok, Epp} = epp:open(File, Inc, Predef), + List = do_epp_parse_file(fun() -> + epp:open(File, Inc, Predef) + end), + List = do_epp_parse_file(fun() -> + Opts = [{name, File}, + {includes, Inc}, + {macros, Predef}], + epp:open(Opts) + end), + {ok, List}. + +do_epp_parse_file(Open) -> + {ok, Epp} = Open(), List = collect_epp_forms(Epp), epp:close(Epp), - {ok, List}. + List. collect_epp_forms(Epp) -> Result = epp_parse_erl_form(Epp), @@ -1413,6 +1426,63 @@ otp_11728(Config) when is_list(Config) -> _ = file:delete(ErlFile), ok. +%% Check the new API for setting the default encoding. +encoding(Config) when is_list(Config) -> + Dir = ?config(priv_dir, Config), + ErlFile = filename:join(Dir, "encoding.erl"), + + %% Try a latin-1 file with no encoding given. + C1 = <<"-module(encoding). + %% ",246," + ">>, + ok = file:write_file(ErlFile, C1), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {error,_}, + {error,{2,epp,cannot_parse}}, + {eof,2}]} = epp:parse_file(ErlFile, []), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {eof,3}]} = + epp:parse_file(ErlFile, [{default_encoding,latin1}]), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {eof,3}],[{encoding,none}]} = + epp:parse_file(ErlFile, [{default_encoding,latin1},extra]), + + %% Try a latin-1 file with encoding given in a comment. + C2 = <<"-module(encoding). + %% encoding: latin-1 + %% ",246," + ">>, + ok = file:write_file(ErlFile, C2), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {eof,4}]} = + epp:parse_file(ErlFile, []), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {eof,4}]} = + epp:parse_file(ErlFile, [{default_encoding,latin1}]), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {eof,4}]} = + epp:parse_file(ErlFile, [{default_encoding,utf8}]), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {eof,4}],[{encoding,latin1}]} = + epp:parse_file(ErlFile, [extra]), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {eof,4}],[{encoding,latin1}]} = + epp:parse_file(ErlFile, [{default_encoding,latin1},extra]), + {ok,[{attribute,1,file,_}, + {attribute,1,module,encoding}, + {eof,4}],[{encoding,latin1}]} = + epp:parse_file(ErlFile, [{default_encoding,utf8},extra]), + ok. + + check(Config, Tests) -> eval_tests(Config, fun check_test/2, Tests). diff --git a/lib/stdlib/test/erl_eval_SUITE.erl b/lib/stdlib/test/erl_eval_SUITE.erl index e6512b7d71..b91d14b5b8 100644 --- a/lib/stdlib/test/erl_eval_SUITE.erl +++ b/lib/stdlib/test/erl_eval_SUITE.erl @@ -1451,6 +1451,8 @@ eep43(Config) when is_list(Config) -> " {Map#{a := B},Map#{a => c},Map#{d => e}} " "end.", {#{a => b},#{a => c},#{a => b,d => e}}), + error_check("[camembert]#{}.", {badarg,[camembert]}), + error_check("#{} = 1.", {badmatch,1}), ok. %% Check the string in different contexts: as is; in fun; from compiled code. diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index 5d189006a1..673a3cf159 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -63,7 +63,7 @@ too_many_arguments/1, basic_errors/1,bin_syntax_errors/1, predef/1, - maps/1 + maps/1,maps_type/1 ]). % Default timetrap timeout (set in init_per_testcase). @@ -91,7 +91,8 @@ all() -> otp_11772, otp_11771, export_all, bif_clash, behaviour_basic, behaviour_multiple, otp_7550, otp_8051, format_warn, {group, on_load}, - too_many_arguments, basic_errors, bin_syntax_errors, predef, maps]. + too_many_arguments, basic_errors, bin_syntax_errors, predef, + maps,maps_type]. groups() -> [{unused_vars_warn, [], @@ -3388,12 +3389,61 @@ maps(Config) -> {error_in_illegal_map_construction, <<"t() -> #{ a := X }.">>, [], - {errors,[{1,erl_lint,illegal_map_construction}, + {errors,[{1,erl_lint,illegal_map_construction}, {1,erl_lint,{unbound_var,'X'}}], - []}}], + []}}, + {errors_in_map_keys, + <<"t(V) -> #{ a => 1, + #{a=>V} => 2, + #{ \"hi\" => wazzup, hi => ho } => yep, + [try a catch _:_ -> b end] => nope, + ok => 1.0, + [3+3] => nope, + 1.0 => yep, + {3.0+3} => nope, + {yep} => yep, + [case a of a -> a end] => nope + }. + ">>, + [], + {errors,[{2,erl_lint,{illegal_map_key_variable,'V'}}, + {4,erl_lint,illegal_map_key}, + {6,erl_lint,illegal_map_key}, + {8,erl_lint,illegal_map_key}, + {10,erl_lint,illegal_map_key}],[]}}], [] = run(Config, Ts), ok. +maps_type(Config) when is_list(Config) -> + Ts = [ + {maps_type1, + <<" + -type m() :: #{a => integer()}. + -spec t1(#{k=>term()}) -> {term(), map()}. + + t1(#{k:=V}=M) -> {V,M}. + + -spec t2(m()) -> integer(). + + t2(#{a:=V}) -> V. + ">>, + [], + []}, + {maps_type2, + <<" + %% Built-in var arity map type: + -type map() :: tuple(). + -type a() :: map(). + + -spec t(a()) -> a(). + t(M) -> M. + ">>, + [], + {errors,[{3,erl_lint,{redefine_type,{map,0}}}],[]}}], + [] = run(Config, Ts), + ok. + + run(Config, Tests) -> F = fun({N,P,Ws,E}, BadL) -> case catch run_test(Config, P, Ws) of diff --git a/lib/test_server/src/ts.unix.config b/lib/test_server/src/ts.unix.config index a34857b9e5..1ba5d9033e 100644 --- a/lib/test_server/src/ts.unix.config +++ b/lib/test_server/src/ts.unix.config @@ -3,4 +3,4 @@ %% Always run a (VNC) X server on host %% {xserver, "xserver.example.com:66"}. -{unix,[{telnet,"belegost"},{username,"bofh"},{password,"root"},{keep_alive,true}]}. +{unix,[{telnet,"belegost"},{username,"telnet-test"},{password,"tset-tenlet"},{keep_alive,true}]}. diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index f007f780eb..ec5a1f4bc5 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -1420,6 +1420,10 @@ Other commands: (if (boundp 'after-change-major-mode-hook) (run-hooks 'after-change-major-mode-hook))) +;;;###autoload +(dolist (r '("\\.erl$" "\\.app\\.src$" "\\.escript" + "\\.hrl$" "\\.xrl$" "\\.yrl" "/ebin/.+\\.app")) + (add-to-list 'auto-mode-alist (cons r 'erlang-mode))) (defun erlang-syntax-table-init () (if (null erlang-mode-syntax-table) @@ -2570,9 +2574,9 @@ Value is list (stack token-start token-type in-what)." (erlang-pop stack)) (if (and stack (memq (car (car stack)) '(icr begin fun try))) (erlang-pop stack)))) - ((looking-at "catch.*of") + ((looking-at "catch\\b.*of") t) - ((looking-at "catch\\s *\\($\\|%\\|.*->\\)") + ((looking-at "catch\\b\\s *\\($\\|%\\|.*->\\)") ;; Must pop top icr layer, `catch' in try/catch ;;will push a new layer next. (progn @@ -2620,9 +2624,9 @@ Value is list (stack token-start token-type in-what)." ;;((looking-at "when\\s *\\($\\|%\\)") ((looking-at "when[^_a-zA-Z0-9]") (erlang-push (list 'when token (current-column)) stack)) - ((looking-at "catch.*of") + ((looking-at "catch\\b.*of") t) - ((looking-at "catch\\s *\\($\\|%\\|.*->\\)") + ((looking-at "catch\\b\\s *\\($\\|%\\|.*->\\)") (erlang-push (list 'icr token (current-column)) stack)) ;;(erlang-push (list '-> token (current-column)) stack)) ;;((looking-at "^of$") @@ -2913,7 +2917,7 @@ Return nil if inside string, t if in a comment." (if stack (erlang-caddr (car stack)) 0)) - ((looking-at "catch\\($\\|[^_a-zA-Z0-9]\\)") + ((looking-at "catch\\b\\($\\|[^_a-zA-Z0-9]\\)") ;; Are we in a try (let ((start (if (eq (car stack-top) '->) (car (cdr stack)) @@ -3124,12 +3128,12 @@ This assumes that the preceding expression is either simple (defun erlang-at-keyword () "Are we looking at an Erlang keyword which will increase indentation?" (looking-at (concat "\\(when\\|if\\|fun\\|case\\|begin\\|" - "of\\|receive\\|after\\|catch\\|try\\)[^_a-zA-Z0-9]"))) + "of\\|receive\\|after\\|catch\\|try\\)\\b"))) (defun erlang-at-operator () "Are we looking at an Erlang operator?" (looking-at - "\\(bnot\\|div\\|mod\\|band\\|bor\\|bxor\\|bsl\\|bsr\\)[^_a-zA-Z0-9]")) + "\\(bnot\\|div\\|mod\\|band\\|bor\\|bxor\\|bsl\\|bsr\\)\\b")) (defun erlang-comment-indent () "Compute Erlang comment indentation. diff --git a/lib/tools/emacs/test.erl.indented b/lib/tools/emacs/test.erl.indented index 0dc1b47f0d..1c1086ca58 100644 --- a/lib/tools/emacs/test.erl.indented +++ b/lib/tools/emacs/test.erl.indented @@ -749,3 +749,14 @@ commas_first() -> %% this used to result in a scan-sexp error [{ }]. + +%% this used to result in 2x the correct indentation within the function +%% body, due to the function name being mistaken for a keyword +catcher(N) -> + try generate_exception(N) of + Val -> {N, normal, Val} + catch + throw:X -> {N, caught, thrown, X}; + exit:X -> {N, caught, exited, X}; + error:X -> {N, caught, error, X} + end. diff --git a/lib/tools/emacs/test.erl.orig b/lib/tools/emacs/test.erl.orig index c7d2dc4ce5..a9d09000d2 100644 --- a/lib/tools/emacs/test.erl.orig +++ b/lib/tools/emacs/test.erl.orig @@ -749,3 +749,14 @@ commas_first() -> %% this used to result in a scan-sexp error [{ }]. + +%% this used to result in 2x the correct indentation within the function +%% body, due to the function name being mistaken for a keyword +catcher(N) -> +try generate_exception(N) of +Val -> {N, normal, Val} +catch +throw:X -> {N, caught, thrown, X}; +exit:X -> {N, caught, exited, X}; +error:X -> {N, caught, error, X} +end. diff --git a/system/doc/reference_manual/character_set.xml b/system/doc/reference_manual/character_set.xml new file mode 100644 index 0000000000..884898eb34 --- /dev/null +++ b/system/doc/reference_manual/character_set.xml @@ -0,0 +1,132 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2014</year><year>2014</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Character Set and Source File Encoding</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>character_set.xml</file> + </header> + + <section> + <title>Character Set</title> + <p>In Erlang 4.8/OTP R5A the syntax of Erlang tokens was extended to + allow the use of the full ISO-8859-1 (Latin-1) character set. This + is noticeable in the following ways:</p> + <list type="bulleted"> + <item> + <p>All the Latin-1 printable characters can be used and are + shown without the escape backslash convention.</p> + </item> + <item> + <p>Atoms and variables can use all Latin-1 letters.</p> + </item> + </list> + <table> + <row> + <cell align="left" valign="middle"><em>Octal</em></cell> + <cell align="left" valign="middle"><em>Decimal</em></cell> + <cell align="left" valign="middle"> </cell> + <cell align="left" valign="middle"><em>Class</em></cell> + </row> + <row> + <cell align="left" valign="middle">200 - 237</cell> + <cell align="left" valign="middle">128 - 159</cell> + <cell align="left" valign="middle"> </cell> + <cell align="left" valign="middle">Control characters</cell> + </row> + <row> + <cell align="left" valign="middle">240 - 277</cell> + <cell align="left" valign="middle">160 - 191</cell> + <cell align="right" valign="middle">- ¿</cell> + <cell align="left" valign="middle">Punctuation characters</cell> + </row> + <row> + <cell align="left" valign="middle">300 - 326</cell> + <cell align="left" valign="middle">192 - 214</cell> + <cell align="center" valign="middle">À - Ö</cell> + <cell align="left" valign="middle">Uppercase letters</cell> + </row> + <row> + <cell align="center" valign="middle">327</cell> + <cell align="center" valign="middle">215</cell> + <cell align="center" valign="middle">×</cell> + <cell align="left" valign="middle">Punctuation character</cell> + </row> + <row> + <cell align="left" valign="middle">330 - 336</cell> + <cell align="left" valign="middle">216 - 222</cell> + <cell align="center" valign="middle">Ø - Þ</cell> + <cell align="left" valign="middle">Uppercase letters</cell> + </row> + <row> + <cell align="left" valign="middle">337 - 366</cell> + <cell align="left" valign="middle">223 - 246</cell> + <cell align="center" valign="middle">ß - ö</cell> + <cell align="left" valign="middle">Lowercase letters</cell> + </row> + <row> + <cell align="center" valign="middle">367</cell> + <cell align="center" valign="middle">247</cell> + <cell align="center" valign="middle">÷</cell> + <cell align="left" valign="middle">Punctuation character</cell> + </row> + <row> + <cell align="left" valign="middle">370 - 377</cell> + <cell align="left" valign="middle">248 - 255</cell> + <cell align="center" valign="middle">ø - ÿ</cell> + <cell align="left" valign="middle">Lowercase letters</cell> + </row> + <tcaption>Character Classes.</tcaption> + </table> + <p>In Erlang/OTP R16B the syntax of Erlang tokens was extended to + handle Unicode. To begin with the support is limited to + strings, but Erlang/OTP 18 is expected to handle Unicode atoms + as well. More about the usage of Unicode in Erlang source files + can be found in <seealso + marker="stdlib:unicode_usage#unicode_in_erlang">STDLIB's User's + Guide</seealso>.</p> + </section> + <section> + <title>Source File Encoding</title> + <p>The Erlang source file <marker + id="encoding">encoding</marker> is selected by a + comment in one of the first two lines of the source file. The + first string that matches the regular expression + <c>coding\s*[:=]\s*([-a-zA-Z0-9])+</c> selects the encoding. If + the matching string is not a valid encoding it is ignored. The + valid encodings are <c>Latin-1</c> and <c>UTF-8</c> where the + case of the characters can be chosen freely.</p> + <p>The following example selects UTF-8 as default encoding:</p> + <pre> +%% coding: utf-8</pre> + <p>Two more examples, both selecting Latin-1 as default encoding:</p> + <pre> +%% For this file we have chosen encoding = Latin-1</pre> + <pre> +%% -*- coding: latin-1 -*-</pre> + <p>The default encoding for Erlang source files was changed from + Latin-1 to UTF-8 in Erlang OTP 17.0.</p> + </section> +</chapter> diff --git a/system/doc/reference_manual/introduction.xml b/system/doc/reference_manual/introduction.xml index aa42967625..36bec17825 100644 --- a/system/doc/reference_manual/introduction.xml +++ b/system/doc/reference_manual/introduction.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2013</year> + <year>2003</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -79,88 +79,5 @@ when xor</p> </section> - <section> - <title>Character Set</title> - <p>In Erlang 4.8/OTP R5A the syntax of Erlang tokens was extended to - allow the use of the full ISO-8859-1 (Latin-1) character set. This - is noticeable in the following ways:</p> - <list type="bulleted"> - <item> - <p>All the Latin-1 printable characters can be used and are - shown without the escape backslash convention.</p> - </item> - <item> - <p>Atoms and variables can use all Latin-1 letters.</p> - </item> - </list> - <table> - <row> - <cell align="left" valign="middle"><em>Octal</em></cell> - <cell align="left" valign="middle"><em>Decimal</em></cell> - <cell align="left" valign="middle"> </cell> - <cell align="left" valign="middle"><em>Class</em></cell> - </row> - <row> - <cell align="left" valign="middle">200 - 237</cell> - <cell align="left" valign="middle">128 - 159</cell> - <cell align="left" valign="middle"> </cell> - <cell align="left" valign="middle">Control characters</cell> - </row> - <row> - <cell align="left" valign="middle">240 - 277</cell> - <cell align="left" valign="middle">160 - 191</cell> - <cell align="right" valign="middle">- ¿</cell> - <cell align="left" valign="middle">Punctuation characters</cell> - </row> - <row> - <cell align="left" valign="middle">300 - 326</cell> - <cell align="left" valign="middle">192 - 214</cell> - <cell align="center" valign="middle">À - Ö</cell> - <cell align="left" valign="middle">Uppercase letters</cell> - </row> - <row> - <cell align="center" valign="middle">327</cell> - <cell align="center" valign="middle">215</cell> - <cell align="center" valign="middle">×</cell> - <cell align="left" valign="middle">Punctuation character</cell> - </row> - <row> - <cell align="left" valign="middle">330 - 336</cell> - <cell align="left" valign="middle">216 - 222</cell> - <cell align="center" valign="middle">Ø - Þ</cell> - <cell align="left" valign="middle">Uppercase letters</cell> - </row> - <row> - <cell align="left" valign="middle">337 - 366</cell> - <cell align="left" valign="middle">223 - 246</cell> - <cell align="center" valign="middle">ß - ö</cell> - <cell align="left" valign="middle">Lowercase letters</cell> - </row> - <row> - <cell align="center" valign="middle">367</cell> - <cell align="center" valign="middle">247</cell> - <cell align="center" valign="middle">÷</cell> - <cell align="left" valign="middle">Punctuation character</cell> - </row> - <row> - <cell align="left" valign="middle">370 - 377</cell> - <cell align="left" valign="middle">248 - 255</cell> - <cell align="center" valign="middle">ø - ÿ</cell> - <cell align="left" valign="middle">Lowercase letters</cell> - </row> - <tcaption>Character Classes.</tcaption> - </table> - <p>In Erlang/OTP R16 the syntax of Erlang tokens was extended to - handle Unicode. To begin with the support is limited to strings, - but Erlang/OTP R18 is expected to handle Unicode atoms as well. - More about the usage of Unicode in Erlang source files can be - found in <seealso - marker="stdlib:unicode_usage#unicode_in_erlang">STDLIB's User'S - Guide</seealso>. The default encoding for Erlang source files - is still Latin-1, but in Erlang/OTP R17 the default encoding - will be UTF-8. The details on how to state the encoding of an - Erlang source file can be found in <seealso - marker="stdlib:epp#encoding">epp(3)</seealso>.</p> - </section> </chapter> diff --git a/system/doc/reference_manual/part.xml b/system/doc/reference_manual/part.xml index b4f114c268..ee8f3dd7eb 100644 --- a/system/doc/reference_manual/part.xml +++ b/system/doc/reference_manual/part.xml @@ -4,7 +4,7 @@ <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2013</year> + <year>2003</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -28,6 +28,7 @@ <rev></rev> </header> <xi:include href="introduction.xml"/> + <xi:include href="character_set.xml"/> <xi:include href="data_types.xml"/> <xi:include href="patterns.xml"/> <xi:include href="modules.xml"/> |