From 92c79e394041f76d8f676cafe9b6af44522497bd Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Wed, 22 Mar 2017 09:41:57 +0100 Subject: stdlib: Add function to io_lib to handle Unicode atoms --- lib/stdlib/doc/src/io_lib.xml | 14 ++++++++++++-- lib/stdlib/src/io_lib.erl | 24 +++++++++++++++++++----- lib/stdlib/src/io_lib_pretty.erl | 14 ++++++++++++-- lib/stdlib/test/io_SUITE.erl | 29 +++++++++++++++++++++++++++-- 4 files changed, 70 insertions(+), 11 deletions(-) diff --git a/lib/stdlib/doc/src/io_lib.xml b/lib/stdlib/doc/src/io_lib.xml index 931e50f6f2..5ae400da62 100644 --- a/lib/stdlib/doc/src/io_lib.xml +++ b/lib/stdlib/doc/src/io_lib.xml @@ -4,7 +4,7 @@
- 19962016 + 19962017 Ericsson AB. All Rights Reserved. @@ -147,7 +147,7 @@ format string (that is, ~ts or ~tc), the resulting list can contain characters beyond the ISO Latin-1 character range (that is, numbers > 255). If so, the - result is not an ordinary Erlang string(), but can well be + result is still an ordinary Erlang string(), and can well be used in any context where Unicode data is allowed.

@@ -383,6 +383,16 @@ + + + Write an atom. + +

Returns the list of characters needed to print atom + Atom. Non-Latin-1 characters + are escaped.

+ + + Write a character. diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl index a91143a764..28e5007e5a 100644 --- a/lib/stdlib/src/io_lib.erl +++ b/lib/stdlib/src/io_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -68,8 +68,8 @@ -export([write_atom/1,write_string/1,write_string/2,write_latin1_string/1, write_latin1_string/2, write_char/1, write_latin1_char/1]). --export([write_string_as_latin1/1, write_string_as_latin1/2, - write_char_as_latin1/1]). +-export([write_atom_as_latin1/1, write_string_as_latin1/1, + write_string_as_latin1/2, write_char_as_latin1/1]). -export([quote_atom/2, char_list/1, latin1_char_list/1, deep_char_list/1, deep_latin1_char_list/1, @@ -344,6 +344,11 @@ write_binary_body(B, _D) -> <> = B, [integer_to_list(X),$:,integer_to_list(L)]. +%%% There are two functions to write Unicode atoms: +%%% - they both escape control characters < 160; +%%% - write_atom() never escapes characters >= 160; +%%% - write_atom_as_latin1() also escapes characters >= 255. + %% write_atom(Atom) -> [Char] %% Generate the list of characters needed to print an atom. @@ -351,17 +356,26 @@ write_binary_body(B, _D) -> Atom :: atom(). write_atom(Atom) -> + write_possibly_quoted_atom(Atom, fun write_string/2). + +-spec write_atom_as_latin1(Atom) -> latin1_string() when + Atom :: atom(). + +write_atom_as_latin1(Atom) -> + write_possibly_quoted_atom(Atom, fun write_string_as_latin1/2). + +write_possibly_quoted_atom(Atom, PFun) -> Chars = atom_to_list(Atom), case quote_atom(Atom, Chars) of true -> - write_string(Chars, $'); %' + PFun(Chars, $'); %' false -> Chars end. %% quote_atom(Atom, CharList) %% Return 'true' if atom with chars in CharList needs to be quoted, else -%% return 'false'. +%% return 'false'. Notice that characters >= 160 are always quoted. -spec quote_atom(atom(), chars()) -> boolean(). diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index aabccfc5d9..ff368d02da 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -105,6 +105,8 @@ print(_, _, _, 0, _M, _RF, _Enc, _Str) -> "..."; print(Term, Col, Ll, D, M, RecDefFun, Enc, Str) when Col =< 0 -> %% ensure Col is at least 1 print(Term, 1, Ll, D, M, RecDefFun, Enc, Str); +print(Atom, _Col, _Ll, _D, _M, _RF, Enc, _Str) when is_atom(Atom) -> + write_atom(Atom, Enc); print(Term, Col, Ll, D, M0, RecDefFun, Enc, Str) when is_tuple(Term); is_list(Term); is_map(Term); @@ -407,6 +409,9 @@ print_length({}, _D, _RF, _Enc, _Str) -> {"{}", 2}; print_length(#{}=M, _D, _RF, _Enc, _Str) when map_size(M) =:= 0 -> {"#{}", 3}; +print_length(Atom, _D, _RF, Enc, _Str) when is_atom(Atom) -> + S = write_atom(Atom, Enc), + {S, lists:flatlength(S)}; print_length(List, D, RF, Enc, Str) when is_list(List) -> %% only flat lists are "printable" case Str andalso printable_list(List, D, Enc) of @@ -500,7 +505,7 @@ print_length_tuple(Tuple, D, RF, Enc, Str) -> print_length_record(_Tuple, 1, _RF, _RDefs, _Enc, _Str) -> {"{...}", 5}; print_length_record(Tuple, D, RF, RDefs, Enc, Str) -> - Name = [$# | io_lib:write_atom(element(1, Tuple))], + Name = [$# | write_atom(element(1, Tuple), Enc)], NameL = length(Name), Elements = tl(tuple_to_list(Tuple)), L = print_length_fields(RDefs, D - 1, Elements, RF, Enc, Str), @@ -515,7 +520,7 @@ print_length_fields([Def | Defs], D, [E | Es], RF, Enc, Str) -> print_length_fields(Defs, D - 1, Es, RF, Enc, Str)]. print_length_field(Def, D, E, RF, Enc, Str) -> - Name = io_lib:write_atom(Def), + Name = write_atom(Def, Enc), {S, L} = print_length(E, D, RF, Enc, Str), NameL = length(Name) + 3, {{field, Name, NameL, {S, L}}, NameL + L}. @@ -664,6 +669,11 @@ printable_char(C,unicode) -> C > 16#DFFF andalso C < 16#FFFE orelse C > 16#FFFF andalso C =< 16#10FFFF. +write_atom(A, latin1) -> + io_lib:write_atom_as_latin1(A); +write_atom(A, _Uni) -> + io_lib:write_atom(A). + write_string(S, latin1) -> io_lib:write_latin1_string(S, $"); %" write_string(S, _Uni) -> diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index d546e8fad2..b2754e47ba 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -30,7 +30,8 @@ io_lib_print_binary_depth_one/1, otp_10302/1, otp_10755/1, otp_10836/1, io_lib_width_too_small/1, io_with_huge_message_queue/1, format_string/1, - maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1]). + maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1, + otp_14285/1]). -export([pretty/2]). @@ -61,7 +62,8 @@ all() -> printable_range, bad_printable_range, io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836, io_lib_width_too_small, io_with_huge_message_queue, - format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175]. + format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175, + otp_14285]. %% Error cases for output. error_1(Config) when is_list(Config) -> @@ -755,6 +757,8 @@ rfd(rrrrr, 3) -> [f1, f2, f3]; rfd(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, 0) -> []; +rfd('\x{400}', 1) -> + ['\x{400}']; rfd(_, _) -> no. @@ -1881,6 +1885,7 @@ otp_10302(Suite) when is_list(Suite) -> pretty(Term, Depth) when is_integer(Depth) -> Opts = [{column, 1}, {line_length, 20}, {depth, Depth}, {max_chars, 60}, + {record_print_fun, fun rfd/2}, {encoding, unicode}], pretty(Term, Opts); pretty(Term, Opts) when is_list(Opts) -> @@ -2324,3 +2329,23 @@ text1([T|Ts]) -> [erl_anno:text(Anno) | text1(Ts)]. -endif. % EXACT + +otp_14285(_Config) -> + UOpts = [{record_print_fun, fun rfd/2}, + {encoding, unicode}], + LOpts = [{record_print_fun, fun rfd/2}, + {encoding, latin1}], + + RT = {'\x{400}','\x{400}'}, + "#'\x{400}'{'\x{400}' = '\x{400}'}" = pretty(RT, UOpts), + "#'\\x{400}'{'\\x{400}' = '\\x{400}'}" = pretty(RT, LOpts), + + Chars = lists:seq(0, 512), + [] = [C || + C <- Chars, + S <- io_lib:write_atom_as_latin1(list_to_atom([C])), + not is_latin1(S)], + L1 = [S || C <- Chars, S <- io_lib:write_atom(list_to_atom([C])), + not is_latin1(S)], + L1 = lists:seq(256, 512), + ok. -- cgit v1.2.3 From 421012d3e8062c9522e8b425cd936009ba83c887 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Tue, 21 Mar 2017 15:32:11 +0100 Subject: stdlib: Fix Erlang pretty printer regarding Unicode atoms --- lib/stdlib/src/erl_pp.erl | 116 ++++++++++++++++++++++++--------------- lib/stdlib/test/erl_pp_SUITE.erl | 41 ++++++++++---- 2 files changed, 104 insertions(+), 53 deletions(-) diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index 6068afb293..ee5e7a11bf 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ | {encoding, latin1 | unicode | utf8}). -type(options() :: hook_function() | [option()]). --record(pp, {string_fun, char_fun}). +-record(pp, {value_fun, string_fun, char_fun}). -record(options, {hook, encoding, opts}). @@ -214,11 +214,15 @@ state(_Hook) -> state(). state() -> - #pp{string_fun = fun io_lib:write_string_as_latin1/1, + Options = [{encoding,latin1}], + #pp{value_fun = fun(V) -> io_lib_pretty:print(V, Options) end, + string_fun = fun io_lib:write_string_as_latin1/1, char_fun = fun io_lib:write_char_as_latin1/1}. unicode_state() -> - #pp{string_fun = fun io_lib:write_string/1, + Options = [{encoding,unicode}], + #pp{value_fun = fun(V) -> io_lib_pretty:print(V, Options) end, + string_fun = fun io_lib:write_string/1, char_fun = fun io_lib:write_char/1}. encoding(Options) -> @@ -253,31 +257,30 @@ lattribute({attribute,_Line,Name,Arg}, Opts) -> lattribute(module, {M,Vs}, _Opts) -> A = a0(), - attr("module",[{var,A,pname(M)}, - foldr(fun(V, C) -> {cons,A,{var,A,V},C} - end, {nil,A}, Vs)]); + attr(module,[{var,A,pname(M)}, + foldr(fun(V, C) -> {cons,A,{var,A,V},C} + end, {nil,A}, Vs)]); lattribute(module, M, _Opts) -> - attr("module", [{var,a0(),pname(M)}]); + attr(module, [{var,a0(),pname(M)}]); lattribute(export, Falist, _Opts) -> - call({var,a0(),"-export"}, [falist(Falist)], 0, options(none)); + attrib(export, falist(Falist)); lattribute(import, Name, _Opts) when is_list(Name) -> - attr("import", [{var,a0(),pname(Name)}]); + attr(import, [{var,a0(),pname(Name)}]); lattribute(import, {From,Falist}, _Opts) -> - attr("import",[{var,a0(),pname(From)},falist(Falist)]); + attrib(import, [leaf(pname(From)),falist(Falist)]); lattribute(export_type, Talist, _Opts) -> - call({var,a0(),"-export_type"}, [falist(Talist)], 0, options(none)); + attrib(export_type, falist(Talist)); lattribute(optional_callbacks, Falist, Opts) -> - ArgL = try falist(Falist) - catch _:_ -> abstract(Falist, Opts) - end, - call({var,a0(),"-optional_callbacks"}, [ArgL], 0, options(none)); + try attrib(optional_callbacks, falist(Falist)) + catch _:_ -> attr(optional_callbacks, [abstract(Falist, Opts)]) + end; lattribute(file, {Name,Line}, _Opts) -> - attr("file", [{string,a0(),Name},{integer,a0(),Line}]); + attr(file, [{string,a0(),Name},{integer,a0(),Line}]); lattribute(record, {Name,Is}, Opts) -> - Nl = leaf(format("-record(~w,", [Name])), + Nl = [leaf("-record("),{atom,Name},$,], [{first,Nl,record_fields(Is, Opts)},$)]; lattribute(Name, Arg, Options) -> - attr(write(Name), [abstract(Arg, Options)]). + attr(Name, [abstract(Arg, Options)]). abstract(Arg, #options{encoding = Encoding}) -> erl_parse:abstract(Arg, [{encoding,Encoding}]). @@ -340,7 +343,7 @@ ltype({user_type,Line,T,Ts}, _) -> ltype({remote_type,Line,[M,F,Ts]}, _) -> simple_type({remote,Line,M,F}, Ts); ltype({atom,_,T}, _) -> - leaf(write(T)); + {atom,T}; ltype(E, P) -> lexpr(E, P, options(none)). @@ -382,12 +385,12 @@ tuple_type(Ts, F) -> specattr(SpecKind, {FuncSpec,TypeSpecs}) -> Func = case FuncSpec of {F,_A} -> - format("~w", [F]); + {atom,F}; {M,F,_A} -> - format("~w:~w", [M, F]) + [{atom,M},$:,{atom,F}] end, {first,leaf(lists:concat(["-", SpecKind, " "])), - {list,[{first,leaf(Func),spec_clauses(TypeSpecs)}]}}. + {list,[{first,Func,spec_clauses(TypeSpecs)}]}}. spec_clauses(TypeSpecs) -> {prefer_nl,[$;],[sig_type(T) || T <- TypeSpecs]}. @@ -429,7 +432,10 @@ ltypes(Ts, F, Prec) -> [F(T, Prec) || T <- Ts]. attr(Name, Args) -> - call({var,a0(),format("-~s", [Name])}, Args, 0, options(none)). + {first,[$-,{atom,Name}],args(Args, options(none))}. + +attrib(Name, Args) -> + {first,[$-,{atom,Name}],[{seq,$(,$),[$,],Args}]}. pname(['' | As]) -> [$. | pname(As)]; @@ -441,10 +447,13 @@ pname(A) when is_atom(A) -> write(A). falist([]) -> - {nil,a0()}; -falist([{Name,Arity}|Falist]) -> - A = a0(), - {cons,A,{var,A,format("~w/~w", [Name,Arity])},falist(Falist)}. + [leaf("[]")]; +falist(Falist) -> + L = [begin + {Name,Arity} = Fa, + [{atom,Name},leaf(format("/~w", [Arity]))] + end || Fa <- Falist], + [{seq,$[,$],$,,L}]. lfunction({function,_Line,Name,_Arity,Cs}, Opts) -> Cll = nl_clauses(fun (C, H) -> func_clause(Name, C, H) end, $;, Opts, Cs), @@ -489,7 +498,7 @@ lexpr({var,_,V}, _, _) -> leaf(format("~ts", [V])); lexpr({char,_,C}, _, _) -> {char,C}; lexpr({integer,_,N}, _, _) -> leaf(write(N)); lexpr({float,_,F}, _, _) -> leaf(write(F)); -lexpr({atom,_,A}, _, _) -> leaf(write(A)); +lexpr({atom,_,A}, _, _) -> {atom,A}; lexpr({string,_,S}, _, _) -> {string,S}; lexpr({nil,_}, _, _) -> '[]'; lexpr({cons,_,H,T}, _, Opts) -> @@ -519,7 +528,7 @@ lexpr({record, _, Name, Fs}, Prec, Opts) -> lexpr({record_field, _, Rec, Name, F}, Prec, Opts) -> {L,P,R} = inop_prec('#'), Rl = lexpr(Rec, L, Opts), - Nl = leaf(format("#~w.", [Name])), + Nl = [$#,{atom,Name},$.], El = [Rl,Nl,lexpr(F, R, Opts)], maybe_paren(P, Prec, El); lexpr({record, _, Rec, Name, Fs}, Prec, Opts) -> @@ -538,12 +547,12 @@ lexpr({record_field, _, Rec, F}, Prec, Opts) -> maybe_paren(P, Prec, El); lexpr({map, _, Fs}, Prec, Opts) -> {P,_R} = preop_prec('#'), - El = {first,leaf("#"),map_fields(Fs, Opts)}, + El = {first,$#,map_fields(Fs, Opts)}, maybe_paren(P, Prec, El); lexpr({map, _, Map, Fs}, Prec, Opts) -> {L,P,_R} = inop_prec('#'), Rl = lexpr(Map, L, Opts), - El = {first,[Rl,leaf("#")],map_fields(Fs, Opts)}, + El = {first,[Rl,$#],map_fields(Fs, Opts)}, maybe_paren(P, Prec, El); lexpr({block,_,Es}, _, Opts) -> {list,[{step,'begin',body(Es, Opts)},'end']}; @@ -563,13 +572,16 @@ lexpr({'receive',_,Cs,To,ToOpt}, _, Opts) -> {step,'after',Al}, 'end']}; lexpr({'fun',_,{function,F,A}}, _Prec, _Opts) -> - leaf(format("fun ~w/~w", [F,A])); -lexpr({'fun',_,{function,F,A},Extra}, _Prec, _Opts) -> - {force_nl,fun_info(Extra),leaf(format("fun ~w/~w", [F,A]))}; -lexpr({'fun',_,{function,M,F,A}}, _Prec, _Opts) + [leaf("fun "),{atom,F},leaf(format("/~w", [A]))]; +lexpr({'fun',L,{function,_,_}=Func,Extra}, Prec, Opts) -> + {force_nl,fun_info(Extra),lexpr({'fun',L,Func}, Prec, Opts)}; +lexpr({'fun',L,{function,M,F,A}}, Prec, Opts) when is_atom(M), is_atom(F), is_integer(A) -> %% For backward compatibility with pre-R15 abstract format. - leaf(format("fun ~w:~w/~w", [M,F,A])); + Mod = erl_parse:abstract(M), + Fun = erl_parse:abstract(F), + Arity = erl_parse:abstract(A), + lexpr({'fun',L,{function,Mod,Fun,Arity}}, Prec, Opts); lexpr({'fun',_,{function,M,F,A}}, _Prec, Opts) -> %% New format in R15. NameItem = lexpr(M, Opts), @@ -660,7 +672,7 @@ lexpr({bin,_,Fs}, _, Opts) -> bit_grp(Fs, Opts); %% Special case for straight values. lexpr({value,_,Val}, _,_) -> - leaf(write(Val)); + {value,Val}; %% Now do the hook. lexpr(Other, _Precedence, #options{hook = none}) -> leaf(format("INVALID-FORM:~w:",[Other])); @@ -676,7 +688,7 @@ call(Name, Args, Prec, Opts) -> maybe_paren(P, Prec, Item). fun_info(Extra) -> - leaf(format("% fun-info: ~w", [Extra])). + [leaf("% fun-info: "),{value,Extra}]. %% BITS: @@ -717,7 +729,7 @@ bit_elem_type(T) -> %% end of BITS record_name(Name) -> - leaf(format("#~w", [Name])). + [$#,{atom,Name}]. record_fields(Fs, Opts) -> tuple(Fs, fun record_field/2, Opts). @@ -919,8 +931,10 @@ frmt(Item, I, PP) -> %%% - {force_nl,ExtraInfo,I}: fun-info (a comment) forces linebreak before I. %%% - {prefer_nl,Sep,IPs}: forces linebreak between Is unlesss negative %%% indentation. +%%% - {atom,A}: an atom %%% - {char,C}: a character %%% - {string,S}: a string. +%%% - {value,T}: a term. %%% - {hook,...}, {ehook,...}: hook expressions. %%% %%% list, first, seq, force_nl, and prefer_nl all accept IPs, where each @@ -981,6 +995,10 @@ f({prefer_nl,Sep,LItems}, I0, ST, WT, PP) -> true -> {insert_newlines(CharsSize2L, I0, ST),nsz(lists:last(Sizes), I0)} end; +f({value,V}, I, ST, WT, PP) -> + f(write_a_value(V, PP), I, ST, WT, PP); +f({atom,A}, I, ST, WT, PP) -> + f(write_an_atom(A, PP), I, ST, WT, PP); f({char,C}, I, ST, WT, PP) -> f(write_a_char(C, PP), I, ST, WT, PP); f({string,S}, I, ST, WT, PP) -> @@ -1119,6 +1137,12 @@ has_nl([C|Cs]) -> has_nl([]) -> false. +write_a_value(V, PP) -> + flat_leaf(write_value(V, PP)). + +write_an_atom(A, PP) -> + flat_leaf(write_atom(A, PP)). + write_a_char(C, PP) -> flat_leaf(write_char(C, PP)). @@ -1135,7 +1159,7 @@ write_a_string([], _N, _Len, _PP) -> write_a_string(S, N, Len, PP) -> SS = string:sub_string(S, 1, N), Sl = write_string(SS, PP), - case (length(Sl) > Len) and (N > ?MIN_SUBSTRING) of + case (chars_size(Sl) > Len) and (N > ?MIN_SUBSTRING) of true -> write_a_string(S, N-1, Len, PP); false -> @@ -1147,11 +1171,17 @@ flat_leaf(S) -> L = lists:flatten(S), {leaf,length(L),L}. +write_value(V, PP) -> + (PP#pp.value_fun)(V). + +write_atom(A, PP) -> + (PP#pp.value_fun)(A). + write_string(S, PP) -> - lists:flatten((PP#pp.string_fun)(S)). + (PP#pp.string_fun)(S). write_char(C, PP) -> - lists:flatten((PP#pp.char_fun)(C)). + (PP#pp.char_fun)(C). %% %% Utilities diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index 1a028204b4..808ba9b4c1 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -51,7 +51,7 @@ otp_6321/1, otp_6911/1, otp_6914/1, otp_8150/1, otp_8238/1, otp_8473/1, otp_8522/1, otp_8567/1, otp_8664/1, otp_9147/1, otp_10302/1, otp_10820/1, otp_11100/1, otp_11861/1, pr_1014/1, - otp_13662/1]). + otp_13662/1, otp_14285/1]). %% Internal export. -export([ehook/6]). @@ -80,7 +80,8 @@ groups() -> {tickets, [], [otp_6321, otp_6911, otp_6914, otp_8150, otp_8238, otp_8473, otp_8522, otp_8567, otp_8664, otp_9147, - otp_10302, otp_10820, otp_11100, otp_11861, pr_1014, otp_13662]}]. + otp_10302, otp_10820, otp_11100, otp_11861, pr_1014, otp_13662, + otp_14285]}]. init_per_suite(Config) -> Config. @@ -627,11 +628,6 @@ do_hook(HookFun) -> true = "{some,value}" =:= lists:flatten(erl_pp:expr({value,A0,{some,value}})), - %% Silly... - true = - "if true -> 0 end" =:= - flat_expr({'if',0,[{clause,0,[],[],[{atom,0,0}]}]}), - %% More compatibility: before R6 OldIf = {'if',A0,[{clause,A0,[],[{atom,A0,true}],[{atom,A0,b}]}]}, NewIf = {'if',A0,[{clause,A0,[],[[{atom,A0,true}]],[{atom,A0,b}]}]}, @@ -1069,9 +1065,6 @@ otp_11100(Config) when is_list(Config) -> %% doesn't make a difference (pp:bit_elem_type/1 is an example). A1 = erl_anno:new(1), - %% Cannot trigger the use of the hook function with export/import. - "-export([{fy,a}/b]).\n" = - pf({attribute,A1,export,[{{fy,a},b}]}), "-type foo() :: integer(INVALID-FORM:{foo,bar}:).\n" = pf({attribute,A1,type,{foo,{type,A1,integer,[{foo,bar}]},[]}}), pf({attribute,A1,type, @@ -1146,6 +1139,34 @@ otp_13662(Config) -> ], compile(Config, Ts). +otp_14285(_Config) -> + pp_forms(<<"-export([t/0, '\\x{400}\\''/0]).">>), + pp_forms(<<"-import(lists, [append/2]).">>), + pp_forms(<<"-optional_callbacks([]).">>), + pp_forms(<<"-optional_callbacks(['\\x{400}\\''/1]).">>), + pp_forms(<<"-'\\x{400}\\''('\\x{400}\\'').">>), + pp_forms(<<"-type '\\x{400}\\''() :: '\\x{400}\\''.">>), + pp_forms(<<"-record('\\x{400}\\'', {'\\x{400}\\''}).">>), + pp_forms(<<"-callback '\\x{400}\\''(_) -> '\\x{400}\\''.">>), + pp_forms(<<"t() -> '\\x{400}\\''('\\x{400}\\'').">>), + pp_forms(<<"'\\x{400}\\''(_) -> '\\x{400}\\''.">>), + pp_forms(<<"-spec '\\x{400}'() -> " + "#'\\x{400}'{'\\x{400}' :: '\\x{400}'}.">>), + pp_forms(<<"'\\x{400}\\''() ->" + "R = #'\\x{400}\\''{}," + "#'\\x{400}\\''{'\\x{400}\\'' =" + "{'\\x{400}\\''," + "fun '\\x{400}\\''/0," + "R#'\\x{400}\\''.'\\x{400}\\''," + "#'\\x{400}\\''.'\\x{400}\\''}}.">>), + + %% Special... + true = + "{some,'\\x{400}\\''}" =:= + lists:flatten(erl_pp:expr({value,erl_anno:new(0),{some,'\x{400}\''}}, + [{encoding,latin1}])), + ok. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% compile(Config, Tests) -> -- cgit v1.2.3 From 46e08e6ac477d1acccb360ad5d616c96dcdfe850 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Tue, 21 Mar 2017 16:34:32 +0100 Subject: stdlib: Fix Erlang shell regarding Unicode atoms --- lib/stdlib/doc/src/shell.xml | 4 ++-- lib/stdlib/src/shell.erl | 17 +++++++++++++---- lib/stdlib/test/shell_SUITE.erl | 23 ++++++++++++++++++++--- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lib/stdlib/doc/src/shell.xml b/lib/stdlib/doc/src/shell.xml index f52bc39deb..ab62c2fcdd 100644 --- a/lib/stdlib/doc/src/shell.xml +++ b/lib/stdlib/doc/src/shell.xml @@ -4,7 +4,7 @@
- 19962016 + 19962017 Ericsson AB. All Rights Reserved. @@ -854,7 +854,7 @@ q - quit erlang {history, N}, where N is the current command number. The function is to return a list of characters or an atom. This constraint is because of the Erlang I/O protocol. Unicode characters - beyond code point 255 are allowed in the list. Notice + beyond code point 255 are allowed in the list and the atom. Notice that in restricted mode the call Mod:Func(L) must be allowed or the default shell prompt function is called.

diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index 28f37ef8bf..394f4f2fa4 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -349,10 +349,16 @@ default_prompt(N) -> %% Don't bother flattening the list irrespective of what the %% I/O-protocol states. case is_alive() of - true -> io_lib:format(<<"(~s)~w> ">>, [node(), N]); + true -> io_lib:format(<<"(~ts)~w> ">>, [node_string(), N]); false -> io_lib:format(<<"~w> ">>, [N]) end. +node_string() -> + case encoding() of + latin1 -> io_lib:write_atom_as_latin1(node()); + _ -> io_lib:write_atom(node()) + end. + %% expand_hist(Expressions, CommandNumber) %% Preprocess the expression list replacing all history list commands %% with their expansions. @@ -967,10 +973,11 @@ local_func(f, [{var,_,Name}], Bs, _Shell, _RT, _Lf, _Ef) -> {value,ok,erl_eval:del_binding(Name, Bs)}; local_func(f, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,f,1}]); -local_func(rd, [{atom,_,RecName},RecDef0], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(rd, [{atom,_,RecName0},RecDef0], Bs, _Shell, RT, _Lf, _Ef) -> RecDef = expand_value(RecDef0), RDs = lists:flatten(erl_pp:expr(RecDef)), - Attr = lists:concat(["-record('", RecName, "',", RDs, ")."]), + RecName = io_lib:write_atom_as_latin1(RecName0), + Attr = lists:concat(["-record(", RecName, ",", RDs, ")."]), {ok, Tokens, _} = erl_scan:string(Attr), case erl_parse:parse_form(Tokens) of {ok,AttrForm} -> @@ -1417,9 +1424,11 @@ columns() -> {ok,N} -> N; _ -> 80 end. + encoding() -> [{encoding, Encoding}] = enc(), Encoding. + enc() -> case lists:keyfind(encoding, 1, io:getopts()) of false -> [{encoding,latin1}]; % should never happen diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index 4864bc3d72..56002dda25 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,7 +30,8 @@ progex_bit_syntax/1, progex_records/1, progex_lc/1, progex_funs/1, otp_5990/1, otp_6166/1, otp_6554/1, - otp_7184/1, otp_7232/1, otp_8393/1, otp_10302/1, otp_13719/1]). + otp_7184/1, otp_7232/1, otp_8393/1, otp_10302/1, otp_13719/1, + otp_14285/1]). -export([ start_restricted_from_shell/1, start_restricted_on_command_line/1,restricted_local/1]). @@ -91,7 +92,7 @@ groups() -> progex_funs]}, {tickets, [], [otp_5990, otp_6166, otp_6554, otp_7184, - otp_7232, otp_8393, otp_10302, otp_13719]}]. + otp_7232, otp_8393, otp_10302, otp_13719, otp_14285]}]. init_per_suite(Config) -> Config. @@ -2824,6 +2825,22 @@ otp_13719(Config) when is_list(Config) -> file:delete(File), ok. +otp_14285(Config) -> + {ok,Node} = start_node(shell_suite_helper_4, + "-pa "++proplists:get_value(priv_dir,Config)++ + " +pc unicode"), + Test1 = + <<"begin + io:setopts([{encoding,utf8}]), + [1024] = atom_to_list('\\x{400}'), + rd('\\x{400}', {'\\x{400}' = '\\x{400}'}), + ok = rl('\\x{400}') + end.">>, + "-record('\x{400}',{'\x{400}' = '\x{400}'}).\nok.\n" = + t({Node,Test1}), + test_server:stop_node(Node), + ok. + scan(B) -> F = fun(Ts) -> case erl_parse:parse_term(Ts) of -- cgit v1.2.3 From 25271fa55aacf0b367ad74532c952352344ed97d Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Thu, 2 Feb 2017 10:16:07 +0100 Subject: parsetools: Fix Leex regarding Unicode atoms Notice that macro names are unquoted atoms. --- lib/parsetools/doc/src/leex.xml | 5 ++- lib/parsetools/src/leex.erl | 92 +++++++++++++++++++++++--------------- lib/parsetools/test/leex_SUITE.erl | 45 +++++++++++++++++-- 3 files changed, 100 insertions(+), 42 deletions(-) diff --git a/lib/parsetools/doc/src/leex.xml b/lib/parsetools/doc/src/leex.xml index 29d546105f..1227625287 100644 --- a/lib/parsetools/doc/src/leex.xml +++ b/lib/parsetools/doc/src/leex.xml @@ -4,7 +4,7 @@
- 20092016 + 20092017 Ericsson AB. All Rights Reserved. @@ -446,7 +446,8 @@ D = [0-9] -

The following examples define Erlang data types:

+

The following examples define simplified versions of a few + Erlang data types:

Atoms [a-z][0-9a-zA-Z_]* diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl index e0f37ae9df..e2e7d7359f 100644 --- a/lib/parsetools/src/leex.erl +++ b/lib/parsetools/src/leex.erl @@ -1548,22 +1548,23 @@ out_action_code(File, XrlFile, {_A,Code,_Vars,Name,Args,ArgsChars}) -> L = erl_scan:line(hd(Code)), output_file_directive(File, XrlFile, L-2), io:fwrite(File, "~s(~s) ->~n", [Name, ArgsChars]), - io:fwrite(File, " ~s\n", [pp_tokens(Code, L)]). + io:fwrite(File, " ~ts\n", [pp_tokens(Code, L, File)]). -%% pp_tokens(Tokens, Line) -> [char()]. +%% pp_tokens(Tokens, Line, File) -> [char()]. %% Prints the tokens keeping the line breaks of the original code. -pp_tokens(Tokens, Line0) -> pp_tokens(Tokens, Line0, none). +pp_tokens(Tokens, Line0, File) -> pp_tokens(Tokens, Line0, File, none). -pp_tokens([], _Line0, _) -> []; -pp_tokens([T | Ts], Line0, Prev) -> +pp_tokens([], _Line0, _, _) -> []; +pp_tokens([T | Ts], Line0, File, Prev) -> Line = erl_scan:line(T), - [pp_sep(Line, Line0, Prev, T), pp_symbol(T) | pp_tokens(Ts, Line, T)]. + [pp_sep(Line, Line0, Prev, T), + pp_symbol(T, File) | pp_tokens(Ts, Line, File, T)]. -pp_symbol({var,_,Var}) -> atom_to_list(Var); -pp_symbol({_,_,Symbol}) -> io_lib:fwrite("~p", [Symbol]); -pp_symbol({dot, _}) -> "."; -pp_symbol({Symbol, _}) -> atom_to_list(Symbol). +pp_symbol({var,_,Var}, _) -> atom_to_list(Var); +pp_symbol({_,_,Symbol}, File) -> format_symbol(Symbol, File); +pp_symbol({dot, _}, _) -> "."; +pp_symbol({Symbol, _}, _) -> atom_to_list(Symbol). pp_sep(Line, Line0, Prev, T) when Line > Line0 -> ["\n " | pp_sep(Line - 1, Line0, Prev, T)]; @@ -1622,17 +1623,17 @@ out_dfa_edges(File, DFA) -> end, orddict:new(), Pt), foreach(fun (T) -> Crs = orddict:fetch(T, Tdict), - Edgelab = dfa_edgelabel(Crs), + Edgelab = dfa_edgelabel(Crs, File), io:fwrite(File, " ~b -> ~b [label=\"~ts\"];~n", [S,T,Edgelab]) end, sort(orddict:fetch_keys(Tdict))) end, DFA). -dfa_edgelabel([C]) when is_integer(C) -> quote(C); -dfa_edgelabel(Cranges) -> +dfa_edgelabel([C], File) when is_integer(C) -> quote(C, File); +dfa_edgelabel(Cranges, File) -> %% io:fwrite("el: ~p\n", [Cranges]), - "[" ++ map(fun ({A,B}) -> [quote(A), "-", quote(B)]; - (C) -> [quote(C)] + "[" ++ map(fun ({A,B}) -> [quote(A, File), "-", quote(B, File)]; + (C) -> [quote(C, File)] end, Cranges) ++ "]". set_encoding(#leex{encoding = none}, File) -> @@ -1651,33 +1652,50 @@ output_file_directive(File, Filename, Line) -> format_filename(Filename0, File) -> Filename = filename:flatten(Filename0), + case enc(File) of + unicode -> io_lib:write_string(Filename); + latin1 -> io_lib:write_string_as_latin1(Filename) + end. + +format_symbol(Symbol, File) -> + Format = case enc(File) of + latin1 -> "~p"; + unicode -> "~tp" + end, + io_lib:fwrite(Format, [Symbol]). + +enc(File) -> case lists:keyfind(encoding, 1, io:getopts(File)) of - {encoding, unicode} -> io_lib:write_string(Filename); - _ -> io_lib:write_string_as_latin1(Filename) + false -> latin1; % should never happen + {encoding, Enc} -> Enc end. -quote($^) -> "\\^"; -quote($.) -> "\\."; -quote($$) -> "\\$"; -quote($-) -> "\\-"; -quote($[) -> "\\["; -quote($]) -> "\\]"; -quote($\s) -> "\\\\s"; -quote($\") -> "\\\""; -quote($\b) -> "\\\\b"; -quote($\f) -> "\\\\f"; -quote($\n) -> "\\\\n"; -quote($\r) -> "\\\\r"; -quote($\t) -> "\\\\t"; -quote($\e) -> "\\\\e"; -quote($\v) -> "\\\\v"; -quote($\d) -> "\\\\d"; -quote($\\) -> "\\\\"; -quote(C) when is_integer(C) -> +quote($^, _File) -> "\\^"; +quote($., _File) -> "\\."; +quote($$, _File) -> "\\$"; +quote($-, _File) -> "\\-"; +quote($[, _File) -> "\\["; +quote($], _File) -> "\\]"; +quote($\s, _File) -> "\\\\s"; +quote($\", _File) -> "\\\""; +quote($\b, _File) -> "\\\\b"; +quote($\f, _File) -> "\\\\f"; +quote($\n, _File) -> "\\\\n"; +quote($\r, _File) -> "\\\\r"; +quote($\t, _File) -> "\\\\t"; +quote($\e, _File) -> "\\\\e"; +quote($\v, _File) -> "\\\\v"; +quote($\d, _File) -> "\\\\d"; +quote($\\, _File) -> "\\\\"; +quote(C, File) when is_integer(C) -> %% Must remove the $ and get the \'s right. - case io_lib:write_char(C) of + S = case enc(File) of + unicode -> io_lib:write_char(C); + latin1 -> io_lib:write_char_as_latin1(C) + end, + case S of [$$,$\\|Cs] -> "\\\\" ++ Cs; [$$|Cs] -> Cs end; -quote(maxchar) -> +quote(maxchar, _File) -> "MAXCHAR". diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index 54602848ec..3f5d9fee3e 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ pt/1, man/1, ex/1, ex2/1, not_yet/1, line_wrap/1, - otp_10302/1, otp_11286/1, unicode/1, otp_13916/1]). + otp_10302/1, otp_11286/1, unicode/1, otp_13916/1, otp_14285/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(1)). @@ -67,7 +67,7 @@ all() -> groups() -> [{checks, [], [file, compile, syntax]}, {examples, [], [pt, man, ex, ex2, not_yet, unicode]}, - {tickets, [], [otp_10302, otp_11286, otp_13916]}, + {tickets, [], [otp_10302, otp_11286, otp_13916, otp_14285]}, {bugs, [], [line_wrap]}]. init_per_suite(Config) -> @@ -1131,6 +1131,45 @@ otp_13916(Config) when is_list(Config) -> ?line run(Config, Ts), ok. +otp_14285(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.xrl"), + + Ts = [{otp_14285_1, + <<"%% encoding: latin-1\n" + "Definitions.\n" + "A = a\n" + "Z = z\n" + "L = [{A}-{Z}]\n" + "U = [\\x{400}]\n" + "Rules.\n" + "{L}+ : {token,l}.\n" + "{U}+ : {token,'\\x{400}'}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,['\\x{400}'],1} = string(\"\\x{400}\"), ok.\n">>, + default, + ok}, + {otp_14285_2, + <<"%% encoding: UTF-8\n" + "Definitions.\n" + "A = a\n" + "Z = z\n" + "L = [{A}-{Z}]\n" + "U = [\x{400}]\n" + "Rules.\n" + "{L}+ : {token,l}.\n" + "{U}+ : {token,'\x{400}'}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,['\x{400}'],1} = string(\"\x{400}\"), ok.\n">>, + default, + ok}], + run(Config, Ts), + ok. + start_node(Name, Args) -> [_,Host] = string:tokens(atom_to_list(node()), "@"), ct:log("Trying to start ~w@~s~n", [Name,Host]), -- cgit v1.2.3 From 9c013d50cc5abf3b0a0dbb5fc2be97c825bc0261 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Thu, 23 Mar 2017 16:36:17 +0100 Subject: parsetools: Fix Yecc regarding Unicode atoms Terminals and non-terminals still need to be quoted, which is a side-effect of using the Erlang scanner. --- lib/parsetools/src/yecc.erl | 116 +++++++++++++++++++------------------ lib/parsetools/test/yecc_SUITE.erl | 88 +++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 57 deletions(-) diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl index 05446c1a85..48559ec402 100644 --- a/lib/parsetools/src/yecc.erl +++ b/lib/parsetools/src/yecc.erl @@ -154,13 +154,13 @@ compile(Input0, Output0, format_error(bad_declaration) -> io_lib:fwrite("unknown or bad declaration, ignored", []); format_error({bad_expect, SymName}) -> - io_lib:fwrite("argument ~s of Expect is not an integer", + io_lib:fwrite("argument ~ts of Expect is not an integer", [format_symbol(SymName)]); format_error({bad_rootsymbol, SymName}) -> - io_lib:fwrite("rootsymbol ~s is not a nonterminal", + io_lib:fwrite("rootsymbol ~ts is not a nonterminal", [format_symbol(SymName)]); format_error({bad_states, SymName}) -> - io_lib:fwrite("argument ~s of States is not an integer", + io_lib:fwrite("argument ~ts of States is not an integer", [format_symbol(SymName)]); format_error({conflict, Conflict}) -> format_conflict(Conflict); @@ -169,19 +169,19 @@ format_error({conflicts, SR, RR}) -> format_error({duplicate_declaration, Tag}) -> io_lib:fwrite("duplicate declaration of ~s", [atom_to_list(Tag)]); format_error({duplicate_nonterminal, Nonterminal}) -> - io_lib:fwrite("duplicate non-terminals ~s", + io_lib:fwrite("duplicate non-terminals ~ts", [format_symbol(Nonterminal)]); format_error({duplicate_precedence, Op}) -> - io_lib:fwrite("duplicate precedence operator ~s", + io_lib:fwrite("duplicate precedence operator ~ts", [format_symbol(Op)]); format_error({duplicate_terminal, Terminal}) -> - io_lib:fwrite("duplicate terminal ~s", + io_lib:fwrite("duplicate terminal ~ts", [format_symbol(Terminal)]); format_error({endsymbol_is_nonterminal, Symbol}) -> - io_lib:fwrite("endsymbol ~s is a nonterminal", + io_lib:fwrite("endsymbol ~ts is a nonterminal", [format_symbol(Symbol)]); format_error({endsymbol_is_terminal, Symbol}) -> - io_lib:fwrite("endsymbol ~s is a terminal", + io_lib:fwrite("endsymbol ~ts is a terminal", [format_symbol(Symbol)]); format_error({error, Module, Error}) -> Module:format_error(Error); @@ -192,7 +192,7 @@ format_error(illegal_empty) -> format_error({internal_error, Error}) -> io_lib:fwrite("internal yecc error: ~w", [Error]); format_error({missing_syntax_rule, Nonterminal}) -> - io_lib:fwrite("no syntax rule for non-terminal symbol ~s", + io_lib:fwrite("no syntax rule for non-terminal symbol ~ts", [format_symbol(Nonterminal)]); format_error({n_states, Exp, N}) -> io_lib:fwrite("expected ~w states, but got ~p states", [Exp, N]); @@ -201,31 +201,31 @@ format_error(no_grammar_rules) -> format_error(nonterminals_missing) -> io_lib:fwrite("Nonterminals is missing", []); format_error({precedence_op_is_endsymbol, SymName}) -> - io_lib:fwrite("precedence operator ~s is endsymbol", + io_lib:fwrite("precedence operator ~ts is endsymbol", [format_symbol(SymName)]); format_error({precedence_op_is_unknown, SymName}) -> - io_lib:fwrite("unknown precedence operator ~s", + io_lib:fwrite("unknown precedence operator ~ts", [format_symbol(SymName)]); format_error({reserved, N}) -> io_lib:fwrite("the use of ~w should be avoided", [N]); format_error({symbol_terminal_and_nonterminal, SymName}) -> - io_lib:fwrite("symbol ~s is both a terminal and nonterminal", + io_lib:fwrite("symbol ~ts is both a terminal and nonterminal", [format_symbol(SymName)]); format_error(rootsymbol_missing) -> io_lib:fwrite("Rootsymbol is missing", []); format_error(terminals_missing) -> io_lib:fwrite("Terminals is missing", []); format_error({undefined_nonterminal, Symbol}) -> - io_lib:fwrite("undefined nonterminal: ~s", [format_symbol(Symbol)]); + io_lib:fwrite("undefined nonterminal: ~ts", [format_symbol(Symbol)]); format_error({undefined_pseudo_variable, Atom}) -> io_lib:fwrite("undefined pseudo variable ~w", [Atom]); format_error({undefined_symbol, SymName}) -> - io_lib:fwrite("undefined rhs symbol ~s", [format_symbol(SymName)]); + io_lib:fwrite("undefined rhs symbol ~ts", [format_symbol(SymName)]); format_error({unused_nonterminal, Nonterminal}) -> - io_lib:fwrite("non-terminal symbol ~s not used", + io_lib:fwrite("non-terminal symbol ~ts not used", [format_symbol(Nonterminal)]); format_error({unused_terminal, Terminal}) -> - io_lib:fwrite("terminal symbol ~s not used", + io_lib:fwrite("terminal symbol ~ts not used", [format_symbol(Terminal)]); format_error({bad_symbol, String}) -> io_lib:fwrite("bad symbol ~ts", [String]); @@ -1809,9 +1809,9 @@ report_conflict(Conflict, St, ActionName, How) -> Formated = format_symbol(ActionName), case How of prec -> - io:fwrite(<<"Resolved in favor of ~s.\n\n">>, [Formated]); + io:fwrite(<<"Resolved in favor of ~ts.\n\n">>, [Formated]); default -> - io:fwrite(<<"Conflict resolved in favor of ~s.\n\n">>, + io:fwrite(<<"Conflict resolved in favor of ~ts.\n\n">>, [Formated]) end; true -> @@ -1856,7 +1856,7 @@ format_conflict({Symbol, N, _, {one_level_up, {L1, RuleN1, {P1, Ass1}}, {L2, RuleN2, {P2, Ass2}}}}) -> S1 = io_lib:fwrite(<<"Conflicting precedences of symbols when " - "scanning ~s in state ~w:\n">>, + "scanning ~ts in state ~w:\n">>, [format_symbol(Symbol), N]), S2 = io_lib:fwrite(<<" ~s ~w (rule ~w at line ~w)\n" " vs.\n">>, @@ -1866,26 +1866,26 @@ format_conflict({Symbol, N, _, {one_level_up, [S1, S2, S3]; format_conflict({Symbol, N, Reduce, Confl}) -> S1 = io_lib:fwrite(<<"Parse action conflict scanning symbol " - "~s in state ~w:\n">>, [format_symbol(Symbol), N]), + "~ts in state ~w:\n">>, [format_symbol(Symbol), N]), S2 = case Reduce of {[HR | TR], RuleNmbr, RuleLine} -> - io_lib:fwrite(<<" Reduce to ~s from ~s (rule ~w at " + io_lib:fwrite(<<" Reduce to ~ts from ~ts (rule ~w at " "line ~w)\n vs.\n">>, [format_symbol(HR), format_symbols(TR), RuleNmbr, RuleLine]) end, S3 = case Confl of {reduce, [HR2|TR2], RuleNmbr2, RuleLine2} -> - io_lib:fwrite(<<" reduce to ~s from ~s " + io_lib:fwrite(<<" reduce to ~ts from ~ts " "(rule ~w at line ~w).">>, [format_symbol(HR2), format_symbols(TR2), RuleNmbr2, RuleLine2]); {shift, NewState, Sym} -> io_lib:fwrite(<<" shift to state ~w, adding right " - "sisters to ~s.">>, + "sisters to ~ts.">>, [NewState, format_symbol(Sym)]); {accept, Rootsymbol} -> - io_lib:fwrite(<<" reduce to rootsymbol ~s.">>, + io_lib:fwrite(<<" reduce to rootsymbol ~ts.">>, [format_symbol(Rootsymbol)]) end, [S1, S2, S3]. @@ -1926,8 +1926,9 @@ format_conflict({Symbol, N, Reduce, Confl}) -> -define(CODE_VERSION, "1.4"). -define(YECC_BUG(M, A), - iolist_to_binary([" erlang:error({yecc_bug,\"",?CODE_VERSION,"\",", - io_lib:fwrite(M, A), "}).\n\n"])). + unicode:characters_to_binary( + [" erlang:error({yecc_bug,\"",?CODE_VERSION,"\",", + io_lib:fwrite(M, A), "}).\n\n"])). %% Returns number of newlines in included files. output_prelude(Outport, Inport, St0) when St0#yecc.includefile =:= [] -> @@ -1980,7 +1981,7 @@ output_header(St0) -> output_goto(St, [{_Nonterminal, []} | Go], StateInfo) -> output_goto(St, Go, StateInfo); output_goto(St0, [{Nonterminal, List} | Go], StateInfo) -> - F = function_name(yeccgoto, Nonterminal), + F = function_name(St0, yeccgoto, Nonterminal), St05 = fwrite(St0, <<"-dialyzer({nowarn_function, ~w/7}).\n">>, [F]), St10 = output_goto1(St05, List, F, StateInfo, true), St = output_goto_fini(F, Nonterminal, St10), @@ -2018,7 +2019,8 @@ output_goto_fini(F, NT, #yecc{includefile_version = {1,1}}=St0) -> St = fwrite(St10, <<"~w(State, _Cat, _Ss, _Stack, _T, _Ts, _Tzr) ->\n">>, [F]), fwrite(St, - ?YECC_BUG(<<"{~w, State, missing_in_goto_table}">>, [NT]), + ?YECC_BUG(<<"{~ts, State, missing_in_goto_table}">>, + [quoted_atom(St0, NT)]), []); output_goto_fini(_F, _NT, St) -> fwrite(St, <<".\n\n">>, []). @@ -2027,7 +2029,7 @@ output_goto_fini(_F, _NT, St) -> find_user_code(ParseActions, St) -> [#user_code{state = State, terminal = Terminal, - funname = inlined_function_name(State, Terminal), + funname = inlined_function_name(St, State, Terminal), action = Action} || {State, La_actions} <- ParseActions, {Action, Terminals, RuleNmbr, NmbrOfDaughters} @@ -2148,14 +2150,14 @@ output_action(St, State, Terminal, #reduce{}=Action, IsFirst, SI) -> output_reduce(St, State, Terminal, Action, IsFirst, SI); output_action(St0, State, Terminal, #shift{state = NewState}, IsFirst, _SI) -> St10 = delim(St0, IsFirst), - St = fwrite(St10, <<"yeccpars2_~w(S, ~s, Ss, Stack, T, Ts, Tzr) ->\n">>, - [State, quoted_atom(Terminal)]), + St = fwrite(St10, <<"yeccpars2_~w(S, ~ts, Ss, Stack, T, Ts, Tzr) ->\n">>, + [State, quoted_atom(St10, Terminal)]), output_call_to_includefile(NewState, St); output_action(St0, State, Terminal, accept, IsFirst, _SI) -> St10 = delim(St0, IsFirst), St = fwrite(St10, - <<"yeccpars2_~w(_S, ~s, _Ss, Stack, _T, _Ts, _Tzr) ->\n">>, - [State, quoted_atom(Terminal)]), + <<"yeccpars2_~w(_S, ~ts, _Ss, Stack, _T, _Ts, _Tzr) ->\n">>, + [State, quoted_atom(St10, Terminal)]), fwrite(St, <<" {ok, hd(Stack)}">>, []); output_action(St, _State, _Terminal, nonassoc, _IsFirst, _SI) -> St. @@ -2174,19 +2176,19 @@ output_state_actions_fini(State, IsFirst, St0) -> St = fwrite(St10, <<"yeccpars2_~w(_, _, _, _, T, _, _) ->\n">>, [State]), fwrite(St, <<" yeccerror(T).\n\n">>, []). -output_reduce(St0, State, Terminal0, +output_reduce(St0, State, Terminal, #reduce{rule_nmbr = RuleNmbr, head = Head, nmbr_of_daughters = NmbrOfDaughters}, IsFirst, StateInfo) -> St10 = delim(St0, IsFirst), - Terminal = if - is_atom(Terminal0) -> quoted_atom(Terminal0); - true -> Terminal0 - end, + QuotedTerminal = if + is_atom(Terminal) -> quoted_atom(St10, Terminal); + true -> Terminal + end, St20 = fwrite(St10, - <<"yeccpars2_~w(_S, ~s, Ss, Stack, T, Ts, Tzr) ->\n">>, - [State, Terminal]), + <<"yeccpars2_~w(_S, ~ts, Ss, Stack, T, Ts, Tzr) ->\n">>, + [State, QuotedTerminal]), St30 = if NmbrOfDaughters < 2 -> @@ -2205,7 +2207,7 @@ output_reduce(St0, State, Terminal0, _ -> NewStack = "NewStack", fwrite(St30, <<" NewStack = ~w(Stack),\n">>, - [inlined_function_name(State, Terminal0)]) + [inlined_function_name(St30, State, Terminal)]) end, if NmbrOfDaughters =:= 0 -> @@ -2221,13 +2223,13 @@ output_reduce(St0, State, Terminal0, St = fwrite(St40, <<"~s">>, [C]), %% Short-circuit call to yeccpars2: fwrite(St, - <<" yeccpars2_~w(~s, ~s, [~w | Ss], ~s, T, Ts, Tzr)">>, - [Repr, NextS, Terminal, State, NewStack]); + <<" yeccpars2_~w(~s, ~ts, [~w | Ss], ~s, T, Ts, Tzr)">>, + [Repr, NextS, QuotedTerminal, State, NewStack]); true -> fwrite(St40, - <<" ~w(hd(~s), ~s, ~s, ~s, T, Ts, Tzr)">>, - [function_name(yeccgoto, Head), Ns, - Terminal, Ns, NewStack]) + <<" ~w(hd(~s), ~ts, ~s, ~s, T, Ts, Tzr)">>, + [function_name(St40, yeccgoto, Head), Ns, + QuotedTerminal, Ns, NewStack]) end. delim(St, true) -> @@ -2235,8 +2237,10 @@ delim(St, true) -> delim(St, false) -> fwrite(St, <<";\n">>, []). -quoted_atom(Atom) -> - io_lib:fwrite(<<"~w">>, [Atom]). +quoted_atom(#yecc{encoding = latin1}, Atom) when is_atom(Atom) -> + io_lib:write_atom_as_latin1(Atom); +quoted_atom(_St, Atomic) -> + io_lib:write(Atomic). output_inlined(St, UserCodeActions, Infile) -> foldl(fun(#user_code{funname = InlinedFunctionName, @@ -2288,14 +2292,16 @@ output_inlined(St0, FunctionName, Reduce, Infile) -> fwrite(St, <<" [begin\n ~ts\n end | ~s].\n\n">>, [pp_tokens(Tokens, Line0, St#yecc.encoding), Stack]). -inlined_function_name(State, "Cat") -> - inlined_function_name(State, ""); -inlined_function_name(State, Terminal) -> - list_to_atom(concat([yeccpars2_, State, '_', Terminal])). +inlined_function_name(St, State, Terminal) -> + End = case Terminal of + "Cat" -> []; + _ -> [quoted_atom(St, Terminal)] + end, + list_to_atom(concat([yeccpars2_, State, '_'] ++ End)). --compile({nowarn_unused_function,function_name/2}). -function_name(Name, Suf) -> - list_to_atom(concat([Name, '_' | quoted_atom(Suf)])). +-compile({nowarn_unused_function,function_name/3}). +function_name(St, Name, Suf) -> + list_to_atom(concat([Name, '_'] ++ [quoted_atom(St, Suf)])). rule(RulePointer, St) -> #rule{n = N, anno = Anno, symbols = Symbols} = diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl index 2c37278d4b..a7166b91ed 100644 --- a/lib/parsetools/test/yecc_SUITE.erl +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -50,7 +50,7 @@ otp_5369/1, otp_6362/1, otp_7945/1, otp_8483/1, otp_8486/1, otp_7292/1, otp_7969/1, otp_8919/1, otp_10302/1, otp_11269/1, - otp_11286/1]). + otp_11286/1, otp_14285/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(1)). @@ -78,7 +78,7 @@ groups() -> {bugs, [], [otp_5369, otp_6362, otp_7945, otp_8483, otp_8486]}, {improvements, [], [otp_7292, otp_7969, otp_8919, otp_10302, - otp_11269, otp_11286]}]. + otp_11269, otp_11286, otp_14285]}]. init_per_suite(Config) -> Config. @@ -2048,6 +2048,90 @@ otp_11286(Config) when is_list(Config) -> true = test_server:stop_node(Node), ok. +otp_14285(Config) -> + Dir = ?privdir, + YeccPre = filename:join(Dir, "yeccpre.hrl"), + ?line ok = file:write_file(YeccPre, + [<<"-export([t/0]).\n">>,my_yeccpre()]), + + T0 = <<" + Nonterminals '\\x{400}'. + Terminals t. + Rootsymbol '\\x{400}'. + '\\x{400}' -> t : '$1'. + Erlang code. + t() -> + L = [{t, 1}], + {ok, R} = parse(L), + {t, 1} = R, + ok.">>, + Ts0 = [{otp_14285_1, + [<<"%% coding: Latin-1\n">>,T0],YeccPre,ok}, + {otp_14285_2, + [<<"%% coding: coding: UTF-8\n">>,T0],YeccPre,ok}], + run(Config, Ts0), + file:delete(YeccPre), + + T1 = <<" + Nonterminals '1\\x{400}' list 'unused\\x{400}'. + Terminals '2\\x{400}'. + Rootsymbol '1\\x{400}'. + + '1\\x{400}' -> list : '$1'. + + list -> '2\\x{400}' : '$1'. + list -> list '2\\x{400}' : {foo,'\\x{400}'}. + + Erlang code. + + -export([t/0]). + + t() -> + L = [{'2\\x{400}', 1}, {'2\\x{400}',2}], + {ok, R} = parse(L), + {foo,A} = R, + '\\x{400}' = A, + [1024] = atom_to_list(A), + ok.">>, + + Ts1 = [{otp_14285_3, + [<<"%% coding: Latin-1\n">>,T1],default,ok}, + {otp_14285_4, + [<<"%% coding: UTF-8\n">>,T1],default,ok}], + run(Config, Ts1), + + T2 = <<" + Nonterminals E. + Terminals '-' '+' '=' id. + Rootsymbol E. + Endsymbol '\\x{400}'. + + E -> E '=' E : {op, '=', '$1', '$3'}. + E -> E '+' E : {op, '+', '$1', '$3'}. + E -> '-' E : {op, '-', '$2'}. + E -> id : '$1'. + + Nonassoc 100 '='. + Right 200 '+' '-'. + + Erlang code. + + -export([t/0]). + + t() -> + {ok,{op,'=',{id,1},{op,'-',{op,'+',{id,4},{id,6}}}}} = + parse([{id,1},{'=',2},{'-',3},{id,4},{'+',5},{id,6}, + {'\\x{400}',1}]), + ok.">>, + + Ts2 = [{otp_14285_5, + [<<"%% coding: Latin-1\n">>,T2],default,ok}, + {otp_14285_6, + [<<"%% coding: UTF-8\n">>,T2],default,ok}], + run(Config, Ts2), + + ok. + start_node(Name, Args) -> [_,Host] = string:tokens(atom_to_list(node()), "@"), ct:log("Trying to start ~w@~s~n", [Name,Host]), -- cgit v1.2.3 From 15a631cb2ec70860bc58492020904b1b16fed5c4 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Wed, 29 Mar 2017 15:48:27 +0200 Subject: edoc: Fix EDoc regarding Unicode atoms Also extended the scanner to handle the \x{...} syntax. Capitalizing now works with a few more characters. --- lib/edoc/src/edoc.erl | 2 +- lib/edoc/src/edoc_doclet.erl | 2 +- lib/edoc/src/edoc_extract.erl | 12 +- lib/edoc/src/edoc_layout.erl | 299 +++++++++++++++-------------- lib/edoc/src/edoc_report.erl | 2 +- lib/edoc/src/edoc_scanner.erl | 30 +++ lib/edoc/src/edoc_specs.erl | 13 +- lib/edoc/test/edoc_SUITE.erl | 20 +- lib/edoc/test/edoc_SUITE_data/un_atom1.erl | 41 ++++ lib/edoc/test/edoc_SUITE_data/un_atom2.erl | 40 ++++ 10 files changed, 311 insertions(+), 150 deletions(-) create mode 100644 lib/edoc/test/edoc_SUITE_data/un_atom1.erl create mode 100644 lib/edoc/test/edoc_SUITE_data/un_atom2.erl diff --git a/lib/edoc/src/edoc.erl b/lib/edoc/src/edoc.erl index 7276a57268..b641118c5d 100644 --- a/lib/edoc/src/edoc.erl +++ b/lib/edoc/src/edoc.erl @@ -197,7 +197,7 @@ application(App, Dir, Options) when is_atom(App) -> ?OVERVIEW_FILE), Opts = Options ++ [{source_path, [Src]}, subpackages, - {title, io_lib:fwrite("The ~s application", [App])}, + {title, io_lib:fwrite("The ~ts application", [App])}, {overview, Overview}, {dir, filename:join(Dir, ?EDOC_DIR)}, {includes, [filename:join(Dir, "include")]}], diff --git a/lib/edoc/src/edoc_doclet.erl b/lib/edoc/src/edoc_doclet.erl index 006b07574b..6e17ec0af0 100644 --- a/lib/edoc/src/edoc_doclet.erl +++ b/lib/edoc/src/edoc_doclet.erl @@ -152,7 +152,7 @@ title(App, Options) -> if App == ?NO_APP -> "Overview"; true -> - io_lib:fwrite("Application: ~s", [App]) + io_lib:fwrite("Application: ~ts", [App]) end). diff --git a/lib/edoc/src/edoc_extract.erl b/lib/edoc/src/edoc_extract.erl index 68edad1a3e..390851e9ef 100644 --- a/lib/edoc/src/edoc_extract.erl +++ b/lib/edoc/src/edoc_extract.erl @@ -488,8 +488,15 @@ find_names([P | Ps], Ns) -> find_names([P1 | Ps], Ns); record_expr -> A = erl_syntax:record_expr_type(P), - N = list_to_atom(capitalize(erl_syntax:atom_name(A))), - find_names(Ps, [N | Ns]); + AtomName = erl_syntax:atom_name(A), + Atom = list_to_atom(AtomName), + case AtomName =:= lists:flatten(io_lib:write_atom(Atom)) of + true -> + N = list_to_atom(capitalize(AtomName)), + find_names(Ps, [N | Ns]); + false -> + find_names(Ps, Ns) + end; infix_expr -> %% this can only be a '++' operation P1 = erl_syntax:infix_expr_right(P), @@ -540,6 +547,7 @@ tidy_name_1(Cs) -> [$_ | Cs]. %% Change initial character from lowercase to uppercase. capitalize([C | Cs]) when C >= $a, C =< $z -> [C - 32 | Cs]; +capitalize([C | Cs]) when C >= $\340, C =< $\376, C /= $\367 -> [C - 32 | Cs]; capitalize(Cs) -> Cs. %% Collects the tags belonging to each entry, checks them, expands diff --git a/lib/edoc/src/edoc_layout.erl b/lib/edoc/src/edoc_layout.erl index 5ef210980c..eafab0588e 100644 --- a/lib/edoc/src/edoc_layout.erl +++ b/lib/edoc/src/edoc_layout.erl @@ -109,14 +109,20 @@ module(Element, Options) -> stylesheet, index_columns, sort_functions, + encoding, pretty_printer}). init_opts(Element, Options) -> + Encoding = case get_attrval(encoding, Element) of + "latin1" -> latin1; + _ -> utf8 + end, R = #opts{root = get_attrval(root, Element), index_columns = proplists:get_value(index_columns, Options, 1), sort_functions = proplists:get_value(sort_functions, Options, true), + encoding = Encoding, pretty_printer = proplists:get_value(pretty_printer, Options, '') }, @@ -183,8 +189,9 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) -> Desc = get_content(description, Es), ShortDesc = get_content(briefDescription, Desc), FullDesc = get_content(fullDescription, Desc), - Functions = [{function_name(E), E} || E <- get_content(functions, Es)], - Types = [{type_name(E), E} || E <- get_content(typedecls, Es)], + Functions = [{function_name(E, Opts), E} || + E <- get_content(functions, Es)], + Types = [{type_name(E, Opts), E} || E <- get_content(typedecls, Es)], SortedFs = if Opts#opts.sort_functions -> lists:sort(Functions); true -> Functions end, @@ -198,7 +205,7 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) -> ++ [?NL] ++ version(Es) ++ since(Es) - ++ behaviours(Es, Name) + ++ behaviours(Es, Name, Opts) ++ authors(Es) ++ references(Es) ++ sees(Es) @@ -215,7 +222,7 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) -> ++ [hr, ?NL] ++ navigation("bottom") ++ footer()), - Encoding = get_attrval(encoding, E), + Encoding = Opts#opts.encoding, xhtml(Title, stylesheet(Opts), Body, Encoding). module_params(Es) -> @@ -367,7 +374,7 @@ function(Name, E=#xmlElement{content = Es}, Opts) -> case typespec(get_content(typespec, Es), Opts) of [] -> signature(get_content(args, Es), - get_attrval(name, E)); + atom(get_attrval(name, E), Opts)); Spec -> Spec end}, ?NL] @@ -387,8 +394,8 @@ function(Name, E=#xmlElement{content = Es}, Opts) -> ++ sees(Es) ++ todos(Es)). -function_name(E) -> - atom(get_attrval(name, E)) ++ "/" ++ get_attrval(arity, E). +function_name(E, Opts) -> + atom(get_attrval(name, E), Opts) ++ "/" ++ get_attrval(arity, E). function_header(Name, E, Private) -> case is_exported(E) of @@ -449,7 +456,7 @@ throws(Es, Opts) -> [] -> []; Es1 -> %% Doesn't use format_type; keep it short! - [{p, (["throws ", {tt, t_utype(get_elem(type, Es1))}] + [{p, (["throws ", {tt, t_utype(get_elem(type, Es1), Opts)}] ++ local_defs(get_elem(localdef, Es1), Opts))}, ?NL] end. @@ -458,7 +465,7 @@ throws(Es, Opts) -> typespec([], _Opts) -> []; typespec(Es, Opts) -> - Name = t_name(get_elem(erlangName, Es)), + Name = t_name(get_elem(erlangName, Es), Opts), Defs = get_elem(localdef, Es), [Type] = get_elem(type, Es), format_spec(Name, Type, Defs, Opts) ++ local_defs(Defs, Opts). @@ -479,12 +486,12 @@ typedecl(Name, E=#xmlElement{content = Es}, Opts) -> ++ [{p, typedef(get_content(typedef, Es), Opts)}, ?NL] ++ fulldesc(Es)). -type_name(#xmlElement{content = Es}) -> - t_name(get_elem(erlangName, get_content(typedef, Es))). +type_name(#xmlElement{content = Es}, Opts) -> + t_name(get_elem(erlangName, get_content(typedef, Es)), Opts). typedef(Es, Opts) -> - Name = ([t_name(get_elem(erlangName, Es)), "("] - ++ seq(fun t_utype_elem/1, get_content(argtypes, Es), [")"])), + Name = ([t_name(get_elem(erlangName, Es), Opts), "("] + ++ seq(t_utype_elem_fun(Opts), get_content(argtypes, Es), [")"])), (case get_elem(type, Es) of [] -> [{b, ["abstract datatype"]}, ": ", {tt, Name}]; Type -> format_type(Name, Name, Type, [], Opts) @@ -505,7 +512,9 @@ local_defs(Es0, Last, Opts) -> localdef(E = #xmlElement{content = Es}, Last, Opts) -> Name = case get_elem(typevar, Es) of [] -> - label_anchor(N0 = t_abstype(get_content(abstype, Es)), E); + label_anchor(N0 = t_abstype(get_content(abstype, Es), + Opts), + E); [V] -> N0 = t_var(V) end, @@ -516,97 +525,99 @@ localdef(E = #xmlElement{content = Es}, Last, Opts) -> %% (fast) Erlang pretty printer). format_spec(Name, Type, Defs, #opts{pretty_printer = erl_pp}=Opts) -> try - L = t_clause(Name, Type), - O = pp_clause(Name, Type), - {R, ".\n"} = etypef(L, O), + L = t_clause(Name, Type, Opts), + O = pp_clause(Name, Type, Opts), + {R, ".\n"} = etypef(L, O, Opts), [{pre, R}] catch _:_ -> %% Should not happen. format_spec(Name, Type, Defs, Opts#opts{pretty_printer=''}) end; -format_spec(Sep, Type, Defs, _Opts) -> +format_spec(Sep, Type, Defs, Opts) -> %% Very limited formatting. Br = if Defs =:= [] -> br; true -> [] end, - [{tt, t_clause(Sep, Type)}, Br]. + [{tt, t_clause(Sep, Type, Opts)}, Br]. -t_clause(Name, Type) -> +t_clause(Name, Type, Opts) -> #xmlElement{content = [#xmlElement{name = 'fun', content = C}]} = Type, - [Name] ++ t_fun(C). + [Name] ++ t_fun(C, Opts). -pp_clause(Pre, Type) -> +pp_clause(Pre, Type, Opts) -> Types = ot_utype([Type]), - Atom = lists:duplicate(iolist_size(Pre), $a), + Atom = lists:duplicate(string:length(Pre), $a), Attr = {attribute,0,spec,{{list_to_atom(Atom),0},[Types]}}, - L1 = erl_pp:attribute(erl_parse:new_anno(Attr)), + L1 = erl_pp:attribute(erl_parse:new_anno(Attr), + [{encoding, Opts#opts.encoding}]), "-spec " ++ L2 = lists:flatten(L1), L3 = Pre ++ lists:nthtail(length(Atom), L2), - re:replace(L3, "\n ", "\n", [{return,list},global]). + re:replace(L3, "\n ", "\n", [{return,list},global,unicode]). format_type(Prefix, Name, Type, Last, #opts{pretty_printer = erl_pp}=Opts) -> try - L = t_utype(Type), - O = pp_type(Name, Type), - {R, ".\n"} = etypef(L, O), + L = t_utype(Type, Opts), + O = pp_type(Name, Type, Opts), + {R, ".\n"} = etypef(L, O, Opts), [{pre, Prefix ++ [" = "] ++ R ++ Last}] catch _:_ -> %% Example: "t() = record(a)." format_type(Prefix, Name, Type, Last, Opts#opts{pretty_printer =''}) end; -format_type(Prefix, _Name, Type, Last, _Opts) -> - [{tt, Prefix ++ [" = "] ++ t_utype(Type) ++ Last}]. +format_type(Prefix, _Name, Type, Last, Opts) -> + [{tt, Prefix ++ [" = "] ++ t_utype(Type, Opts) ++ Last}]. -pp_type(Prefix, Type) -> - Atom = list_to_atom(lists:duplicate(iolist_size(Prefix), $a)), +pp_type(Prefix, Type, Opts) -> + Atom = list_to_atom(lists:duplicate(string:length(Prefix), $a)), Attr = {attribute,0,type,{Atom,ot_utype(Type),[]}}, - L1 = erl_pp:attribute(erl_parse:new_anno(Attr)), + L1 = erl_pp:attribute(erl_parse:new_anno(Attr), + [{encoding, Opts#opts.encoding}]), {L2,N} = case lists:dropwhile(fun(C) -> C =/= $: end, lists:flatten(L1)) of ":: " ++ L3 -> {L3,9}; % compensation for extra "()" and ":" "::\n" ++ L3 -> {"\n"++L3,6} end, Ss = lists:duplicate(N, $\s), - re:replace(L2, "\n"++Ss, "\n", [{return,list},global]). + re:replace(L2, "\n"++Ss, "\n", [{return,list},global,unicode]). -etypef(L, O0) -> - {R, O} = etypef(L, [], O0, []), +etypef(L, O0, Opts) -> + {R, O} = etypef(L, [], O0, [], Opts), {lists:reverse(R), O}. -etypef([C | L], St, [C | O], R) -> - etypef(L, St, O, [[C] | R]); -etypef(" "++L, St, O, R) -> - etypef(L, St, O, R); -etypef("", [Cs | St], O, R) -> - etypef(Cs, St, O, R); -etypef("", [], O, R) -> +etypef([C | L], St, [C | O], R, Opts) -> + etypef(L, St, O, [[C] | R], Opts); +etypef(" "++L, St, O, R, Opts) -> + etypef(L, St, O, R, Opts); +etypef("", [Cs | St], O, R, Opts) -> + etypef(Cs, St, O, R, Opts); +etypef("", [], O, R, _Opts) -> {R, O}; -etypef(L, St, " "++O, R) -> - etypef(L, St, O, [" " | R]); -etypef(L, St, "\n"++O, R) -> +etypef(L, St, " "++O, R, Opts) -> + etypef(L, St, O, [" " | R], Opts); +etypef(L, St, "\n"++O, R, Opts) -> Ss = lists:takewhile(fun(C) -> C =:= $\s end, O), - etypef(L, St, lists:nthtail(length(Ss), O), ["\n"++Ss | R]); -etypef([{a, HRef, S0} | L], St, O0, R) -> - {S, O} = etypef(S0, app_fix(O0)), - etypef(L, St, O, [{a, HRef, S} | R]); -etypef("="++L, St, "::"++O, R) -> + etypef(L, St, lists:nthtail(length(Ss), O), ["\n"++Ss | R], Opts); +etypef([{a, HRef, S0} | L], St, O0, R, Opts) -> + {S, O} = etypef(S0, app_fix(O0, Opts), Opts), + etypef(L, St, O, [{a, HRef, S} | R], Opts); +etypef("="++L, St, "::"++O, R, Opts) -> %% EDoc uses "=" for record field types; Erlang types use "::". %% Maybe there should be an option for this, possibly affecting %% other similar discrepancies. - etypef(L, St, O, ["=" | R]); -etypef([Cs | L], St, O, R) -> - etypef(Cs, [L | St], O, R). + etypef(L, St, O, ["=" | R], Opts); +etypef([Cs | L], St, O, R, Opts) -> + etypef(Cs, [L | St], O, R, Opts). -app_fix(L) -> +app_fix(L, Opts) -> try - {"//" ++ R1,L2} = app_fix(L, 1), + {"//" ++ R1,L2} = app_fix1(L, 1), [App, Mod] = string:tokens(R1, "/"), - "//" ++ atom(App) ++ "/" ++ atom(Mod) ++ L2 + "//" ++ atom(App, Opts) ++ "/" ++ atom(Mod, Opts) ++ L2 catch _:_ -> L end. -app_fix(L, I) -> % a bit slow +app_fix1(L, I) -> % a bit slow {L1, L2} = lists:split(I, L), case erl_scan:tokens([], L1 ++ ". ", 1) of {done, {ok,[{atom,_,Atom}|_],_}, _} -> {atom_to_list(Atom), L2}; - _ -> app_fix(L, I+1) + _ -> app_fix1(L, I+1) end. fulldesc(Es) -> @@ -703,7 +714,7 @@ deprecated(Es, S) -> ?NL] end. -behaviours(Es, Name) -> +behaviours(Es, Name, Opts) -> CBs = get_content(callbacks, Es), OCBs = get_content(optional_callbacks, Es), (case get_elem(behaviour, Es) of @@ -717,17 +728,18 @@ behaviours(Es, Name) -> if CBs =:= [], OCBs =:= [] -> []; true -> + CBFun = fun(E) -> callback(E, Opts) end, Req = if CBs =:= [] -> []; true -> [br, " Required callback functions: "] - ++ seq(fun callback/1, CBs, ["."]) + ++ seq(CBFun, CBs, ["."]) end, Opt = if OCBs =:= [] -> []; true -> [br, " Optional callback functions: "] - ++ seq(fun callback/1, OCBs, ["."]) + ++ seq(CBFun, OCBs, ["."]) end, [{p, ([{b, ["This module defines the ", {tt, [Name]}, " behaviour."]}] @@ -738,10 +750,10 @@ behaviours(Es, Name) -> behaviour(E=#xmlElement{content = Es}) -> see(E, [{tt, Es}]). -callback(E=#xmlElement{}) -> +callback(E=#xmlElement{}, Opts) -> Name = get_attrval(name, E), Arity = get_attrval(arity, E), - [{tt, [Name, "/", Arity]}]. + [{tt, [atom(Name, Opts), "/", Arity]}]. authors(Es) -> case get_elem(author, Es) of @@ -751,7 +763,9 @@ authors(Es) -> ?NL] end. -atom(String) -> +atom(String, #opts{encoding = latin1}) -> + io_lib:write_atom_as_latin1(list_to_atom(String)); +atom(String, #opts{encoding = utf8}) -> io_lib:write_atom(list_to_atom(String)). %% ?NL] end. -t_name([E]) -> +t_name([E], Opts) -> N = get_attrval(name, E), case get_attrval(module, E) of - "" -> atom(N); + "" -> atom(N, Opts); M -> - S = atom(M) ++ ":" ++ atom(N), + S = atom(M, Opts) ++ ":" ++ atom(N, Opts), case get_attrval(app, E) of "" -> S; - A -> "//" ++ atom(A) ++ "/" ++ S + A -> "//" ++ atom(A, Opts) ++ "/" ++ S end end. -t_utype([E]) -> - t_utype_elem(E). +t_utype([E], Opts) -> + t_utype_elem(E, Opts). + +t_utype_elem_fun(Opts) -> + fun(E) -> t_utype_elem(E, Opts) end. -t_utype_elem(E=#xmlElement{content = Es}) -> +t_utype_elem(E=#xmlElement{content = Es}, Opts) -> case get_attrval(name, E) of - "" -> t_type(Es); + "" -> t_type(Es, Opts); Name -> - T = t_type(Es), + T = t_type(Es, Opts), case T of [Name] -> T; % avoid generating "Foo::Foo" T -> [Name] ++ ["::"] ++ T end end. -t_type([E=#xmlElement{name = typevar}]) -> +t_type([E=#xmlElement{name = typevar}], _Opts) -> t_var(E); -t_type([E=#xmlElement{name = atom}]) -> - t_atom(E); -t_type([E=#xmlElement{name = integer}]) -> +t_type([E=#xmlElement{name = atom}], Opts) -> + t_atom(E, Opts); +t_type([E=#xmlElement{name = integer}], _Opts) -> t_integer(E); -t_type([E=#xmlElement{name = range}]) -> +t_type([E=#xmlElement{name = range}], _Opts) -> t_range(E); -t_type([E=#xmlElement{name = binary}]) -> +t_type([E=#xmlElement{name = binary}], _Opts) -> t_binary(E); -t_type([E=#xmlElement{name = float}]) -> +t_type([E=#xmlElement{name = float}], _Opts) -> t_float(E); -t_type([#xmlElement{name = nil}]) -> +t_type([#xmlElement{name = nil}], _Opts) -> t_nil(); -t_type([#xmlElement{name = paren, content = Es}]) -> - t_paren(Es); -t_type([#xmlElement{name = list, content = Es}]) -> - t_list(Es); -t_type([#xmlElement{name = nonempty_list, content = Es}]) -> - t_nonempty_list(Es); -t_type([#xmlElement{name = map, content = Es}]) -> - t_map(Es); -t_type([#xmlElement{name = tuple, content = Es}]) -> - t_tuple(Es); -t_type([#xmlElement{name = 'fun', content = Es}]) -> - ["fun("] ++ t_fun(Es) ++ [")"]; -t_type([E = #xmlElement{name = record, content = Es}]) -> - t_record(E, Es); -t_type([E = #xmlElement{name = abstype, content = Es}]) -> - t_abstype(E, Es); -t_type([#xmlElement{name = union, content = Es}]) -> - t_union(Es). +t_type([#xmlElement{name = paren, content = Es}], Opts) -> + t_paren(Es, Opts); +t_type([#xmlElement{name = list, content = Es}], Opts) -> + t_list(Es, Opts); +t_type([#xmlElement{name = nonempty_list, content = Es}], Opts) -> + t_nonempty_list(Es, Opts); +t_type([#xmlElement{name = map, content = Es}], Opts) -> + t_map(Es, Opts); +t_type([#xmlElement{name = tuple, content = Es}], Opts) -> + t_tuple(Es, Opts); +t_type([#xmlElement{name = 'fun', content = Es}], Opts) -> + ["fun("] ++ t_fun(Es, Opts) ++ [")"]; +t_type([E = #xmlElement{name = record, content = Es}], Opts) -> + t_record(E, Es, Opts); +t_type([E = #xmlElement{name = abstype, content = Es}], Opts) -> + t_abstype(E, Es, Opts); +t_type([#xmlElement{name = union, content = Es}], Opts) -> + t_union(Es, Opts). t_var(E) -> [get_attrval(name, E)]. -t_atom(E) -> - [get_attrval(value, E)]. +t_atom(E, Opts) -> + [atom(get_attrval(value, E), Opts)]. t_integer(E) -> [get_attrval(value, E)]. @@ -879,62 +896,64 @@ t_float(E) -> t_nil() -> ["[]"]. -t_paren(Es) -> - ["("] ++ t_utype(get_elem(type, Es)) ++ [")"]. +t_paren(Es, Opts) -> + ["("] ++ t_utype(get_elem(type, Es), Opts) ++ [")"]. -t_list(Es) -> - ["["] ++ t_utype(get_elem(type, Es)) ++ ["]"]. +t_list(Es, Opts) -> + ["["] ++ t_utype(get_elem(type, Es), Opts) ++ ["]"]. -t_nonempty_list(Es) -> - ["["] ++ t_utype(get_elem(type, Es)) ++ [", ...]"]. +t_nonempty_list(Es, Opts) -> + ["["] ++ t_utype(get_elem(type, Es), Opts) ++ [", ...]"]. -t_tuple(Es) -> - ["{"] ++ seq(fun t_utype_elem/1, Es, ["}"]). +t_tuple(Es, Opts) -> + ["{"] ++ seq(t_utype_elem_fun(Opts), Es, ["}"]). -t_fun(Es) -> - ["("] ++ seq(fun t_utype_elem/1, get_content(argtypes, Es), - [") -> "] ++ t_utype(get_elem(type, Es))). +t_fun(Es, Opts) -> + ["("] ++ seq(t_utype_elem_fun(Opts), get_content(argtypes, Es), + [") -> "] ++ t_utype(get_elem(type, Es), Opts)). -t_map(Es) -> +t_map(Es, Opts) -> Fs = get_elem(map_field, Es), - ["#{"] ++ seq(fun t_map_field/1, Fs, ["}"]). + ["#{"] ++ seq(fun(E) -> t_map_field(E, Opts) end, Fs, ["}"]). -t_map_field(#xmlElement{content = [K,V]}=E) -> - KElem = t_utype_elem(K), - VElem = t_utype_elem(V), +t_map_field(#xmlElement{content = [K,V]}=E, Opts) -> + KElem = t_utype_elem(K, Opts), + VElem = t_utype_elem(V, Opts), AS = case get_attrval(assoc_type, E) of "assoc" -> " => "; "exact" -> " := " end, KElem ++ [AS] ++ VElem. -t_record(E, Es) -> - Name = ["#"] ++ t_type(get_elem(atom, Es)), +t_record(E, Es, Opts) -> + Name = ["#"] ++ t_type(get_elem(atom, Es), Opts), case get_elem(field, Es) of [] -> see(E, [Name, "{}"]); Fs -> - see(E, Name) ++ ["{"] ++ seq(fun t_field/1, Fs, ["}"]) + see(E, Name) ++ ["{"] ++ seq(fun(F) -> t_field(F, Opts) end, + Fs, ["}"]) end. -t_field(#xmlElement{content = Es}) -> - t_type(get_elem(atom, Es)) ++ [" = "] ++ t_utype(get_elem(type, Es)). +t_field(#xmlElement{content = Es}, Opts) -> + (t_type(get_elem(atom, Es), Opts) ++ [" = "] ++ + t_utype(get_elem(type, Es), Opts)). -t_abstype(E, Es) -> - Name = t_name(get_elem(erlangName, Es)), +t_abstype(E, Es, Opts) -> + Name = t_name(get_elem(erlangName, Es), Opts), case get_elem(type, Es) of [] -> see(E, [Name, "()"]); Ts -> - see(E, [Name]) ++ ["("] ++ seq(fun t_utype_elem/1, Ts, [")"]) + see(E, [Name]) ++ ["("] ++ seq(t_utype_elem_fun(Opts), Ts, [")"]) end. -t_abstype(Es) -> - ([t_name(get_elem(erlangName, Es)), "("] - ++ seq(fun t_utype_elem/1, get_elem(type, Es), [")"])). +t_abstype(Es, Opts) -> + ([t_name(get_elem(erlangName, Es), Opts), "("] + ++ seq(t_utype_elem_fun(Opts), get_elem(type, Es), [")"])). -t_union(Es) -> - seq(fun t_utype_elem/1, Es, " | ", []). +t_union(Es, Opts) -> + seq(t_utype_elem_fun(Opts), Es, " | ", []). seq(F, Es) -> seq(F, Es, []). @@ -989,8 +1008,8 @@ local_label(R) -> xhtml(Title, CSS, Body, Encoding) -> EncString = case Encoding of - "latin1" -> "ISO-8859-1"; - _ -> "UTF-8" + latin1 -> "ISO-8859-1"; + utf8 -> "UTF-8" end, [{html, [?NL, {head, [?NL, @@ -1009,11 +1028,11 @@ xhtml(Title, CSS, Body, Encoding) -> %% --------------------------------------------------------------------- type(E) -> - type(E, []). + Opts = init_opts(E, []), + type(E, [], Opts). -type(E, Ds) -> - Opts = [], - xmerl:export_simple_content(t_utype_elem(E) ++ local_defs(Ds, Opts), +type(E, Ds, Opts) -> + xmerl:export_simple_content(t_utype_elem(E, Opts) ++ local_defs(Ds, Opts), ?HTML_EXPORT). overview(E=#xmlElement{name = overview, content = Es}, Options) -> @@ -1036,7 +1055,7 @@ overview(E=#xmlElement{name = overview, content = Es}, Options) -> ++ [?NL, hr] ++ navigation("bottom") ++ footer()), - Encoding = get_attrval(encoding, E), + Encoding = Opts#opts.encoding, XML = xhtml(Title, stylesheet(Opts), Body, Encoding), xmerl:export_simple(XML, ?HTML_EXPORT, []). @@ -1094,8 +1113,8 @@ ot_var(E) -> {var,0,list_to_atom(get_attrval(name, E))}. ot_atom(E) -> - {ok, [{atom,A,Name}], _} = erl_scan:string(get_attrval(value, E), 0), - {atom,erl_anno:line(A),Name}. + Name = list_to_atom(get_attrval(value, E)), + {atom,erl_anno:new(0),Name}. ot_integer(E) -> {integer,0,list_to_integer(get_attrval(value, E))}. diff --git a/lib/edoc/src/edoc_report.erl b/lib/edoc/src/edoc_report.erl index ed778c8112..76557ef483 100644 --- a/lib/edoc/src/edoc_report.erl +++ b/lib/edoc/src/edoc_report.erl @@ -94,7 +94,7 @@ where({File, footer}) -> where({File, header}) -> io_lib:fwrite("~ts, in header file: ", [File]); where({File, {F, A}}) -> - io_lib:fwrite("~ts, function ~s/~w: ", [File, F, A]); + io_lib:fwrite("~ts, function ~ts/~w: ", [File, F, A]); where([]) -> io_lib:fwrite("~s: ", [?APPLICATION]); where(File) when is_list(File) -> diff --git a/lib/edoc/src/edoc_scanner.erl b/lib/edoc/src/edoc_scanner.erl index f1d5e1d4b9..35d00c6c0e 100644 --- a/lib/edoc/src/edoc_scanner.erl +++ b/lib/edoc/src/edoc_scanner.erl @@ -86,6 +86,8 @@ scan1([C|Cs], Toks, Pos) when C >= 0, C =< $ -> % Skip blanks scan1(Cs, Toks, Pos); scan1([C|Cs], Toks, Pos) when C >= $a, C =< $z -> % Unquoted atom scan_atom(C, Cs, Toks, Pos); +scan1([C|Cs], Toks, Pos) when C >= $\337, C =< $\377, C /= $\367 -> + scan_atom(C, Cs, Toks, Pos); scan1([C|Cs], Toks, Pos) when C >= $0, C =< $9 -> % Numbers scan_number(C, Cs, Toks, Pos); scan1([$-,C| Cs], Toks, Pos) when C >= $0, C =< $9 -> % Signed numbers @@ -96,6 +98,8 @@ scan1([C|Cs], Toks, Pos) when C >= $A, C =< $Z -> % Variables scan_variable(C, Cs, Toks, Pos); scan1([$_|Cs], Toks, Pos) -> % Variables scan_variable($_, Cs, Toks, Pos); +scan1([C|Cs], Toks, Pos) when C >= $\300, C =< $\336, C /= $\327 -> + scan_variable(C, Cs, Toks, Pos); scan1([$$|Cs], Toks, Pos) -> % Character constant case scan_char_const(Cs, Toks, Pos) of {ok, Result} -> @@ -261,6 +265,15 @@ scan_char([], _Pos) -> %% The following conforms to Standard Erlang escape sequences. +-define(HEX(C), C >= $0 andalso C =< $9 orelse + C >= $A andalso C =< $F orelse + C >= $a andalso C =< $f). + +-define(UNICODE(C), + (C >= 0 andalso C < 16#D800 orelse + C > 16#DFFF andalso C < 16#FFFE orelse + C > 16#FFFF andalso C =< 16#10FFFF)). + scan_escape([O1, O2, O3 | Cs], Pos) when % \<1-3> octal digits O1 >= $0, O1 =< $3, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> Val = (O1*8 + O2)*8 + O3 - 73*$0, @@ -272,6 +285,11 @@ scan_escape([O1, O2 | Cs], Pos) when scan_escape([O1 | Cs], Pos) when O1 >= $0, O1 =< $7 -> {O1 - $0,Cs,Pos}; +scan_escape([$x, ${ | Cs], Pos) -> + scan_hex(Cs, Pos, []); +scan_escape([$x, H1, H2 | Cs], Pos) when ?HEX(H1), ?HEX(H2) -> + Val = (H1*16 + H2) - 17*$0, + {Val,Cs,Pos}; scan_escape([$^, C | Cs], Pos) -> % \^X -> CTL-X if C >= $\100, C =< $\137 -> {C - $\100,Cs,Pos}; @@ -285,6 +303,18 @@ scan_escape([C | Cs], Pos) -> scan_escape([], _Pos) -> {error, truncated_char}. +scan_hex([C | Cs], Pos, HCs) when ?HEX(C) -> + scan_hex(Cs, Pos, [C | HCs]); +scan_hex([$} | Cs], Pos, HCs) -> + case catch erlang:list_to_integer(lists:reverse(HCs), 16) of + Val when ?UNICODE(Val) -> + {Val,Cs,Pos}; + _ -> + {error, undefined_escape_sequence} + end; +scan_hex(_Cs, _Pos, _HCs) -> + {error, undefined_escape_sequence}. + %% Note that we return $\000 for undefined escapes. escape_char($b) -> $\010; % \b = BS escape_char($d) -> $\177; % \d = DEL diff --git a/lib/edoc/src/edoc_specs.erl b/lib/edoc/src/edoc_specs.erl index c15dfd328f..fb04bfce0e 100644 --- a/lib/edoc/src/edoc_specs.erl +++ b/lib/edoc/src/edoc_specs.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -274,12 +274,19 @@ parms([A | As], [D | Ds]) -> param(#t_paren{type = Type}, Default) -> param(Type, Default); -param(#t_record{name = #t_atom{val = Name}}, _Default) -> - list_to_atom(capitalize(atom_to_list(Name))); +param(#t_record{name = #t_atom{val = Name}}=T, Default) -> + AtomList = atom_to_list(Name), + case AtomList =:= lists:flatten(io_lib:write_atom(Name)) of + true -> + list_to_atom(capitalize(AtomList)); + false -> + arg_name(?t_ann(T), Default) + end; param(T, Default) -> arg_name(?t_ann(T), Default). capitalize([C | Cs]) when C >= $a, C =< $z -> [C - 32 | Cs]; +capitalize([C | Cs]) when C >= $\340, C =< $\376, C /= $\367 -> [C - 32 | Cs]; capitalize(Cs) -> Cs. %% Like edoc_types:arg_name/1 diff --git a/lib/edoc/test/edoc_SUITE.erl b/lib/edoc/test/edoc_SUITE.erl index 4d846ad63d..29ca9d1203 100644 --- a/lib/edoc/test/edoc_SUITE.erl +++ b/lib/edoc/test/edoc_SUITE.erl @@ -23,12 +23,13 @@ init_per_group/2,end_per_group/2]). %% Test cases --export([app/1,appup/1,build_std/1,build_map_module/1,otp_12008/1, build_app/1]). +-export([app/1,appup/1,build_std/1,build_map_module/1,otp_12008/1, + build_app/1, otp_14285/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [app,appup,build_std,build_map_module,otp_12008, build_app]. + [app,appup,build_std,build_map_module,otp_12008, build_app, otp_14285]. groups() -> []. @@ -113,3 +114,18 @@ build_app(Config) -> true = filelib:is_regular(filename:join(OutDir, "a.html")), true = filelib:is_regular(filename:join(OutDir, "b.html")), ok. + +otp_14285(Config) -> + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + Un1 = filename:join(DataDir, "un_atom1.erl"), + Un2 = filename:join(DataDir, "un_atom2.erl"), + %% epp_dodger + Opts1 = [{dir, PrivDir}], + ok = edoc:files([Un1], Opts1), + ok = edoc:files([Un2], Opts1), + %% epp + Opts2 = [{preprocess, true}, {dir, PrivDir}], + ok = edoc:files([Un1], Opts2), + ok = edoc:files([Un2], Opts2), + ok. diff --git a/lib/edoc/test/edoc_SUITE_data/un_atom1.erl b/lib/edoc/test/edoc_SUITE_data/un_atom1.erl new file mode 100644 index 0000000000..20ca50d5d2 --- /dev/null +++ b/lib/edoc/test/edoc_SUITE_data/un_atom1.erl @@ -0,0 +1,41 @@ +%% coding:latin-1 + +-module(un_atom1). + +-export(['\x{aaa}memory'/0, 'func-\x{400}'/1, func/1, äfunc/1]). + +-record('rec-\x{400}', {'field-\x{400}'}). + +-type cs() :: $\x{a} + | $\x{aa} + | $\x{aaa} + | $\xaa. + +-callback 'callback-\x{400}'() -> 'apa'. + +-type 'type-\x{400}'() :: 'atom-\x{400}' + | cs() + | #'rec-\x{400}'{'field-\x{400}' :: 'type-\x{400}'()}. + +-spec '\x{aaa}memory'() -> 'type-\x{400}'(). + +'\x{aaa}memory'() -> + apa:foo(). + +%% @deprecated Please use {@link m:f/1}. +-spec 'func-\x{400}'(#'rec-\x{400}'{}) -> #'rec-\x{400}'{}. + +'func-\x{400}'(_T) -> + foo:bar(#'rec-\x{400}'{}). + +-record(rec, {}). + +-spec func(#rec{}) -> #rec{}. + +func(#rec{}) -> #rec{}. + +-record(ärec, {}). + +-spec äfunc(#ärec{}) -> #ärec{}. + +äfunc(#ärec{}) -> #ärec{}. diff --git a/lib/edoc/test/edoc_SUITE_data/un_atom2.erl b/lib/edoc/test/edoc_SUITE_data/un_atom2.erl new file mode 100644 index 0000000000..66c83e30e0 --- /dev/null +++ b/lib/edoc/test/edoc_SUITE_data/un_atom2.erl @@ -0,0 +1,40 @@ +%% coding:utf-8 +-module(un_atom2). + +-export(['\x{aaa}memory'/0, 'func-\x{400}'/1, func/1, äfunc/1]). + +-record('rec-\x{400}', {'field-\x{400}'}). + +-type cs() :: $\x{a} + | $\x{aa} + | $\x{aaa} + | $\xaa. + +-callback 'callback-\x{400}'() -> 'apa'. + +-type 'type-\x{400}'() :: 'atom-\x{400}' + | cs() + | #'rec-\x{400}'{'field-\x{400}' :: 'type-\x{400}'()}. + +-spec '\x{aaa}memory'() -> 'type-\x{400}'(). + +'\x{aaa}memory'() -> + apa:foo(). + +%% @deprecated Please use {@link m:f/1}. +-spec 'func-\x{400}'(#'rec-\x{400}'{}) -> #'rec-\x{400}'{}. + +'func-\x{400}'(_T) -> + foo:bar(#'rec-\x{400}'{}). + +-record(rec, {}). + +-spec func(#rec{}) -> #rec{}. + +func(#rec{}) -> #rec{}. + +-record(ärec, {}). + +-spec äfunc(#ärec{}) -> #ärec{}. + +äfunc(#ärec{}) -> #ärec{}. -- cgit v1.2.3 From 55f4e2b93c92607a5a1c798541f101bae3e9247d Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Thu, 30 Mar 2017 15:30:54 +0200 Subject: debugger: Show Latin-1 code correctly --- lib/debugger/src/dbg_wx_code.erl | 4 ++-- lib/debugger/src/dbg_wx_src_view.erl | 4 ++-- lib/debugger/src/int.erl | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/debugger/src/dbg_wx_code.erl b/lib/debugger/src/dbg_wx_code.erl index 473963500a..bca8a0d241 100644 --- a/lib/debugger/src/dbg_wx_code.erl +++ b/lib/debugger/src/dbg_wx_code.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -190,6 +190,6 @@ find(Ed, Str, Case, Next) -> keyWords() -> L = ["after","begin","case","try","cond","catch","andalso","orelse", - "end","fun","if","let","of","query","receive","when","bnot","not", + "end","fun","if","let","of","receive","when","bnot","not", "div","rem","band","and","bor","bxor","bsl","bsr","or","xor"], lists:flatten([K ++ " " || K <- L] ++ [0]). diff --git a/lib/debugger/src/dbg_wx_src_view.erl b/lib/debugger/src/dbg_wx_src_view.erl index 207c407fbc..ee8eb72407 100644 --- a/lib/debugger/src/dbg_wx_src_view.erl +++ b/lib/debugger/src/dbg_wx_src_view.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -62,6 +62,6 @@ code_area(Parent, Sizer) -> keyWords() -> L = ["after","begin","case","try","cond","catch","andalso","orelse", - "end","fun","if","let","of","query","receive","when","bnot","not", + "end","fun","if","let","of","receive","when","bnot","not", "div","rem","band","and","bor","bxor","bsl","bsr","or","xor"], lists:flatten([K ++ " " || K <- L] ++ [0]). diff --git a/lib/debugger/src/int.erl b/lib/debugger/src/int.erl index e5bade9abe..fdf5957182 100644 --- a/lib/debugger/src/int.erl +++ b/lib/debugger/src/int.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% Copyright Ericsson AB 1998-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -533,7 +533,9 @@ load({Mod, Src, Beam, BeamBin, Exp, Abst}, Dist) -> case erl_prim_loader:get_file(filename:absname(Src)) of {ok, SrcBin, _} -> MD5 = code:module_md5(BeamBin), - Bin = term_to_binary({interpreter_module,Exp,Abst,SrcBin,MD5}), + SrcBin1 = unicode:characters_to_binary(SrcBin, enc(SrcBin)), + true = is_binary(SrcBin1), + Bin = term_to_binary({interpreter_module,Exp,Abst,SrcBin1,MD5}), {module, Mod} = dbg_iserver:safe_call({load, Mod, Src, Bin}), _ = everywhere(Dist, fun() -> -- cgit v1.2.3 From 73657a28e74f8ad12ddb4fea49272a74f5b823c3 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Fri, 31 Mar 2017 10:17:05 +0200 Subject: syntax_tools: Fix Syntax Tools regarding Unicode atoms --- lib/syntax_tools/src/erl_prettypr.erl | 2 +- lib/syntax_tools/src/erl_syntax.erl | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tools/src/erl_prettypr.erl b/lib/syntax_tools/src/erl_prettypr.erl index 378d69095d..40ddd2b22a 100644 --- a/lib/syntax_tools/src/erl_prettypr.erl +++ b/lib/syntax_tools/src/erl_prettypr.erl @@ -452,7 +452,7 @@ lay_2(Node, Ctxt) -> text(erl_syntax:variable_literal(Node)); atom -> - text(erl_syntax:atom_literal(Node)); + text(erl_syntax:atom_literal(Node, Ctxt#ctxt.encoding)); integer -> text(erl_syntax:integer_literal(Node)); diff --git a/lib/syntax_tools/src/erl_syntax.erl b/lib/syntax_tools/src/erl_syntax.erl index 4347cc46c1..9b2b503762 100644 --- a/lib/syntax_tools/src/erl_syntax.erl +++ b/lib/syntax_tools/src/erl_syntax.erl @@ -139,6 +139,7 @@ is_atom/2, atom_value/1, atom_literal/1, + atom_literal/2, atom_name/1, attribute/1, attribute/2, @@ -1841,7 +1842,7 @@ char_literal(Node) -> %% @doc Returns the literal string represented by a `char' %% node. This includes the leading "`$'" character. %% Depending on the encoding a character beyond 255 will be escaped -%% ('latin1') or copied as is ('utf8'). +%% (`latin1') or copied as is (`utf8'). %% %% @see char/1 @@ -1944,7 +1945,7 @@ string_literal(Node) -> %% @doc Returns the literal string represented by a `string' %% node. This includes surrounding double-quote characters. %% Depending on the encoding characters beyond 255 will be escaped -%% ('latin1') or copied as is ('utf8'). +%% (`latin1') or copied as is (`utf8'). %% %% @see string/1 @@ -1965,6 +1966,7 @@ string_literal(Node, latin1) -> %% @see atom_value/1 %% @see atom_name/1 %% @see atom_literal/1 +%% @see atom_literal/2 %% @see is_atom/2 %% type(Node) = atom @@ -2037,6 +2039,7 @@ atom_name(Node) -> %% ===================================================================== %% @doc Returns the literal string represented by an `atom' %% node. This includes surrounding single-quote characters if necessary. +%% Characters beyond 255 will be escaped. %% %% Note that e.g. the result of `atom("x\ny")' represents %% any and all of `'x\ny'', `'x\12y'', @@ -2048,8 +2051,24 @@ atom_name(Node) -> -spec atom_literal(syntaxTree()) -> string(). atom_literal(Node) -> - io_lib:write_atom(atom_value(Node)). + atom_literal(Node, latin1). + +%% ===================================================================== +%% @doc Returns the literal string represented by an `atom' +%% node. This includes surrounding single-quote characters if necessary. +%% Depending on the encoding a character beyond 255 will be escaped +%% (`latin1') or copied as is (`utf8'). +%% +%% @see atom/1 +%% @see atom_literal/1 +%% @see string/1 +atom_literal(Node, utf8) -> + io_lib:write_atom(atom_value(Node)); +atom_literal(Node, unicode) -> + io_lib:write_atom(atom_value(Node)); +atom_literal(Node, latin1) -> + io_lib:write_atom_as_latin1(atom_value(Node)). %% ===================================================================== %% @equiv map_expr(none, Fields) -- cgit v1.2.3