diff options
Diffstat (limited to 'lib/syntax_tools/src')
| -rw-r--r-- | lib/syntax_tools/src/epp_dodger.erl | 4 | ||||
| -rw-r--r-- | lib/syntax_tools/src/erl_comment_scan.erl | 12 | ||||
| -rw-r--r-- | lib/syntax_tools/src/erl_prettypr.erl | 38 | ||||
| -rw-r--r-- | lib/syntax_tools/src/erl_syntax.erl | 164 | ||||
| -rw-r--r-- | lib/syntax_tools/src/erl_syntax_lib.erl | 5 | ||||
| -rw-r--r-- | lib/syntax_tools/src/erl_tidy.erl | 11 | ||||
| -rw-r--r-- | lib/syntax_tools/src/igor.erl | 55 |
7 files changed, 123 insertions, 166 deletions
diff --git a/lib/syntax_tools/src/epp_dodger.erl b/lib/syntax_tools/src/epp_dodger.erl index b3ced34c14..70395848a1 100644 --- a/lib/syntax_tools/src/epp_dodger.erl +++ b/lib/syntax_tools/src/epp_dodger.erl @@ -186,6 +186,7 @@ quick_parse_file(File, Options) -> parse_file(File, Parser, Options) -> case file:open(File, [read]) of {ok, Dev} -> + _ = epp:set_encoding(Dev), try Parser(Dev, 1, Options) after ok = file:close(Dev) end; @@ -400,7 +401,7 @@ quick_parse_form(Dev, L0, Options) -> parse_form(Dev, L0, Parser, Options) -> NoFail = proplists:get_bool(no_fail, Options), Opt = #opt{clever = proplists:get_bool(clever, Options)}, - case io:scan_erl_form(Dev, "", L0) of + case io:scan_erl_form(Dev, "", L0, [unicode]) of {ok, Ts, L1} -> case catch {ok, Parser(Ts, Opt)} of {'EXIT', Term} -> @@ -419,6 +420,7 @@ parse_form(Dev, L0, Parser, Options) -> {ok, F, L1} end; {error, _IoErr, _L1} = Err -> Err; + {error, _Reason} -> {eof, L0}; % This is probably encoding problem {eof, _L1} = Eof -> Eof end. diff --git a/lib/syntax_tools/src/erl_comment_scan.erl b/lib/syntax_tools/src/erl_comment_scan.erl index b833e1c069..a70e7ba413 100644 --- a/lib/syntax_tools/src/erl_comment_scan.erl +++ b/lib/syntax_tools/src/erl_comment_scan.erl @@ -72,7 +72,17 @@ file(Name) -> {ok, V} -> case V of {ok, B} -> - string(binary_to_list(B)); + Enc = case epp:read_encoding(Name) of + none -> epp:default_encoding(); + Enc0 -> Enc0 + end, + case catch unicode:characters_to_list(B, Enc) of + String when is_list(String) -> + string(String); + R -> + error_read_file(Name1), + exit(R) + end; {error, E} -> error_read_file(Name1), exit({read, E}) diff --git a/lib/syntax_tools/src/erl_prettypr.erl b/lib/syntax_tools/src/erl_prettypr.erl index f4bbf975c3..1a55a5e71c 100644 --- a/lib/syntax_tools/src/erl_prettypr.erl +++ b/lib/syntax_tools/src/erl_prettypr.erl @@ -60,7 +60,9 @@ hook = ?NOHOOK :: hook(), paper = ?PAPER :: integer(), ribbon = ?RIBBON :: integer(), - user = ?NOUSER :: term()}). + user = ?NOUSER :: term(), + encoding = epp:default_encoding() :: epp:source_encoding()}). + -type context() :: #ctxt{}. %% ===================================================================== @@ -231,6 +233,8 @@ format(Node) -> %% <dt>{user, term()}</dt> %% <dd>User-specific data for use in hook functions. The default %% value is `undefined'.</dd> +%% <dt>{encoding, epp:source_encoding()}</dt> +%% <dd>Specifies the encoding of the generated file.</dd> %% </dl> %% %% A hook function (cf. the {@link hook()} type) is passed the current @@ -342,7 +346,9 @@ layout(Node, Options) -> #ctxt{hook = proplists:get_value(hook, Options, ?NOHOOK), paper = proplists:get_value(paper, Options, ?PAPER), ribbon = proplists:get_value(ribbon, Options, ?RIBBON), - user = proplists:get_value(user, Options)}). + user = proplists:get_value(user, Options), + encoding = proplists:get_value(encoding, Options, + epp:default_encoding())}). lay(Node, Ctxt) -> case erl_syntax:get_ann(Node) of @@ -445,10 +451,10 @@ lay_2(Node, Ctxt) -> text(tidy_float(erl_syntax:float_literal(Node))); char -> - text(erl_syntax:char_literal(Node)); + text(erl_syntax:char_literal(Node, Ctxt#ctxt.encoding)); string -> - lay_string(erl_syntax:string_literal(Node), Ctxt); + lay_string(erl_syntax:string_literal(Node, Ctxt#ctxt.encoding), Ctxt); nil -> text("[]"); @@ -639,10 +645,6 @@ lay_2(Node, Ctxt) -> set_prec(Ctxt, PrecR)), beside(D1, beside(text(":"), D2)); - qualified_name -> - Ss = erl_syntax:qualified_name_segments(Node), - lay_qualified_name(Ss, Ctxt); - %% %% The rest is in alphabetical order %% @@ -966,26 +968,6 @@ maybe_parentheses(D, Prec, Ctxt) -> D end. -lay_qualified_name([S | Ss1] = Ss, Ctxt) -> - case erl_syntax:type(S) of - atom -> - case erl_syntax:atom_value(S) of - '' -> - beside(text("."), - lay_qualified_name_1(Ss1, Ctxt)); - _ -> - lay_qualified_name_1(Ss, Ctxt) - end; - _ -> - lay_qualified_name_1(Ss, Ctxt) - end. - -lay_qualified_name_1([S], Ctxt) -> - lay(S, Ctxt); -lay_qualified_name_1([S | Ss], Ctxt) -> - beside(lay(S, Ctxt), beside(text("."), - lay_qualified_name_1(Ss, Ctxt))). - lay_string(S, Ctxt) -> %% S includes leading/trailing double-quote characters. The segment %% width is 2/3 of the ribbon width - this seems to work well. diff --git a/lib/syntax_tools/src/erl_syntax.erl b/lib/syntax_tools/src/erl_syntax.erl index 151f04b03b..f7420030c3 100644 --- a/lib/syntax_tools/src/erl_syntax.erl +++ b/lib/syntax_tools/src/erl_syntax.erl @@ -161,6 +161,7 @@ is_char/2, char_value/1, char_literal/1, + char_literal/2, clause/2, clause/3, clause_body/1, @@ -234,8 +235,6 @@ prefix_expr/2, prefix_expr_argument/1, prefix_expr_operator/1, - qualified_name/1, - qualified_name_segments/1, query_expr/1, query_expr_body/1, receive_expr/1, @@ -271,6 +270,7 @@ is_string/2, string_value/1, string_literal/1, + string_literal/2, text/1, text_string/1, try_expr/2, @@ -449,7 +449,6 @@ %% <td>parentheses</td> %% <td>prefix_expr</td> %% </tr><tr> -%% <td>qualified_name</td> %% <td>query_expr</td> %% <td>receive_expr</td> %% <td>record_access</td> @@ -514,7 +513,6 @@ %% @see operator/1 %% @see parentheses/1 %% @see prefix_expr/2 -%% @see qualified_name/1 %% @see query_expr/1 %% @see receive_expr/3 %% @see record_access/3 @@ -584,11 +582,7 @@ type(Node) -> {record, _, _, _, _} -> record_expr; {record, _, _, _} -> record_expr; {record_field, _, _, _, _} -> record_access; - {record_field, _, _, _} -> - case is_qualified_name(Node) of - true -> qualified_name; - false -> record_access - end; + {record_field, _, _, _} -> record_access; {record_index, _, _, _} -> record_index_expr; {remote, _, _, _} -> module_qualifier; {rule, _, _, _, _} -> rule; @@ -1628,6 +1622,7 @@ float_literal(Node) -> %% %% @see char_value/1 %% @see char_literal/1 +%% @see char_literal/2 %% @see is_char/2 %% type(Node) = char @@ -1687,13 +1682,34 @@ char_value(Node) -> %% ===================================================================== %% @doc Returns the literal string represented by a `char' %% node. This includes the leading "`$'" character. +%% Characters beyond 255 will be escaped. %% %% @see char/1 -spec char_literal(syntaxTree()) -> nonempty_string(). char_literal(Node) -> - io_lib:write_char(char_value(Node)). + char_literal(Node, latin1). + + +%% ===================================================================== +%% @doc Returns the literal string represented by a `char' +%% node. This includes the leading "`$'" character. +%% Depending on the encoding a character beyond 255 will be escaped +%% ('latin1') or copied as is ('utf8'). +%% +%% @see char/1 + +-type encoding() :: 'utf8' | 'unicode' | 'latin1'. + +-spec char_literal(syntaxTree(), encoding()) -> nonempty_string(). + +char_literal(Node, unicode) -> + io_lib:write_unicode_char(char_value(Node)); +char_literal(Node, utf8) -> + io_lib:write_unicode_char(char_value(Node)); +char_literal(Node, latin1) -> + io_lib:write_unicode_char_as_latin1(char_value(Node)). %% ===================================================================== @@ -1708,6 +1724,7 @@ char_literal(Node) -> %% %% @see string_value/1 %% @see string_literal/1 +%% @see string_literal/2 %% @see is_string/2 %% @see char/1 @@ -1768,13 +1785,32 @@ string_value(Node) -> %% ===================================================================== %% @doc Returns the literal string represented by a `string' %% node. This includes surrounding double-quote characters. +%% Characters beyond 255 will be escaped. %% %% @see string/1 -spec string_literal(syntaxTree()) -> nonempty_string(). string_literal(Node) -> - io_lib:write_string(string_value(Node)). + string_literal(Node, latin1). + + +%% ===================================================================== +%% @doc Returns the literal string represented by a `string' +%% node. This includes surrounding double-quote characters. +%% Depending on the encoding characters beyond 255 will be escaped +%% ('latin1') or copied as is ('utf8'). +%% +%% @see string/1 + +-spec string_literal(syntaxTree(), encoding()) -> nonempty_string(). + +string_literal(Node, utf8) -> + io_lib:write_unicode_string(string_value(Node)); +string_literal(Node, unicode) -> + io_lib:write_unicode_string(string_value(Node)); +string_literal(Node, latin1) -> + io_lib:write_unicode_string_as_latin1(string_value(Node)). %% ===================================================================== @@ -3003,9 +3039,6 @@ revert_module_name(A) -> case type(A) of atom -> {ok, concrete(A)}; - qualified_name -> - Ss = qualified_name_segments(A), - {ok, [concrete(S) || S <- Ss]}; _ -> error end. @@ -3051,11 +3084,7 @@ attribute_arguments(Node) -> M0 -> {M0, none} end, - M2 = if is_list(M1) -> - qualified_name([atom(A) || A <- M1]); - true -> - atom(M1) - end, + M2 = atom(M1), M = set_pos(M2, Pos), if Vs == none -> [M]; true -> [M, set_pos(list(Vs), Pos)] @@ -3065,20 +3094,11 @@ attribute_arguments(Node) -> list(unfold_function_names(Data, Pos)), Pos)]; import -> - case Data of - {Module, Imports} -> - [if is_list(Module) -> - qualified_name([atom(A) - || A <- Module]); - true -> - set_pos(atom(Module), Pos) - end, - set_pos( - list(unfold_function_names(Imports, Pos)), - Pos)]; - _ -> - [qualified_name([atom(A) || A <- Data])] - end; + {Module, Imports} = Data, + [set_pos(atom(Module), Pos), + set_pos( + list(unfold_function_names(Imports, Pos)), + Pos)]; file -> {File, Line} = Data, [set_pos(string(File), Pos), @@ -3210,53 +3230,6 @@ module_qualifier_body(Node) -> %% ===================================================================== -%% @doc Creates an abstract qualified name. The result represents -%% "<code><em>S1</em>.<em>S2</em>. ... .<em>Sn</em></code>", if -%% `Segments' is `[S1, S2, ..., Sn]'. -%% -%% @see qualified_name_segments/1 - -%% type(Node) = qualified_name -%% data(Node) = [syntaxTree()] -%% -%% `erl_parse' representation: -%% -%% {record_field, Pos, Node, Node} -%% -%% Node = {atom, Pos, Value} | {record_field, Pos, Node, Node} -%% -%% Note that if not all leaf subnodes are (abstract) atoms, then Node -%% represents a Mnemosyne query record field access ('record_access'); -%% see type/1 for details. - --spec qualified_name([syntaxTree()]) -> syntaxTree(). - -qualified_name(Segments) -> - tree(qualified_name, Segments). - -revert_qualified_name(Node) -> - Pos = get_pos(Node), - fold_qualified_name(qualified_name_segments(Node), Pos). - - -%% ===================================================================== -%% @doc Returns the list of name segments of a -%% `qualified_name' node. -%% -%% @see qualified_name/1 - --spec qualified_name_segments(syntaxTree()) -> [syntaxTree()]. - -qualified_name_segments(Node) -> - case unwrap(Node) of - {record_field, _, _, _} = Node1 -> - unfold_qualified_name(Node1); - Node1 -> - data(Node1) - end. - - -%% ===================================================================== %% @doc Creates an abstract function definition. If `Clauses' %% is `[C1, ..., Cn]', the result represents %% "<code><em>Name</em> <em>C1</em>; ...; <em>Name</em> @@ -6068,8 +6041,6 @@ revert_root(Node) -> revert_parentheses(Node); prefix_expr -> revert_prefix_expr(Node); - qualified_name -> - revert_qualified_name(Node); query_expr -> revert_query_expr(Node); receive_expr -> @@ -6312,8 +6283,6 @@ subtrees(T) -> prefix_expr -> [[prefix_expr_operator(T)], [prefix_expr_argument(T)]]; - qualified_name -> - [qualified_name_segments(T)]; query_expr -> [[query_expr_body(T)]]; receive_expr -> @@ -6444,7 +6413,6 @@ make_tree(match_expr, [[P], [E]]) -> match_expr(P, E); make_tree(module_qualifier, [[M], [N]]) -> module_qualifier(M, N); make_tree(parentheses, [[E]]) -> parentheses(E); make_tree(prefix_expr, [[F], [A]]) -> prefix_expr(F, A); -make_tree(qualified_name, [S]) -> qualified_name(S); make_tree(query_expr, [[B]]) -> query_expr(B); make_tree(receive_expr, [C]) -> receive_expr(C); make_tree(receive_expr, [C, [E], A]) -> receive_expr(C, E, A); @@ -6788,32 +6756,6 @@ fold_variable_names(Vs) -> unfold_variable_names(Vs, Pos) -> [set_pos(variable(V), Pos) || V <- Vs]. -%% Support functions for qualified names ("foo.bar.baz", -%% "erl.lang.lists", etc.). The representation overlaps with the weird -%% "Mnesia query record access" operators. The '.' operator is left -%% associative, so folding should nest on the left. - -is_qualified_name({record_field, _, L, R}) -> - is_qualified_name(L) andalso is_qualified_name(R); -is_qualified_name({atom, _, _}) -> true; -is_qualified_name(_) -> false. - -unfold_qualified_name(Node) -> - lists:reverse(unfold_qualified_name(Node, [])). - -unfold_qualified_name({record_field, _, L, R}, Ss) -> - unfold_qualified_name(R, unfold_qualified_name(L, Ss)); -unfold_qualified_name(S, Ss) -> [S | Ss]. - -fold_qualified_name([S | Ss], Pos) -> - fold_qualified_name(Ss, Pos, {atom, Pos, atom_value(S)}). - -fold_qualified_name([S | Ss], Pos, Ack) -> - fold_qualified_name(Ss, Pos, {record_field, Pos, Ack, - {atom, Pos, atom_value(S)}}); -fold_qualified_name([], _Pos, Ack) -> - Ack. - %% Support functions for transforming lists of record field definitions. %% %% There is no unique representation for field definitions in the diff --git a/lib/syntax_tools/src/erl_syntax_lib.erl b/lib/syntax_tools/src/erl_syntax_lib.erl index 36cd35f15d..2c94ac776d 100644 --- a/lib/syntax_tools/src/erl_syntax_lib.erl +++ b/lib/syntax_tools/src/erl_syntax_lib.erl @@ -2223,11 +2223,6 @@ module_name_to_atom(M) -> case erl_syntax:type(M) of atom -> erl_syntax:atom_value(M); - qualified_name -> - list_to_atom(packages:concat( - [erl_syntax:atom_value(A) - || A <- erl_syntax:qualified_name_segments(M)]) - ); _ -> throw(syntax_error) end. diff --git a/lib/syntax_tools/src/erl_tidy.erl b/lib/syntax_tools/src/erl_tidy.erl index 59cf6c0a92..e9a88caff3 100644 --- a/lib/syntax_tools/src/erl_tidy.erl +++ b/lib/syntax_tools/src/erl_tidy.erl @@ -375,6 +375,8 @@ write_module(Tree, Name, Opts) -> end, filename(filename:join(Dir, Name1)) end, + Encoding = [{encoding,Enc} || Enc <- [epp:read_encoding(Name)], + Enc =/= none], case proplists:get_bool(backups, Opts) of true -> backup_file(File, Opts); @@ -382,9 +384,9 @@ write_module(Tree, Name, Opts) -> ok end, Printer = proplists:get_value(printer, Opts), - FD = open_output_file(File), + FD = open_output_file(File, Encoding), verbose("writing to file `~s'.", [File], Opts), - V = (catch {ok, output(FD, Printer, Tree, Opts)}), + V = (catch {ok, output(FD, Printer, Tree, Opts++Encoding)}), ok = file:close(FD), case V of {ok, _} -> @@ -432,8 +434,9 @@ file_type(Name, Links) -> throw(R) end. -open_output_file(FName) -> - case catch file:open(FName, [write]) of +open_output_file(FName, Options) -> +io:format("Options ~p~n", [Options]), + case catch file:open(FName, [write]++Options) of {ok, FD} -> FD; {error, R} -> diff --git a/lib/syntax_tools/src/igor.erl b/lib/syntax_tools/src/igor.erl index 37e561cbbe..8abc3f41cb 100644 --- a/lib/syntax_tools/src/igor.erl +++ b/lib/syntax_tools/src/igor.erl @@ -341,10 +341,12 @@ merge(Name, Files) -> merge(Name, Files, Opts) -> Opts1 = Opts ++ ?DEFAULT_MERGE_OPTS, - {Tree, Stubs} = merge_files(Name, Files, Opts1), + {Sources, Enc} = merge_files1(Files, Opts1), + {Tree, Stubs} = merge_sources(Name, Sources, Opts1), Dir = proplists:get_value(dir, Opts1, ""), Filename = proplists:get_value(outfile, Opts1, Name), - File = write_module(Tree, Filename, Dir, Opts1), + Encoding = [{encoding, Enc} || Enc =/= none], + File = write_module(Tree, Filename, Dir, Encoding ++ Opts1), [File | maybe_create_stubs(Stubs, Opts1)]. @@ -459,16 +461,21 @@ merge_files(Name, Files, Options) -> -spec merge_files(atom(), erl_syntax:forms(), [file:filename()], [option()]) -> {erl_syntax:syntaxTree(), [stubDescriptor()]}. -merge_files(_, _Trees, [], _) -> +merge_files(Name, Trees, Files, Opts) -> + {Sources, _Encoding} = merge_files1(Files, Opts), + merge_sources(Name, Trees ++ Sources, Opts). + +merge_files1([], _) -> report_error("no files to merge."), exit(badarg); -merge_files(Name, Trees, Files, Opts) -> +merge_files1(Files, Opts) -> Opts1 = Opts ++ [{includes, ?DEFAULT_INCLUDES}, {macros, ?DEFAULT_MACROS}, {preprocess, false}, comments], - Sources = [read_module(F, Opts1) || F <- Files], - merge_sources(Name, Trees ++ Sources, Opts1). + SourceEncodings = [read_module(F, Opts1) || F <- Files], + {Sources, [Encoding | _]} = lists:unzip(SourceEncodings), + {Sources, Encoding}. %% ===================================================================== @@ -2512,7 +2519,11 @@ rename(Files, Renamings, Opts) -> lists:flatmap(fun (F) -> rename_file(F, Dict, Opts1) end, Files). rename_file(File, Dict, Opts) -> - S = read_module(File, Opts), + {S, Enc} = read_module(File, Opts), + %% Try to avoid *two* coding: comments: + Encoding = [{encoding, Enc} || + Enc =/= none, + not proplists:get_bool(comments, Opts)], M = get_module_info(S), Name = M#module.name, Name1 = case dict:find(Name, Dict) of @@ -2526,10 +2537,10 @@ rename_file(File, Dict, Opts) -> Opts1 = [no_headers, {export, [Name]}, {static, [Name]}, - {redirect, dict:to_list(Dict1)}] ++ Opts, + {redirect, dict:to_list(Dict1)}] ++ Encoding ++ Opts, {Tree, Stubs} = merge_sources(Name1, [S], Opts1), Dir = filename:dirname(filename(File)), - File1 = write_module(Tree, Name1, Dir, Opts), + File1 = write_module(Tree, Name1, Dir, Opts++Encoding), %% We create the stub file in the same directory as the source file %% and the target file. @@ -2648,7 +2659,7 @@ error_text(D, Name) -> {L, M, E} when is_integer(L), is_atom(M) -> case catch M:format_error(E) of S when is_list(S) -> - io_lib:fwrite("`~w', line ~w: ~s.", + io_lib:fwrite("`~w', line ~w: ~ts.", [Name, L, S]); _ -> error_text_1(D, Name) @@ -2706,7 +2717,17 @@ open_output_file(FName) -> exit(R) end. -%% read_module(Name, Options) -> syntaxTree() +output_encoding(FD, Opts) -> + case proplists:get_value(encoding, Opts) of + undefined -> + ok = io:setopts(FD, [{encoding, epp:default_encoding()}]); + Encoding -> + ok = io:setopts(FD, [{encoding, Encoding}]), + EncS = epp:encoding_to_string(Encoding), + ok = io:fwrite(FD, <<"%% ~s\n">>, [EncS]) + end. + +%% read_module(Name, Options) -> {syntaxTree(), epp:source_encoding()} %% %% This also tries to locate the real source file, if "Name" does not %% point directly to a particular file. @@ -2729,20 +2750,21 @@ read_module(Name, Options) -> read_module_1(Name, Options) -> verbose("reading module `~s'.", [filename(Name)], Options), - Forms = read_module_2(Name, Options), + {Forms, Enc} = read_module_2(Name, Options), case proplists:get_bool(comments, Options) of false -> - Forms; + {Forms, Enc}; true -> Comments = erl_comment_scan:file(Name), - erl_recomment:recomment_forms(Forms, Comments) + {erl_recomment:recomment_forms(Forms, Comments), Enc} end. read_module_2(Name, Options) -> case read_module_3(Name, Options) of {ok, Forms} -> check_forms(Forms, Name), - Forms; + Enc = epp:read_encoding(Name), + {Forms, Enc}; {error, _} = Error -> error_read_file(Name), exit(Error) @@ -2772,7 +2794,7 @@ check_forms([F | Fs], File) -> _ -> "unknown error" end, - report_error("in file `~s' at line ~w:\n ~s", + report_error("in file `~s' at line ~w:\n ~ts", [filename(File), erl_syntax:get_pos(F), S]), exit(error); _ -> @@ -2847,6 +2869,7 @@ write_module(Tree, Name, Dir, Opts) -> end, Printer = proplists:get_value(printer, Opts), FD = open_output_file(File), + ok = output_encoding(FD, Opts), verbose("writing to file `~s'.", [File], Opts), V = (catch {ok, output(FD, Printer, Tree, Opts)}), ok = file:close(FD), |