aboutsummaryrefslogtreecommitdiffstats
path: root/lib/parsetools/src
diff options
context:
space:
mode:
Diffstat (limited to 'lib/parsetools/src')
-rw-r--r--lib/parsetools/src/leex.erl99
-rw-r--r--lib/parsetools/src/yecc.erl120
-rw-r--r--lib/parsetools/src/yeccscan.erl6
3 files changed, 146 insertions, 79 deletions
diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl
index cdf20461d9..bbef4053b4 100644
--- a/lib/parsetools/src/leex.erl
+++ b/lib/parsetools/src/leex.erl
@@ -58,6 +58,7 @@
gfile=[], % Graph file
module, % Module name
opts=[], % Options
+ encoding=none, % Encoding of Xrl file
% posix=false, % POSIX regular expressions
errors=[],
warnings=[]
@@ -146,7 +147,9 @@ format_error({regexp,E})->
end,
["bad regexp `",Es,"'"];
format_error(ignored_characters) ->
- "ignored characters".
+ "ignored characters";
+format_error(cannot_parse) ->
+ io_lib:fwrite("cannot parse; probably encoding mismatch", []).
%%%
%%% Local functions
@@ -298,10 +301,10 @@ pack_warnings([]) ->
report_errors(St) ->
when_opt(fun () ->
foreach(fun({File,{none,Mod,E}}) ->
- io:fwrite("~s: ~s\n",
+ io:fwrite("~s: ~ts\n",
[File,Mod:format_error(E)]);
({File,{Line,Mod,E}}) ->
- io:fwrite("~s:~w: ~s\n",
+ io:fwrite("~s:~w: ~ts\n",
[File,Line,Mod:format_error(E)])
end, sort(St#leex.errors))
end, report_errors, St#leex.opts).
@@ -316,11 +319,11 @@ report_warnings(St) ->
ShouldReport = member(report_warnings, St#leex.opts) orelse ReportWerror,
when_bool(fun () ->
foreach(fun({File,{none,Mod,W}}) ->
- io:fwrite("~s: ~s~s\n",
+ io:fwrite("~s: ~s~ts\n",
[File,Prefix,
Mod:format_error(W)]);
({File,{Line,Mod,W}}) ->
- io:fwrite("~s:~w: ~s~s\n",
+ io:fwrite("~s:~w: ~s~ts\n",
[File,Line,Prefix,
Mod:format_error(W)])
end, sort(St#leex.warnings))
@@ -396,17 +399,18 @@ verbose_print(St, Format, Args) ->
parse_file(St0) ->
case file:open(St0#leex.xfile, [read]) of
{ok,Xfile} ->
+ St1 = St0#leex{encoding = epp:set_encoding(Xfile)},
try
- verbose_print(St0, "Parsing file ~s, ", [St0#leex.xfile]),
+ verbose_print(St1, "Parsing file ~s, ", [St1#leex.xfile]),
%% We KNOW that errors throw so we can ignore them here.
- {ok,Line1,St1} = parse_head(Xfile, St0),
- {ok,Line2,Macs,St2} = parse_defs(Xfile, Line1, St1),
- {ok,Line3,REAs,Actions,St3} =
- parse_rules(Xfile, Line2, Macs, St2),
- {ok,Code,St4} = parse_code(Xfile, Line3, St3),
- verbose_print(St1, "contained ~w rules.~n", [length(REAs)]),
- {ok,REAs,Actions,Code,St4}
- after file:close(Xfile)
+ {ok,Line1,St2} = parse_head(Xfile, St1),
+ {ok,Line2,Macs,St3} = parse_defs(Xfile, Line1, St2),
+ {ok,Line3,REAs,Actions,St4} =
+ parse_rules(Xfile, Line2, Macs, St3),
+ {ok,Code,St5} = parse_code(Xfile, Line3, St4),
+ verbose_print(St5, "contained ~w rules.~n", [length(REAs)]),
+ {ok,REAs,Actions,Code,St5}
+ after ok = file:close(Xfile)
end;
{error,Error} ->
add_error({none,leex,{file_error,Error}}, St0)
@@ -415,7 +419,7 @@ parse_file(St0) ->
%% parse_head(File, State) -> {ok,NextLine,State}.
%% Parse the head of the file. Skip all comments and blank lines.
-parse_head(Ifile, St) -> {ok,nextline(Ifile, 0),St}.
+parse_head(Ifile, St) -> {ok,nextline(Ifile, 0, St),St}.
%% parse_defs(File, Line, State) -> {ok,NextLine,Macros,State}.
%% Parse the macro definition section of a file. This must exist.
@@ -423,7 +427,7 @@ parse_head(Ifile, St) -> {ok,nextline(Ifile, 0),St}.
parse_defs(Ifile, {ok,?DEFS_HEAD ++ Rest,L}, St) ->
St1 = warn_ignored_chars(L, Rest, St),
- parse_defs(Ifile, nextline(Ifile, L), [], St1);
+ parse_defs(Ifile, nextline(Ifile, L, St), [], St1);
parse_defs(_, {ok,_,L}, St) ->
add_error({L,leex,missing_defs}, St);
parse_defs(_, {eof,L}, St) ->
@@ -435,7 +439,7 @@ parse_defs(Ifile, {ok,Chars,L}=Line, Ms, St) ->
case re:run(Chars, MS, [{capture,all_but_first,list}]) of
{match,[Name,Def]} ->
%%io:fwrite("~p = ~p\n", [Name,Def]),
- parse_defs(Ifile, nextline(Ifile, L), [{Name,Def}|Ms], St);
+ parse_defs(Ifile, nextline(Ifile, L, St), [{Name,Def}|Ms], St);
_ -> {ok,Line,Ms,St} % Anything else
end;
parse_defs(_, Line, Ms, St) ->
@@ -446,7 +450,7 @@ parse_defs(_, Line, Ms, St) ->
parse_rules(Ifile, {ok,?RULE_HEAD ++ Rest,L}, Ms, St) ->
St1 = warn_ignored_chars(L, Rest, St),
- parse_rules(Ifile, nextline(Ifile, L), Ms, [], [], 0, St1);
+ parse_rules(Ifile, nextline(Ifile, L, St), Ms, [], [], 0, St1);
parse_rules(_, {ok,_,L}, _, St) ->
add_error({L,leex,missing_rules}, St);
parse_rules(_, {eof,L}, _, St) ->
@@ -464,7 +468,7 @@ parse_rules(Ifile, NextLine, Ms, REAs, As, N, St) ->
case collect_rule(Ifile, Chars, L0) of
{ok,Re,Atoks,L1} ->
{ok,REA,A,St1} = parse_rule(Re, L0, Atoks, Ms, N, St),
- parse_rules(Ifile, nextline(Ifile, L1), Ms,
+ parse_rules(Ifile, nextline(Ifile, L1, St), Ms,
[REA|REAs], [A|As], N+1, St1);
{error,E} -> add_error(E, St)
end;
@@ -497,8 +501,10 @@ collect_rule(Ifile, Chars, L0) ->
{error,E,_} -> {error,E}
end.
+collect_action(_Ifile, {error, _}, L, _Cont0) ->
+ {error, {L, leex, cannot_parse}, ignored_end_line};
collect_action(Ifile, Chars, L0, Cont0) ->
- case erl_scan:tokens(Cont0, Chars, L0) of
+ case erl_scan:tokens(Cont0, Chars, L0, [unicode]) of
{done,{ok,Toks,_},_} -> {ok,Toks,L0};
{done,{eof,_},_} -> {eof,L0};
{done,{error,E,_},_} -> {error,E,L0};
@@ -560,29 +566,32 @@ parse_code(Ifile, {ok,?CODE_HEAD ++ Rest,CodeL}, St) ->
St1 = warn_ignored_chars(CodeL, Rest, St),
{ok, CodePos} = file:position(Ifile, cur),
%% Just count the lines; copy the code from file to file later.
- NCodeLines = count_lines(Ifile, 0),
+ EndCodeLine = count_lines(Ifile, CodeL, St),
+ NCodeLines = EndCodeLine - CodeL,
{ok,{CodeL,CodePos,NCodeLines},St1};
parse_code(_, {ok,_,L}, St) ->
add_error({L,leex,missing_code}, St);
parse_code(_, {eof,L}, St) ->
add_error({L,leex,missing_code}, St).
-count_lines(File, N) ->
+count_lines(File, N, St) ->
case io:get_line(File, leex) of
eof -> N;
- _Line -> count_lines(File, N+1)
+ {error, _} -> add_error({N+1, leex, cannot_parse}, St);
+ _Line -> count_lines(File, N+1, St)
end.
-%% nextline(InputFile, PrevLineNo) -> {ok,Chars,LineNo} | {eof,LineNo}.
+%% nextline(InputFile, PrevLineNo, State) -> {ok,Chars,LineNo} | {eof,LineNo}.
%% Get the next line skipping comment lines and blank lines.
-nextline(Ifile, L) ->
+nextline(Ifile, L, St) ->
case io:get_line(Ifile, leex) of
eof -> {eof,L};
+ {error, _} -> add_error({L+1, leex, cannot_parse}, St);
Chars ->
case substr(Chars, span(Chars, " \t\n")+1) of
- [$%|_Rest] -> nextline(Ifile, L+1);
- [] -> nextline(Ifile, L+1);
+ [$%|_Rest] -> nextline(Ifile, L+1, St);
+ [] -> nextline(Ifile, L+1, St);
_Other -> {ok,Chars,L+1}
end
end.
@@ -1289,19 +1298,21 @@ out_file(St0, DFA, DF, Actions, Code) ->
try
case file:open(St0#leex.efile, [write]) of
{ok,Ofile} ->
+ set_encoding(St0, Ofile),
try
+ output_encoding_comment(Ofile, St0),
output_file_directive(Ofile, St0#leex.ifile, 0),
out_file(Ifile, Ofile, St0, DFA, DF, Actions,
Code, 1),
verbose_print(St0, "ok~n", []),
St0
- after file:close(Ofile)
+ after ok = file:close(Ofile)
end;
{error,Error} ->
verbose_print(St0, "error~n", []),
add_error({none,leex,{file_error,Error}}, St0)
end
- after file:close(Ifile)
+ after ok = file:close(Ifile)
end;
{{error,Error},Ifile} ->
add_error(Ifile, {none,leex,{file_error,Error}}, St0)
@@ -1310,7 +1321,9 @@ out_file(St0, DFA, DF, Actions, Code) ->
open_inc_file(State) ->
Ifile = State#leex.ifile,
case file:open(Ifile, [read]) of
- {ok,F} -> {ok,F};
+ {ok,F} ->
+ _ = epp:set_encoding(F),
+ {ok,F};
Error -> {Error,Ifile}
end.
@@ -1328,6 +1341,7 @@ inc_file_name(Filename) ->
out_file(Ifile, Ofile, St, DFA, DF, Actions, Code, L) ->
case io:get_line(Ifile, leex) of
eof -> output_file_directive(Ofile, St#leex.ifile, L);
+ {error, _} -> add_error(St#leex.ifile, {L, leex, cannot_parse}, St);
Line ->
case substr(Line, 1, 5) of
"##mod" -> out_module(Ofile, St);
@@ -1347,14 +1361,23 @@ out_erlang_code(File, St, Code, L) ->
output_file_directive(File, St#leex.xfile, CodeL),
{ok,Xfile} = file:open(St#leex.xfile, [read]),
try
+ set_encoding(St, Xfile),
{ok,_} = file:position(Xfile, CodePos),
- {ok,_} = file:copy(Xfile, File)
+ ok = file_copy(Xfile, File)
after
- file:close(Xfile)
+ ok = file:close(Xfile)
end,
io:nl(File),
output_file_directive(File, St#leex.ifile, L).
+file_copy(From, To) ->
+ case io:get_line(From, leex) of
+ eof -> ok;
+ Line when is_list(Line) ->
+ io:fwrite(To, "~ts", [Line]),
+ file_copy(From, To)
+ end.
+
out_dfa(File, St, DFA, Code, DF, L) ->
{_CodeL,_CodePos,NCodeLines} = Code,
%% Three file attributes before this one...
@@ -1569,7 +1592,7 @@ out_dfa_graph(St, DFA, DF) ->
io:fwrite(Gfile, "}~n", []),
verbose_print(St, "ok~n", []),
St
- after file:close(Gfile)
+ after ok = file:close(Gfile)
end;
{error,Error} ->
verbose_print(St, "error~n", []),
@@ -1610,6 +1633,16 @@ dfa_edgelabel(Cranges) ->
(C) -> [quote(C)]
end, Cranges) ++ "]".
+set_encoding(#leex{encoding = none}, File) ->
+ ok = io:setopts(File, [{encoding, epp:default_encoding()}]);
+set_encoding(#leex{encoding = E}, File) ->
+ ok = io:setopts(File, [{encoding, E}]).
+
+output_encoding_comment(_File, #leex{encoding = none}) ->
+ ok;
+output_encoding_comment(File, #leex{encoding = Encoding}) ->
+ io:fwrite(File, <<"%% ~s\n">>, [epp:encoding_to_string(Encoding)]).
+
output_file_directive(File, Filename, Line) ->
io:fwrite(File, <<"-file(~s, ~w).\n">>,
[format_filename(Filename), Line]).
diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl
index b0792a6ed8..dbb7d025ae 100644
--- a/lib/parsetools/src/yecc.erl
+++ b/lib/parsetools/src/yecc.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1996-2011. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2012. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -42,6 +42,7 @@
includefile,
includefile_version,
module,
+ encoding = none,
options = [],
verbose = false,
file_attrs = true,
@@ -224,7 +225,11 @@ format_error({unused_nonterminal, Nonterminal}) ->
[format_symbol(Nonterminal)]);
format_error({unused_terminal, Terminal}) ->
io_lib:fwrite("terminal symbol ~s not used",
- [format_symbol(Terminal)]).
+ [format_symbol(Terminal)]);
+format_error({bad_symbol, String}) ->
+ io_lib:fwrite("bad symbol ~ts", [String]);
+format_error(cannot_parse) ->
+ io_lib:fwrite("cannot parse; possibly encoding mismatch", []).
file(File) ->
file(File, [report_errors, report_warnings]).
@@ -257,7 +262,7 @@ yecc(Infile, Outfile, Verbose) ->
yecc(Infile, Outfile, Verbose, []).
yecc(Infilex, Outfilex, Verbose, Includefilex) ->
- statistics(runtime),
+ _ = statistics(runtime),
case file(Infilex, [{parserfile, Outfilex},
{verbose, Verbose},
{report, true},
@@ -407,7 +412,9 @@ infile(Parent, Infilex, Options) ->
St = case file:open(St0#yecc.infile, [read, read_ahead]) of
{ok, Inport} ->
try
- outfile(St0#yecc{inport = Inport})
+ Encoding = epp:set_encoding(Inport),
+ St1 = St0#yecc{inport = Inport, encoding = Encoding},
+ outfile(St1)
after
ok = file:close(Inport)
end;
@@ -428,6 +435,8 @@ outfile(St0) ->
case file:open(St0#yecc.outfile, [write, delayed_write]) of
{ok, Outport} ->
try
+ %% Set the same encoding as infile:
+ set_encoding(St0, Outport),
generate(St0#yecc{outport = Outport, line = 1})
catch
throw: St1 ->
@@ -466,13 +475,14 @@ timeit(Name, Fun, St0) ->
-define(PASS(P), {P, fun P/1}).
generate(St0) ->
+ St1 = output_encoding_comment(St0),
Passes = [?PASS(parse_grammar), ?PASS(check_grammar),
?PASS(states_and_goto_table), ?PASS(parse_actions),
?PASS(action_conflicts), ?PASS(write_file)],
- F = case member(time, St0#yecc.options) of
+ F = case member(time, St1#yecc.options) of
true ->
io:fwrite(<<"Generating parser from grammar in ~s\n">>,
- [format_filename(St0#yecc.infile)]),
+ [format_filename(St1#yecc.infile)]),
fun timeit/3;
false ->
fun(_Name, Fn, St) -> Fn(St) end
@@ -484,13 +494,13 @@ generate(St0) ->
true -> throw(St2)
end
end,
- foldl(Fun, St0, Passes).
+ foldl(Fun, St1, Passes).
parse_grammar(St) ->
parse_grammar(St#yecc.inport, 1, St).
parse_grammar(Inport, Line, St) ->
- {NextLine, Grammar} = read_grammar(Inport, Line),
+ {NextLine, Grammar} = read_grammar(Inport, St, Line),
parse_grammar(Grammar, Inport, NextLine, St).
parse_grammar(eof, _Inport, _NextLine, St) ->
@@ -523,6 +533,8 @@ parse_grammar({rule, Rule, Tokens}, St0) ->
St#yecc{rules_list = [RuleDef | St#yecc.rules_list]};
parse_grammar({prec, Prec}, St) ->
St#yecc{prec = Prec ++ St#yecc.prec};
+parse_grammar({#symbol{}, [{string,Line,String}]}, St) ->
+ add_error(Line, {bad_symbol, String}, St);
parse_grammar({#symbol{line = Line, name = Name}, Symbols}, St) ->
CF = fun(I) ->
case element(I, St) of
@@ -543,12 +555,17 @@ parse_grammar({#symbol{line = Line, name = Name}, Symbols}, St) ->
_ -> add_warning(Line, bad_declaration, St)
end.
-read_grammar(Inport, Line) ->
+read_grammar(Inport, St, Line) ->
case yeccscan:scan(Inport, '', Line) of
{eof, NextLine} ->
{NextLine, eof};
{error, {ErrorLine, Mod, What}, NextLine} ->
{NextLine, {error, ErrorLine, {error, Mod, What}}};
+ {error, terminated} ->
+ throw(St);
+ {error, _} ->
+ File = St#yecc.infile,
+ throw(add_error(File, none, cannot_parse, St));
{ok, Input, NextLine} ->
{NextLine, case yeccparser:parse(Input) of
{error, {ErrorLine, Mod, Message}} ->
@@ -738,9 +755,9 @@ states_and_goto_table(St0) ->
create_precedence_table(St).
parse_actions(St) ->
- erase(), % the pd is used when decoding lookahead sets
+ _ = erase(), % the pd is used when decoding lookahead sets
ParseActions = compute_parse_actions(St#yecc.n_states, St, []),
- erase(),
+ _ = erase(),
St#yecc{parse_actions = ParseActions, state_tab = []}.
action_conflicts(St0) ->
@@ -841,10 +858,10 @@ report_errors(St) ->
case member(report_errors, St#yecc.options) of
true ->
foreach(fun({File,{none,Mod,E}}) ->
- io:fwrite(<<"~s: ~s\n">>,
+ io:fwrite(<<"~s: ~ts\n">>,
[File,Mod:format_error(E)]);
({File,{Line,Mod,E}}) ->
- io:fwrite(<<"~s:~w: ~s\n">>,
+ io:fwrite(<<"~s:~w: ~ts\n">>,
[File,Line,Mod:format_error(E)])
end, sort(St#yecc.errors));
false ->
@@ -861,11 +878,11 @@ report_warnings(St) ->
case member(report_warnings, St#yecc.options) orelse ReportWerror of
true ->
foreach(fun({File,{none,Mod,W}}) ->
- io:fwrite(<<"~s: ~s~s\n">>,
+ io:fwrite(<<"~s: ~s~ts\n">>,
[File,Prefix,
Mod:format_error(W)]);
({File,{Line,Mod,W}}) ->
- io:fwrite(<<"~s:~w: ~s~s\n">>,
+ io:fwrite(<<"~s:~w: ~s~ts\n">>,
[File,Line,Prefix,
Mod:format_error(W)])
end, sort(St#yecc.warnings));
@@ -1024,7 +1041,7 @@ compute_states(St0) ->
rp_info = RulePointerInfo,
goto = GotoTab},
- erase(),
+ _ = erase(),
EndsymCode = code_terminal(StC#yecc.endsymbol, StC#yecc.symbol_tab),
{StateId, State0} = compute_state([{EndsymCode, 1}], Tables),
@@ -1923,9 +1940,10 @@ output_prelude(Outport, Inport, St0) when St0#yecc.includefile =:= [] ->
{St20, 0, no_erlang_code};
Next_line ->
St_10 = output_file_directive(St20, Infile, Next_line-1),
- Nmbr_of_lines = include1([], Inport, Outport),
- {St_10, Nmbr_of_lines,
- {last_erlang_code_line, Next_line+Nmbr_of_lines}}
+ Last_line = include1([], Inport, Outport, Infile,
+ Next_line, St_10),
+ Nmbr_of_lines = Last_line - Next_line,
+ {St_10, Nmbr_of_lines, {last_erlang_code_line, Last_line}}
end,
St30 = nl(St25),
IncludeFile =
@@ -1946,13 +1964,13 @@ output_prelude(Outport, Inport, St0) ->
{St30, N_lines_1, no_erlang_code};
Next_line ->
St = output_file_directive(St30, Infile, Next_line-1),
- Nmbr_of_lines = include1([], Inport, Outport),
- {St, Nmbr_of_lines + N_lines_1,
- {last_erlang_code_line, Next_line+Nmbr_of_lines}}
+ Last_line = include1([], Inport, Outport, Infile, Next_line, St),
+ Nmbr_of_lines = Last_line - Next_line,
+ {St, Nmbr_of_lines + N_lines_1, {last_erlang_code_line, Last_line}}
end.
output_header(St0) ->
- lists:foldl(fun(Str, St) -> fwrite(St, <<"~s\n">>, [Str])
+ lists:foldl(fun(Str, St) -> fwrite(St, <<"~ts\n">>, [Str])
end, St0, St0#yecc.header).
output_goto(St, [{_Nonterminal, []} | Go], StateInfo) ->
@@ -2250,8 +2268,8 @@ output_inlined(St0, FunctionName, Reduce, Infile) ->
[append(["[", tl(A), " | __Stack]"])])
end,
St = St40#yecc{line = St40#yecc.line + NLines},
- fwrite(St, <<" [begin\n ~s\n end | ~s].\n\n">>,
- [pp_tokens(Tokens, Line0), Stack]).
+ fwrite(St, <<" [begin\n ~ts\n end | ~s].\n\n">>,
+ [pp_tokens(Tokens, Line0, St#yecc.encoding), Stack]).
inlined_function_name(State, "Cat") ->
inlined_function_name(State, "");
@@ -2421,24 +2439,24 @@ include(St, File, Outport) ->
{error, Reason} ->
throw(add_error(File, none, {file_error, Reason}, St));
{ok, Inport} ->
+ _ = epp:set_encoding(Inport),
Line = io:get_line(Inport, ''),
- N_lines = include1(Line, Inport, Outport),
- file:close(Inport),
- N_lines
+ try include1(Line, Inport, Outport, File, 1, St) - 1
+ after ok = file:close(Inport)
+ end
end.
-include1(Line, Inport, Outport) ->
- include1(Line, Inport, Outport, 0).
-
-include1(eof, _, _, Nmbr_of_lines) ->
- Nmbr_of_lines;
-include1(Line, Inport, Outport, Nmbr_of_lines) ->
+include1(eof, _, _, _File, L, _St) ->
+ L;
+include1({error, _}=_Error, _Inport, _Outport, File, L, St) ->
+ throw(add_error(File, L, cannot_parse, St));
+include1(Line, Inport, Outport, File, L, St) ->
Incr = case member($\n, Line) of
true -> 1;
false -> 0
end,
io:put_chars(Outport, Line),
- include1(io:get_line(Inport, ''), Inport, Outport, Nmbr_of_lines + Incr).
+ include1(io:get_line(Inport, ''), Inport, Outport, File, L + Incr, St).
includefile_version([]) ->
{1,4};
@@ -2465,18 +2483,22 @@ parse_file(Epp) ->
end.
%% Keeps the line breaks of the original code.
-pp_tokens(Tokens, Line0) ->
- concat(pp_tokens1(Tokens, Line0, [])).
+pp_tokens(Tokens, Line0, Enc) ->
+ concat(pp_tokens1(Tokens, Line0, Enc, [])).
-pp_tokens1([], _Line0, _T0) ->
+pp_tokens1([], _Line0, _Enc, _T0) ->
[];
-pp_tokens1([T | Ts], Line0, T0) ->
+pp_tokens1([T | Ts], Line0, Enc, T0) ->
Line = element(2, T),
- [pp_sep(Line, Line0, T0), pp_symbol(T) | pp_tokens1(Ts, Line, T)].
+ [pp_sep(Line, Line0, T0), pp_symbol(T, Enc)|pp_tokens1(Ts, Line, Enc, T)].
-pp_symbol({var,_,Var}) -> Var;
-pp_symbol({_,_,Symbol}) -> io_lib:fwrite(<<"~p">>, [Symbol]);
-pp_symbol({Symbol, _}) -> Symbol.
+pp_symbol({var,_,Var}, _Enc) -> Var;
+pp_symbol({string,_,String}, latin1) ->
+ io_lib:write_unicode_string_as_latin1(String);
+pp_symbol({string,_,String}, _Enc) -> io_lib:write_unicode_string(String);
+pp_symbol({_,_,Symbol}, latin1) -> io_lib:fwrite(<<"~p">>, [Symbol]);
+pp_symbol({_,_,Symbol}, _Enc) -> io_lib:fwrite(<<"~tp">>, [Symbol]);
+pp_symbol({Symbol, _}, _Enc) -> Symbol.
pp_sep(Line, Line0, T0) when Line > Line0 ->
["\n " | pp_sep(Line - 1, Line0, T0)];
@@ -2485,6 +2507,16 @@ pp_sep(_Line, _Line0, {'.',_}) ->
pp_sep(_Line, _Line0, _T0) ->
" ".
+set_encoding(#yecc{encoding = none}, Port) ->
+ ok = io:setopts(Port, [{encoding, epp:default_encoding()}]);
+set_encoding(#yecc{encoding = E}, Port) ->
+ ok = io:setopts(Port, [{encoding, E}]).
+
+output_encoding_comment(#yecc{encoding = none}=St) ->
+ St;
+output_encoding_comment(#yecc{encoding = Encoding}=St) ->
+ fwrite(St, <<"%% ~s\n">>, [epp:encoding_to_string(Encoding)]).
+
output_file_directive(St, Filename, Line) when St#yecc.file_attrs ->
fwrite(St, <<"-file(~s, ~w).\n">>,
[format_filename(Filename), Line]);
@@ -2529,7 +2561,7 @@ format_assoc(nonassoc) ->
format_symbol(Symbol) ->
String = concat([Symbol]),
- case erl_scan:string(String) of
+ case erl_scan:string(String, 1, [unicode]) of
{ok, [{atom, _, _}], _} ->
io_lib:fwrite(<<"~w">>, [Symbol]);
{ok, [{Word, _}], _} when Word =/= ':', Word =/= '->' ->
diff --git a/lib/parsetools/src/yeccscan.erl b/lib/parsetools/src/yeccscan.erl
index d7ec3ba8d3..9e0e85143a 100644
--- a/lib/parsetools/src/yeccscan.erl
+++ b/lib/parsetools/src/yeccscan.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1996-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2012. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -24,7 +24,7 @@ scan(Inport) ->
scan(Inport, '', 1).
scan(Inport, Prompt, Line1) ->
- case catch io:scan_erl_form(Inport, Prompt, Line1) of
+ case catch io:scan_erl_form(Inport, Prompt, Line1, [unicode]) of
{eof, Line2} ->
{eof, Line2};
{ok, Tokens, Line2} ->
@@ -34,6 +34,8 @@ scan(Inport, Prompt, Line1) ->
_ ->
{ok, lex(Tokens), Line2}
end;
+ {error, Reason} ->
+ {error, Reason};
{error, Descriptor, Line2} ->
{error, Descriptor, Line2};
{'EXIT', Why} ->
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 20:00:30 +0000