aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/compiler/src/compile.erl59
-rw-r--r--lib/compiler/test/error_SUITE.erl28
-rw-r--r--lib/compiler/test/warnings_SUITE.erl38
3 files changed, 111 insertions, 14 deletions
diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl
index 9030dd998b..c7d91070f6 100644
--- a/lib/compiler/src/compile.erl
+++ b/lib/compiler/src/compile.erl
@@ -234,7 +234,9 @@ format_error({crash,Pass,Reason}) ->
format_error({bad_return,Pass,Reason}) ->
io_lib:format("internal error in ~p;\nbad return value: ~ts", [Pass,format_error_reason(Reason)]);
format_error({module_name,Mod,Filename}) ->
- io_lib:format("Module name '~s' does not match file name '~ts'", [Mod,Filename]).
+ io_lib:format("Module name '~s' does not match file name '~ts'", [Mod,Filename]);
+format_error(reparsing_invalid_unicode) ->
+ "Non-UTF-8 character(s) detected, but no encoding declared. Encode the file in UTF-8 or add \"%% coding: latin-1\" at the beginning of the file. Retrying with latin-1 encoding.".
format_error_reason({Reason, Stack}) when is_list(Stack) ->
StackFun = fun
@@ -792,20 +794,59 @@ no_native_compilation(BeamFile, #compile{options=Opts0}) ->
_ -> false
end.
-parse_module(St) ->
- Opts = St#compile.options,
- Cwd = ".",
- IncludePath = [Cwd, St#compile.dir|inc_paths(Opts)],
- R = epp:parse_file(St#compile.ifile, IncludePath, pre_defs(Opts)),
+parse_module(St0) ->
+ case do_parse_module(utf8, St0) of
+ {ok,_}=Ret ->
+ Ret;
+ {error,_}=Ret ->
+ Ret;
+ {invalid_unicode,File,Line} ->
+ case do_parse_module(latin1, St0) of
+ {ok,St} ->
+ Es = [{File,[{Line,?MODULE,reparsing_invalid_unicode}]}],
+ {ok,St#compile{warnings=Es++St#compile.warnings}};
+ {error,St} ->
+ Es = [{File,[{Line,?MODULE,reparsing_invalid_unicode}]}],
+ {error,St#compile{errors=Es++St#compile.errors}}
+ end
+ end.
+
+do_parse_module(DefEncoding, #compile{ifile=File,options=Opts,dir=Dir}=St) ->
+ R = epp:parse_file(File,
+ [{includes,[".",Dir|inc_paths(Opts)]},
+ {macros,pre_defs(Opts)},
+ {default_encoding,DefEncoding},
+ extra]),
case R of
- {ok,Forms} ->
- Encoding = epp:read_encoding(St#compile.ifile),
- {ok,St#compile{code=Forms,encoding=Encoding}};
+ {ok,Forms,Extra} ->
+ Encoding = proplists:get_value(encoding, Extra),
+ case find_invalid_unicode(Forms, File) of
+ none ->
+ {ok,St#compile{code=Forms,encoding=Encoding}};
+ {invalid_unicode,_,_}=Ret ->
+ case Encoding of
+ none ->
+ Ret;
+ _ ->
+ {ok,St#compile{code=Forms,encoding=Encoding}}
+ end
+ end;
{error,E} ->
Es = [{St#compile.ifile,[{none,?MODULE,{epp,E}}]}],
{error,St#compile{errors=St#compile.errors ++ Es}}
end.
+find_invalid_unicode([H|T], File0) ->
+ case H of
+ {attribute,_,file,{File,_}} ->
+ find_invalid_unicode(T, File);
+ {error,{Line,file_io_server,invalid_unicode}} ->
+ {invalid_unicode,File0,Line};
+ _Other ->
+ find_invalid_unicode(T, File0)
+ end;
+find_invalid_unicode([], _) -> none.
+
parse_core(St) ->
case file:read_file(St#compile.ifile) of
{ok,Bin} ->
diff --git a/lib/compiler/test/error_SUITE.erl b/lib/compiler/test/error_SUITE.erl
index 5cdf429a5f..bd877bb528 100644
--- a/lib/compiler/test/error_SUITE.erl
+++ b/lib/compiler/test/error_SUITE.erl
@@ -23,7 +23,7 @@
-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
init_per_group/2,end_per_group/2,
head_mismatch_line/1,warnings_as_errors/1, bif_clashes/1,
- transforms/1,forbidden_maps/1]).
+ transforms/1,forbidden_maps/1,bad_utf8/1]).
%% Used by transforms/1 test case.
-export([parse_transform/2]).
@@ -36,7 +36,8 @@ all() ->
groups() ->
[{p,test_lib:parallel(),
- [head_mismatch_line,warnings_as_errors,bif_clashes,transforms,forbidden_maps]}].
+ [head_mismatch_line,warnings_as_errors,bif_clashes,
+ transforms,forbidden_maps,bad_utf8]}].
init_per_suite(Config) ->
Config.
@@ -254,6 +255,23 @@ forbidden_maps(Config) when is_list(Config) ->
[] = run2(Config, Ts1),
ok.
+bad_utf8(Config) ->
+ Ts = [{bad_utf8,
+ %% If coding is specified explicitly as utf-8, there should be
+ %% a compilation error; we must not fallback to parsing the
+ %% file in latin-1 mode.
+ <<"%% coding: utf-8
+ %% Bj",246,"rn
+ t() -> \"",246,"\".
+ ">>,
+ [],
+ {error,[{2,epp,cannot_parse},
+ {2,file_io_server,invalid_unicode}],
+ []}
+ }],
+ [] = run2(Config, Ts),
+ ok.
+
run(Config, Tests) ->
?line File = test_filename(Config),
@@ -318,6 +336,7 @@ run_test(Test0, File, Warnings, WriteBeam) ->
?line compile:file(File, [binary,report|Warnings]),
%% Test result of compilation.
+ io:format("~p\n", [Opts]),
?line Res = case compile:file(File, Opts) of
{ok,Mod,_,[{_File,Ws}]} ->
%io:format("compile:file(~s,~p) ->~n~p~n",
@@ -335,6 +354,11 @@ run_test(Test0, File, Warnings, WriteBeam) ->
%io:format("compile:file(~s,~p) ->~n~p~n",
% [File,Opts,_ZZ]),
{error,Es,Ws};
+ {error,[{XFile,Es1},{XFile,Es2}],Ws} = _ZZ
+ when is_list(XFile) ->
+ %io:format("compile:file(~s,~p) ->~n~p~n",
+ % [File,Opts,_ZZ]),
+ {error,Es1++Es2,Ws};
{error,Es,[{_File,Ws}]} = _ZZ->
%io:format("compile:file(~s,~p) ->~n~p~n",
% [File,Opts,_ZZ]),
diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl
index de56a59e12..c3b02819f9 100644
--- a/lib/compiler/test/warnings_SUITE.erl
+++ b/lib/compiler/test/warnings_SUITE.erl
@@ -37,8 +37,9 @@
-export([pattern/1,pattern2/1,pattern3/1,pattern4/1,
guard/1,bad_arith/1,bool_cases/1,bad_apply/1,
- files/1,effect/1,bin_opt_info/1,bin_construction/1, comprehensions/1,
- maps/1,redundant_boolean_clauses/1]).
+ files/1,effect/1,bin_opt_info/1,bin_construction/1,
+ comprehensions/1,maps/1,redundant_boolean_clauses/1,
+ latin1_fallback/1]).
% Default timetrap timeout (set in init_per_testcase).
-define(default_timeout, ?t:minutes(2)).
@@ -63,7 +64,7 @@ groups() ->
[pattern,pattern2,pattern3,pattern4,guard,
bad_arith,bool_cases,bad_apply,files,effect,
bin_opt_info,bin_construction,comprehensions,maps,
- redundant_boolean_clauses]}].
+ redundant_boolean_clauses,latin1_fallback]}].
init_per_suite(Config) ->
Config.
@@ -591,6 +592,37 @@ redundant_boolean_clauses(Config) when is_list(Config) ->
run(Config, Ts),
ok.
+latin1_fallback(Conf) when is_list(Conf) ->
+ DataDir = ?privdir,
+ IncFile = filename:join(DataDir, "include_me.hrl"),
+ file:write_file(IncFile, <<"%% ",246," in include file\n">>),
+ Ts1 = [{latin1_fallback1,
+ %% Test that the compiler fall backs to latin-1 with
+ %% a warning if a file has no encoding and does not
+ %% contain correct UTF-8 sequences.
+ <<"%% Bj",246,"rn
+ t(_) -> \"",246,"\";
+ t(x) -> ok.
+ ">>,
+ [],
+ {warnings,[{1,compile,reparsing_invalid_unicode},
+ {3,sys_core_fold,{nomatch_shadow,2}}]}}],
+ [] = run(Conf, Ts1),
+
+ Ts2 = [{latin1_fallback2,
+ %% Test that the compiler fall backs to latin-1 with
+ %% a warning if a file has no encoding and does not
+ %% contain correct UTF-8 sequences.
+ <<"
+
+ -include(\"include_me.hrl\").
+ ">>,
+ [],
+ {warnings,[{1,compile,reparsing_invalid_unicode}]}
+ }],
+ [] = run(Conf, Ts2),
+ ok.
+
%%%
%%% End of test cases.
%%%
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 07:23:16 +0000