Don't fail compilation for modules that contain invalid UTF-8

The default encoding for Erlang modules is now UTF-8, and the
compilation would fail if a module contained byte sequences that
are not valid UTF-8 sequences.

In a large project with say many hundreds of Erlang modules
with names of developers such as "Björn" or "Håkan" encoded in
latin-1, that could mean that many hundreds of files would need
to be modified just to get started testing OTP 17.

As a temporary measure to ease the transition, automatically
fall back to the latin-1 encoding with a warning for any module
that contains invalid byte sequences and for which no encoding
has been specified.

The intention is to remove this workaround in OTP 18 or 19.

2 files changed, 61 insertions, 5 deletions

diff --git a/lib/compiler/test/error_SUITE.erl b/lib/compiler/test/error_SUITE.erl
index 5cdf429a5f..bd877bb528 100644
--- a/lib/compiler/test/error_SUITE.erl
+++ b/lib/compiler/test/error_SUITE.erl
@@ -23,7 +23,7 @@
 -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
 	 init_per_group/2,end_per_group/2,
 	 head_mismatch_line/1,warnings_as_errors/1, bif_clashes/1,
-	 transforms/1,forbidden_maps/1]).
+	 transforms/1,forbidden_maps/1,bad_utf8/1]).
 
 %% Used by transforms/1 test case.
 -export([parse_transform/2]).
@@ -36,7 +36,8 @@ all() ->
 
 groups() -> 
     [{p,test_lib:parallel(),
-      [head_mismatch_line,warnings_as_errors,bif_clashes,transforms,forbidden_maps]}].
+      [head_mismatch_line,warnings_as_errors,bif_clashes,
+       transforms,forbidden_maps,bad_utf8]}].
 
 init_per_suite(Config) ->
     Config.
@@ -254,6 +255,23 @@ forbidden_maps(Config) when is_list(Config) ->
     [] = run2(Config, Ts1),
     ok.
 
+bad_utf8(Config) ->
+    Ts = [{bad_utf8,
+	   %% If coding is specified explicitly as utf-8, there should be
+	   %% a compilation error; we must not fallback to parsing the
+	   %% file in latin-1 mode.
+	   <<"%% coding: utf-8
+              %% Bj",246,"rn
+              t() -> \"",246,"\".
+             ">>,
+	   [],
+	   {error,[{2,epp,cannot_parse},
+		   {2,file_io_server,invalid_unicode}],
+	    []}
+	  }],
+    [] = run2(Config, Ts),
+    ok.
+
 
 run(Config, Tests) ->
     ?line File = test_filename(Config),
@@ -318,6 +336,7 @@ run_test(Test0, File, Warnings, WriteBeam) ->
     ?line compile:file(File, [binary,report|Warnings]),
 
     %% Test result of compilation.
+    io:format("~p\n", [Opts]),
     ?line Res = case compile:file(File, Opts) of
 		    {ok,Mod,_,[{_File,Ws}]} ->
 			%io:format("compile:file(~s,~p) ->~n~p~n",
@@ -335,6 +354,11 @@ run_test(Test0, File, Warnings, WriteBeam) ->
 			%io:format("compile:file(~s,~p) ->~n~p~n",
 			%	  [File,Opts,_ZZ]),
 			{error,Es,Ws};
+		    {error,[{XFile,Es1},{XFile,Es2}],Ws} = _ZZ
+		      when is_list(XFile) ->
+			%io:format("compile:file(~s,~p) ->~n~p~n",
+			%	  [File,Opts,_ZZ]),
+			{error,Es1++Es2,Ws};
 		    {error,Es,[{_File,Ws}]} = _ZZ->
 			%io:format("compile:file(~s,~p) ->~n~p~n",
 			%	  [File,Opts,_ZZ]),
diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl
index de56a59e12..c3b02819f9 100644
--- a/lib/compiler/test/warnings_SUITE.erl
+++ b/lib/compiler/test/warnings_SUITE.erl
@@ -37,8 +37,9 @@
 
 -export([pattern/1,pattern2/1,pattern3/1,pattern4/1,
 	 guard/1,bad_arith/1,bool_cases/1,bad_apply/1,
-         files/1,effect/1,bin_opt_info/1,bin_construction/1, comprehensions/1,
-         maps/1,redundant_boolean_clauses/1]).
+         files/1,effect/1,bin_opt_info/1,bin_construction/1,
+	 comprehensions/1,maps/1,redundant_boolean_clauses/1,
+	 latin1_fallback/1]).
 
 % Default timetrap timeout (set in init_per_testcase).
 -define(default_timeout, ?t:minutes(2)).
@@ -63,7 +64,7 @@ groups() ->
       [pattern,pattern2,pattern3,pattern4,guard,
        bad_arith,bool_cases,bad_apply,files,effect,
        bin_opt_info,bin_construction,comprehensions,maps,
-       redundant_boolean_clauses]}].
+       redundant_boolean_clauses,latin1_fallback]}].
 
 init_per_suite(Config) ->
     Config.
@@ -591,6 +592,37 @@ redundant_boolean_clauses(Config) when is_list(Config) ->
     run(Config, Ts),
     ok.
 
+latin1_fallback(Conf) when is_list(Conf) ->
+    DataDir = ?privdir,
+    IncFile = filename:join(DataDir, "include_me.hrl"),
+    file:write_file(IncFile, <<"%% ",246," in include file\n">>),
+    Ts1 = [{latin1_fallback1,
+	    %% Test that the compiler fall backs to latin-1 with
+	    %% a warning if a file has no encoding and does not
+	    %% contain correct UTF-8 sequences.
+	    <<"%% Bj",246,"rn
+              t(_) -> \"",246,"\";
+              t(x) -> ok.
+              ">>,
+	    [],
+	    {warnings,[{1,compile,reparsing_invalid_unicode},
+		       {3,sys_core_fold,{nomatch_shadow,2}}]}}],
+    [] = run(Conf, Ts1),
+
+    Ts2 = [{latin1_fallback2,
+	    %% Test that the compiler fall backs to latin-1 with
+	    %% a warning if a file has no encoding and does not
+	    %% contain correct UTF-8 sequences.
+	    <<"
+
+	      -include(\"include_me.hrl\").
+              ">>,
+	    [],
+	    {warnings,[{1,compile,reparsing_invalid_unicode}]}
+	   }],
+    [] = run(Conf, Ts2),
+    ok.
+
 %%%
 %%% End of test cases.
 %%%