diff options
Diffstat (limited to 'lib/compiler/src/core_scan.erl')
| -rw-r--r-- | lib/compiler/src/core_scan.erl | 65 |
1 files changed, 47 insertions, 18 deletions
diff --git a/lib/compiler/src/core_scan.erl b/lib/compiler/src/core_scan.erl index b7799b373a..a50a2ffa8d 100644 --- a/lib/compiler/src/core_scan.erl +++ b/lib/compiler/src/core_scan.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2013. All Rights Reserved. +%% Copyright Ericsson AB 2000-2017. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -48,13 +49,37 @@ -import(lists, [reverse/1]). +-type location() :: integer(). +-type category() :: atom(). +-type symbol() :: atom() | float() | integer() | string(). +-type token() :: {category(), Anno :: location(), symbol()} + | {category(), Anno :: location()}. +-type tokens() :: [token()]. +-type error_description() :: term(). +-type error_info() :: {erl_anno:location(), module(), error_description()}. + %% string([Char]) -> %% string([Char], StartPos) -> %% {ok, [Tok], EndPos} | %% {error, {Pos,core_scan,What}, EndPos} +-spec string(String) -> Return when + String :: string(), + Return :: {'ok', Tokens :: tokens(), EndLocation} + | {'error', ErrorInfo :: error_info(), ErrorLocation}, + EndLocation :: location(), + ErrorLocation :: location(). + string(Cs) -> string(Cs, 1). +-spec string(String, StartLocation) -> Return when + String :: string(), + Return :: {'ok', Tokens :: tokens(), EndLocation} + | {'error', ErrorInfo :: error_info(), ErrorLocation}, + StartLocation :: location(), + EndLocation :: location(), + ErrorLocation :: location(). + string(Cs, Sp) -> %% Add an 'eof' to always get correct handling. case string_pre_scan(Cs, [], Sp) of @@ -175,8 +200,8 @@ pre_string(eof, Q, _, Sp, SoFar, Pos) -> pre_string_error(Q, Sp, SoFar, Pos). pre_string_error(Q, Sp, SoFar, Pos) -> - S = reverse(string:substr(SoFar, 1, string:chr(SoFar, Q)-1)), - pre_error({string,Q,string:substr(S, 1, 16)}, Sp, Pos). + [S,_] = string:split(SoFar, [Q]), + pre_error({string,Q,string:slice(string:reverse(S), 0, 16)}, Sp, Pos). pre_char([C|Cs], SoFar) -> pre_char(C, Cs, SoFar); pre_char([], _) -> more; @@ -258,10 +283,12 @@ scan1([$$|Cs0], Toks, Pos) -> %Character constant scan1(Cs, [{char,Pos,C}|Toks], Pos1); scan1([$'|Cs0], Toks, Pos) -> %Atom (always quoted) {S,Cs1,Pos1} = scan_string(Cs0, $', Pos), - case catch list_to_atom(S) of + try binary_to_atom(list_to_binary(S), utf8) of A when is_atom(A) -> - scan1(Cs1, [{atom,Pos,A}|Toks], Pos1); - _Error -> scan_error({illegal,atom}, Pos) + scan1(Cs1, [{atom,Pos,A}|Toks], Pos1) + catch + error:_ -> + scan_error({illegal,atom}, Pos) end; scan1([$"|Cs0], Toks, Pos) -> %String {S,Cs1,Pos1} = scan_string(Cs0, $", Pos), @@ -271,8 +298,10 @@ scan1("->" ++ Cs, Toks, Pos) -> scan1(Cs, [{'->',Pos}|Toks], Pos); scan1("-|" ++ Cs, Toks, Pos) -> scan1(Cs, [{'-|',Pos}|Toks], Pos); -scan1("::" ++ Cs, Toks, Pos) -> - scan1(Cs, [{'::',Pos}|Toks], Pos); +scan1(":=" ++ Cs, Toks, Pos) -> + scan1(Cs, [{':=',Pos}|Toks], Pos); +scan1("=>" ++ Cs, Toks, Pos) -> + scan1(Cs, [{'=>',Pos}|Toks], Pos); scan1([C|Cs], Toks, Pos) -> %Punctuation character P = list_to_atom([C]), scan1(Cs, [{P,Pos}|Toks], Pos); |