The R13B03 release.OTP_R13B03

1 files changed, 357 insertions, 0 deletions

diff --git a/lib/snmp/src/compile/snmpc_tok.erl b/lib/snmp/src/compile/snmpc_tok.erl
new file mode 100644
index 0000000000..6b99e7ae43
--- /dev/null
+++ b/lib/snmp/src/compile/snmpc_tok.erl
@@ -0,0 +1,357 @@
+%% 
+%% %CopyrightBegin%
+%% 
+%% Copyright Ericsson AB 1996-2009. All Rights Reserved.
+%% 
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%% 
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%% 
+%% %CopyrightEnd%
+%% 
+
+-module(snmpc_tok).
+
+%% c(snmpc_tok).
+
+%%----------------------------------------------------------------------
+%% Generic (?) Tokenizer.
+%%----------------------------------------------------------------------
+%% Token: {Category, Line, Value}|{Category, Line}
+%% (Category == <KeyWord> ==> 2-tuple (otherwise 3-tuple)
+%% Category: integer | quote | string 
+%%         | variable | atom | <keyword> | <single_char>
+%%----------------------------------------------------------------------
+
+%% API
+-export([start_link/2, stop/1, get_token/1, get_all_tokens/1, tokenize/2]).
+
+%% Internal exports
+-export([null_get_line/0, format_error/1, terminate/2, handle_call/3, init/1,
+	 test/0]).
+
+
+%%----------------------------------------------------------------------
+%% Reserved_words: list of KeyWords. Example: ['IF', 'BEGIN', ..., 'GOTO']
+%% Options: list of Option
+%% Option: {file, <filename>},
+%% or:     {get_line_function, {Mod, Func, Arg} (default io, get_line, [Fid])},
+%%         get_line_function shall behave as io:get_line.
+%%         {print_lineno, L}
+%% Returns: {ok, Pid} | {error, Reason}
+%%----------------------------------------------------------------------
+start_link(Reserved_words, Options) ->
+    case lists:keysearch(file, 1, Options) of
+	{value, {file, Filename}} ->
+	    case file:open(Filename, [read]) of
+		{ok, Fid} ->
+		    gen_server:start_link(?MODULE,
+					  {Reserved_words, Options,
+					   {io, get_line, [Fid, prompt]}}, []);
+		Error ->
+		    Str = format_error({"Cannot open file '~s' (~800p).~n",
+					[Filename, Error]}),
+		    {error,Str}
+	    end;
+	false ->
+	    MFA = case lists:keysearch(get_line_function, 1, Options) of
+		      {value, {get_line_function, {M, F, A}}} ->
+			  {M,F,A};
+		      false -> {?MODULE, null_get_line, []}
+		  end,
+	    gen_server:start_link(?MODULE,{Reserved_words,Options,MFA}, [])
+    end.
+    
+%%--------------------------------------------------
+%% Returns:
+%% {ok, [Token], LineNo} | {eof, LineNo} | {error, Error_description, Endline}
+%% For more information, see manual page for yecc (and its requirements on a
+%% tokenizer).
+%%--------------------------------------------------
+get_token(TokPid) ->
+    V = gen_server:call(TokPid, get_token, infinity),
+    %%  io:format("tok:~w~n", [V]),
+    V.
+
+get_all_tokens(TokPid) ->
+    V = gen_server:call(TokPid, get_all_tokens, infinity),
+    %%    io:format("tok:~w~n", [V]),
+    V.
+
+
+
+%%--------------------------------------------------
+%% Returns: {ok, Tokens, EndLine} | {error, Error_description, Endline}
+%% Comment: Tokeniser must be started since all options reside in
+%%          the process dictionary of the tokeniser process.
+%%--------------------------------------------------
+tokenize(TokPid, String) ->
+    gen_server:call(TokPid, {tokenize, String}, infinity).
+
+stop(TokPid) ->
+    gen_server:call(TokPid,stop, infinity).
+
+%%----------------------------------------------------------------------
+%% Implementation
+%%----------------------------------------------------------------------
+insert_keywords_into_ets(_DB, []) -> done;
+insert_keywords_into_ets(DB, [Word | T]) ->
+    ets:insert(DB, {Word, reserved_word}),
+    insert_keywords_into_ets(DB, T).
+
+reserved_word(X) ->
+    case ets:lookup(get(db), X) of
+	[{X, reserved_word}] -> 
+	    true;
+	_ -> 
+	    false
+    end.
+
+%% If you only need to tokenize strings
+null_get_line() -> eof.
+
+format_error({Format, Data}) ->
+    io_lib:format(lists:append("Tokeniser error: ", Format), Data).
+
+test() ->
+    start_link(['AUGMENTS','BEGIN','CONTACT-INFO','DEFINITIONS','DEFVAL',
+		'DESCRIPTION','DISPLAY-HINT','END','IDENTIFIER','IMPLIED',
+		'INDEX','INTEGER','LAST-UPDATED','MAX-ACCESS','MODULE-IDENTITY',
+		'NOTIFICATION-TYPE','OBJECT','OBJECT-IDENTITY','OBJECT-TYPE',
+		'OBJECTS','ORGANIZATION','REFERENCE','REVISION',
+		'SIZE','STATUS','SYNTAX','TEXTUAL-CONVENTION','UNITS',
+		'current','deprecated','not-accessible','obsolete', 
+		'read-create','read-only','read-write', 'IMPORTS', 'FROM',
+		'MODULE-COMPLIANCE',
+		'DisplayString', 
+		'PhysAddress', 
+		'MacAddress', 
+		'TruthValue', 
+		'TestAndIncr', 
+		'AutonomousType', 
+		'InstancePointer', 
+		'VariablePointer', 
+		'RowPointer', 
+		'RowStatus', 
+		'TimeStamp', 
+		'TimeInterval', 
+		'DateAndTime', 
+		'StorageType', 
+		'TDomain', 
+		'TAddress'],
+	       [{file, "modemmib.mib"}]).
+
+init({Reserved_words, Options, GetLineMFA}) ->
+    put(get_line_function,GetLineMFA),
+    put(print_lineno,
+	case lists:keysearch(print_lineno, 1, Options) of
+	    {value, {print_lineno, L}} -> L;
+	    false -> undefined
+	end),
+    DB = ets:new(reserved_words, [set, private]),
+    insert_keywords_into_ets(DB, Reserved_words),
+    put(db, DB),
+    put(line, 0),
+    {ok, ""}.
+
+getLine() ->
+    OldLine = put(line, 1 + get(line)),
+    {M,F,A} = get(get_line_function),
+    case get(print_lineno) of
+	undefined -> true;
+	X -> case OldLine rem X of
+		 0 -> io:format('~w..',[OldLine]);
+		 _ -> true
+	     end
+    end,
+    apply(M,F,A).
+
+%% You can only do this when no file is open.
+handle_call({tokenizeString, String}, _From, "") ->
+    {reply, safe_tokenize_whole_string(String), ""};
+
+handle_call(get_token, _From, String) ->
+    {ReplyToken, RestChars} = safe_tokenise(String),
+    {reply, ReplyToken, RestChars};
+
+handle_call(get_all_tokens, _From, String) ->
+    Toks = get_all_tokens(String,[]),
+    {reply, Toks, []};
+
+
+handle_call(stop, _From, String) ->
+    {stop, normal, ok, String}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+%% ErrorInfo = {ErrorLine,  Module,  ErrorDescriptor}
+%% will be used as 
+%% apply(Module, format_error, [ErrorDescriptor]). shall return a string.
+
+%% Returns a reply
+tokenize_whole_string(eof) -> [];
+tokenize_whole_string(String) ->
+    {Token, RestChars} =  tokenise(String),
+    [Token | tokenize_whole_string(RestChars)].
+
+safe_tokenize_whole_string(String) ->
+    case catch tokenize_whole_string(String) of
+	{error, ErrorInfo} -> {error, ErrorInfo, get(line)};
+	Tokens -> {ok, Tokens, get(line)}
+    end.
+
+%%    throw({error, {get(line), ?MODULE, "Unexpected eof~n"}}).
+
+%% Returns: {ReplyToken, NewState}
+safe_tokenise(eof) -> {{eof, get(line)}, eof};
+safe_tokenise(Chars) when is_list(Chars) ->
+    case catch tokenise(Chars) of
+	{error, ErrorInfo} -> {{error, ErrorInfo, get(line)}, {[], eof}};
+	{Token, RestChars} when is_tuple(Token) ->
+	    {{ok, [Token], get(line)}, RestChars}
+    end.
+
+get_all_tokens(eof,Toks) -> 
+    lists:reverse(Toks);
+get_all_tokens(Str,Toks) ->
+    case catch tokenise(Str) of
+	{error, ErrorInfo} -> {error, ErrorInfo};
+	{Token, RestChars} when is_tuple(Token) -> 
+	    get_all_tokens(RestChars, [Token|Toks])
+    end.
+
+    
+
+%%--------------------------------------------------
+%% Returns: {Token, Rest}
+%%--------------------------------------------------
+tokenise([H|T]) when ($a =< H) andalso (H =< $z) ->
+    get_name(atom, [H], T);
+
+tokenise([H|T]) when ($A =< H) andalso (H =< $Z) ->
+    get_name(variable, [H], T);
+
+tokenise([$:,$:,$=|T]) ->
+    {{'::=', get(line)}, T};
+
+tokenise([$-,$-|T]) ->
+    tokenise(skip_comment(T));
+
+tokenise([$-,H|T]) when ($0 =< H ) andalso (H =< $9) ->
+    {Val, Rest} = get_integer(T, [H]),
+    {{integer, get(line), -1 * Val}, Rest};
+
+tokenise([H|T]) when ($0 =< H) andalso (H =< $9) ->
+    {Val, Rest} = get_integer(T, [H]),
+    {{integer, get(line), Val}, Rest};
+
+tokenise([$"|T]) ->
+    collect_string($", T, []);
+
+tokenise([$'|T]) ->
+    collect_string($', T, []);
+
+%% Read away white spaces
+tokenise([9| T]) -> tokenise(T);
+tokenise([10| T]) -> tokenise(T);
+tokenise([13| T]) -> tokenise(T);
+tokenise([32| T]) -> tokenise(T);
+
+%% Handle singe characters like { } [ ] + = ...
+tokenise([Ch | T]) ->
+    Atm = list_to_atom([Ch]),
+    {{Atm, get(line)}, T};
+
+tokenise([]) ->
+    tokenise(getLine());
+
+tokenise(eof) ->
+    {{'$end', get(line)}, eof}.
+
+collect_string($", [$"|T],BackwardsStr) ->
+    {{string, get(line), BackwardsStr}, T};
+
+collect_string($', [$'|T],BackwardsStr) ->
+    {{quote, get(line), BackwardsStr}, T};
+
+collect_string(StopChar, [Ch|T], Str) ->
+    collect_string(StopChar,T,[Ch|Str]);
+
+collect_string(StopChar, [],Str) ->
+    collect_string(StopChar, getLine(), Str);
+
+collect_string(StopChar, eof, Str) ->
+    throw({error, {get(line), ?MODULE,
+		   {"Missing ~s in string:~n \"~s\"~n",
+		    [[StopChar], lists:reverse(Str)]}}}).
+
+get_name(Category, Name, [Char|T]) ->
+    case isInName(Char) of
+	true ->
+	    get_name(Category, [Char|Name], T);
+	false ->
+	    makeNameRespons(Category, Name, [Char | T])
+    end;
+get_name(Category, Name, []) ->
+    makeNameRespons(Category, Name, []).
+
+makeNameRespons(Category, Name, RestChars) ->
+    Atm = list_to_atom(lists:reverse(Name)),
+    case reserved_word(Atm) of
+	true -> {{Atm, get(line)}, RestChars};
+	false -> {{Category, get(line), Atm}, RestChars}
+    end.
+
+isInName($-) -> true;
+isInName(Ch) -> isalnum(Ch).
+	    
+isalnum(H) when ($A =< H) andalso (H =< $Z) ->
+    true;
+isalnum(H) when ($a =< H) andalso (H =< $z) ->
+    true;
+isalnum(H) when ($0 =< H) andalso (H =< $9) ->
+    true;
+isalnum(_) ->
+    false.
+
+isdigit(H) when ($0 =< H) andalso (H =< $9) ->
+    true;
+isdigit(_) ->
+    false.
+
+get_integer([H|T], "0") ->
+    case isdigit(H) of
+	true ->
+	    throw({error, {get(line), ?MODULE,
+			   {"Unexpected ~w~n",
+			    [list_to_atom([H])]}}});
+	false ->
+	    {0, [H|T]}
+    end;
+get_integer([H|T], L) ->
+    case isdigit(H) of
+	true ->
+	    get_integer(T, [H|L]);
+	false ->
+	    {list_to_integer(lists:reverse(L)), [H|T]}
+    end;
+get_integer([], L) ->
+    {list_to_integer(lists:reverse(L)), []}.
+
+%%--------------------------------------------------
+%% ASN.1 type of comments. "--" is comment to eoln or next "--"
+%%--------------------------------------------------
+skip_comment([]) ->
+    [];
+skip_comment([$-,$-|T]) ->
+    T;
+skip_comment([_|T]) ->
+    skip_comment(T).