aboutsummaryrefslogtreecommitdiffstats
path: root/lib/snmp/src/compile/snmpc_tok.erl
diff options
context:
space:
mode:
authorErlang/OTP <[email protected]>2009-11-20 14:54:40 +0000
committerErlang/OTP <[email protected]>2009-11-20 14:54:40 +0000
commit84adefa331c4159d432d22840663c38f155cd4c1 (patch)
treebff9a9c66adda4df2106dfd0e5c053ab182a12bd /lib/snmp/src/compile/snmpc_tok.erl
downloadotp-84adefa331c4159d432d22840663c38f155cd4c1.tar.gz
otp-84adefa331c4159d432d22840663c38f155cd4c1.tar.bz2
otp-84adefa331c4159d432d22840663c38f155cd4c1.zip
The R13B03 release.OTP_R13B03
Diffstat (limited to 'lib/snmp/src/compile/snmpc_tok.erl')
-rw-r--r--lib/snmp/src/compile/snmpc_tok.erl357
1 files changed, 357 insertions, 0 deletions
diff --git a/lib/snmp/src/compile/snmpc_tok.erl b/lib/snmp/src/compile/snmpc_tok.erl
new file mode 100644
index 0000000000..6b99e7ae43
--- /dev/null
+++ b/lib/snmp/src/compile/snmpc_tok.erl
@@ -0,0 +1,357 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1996-2009. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(snmpc_tok).
+
+%% c(snmpc_tok).
+
+%%----------------------------------------------------------------------
+%% Generic (?) Tokenizer.
+%%----------------------------------------------------------------------
+%% Token: {Category, Line, Value}|{Category, Line}
+%% (Category == <KeyWord> ==> 2-tuple (otherwise 3-tuple)
+%% Category: integer | quote | string
+%% | variable | atom | <keyword> | <single_char>
+%%----------------------------------------------------------------------
+
+%% API
+-export([start_link/2, stop/1, get_token/1, get_all_tokens/1, tokenize/2]).
+
+%% Internal exports
+-export([null_get_line/0, format_error/1, terminate/2, handle_call/3, init/1,
+ test/0]).
+
+
+%%----------------------------------------------------------------------
+%% Reserved_words: list of KeyWords. Example: ['IF', 'BEGIN', ..., 'GOTO']
+%% Options: list of Option
+%% Option: {file, <filename>},
+%% or: {get_line_function, {Mod, Func, Arg} (default io, get_line, [Fid])},
+%% get_line_function shall behave as io:get_line.
+%% {print_lineno, L}
+%% Returns: {ok, Pid} | {error, Reason}
+%%----------------------------------------------------------------------
+start_link(Reserved_words, Options) ->
+ case lists:keysearch(file, 1, Options) of
+ {value, {file, Filename}} ->
+ case file:open(Filename, [read]) of
+ {ok, Fid} ->
+ gen_server:start_link(?MODULE,
+ {Reserved_words, Options,
+ {io, get_line, [Fid, prompt]}}, []);
+ Error ->
+ Str = format_error({"Cannot open file '~s' (~800p).~n",
+ [Filename, Error]}),
+ {error,Str}
+ end;
+ false ->
+ MFA = case lists:keysearch(get_line_function, 1, Options) of
+ {value, {get_line_function, {M, F, A}}} ->
+ {M,F,A};
+ false -> {?MODULE, null_get_line, []}
+ end,
+ gen_server:start_link(?MODULE,{Reserved_words,Options,MFA}, [])
+ end.
+
+%%--------------------------------------------------
+%% Returns:
+%% {ok, [Token], LineNo} | {eof, LineNo} | {error, Error_description, Endline}
+%% For more information, see manual page for yecc (and its requirements on a
+%% tokenizer).
+%%--------------------------------------------------
+get_token(TokPid) ->
+ V = gen_server:call(TokPid, get_token, infinity),
+ %% io:format("tok:~w~n", [V]),
+ V.
+
+get_all_tokens(TokPid) ->
+ V = gen_server:call(TokPid, get_all_tokens, infinity),
+ %% io:format("tok:~w~n", [V]),
+ V.
+
+
+
+%%--------------------------------------------------
+%% Returns: {ok, Tokens, EndLine} | {error, Error_description, Endline}
+%% Comment: Tokeniser must be started since all options reside in
+%% the process dictionary of the tokeniser process.
+%%--------------------------------------------------
+tokenize(TokPid, String) ->
+ gen_server:call(TokPid, {tokenize, String}, infinity).
+
+stop(TokPid) ->
+ gen_server:call(TokPid,stop, infinity).
+
+%%----------------------------------------------------------------------
+%% Implementation
+%%----------------------------------------------------------------------
+insert_keywords_into_ets(_DB, []) -> done;
+insert_keywords_into_ets(DB, [Word | T]) ->
+ ets:insert(DB, {Word, reserved_word}),
+ insert_keywords_into_ets(DB, T).
+
+reserved_word(X) ->
+ case ets:lookup(get(db), X) of
+ [{X, reserved_word}] ->
+ true;
+ _ ->
+ false
+ end.
+
+%% If you only need to tokenize strings
+null_get_line() -> eof.
+
+format_error({Format, Data}) ->
+ io_lib:format(lists:append("Tokeniser error: ", Format), Data).
+
+test() ->
+ start_link(['AUGMENTS','BEGIN','CONTACT-INFO','DEFINITIONS','DEFVAL',
+ 'DESCRIPTION','DISPLAY-HINT','END','IDENTIFIER','IMPLIED',
+ 'INDEX','INTEGER','LAST-UPDATED','MAX-ACCESS','MODULE-IDENTITY',
+ 'NOTIFICATION-TYPE','OBJECT','OBJECT-IDENTITY','OBJECT-TYPE',
+ 'OBJECTS','ORGANIZATION','REFERENCE','REVISION',
+ 'SIZE','STATUS','SYNTAX','TEXTUAL-CONVENTION','UNITS',
+ 'current','deprecated','not-accessible','obsolete',
+ 'read-create','read-only','read-write', 'IMPORTS', 'FROM',
+ 'MODULE-COMPLIANCE',
+ 'DisplayString',
+ 'PhysAddress',
+ 'MacAddress',
+ 'TruthValue',
+ 'TestAndIncr',
+ 'AutonomousType',
+ 'InstancePointer',
+ 'VariablePointer',
+ 'RowPointer',
+ 'RowStatus',
+ 'TimeStamp',
+ 'TimeInterval',
+ 'DateAndTime',
+ 'StorageType',
+ 'TDomain',
+ 'TAddress'],
+ [{file, "modemmib.mib"}]).
+
+init({Reserved_words, Options, GetLineMFA}) ->
+ put(get_line_function,GetLineMFA),
+ put(print_lineno,
+ case lists:keysearch(print_lineno, 1, Options) of
+ {value, {print_lineno, L}} -> L;
+ false -> undefined
+ end),
+ DB = ets:new(reserved_words, [set, private]),
+ insert_keywords_into_ets(DB, Reserved_words),
+ put(db, DB),
+ put(line, 0),
+ {ok, ""}.
+
+getLine() ->
+ OldLine = put(line, 1 + get(line)),
+ {M,F,A} = get(get_line_function),
+ case get(print_lineno) of
+ undefined -> true;
+ X -> case OldLine rem X of
+ 0 -> io:format('~w..',[OldLine]);
+ _ -> true
+ end
+ end,
+ apply(M,F,A).
+
+%% You can only do this when no file is open.
+handle_call({tokenizeString, String}, _From, "") ->
+ {reply, safe_tokenize_whole_string(String), ""};
+
+handle_call(get_token, _From, String) ->
+ {ReplyToken, RestChars} = safe_tokenise(String),
+ {reply, ReplyToken, RestChars};
+
+handle_call(get_all_tokens, _From, String) ->
+ Toks = get_all_tokens(String,[]),
+ {reply, Toks, []};
+
+
+handle_call(stop, _From, String) ->
+ {stop, normal, ok, String}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+%% ErrorInfo = {ErrorLine, Module, ErrorDescriptor}
+%% will be used as
+%% apply(Module, format_error, [ErrorDescriptor]). shall return a string.
+
+%% Returns a reply
+tokenize_whole_string(eof) -> [];
+tokenize_whole_string(String) ->
+ {Token, RestChars} = tokenise(String),
+ [Token | tokenize_whole_string(RestChars)].
+
+safe_tokenize_whole_string(String) ->
+ case catch tokenize_whole_string(String) of
+ {error, ErrorInfo} -> {error, ErrorInfo, get(line)};
+ Tokens -> {ok, Tokens, get(line)}
+ end.
+
+%% throw({error, {get(line), ?MODULE, "Unexpected eof~n"}}).
+
+%% Returns: {ReplyToken, NewState}
+safe_tokenise(eof) -> {{eof, get(line)}, eof};
+safe_tokenise(Chars) when is_list(Chars) ->
+ case catch tokenise(Chars) of
+ {error, ErrorInfo} -> {{error, ErrorInfo, get(line)}, {[], eof}};
+ {Token, RestChars} when is_tuple(Token) ->
+ {{ok, [Token], get(line)}, RestChars}
+ end.
+
+get_all_tokens(eof,Toks) ->
+ lists:reverse(Toks);
+get_all_tokens(Str,Toks) ->
+ case catch tokenise(Str) of
+ {error, ErrorInfo} -> {error, ErrorInfo};
+ {Token, RestChars} when is_tuple(Token) ->
+ get_all_tokens(RestChars, [Token|Toks])
+ end.
+
+
+
+%%--------------------------------------------------
+%% Returns: {Token, Rest}
+%%--------------------------------------------------
+tokenise([H|T]) when ($a =< H) andalso (H =< $z) ->
+ get_name(atom, [H], T);
+
+tokenise([H|T]) when ($A =< H) andalso (H =< $Z) ->
+ get_name(variable, [H], T);
+
+tokenise([$:,$:,$=|T]) ->
+ {{'::=', get(line)}, T};
+
+tokenise([$-,$-|T]) ->
+ tokenise(skip_comment(T));
+
+tokenise([$-,H|T]) when ($0 =< H ) andalso (H =< $9) ->
+ {Val, Rest} = get_integer(T, [H]),
+ {{integer, get(line), -1 * Val}, Rest};
+
+tokenise([H|T]) when ($0 =< H) andalso (H =< $9) ->
+ {Val, Rest} = get_integer(T, [H]),
+ {{integer, get(line), Val}, Rest};
+
+tokenise([$"|T]) ->
+ collect_string($", T, []);
+
+tokenise([$'|T]) ->
+ collect_string($', T, []);
+
+%% Read away white spaces
+tokenise([9| T]) -> tokenise(T);
+tokenise([10| T]) -> tokenise(T);
+tokenise([13| T]) -> tokenise(T);
+tokenise([32| T]) -> tokenise(T);
+
+%% Handle singe characters like { } [ ] + = ...
+tokenise([Ch | T]) ->
+ Atm = list_to_atom([Ch]),
+ {{Atm, get(line)}, T};
+
+tokenise([]) ->
+ tokenise(getLine());
+
+tokenise(eof) ->
+ {{'$end', get(line)}, eof}.
+
+collect_string($", [$"|T],BackwardsStr) ->
+ {{string, get(line), BackwardsStr}, T};
+
+collect_string($', [$'|T],BackwardsStr) ->
+ {{quote, get(line), BackwardsStr}, T};
+
+collect_string(StopChar, [Ch|T], Str) ->
+ collect_string(StopChar,T,[Ch|Str]);
+
+collect_string(StopChar, [],Str) ->
+ collect_string(StopChar, getLine(), Str);
+
+collect_string(StopChar, eof, Str) ->
+ throw({error, {get(line), ?MODULE,
+ {"Missing ~s in string:~n \"~s\"~n",
+ [[StopChar], lists:reverse(Str)]}}}).
+
+get_name(Category, Name, [Char|T]) ->
+ case isInName(Char) of
+ true ->
+ get_name(Category, [Char|Name], T);
+ false ->
+ makeNameRespons(Category, Name, [Char | T])
+ end;
+get_name(Category, Name, []) ->
+ makeNameRespons(Category, Name, []).
+
+makeNameRespons(Category, Name, RestChars) ->
+ Atm = list_to_atom(lists:reverse(Name)),
+ case reserved_word(Atm) of
+ true -> {{Atm, get(line)}, RestChars};
+ false -> {{Category, get(line), Atm}, RestChars}
+ end.
+
+isInName($-) -> true;
+isInName(Ch) -> isalnum(Ch).
+
+isalnum(H) when ($A =< H) andalso (H =< $Z) ->
+ true;
+isalnum(H) when ($a =< H) andalso (H =< $z) ->
+ true;
+isalnum(H) when ($0 =< H) andalso (H =< $9) ->
+ true;
+isalnum(_) ->
+ false.
+
+isdigit(H) when ($0 =< H) andalso (H =< $9) ->
+ true;
+isdigit(_) ->
+ false.
+
+get_integer([H|T], "0") ->
+ case isdigit(H) of
+ true ->
+ throw({error, {get(line), ?MODULE,
+ {"Unexpected ~w~n",
+ [list_to_atom([H])]}}});
+ false ->
+ {0, [H|T]}
+ end;
+get_integer([H|T], L) ->
+ case isdigit(H) of
+ true ->
+ get_integer(T, [H|L]);
+ false ->
+ {list_to_integer(lists:reverse(L)), [H|T]}
+ end;
+get_integer([], L) ->
+ {list_to_integer(lists:reverse(L)), []}.
+
+%%--------------------------------------------------
+%% ASN.1 type of comments. "--" is comment to eoln or next "--"
+%%--------------------------------------------------
+skip_comment([]) ->
+ [];
+skip_comment([$-,$-|T]) ->
+ T;
+skip_comment([_|T]) ->
+ skip_comment(T).