Merge branch 'egil/maps/syntax-tools/OTP-12265' into maint

* egil/maps/syntax-tools/OTP-12265:
  syntax_tools: Strengthen igor tests
  syntax_tools: Strengthen erl_recomment tests
  syntax_tools: Strengthen erl_syntax comments tests
  syntax_tools: Teach Maps to erl_syntax
  syntax_tools: Fix is_literal/1 for binary
  syntax_tools: Strengthen epp_dodger tests
  syntax_tools: Strengthen erl_syntax tests
  syntax_tools: Refactor away ?line macro

7 files changed, 1047 insertions, 21 deletions

diff --git a/lib/syntax_tools/src/erl_syntax.erl b/lib/syntax_tools/src/erl_syntax.erl
index 46a5ca48df..40372a2106 100644
--- a/lib/syntax_tools/src/erl_syntax.erl
+++ b/lib/syntax_tools/src/erl_syntax.erl
@@ -669,6 +669,9 @@ is_leaf(Node) ->
 	operator -> true;	% nonstandard type
 	string -> true;
 	text -> true;		% nonstandard type
+	map_expr ->
+	    map_expr_fields(Node) =:= [] andalso
+	    map_expr_argument(Node) =:= none;
 	tuple -> tuple_elements(Node) =:= [];
 	underscore -> true;
 	variable -> true;
@@ -6093,6 +6096,9 @@ abstract([]) ->
     nil();
 abstract(T) when is_tuple(T) ->
     tuple(abstract_list(tuple_to_list(T)));
+abstract(T) when is_map(T) ->
+    map_expr([map_field_assoc(abstract(Key),abstract(Value))
+	      || {Key,Value} <- maps:to_list(T)]);
 abstract(T) when is_binary(T) ->
     binary([binary_field(integer(B)) || B <- binary_to_list(T)]);
 abstract(T) ->
@@ -6154,6 +6160,14 @@ concrete(Node) ->
 	     | concrete(list_tail(Node))];
 	tuple ->
 	    list_to_tuple(concrete_list(tuple_elements(Node)));
+	map_expr ->
+	    As = [tuple([map_field_assoc_name(F),
+			 map_field_assoc_value(F)]) || F <- map_expr_fields(Node)],
+	    M0 = maps:from_list(concrete_list(As)),
+	    case map_expr_argument(Node) of
+		none  -> M0;
+		Node0 -> maps:merge(concrete(Node0),M0)
+	    end;
 	binary ->
 	    Fs = [revert_binary_field(
 		    binary_field(binary_field_body(F),
@@ -6209,10 +6223,31 @@ is_literal(T) ->
 	    is_literal(list_head(T)) andalso is_literal(list_tail(T));
 	tuple ->
 	    lists:all(fun is_literal/1, tuple_elements(T));
+	map_expr ->
+	    case map_expr_argument(T) of
+		none -> true;
+		Arg  -> is_literal(Arg)
+	    end andalso lists:all(fun is_literal_map_field/1, map_expr_fields(T));
+	binary ->
+	    lists:all(fun is_literal_binary_field/1, binary_fields(T));
 	_ ->
 	    false
     end.
 
+is_literal_binary_field(F) ->
+    case binary_field_types(F) of
+	[] -> is_literal(binary_field_body(F));
+	_  -> false
+    end.
+
+is_literal_map_field(F) ->
+    case type(F) of
+	map_field_assoc ->
+	    is_literal(map_field_assoc_name(F)) andalso
+	    is_literal(map_field_assoc_value(F));
+	map_field_exact ->
+	    false
+    end.
 
 %% =====================================================================
 %% @doc Returns an `erl_parse'-compatible representation of a
diff --git a/lib/syntax_tools/test/Makefile b/lib/syntax_tools/test/Makefile
index d4733b9a42..f67e3f8984 100644
--- a/lib/syntax_tools/test/Makefile
+++ b/lib/syntax_tools/test/Makefile
@@ -61,5 +61,6 @@ release_tests_spec: make_emakefile
 	$(INSTALL_DATA) $(EMAKEFILE) $(ERL_FILES) "$(RELSYSDIR)"
 	$(INSTALL_DATA) syntax_tools.spec syntax_tools.cover "$(RELSYSDIR)"
 	chmod -R u+w "$(RELSYSDIR)"
+	@tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
 
 release_docs_spec:
diff --git a/lib/syntax_tools/test/syntax_tools_SUITE.erl b/lib/syntax_tools/test/syntax_tools_SUITE.erl
index 6fb3e5ccfb..3c6b33f459 100644
--- a/lib/syntax_tools/test/syntax_tools_SUITE.erl
+++ b/lib/syntax_tools/test/syntax_tools_SUITE.erl
@@ -24,12 +24,16 @@
 	 init_per_group/2,end_per_group/2]).
 
 %% Test cases
--export([app_test/1,appup_test/1,smoke_test/1,revert/1,revert_map/1]).
+-export([app_test/1,appup_test/1,smoke_test/1,revert/1,revert_map/1,
+	t_abstract_type/1,t_erl_parse_type/1,t_epp_dodger/1,
+	t_comment_scan/1,t_igor/1]).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [app_test,appup_test,smoke_test,revert,revert_map].
+    [app_test,appup_test,smoke_test,revert,revert_map,
+    t_abstract_type,t_erl_parse_type,t_epp_dodger,
+    t_comment_scan,t_igor].
 
 groups() -> 
     [].
@@ -54,15 +58,15 @@ appup_test(Config) when is_list(Config) ->
 
 %% Read and parse all source in the OTP release.
 smoke_test(Config) when is_list(Config) ->
-    ?line Dog = ?t:timetrap(?t:minutes(12)),
-    ?line Wc = filename:join([code:lib_dir(),"*","src","*.erl"]),
-    ?line Fs = filelib:wildcard(Wc),
-    ?line io:format("~p files\n", [length(Fs)]),
-    ?line case p_run(fun smoke_test_file/1, Fs) of
-	      0 -> ok;
-	      N -> ?line ?t:fail({N,errors})
-	  end,
-    ?line ?t:timetrap_cancel(Dog).
+    Dog = ?t:timetrap(?t:minutes(12)),
+    Wc = filename:join([code:lib_dir(),"*","src","*.erl"]),
+    Fs = filelib:wildcard(Wc),
+    io:format("~p files\n", [length(Fs)]),
+    case p_run(fun smoke_test_file/1, Fs) of
+        0 -> ok;
+        N -> ?t:fail({N,errors})
+    end,
+    ?t:timetrap_cancel(Dog).
 
 smoke_test_file(File) ->
     case epp_dodger:parse_file(File) of
@@ -94,9 +98,9 @@ revert(Config) when is_list(Config) ->
     io:format("~p files\n", [length(Fs)]),
     case p_run(fun (File) -> revert_file(File, Path) end, Fs) of
         0 -> ok;
-        N -> ?line ?t:fail({N,errors})
+        N -> ?t:fail({N,errors})
         end,
-    ?line ?t:timetrap_cancel(Dog).
+    ?t:timetrap_cancel(Dog).
 
 revert_file(File, Path) ->
     case epp:parse_file(File, Path, []) of
@@ -110,14 +114,298 @@ revert_file(File, Path) ->
     end.
 
 %% Testing bug fix for reverting map_field_assoc
-revert_map(Config) ->
+revert_map(Config) when is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(1)),
-    ?line [{map_field_assoc,16,{atom,17,name},{var,18,'Value'}}] =
-        erl_syntax:revert_forms([{tree,map_field_assoc,
-                                  {attr,16,[],none},
-                                  {map_field_assoc,
-                                   {atom,17,name},{var,18,'Value'}}}]),
-    ?line ?t:timetrap_cancel(Dog).
+    [{map_field_assoc,16,{atom,17,name},{var,18,'Value'}}] =
+    erl_syntax:revert_forms([{tree,map_field_assoc,
+                             {attr,16,[],none},
+			     {map_field_assoc,{atom,17,name},{var,18,'Value'}}}]),
+    ?t:timetrap_cancel(Dog).
+
+
+
+%% api tests
+
+t_abstract_type(Config) when is_list(Config) ->
+    F = fun validate_abstract_type/1,
+    ok = validate(F,[{hi,atom},
+		     {1,integer},
+		     {1.0,float},
+		     {$a,integer},
+		     {[],nil},
+		     {[<<1,2>>,a,b],list},
+		     {[2,3,<<1,2>>,a,b],list},
+		     {[$a,$b,$c],string},
+		     {"hello world",string},
+		     {<<1,2,3>>,binary},
+		     {#{a=>1,"b"=>2},map_expr},
+		     {#{#{i=>1}=>1,"b"=>#{v=>2}},map_expr},
+		     {{a,b,c},tuple}]),
+    ok.
+
+t_erl_parse_type(Config) when is_list(Config) ->
+    F = fun validate_erl_parse_type/1,
+    %% leaf types
+    ok = validate(F,[{"1",integer,true},
+		     {"123456789",integer,true},
+		     {"$h", char,true},
+		     {"3.1415", float,true},
+		     {"1.33e36", float,true},
+		     {"\"1.33e36: hello\"", string,true},
+		     {"Var1", variable,true},
+		     {"_", underscore,true},
+		     {"[]", nil,true},
+		     {"{}", tuple,true},
+		     {"#{}",map_expr,true},
+		     {"'some atom'", atom, true}]),
+    %% composite types
+    ok = validate(F,[{"case X of t -> t; f -> f end", case_expr,false},
+		     {"try X of t -> t catch C:R -> error end", try_expr,false},
+		     {"receive X -> X end", receive_expr,false},
+		     {"receive M -> X1 after T -> X2 end", receive_expr,false},
+		     {"catch (X)", catch_expr,false},
+		     {"fun(X) -> X end", fun_expr,false},
+		     {"fun Foo(X) -> X end", named_fun_expr,false},
+		     {"fun foo/2", implicit_fun,false},
+		     {"fun bar:foo/2", implicit_fun,false},
+		     {"if X -> t; true -> f end", if_expr,false},
+		     {"<<1,2,3,4>>", binary,false},
+		     {"<<1,2,3,4:5>>", binary,false},
+		     {"<<V1:63,V2:22/binary, V3/bits>>", binary,false},
+		     {"begin X end", block_expr,false},
+		     {"foo(X1,X2)", application,false},
+		     {"bar:foo(X1,X2)", application,false},
+		     {"[1,2,3,4]", list,false},
+		     {"[1|4]", list, false},
+		     {"[<<1>>,<<2>>,-2,<<>>,[more,list]]", list,false},
+		     {"[1|[2|[3|[4|[]]]]]", list,false},
+		     {"#{ a=>1, b=>2 }", map_expr,false},
+		     {"#{3=>3}#{ a=>1, b=>2 }", map_expr,false},
+		     {"#{ a:=1, b:=2 }", map_expr,false},
+		     {"M#{ a=>1, b=>2 }", map_expr,false},
+		     {"[V||V <- Vs]", list_comp,false},
+		     {"<< <<B>> || <<B>> <= Bs>>", binary_comp,false},
+		     {"#state{ a = A, b = B}", record_expr,false},
+		     {"#state{}", record_expr,false},
+		     {"#s{ a = #def{ a=A }, b = B}", record_expr,false},
+		     {"State#state{ a = A, b = B}", record_expr,false},
+		     {"State#state.a", record_access,false},
+		     {"#state.a", record_index_expr,false},
+		     {"-X", prefix_expr,false},
+		     {"X1 + X2", infix_expr,false},
+		     {"(X1 + X2) * X3", infix_expr,false},
+		     {"X1 = X2", match_expr,false},
+		     {"{a,b,c}", tuple,false}]),
+    ok.
+
+%% the macro ?MODULE seems faulty
+t_epp_dodger(Config) when is_list(Config) ->
+    DataDir   = ?config(data_dir, Config),
+    PrivDir   = ?config(priv_dir, Config),
+    Filenames = ["syntax_tools_SUITE_test_module.erl",
+		 "syntax_tools_test.erl"],
+    ok = test_epp_dodger(Filenames,DataDir,PrivDir),
+    ok.
+
+t_comment_scan(Config) when is_list(Config) ->
+    DataDir   = ?config(data_dir, Config),
+    Filenames = ["syntax_tools_SUITE_test_module.erl",
+		 "syntax_tools_test.erl"],
+    ok = test_comment_scan(Filenames,DataDir),
+    ok.
+
+t_igor(Config) when is_list(Config) ->
+    DataDir   = ?config(data_dir, Config),
+    PrivDir   = ?config(priv_dir, Config),
+    FileM1  = filename:join(DataDir,"m1.erl"),
+    FileM2  = filename:join(DataDir,"m2.erl"),
+    ["m.erl",_]=R = igor:merge(m,[FileM1,FileM2],[{outdir,PrivDir}]),
+    io:format("igor:merge/3 = ~p~n", [R]),
+    ok.
+
+test_comment_scan([],_) -> ok;
+test_comment_scan([File|Files],DataDir) ->
+    Filename  = filename:join(DataDir,File),
+    {ok, Fs0} = epp:parse_file(Filename, [], []),
+    Comments  = erl_comment_scan:file(Filename),
+    Fun = fun(Node) ->
+		  case erl_syntax:is_form(Node) of
+		      true ->
+			  C1    = erl_syntax:comment(2,[" This is a form."]),
+			  Node1 = erl_syntax:add_precomments([C1],Node),
+			  Node1;
+		      false ->
+			  Node
+		  end
+	  end,
+    Fs1 = erl_recomment:recomment_forms(Fs0, Comments),
+    Fs2 = erl_syntax_lib:map(Fun, Fs1),
+    io:format("File: ~s~n", [Filename]),
+    io:put_chars(erl_prettypr:format(Fs2, [{paper,  120},
+					   {ribbon, 110}])),
+    test_comment_scan(Files,DataDir).
+
+
+test_epp_dodger([], _, _) -> ok;
+test_epp_dodger([Filename|Files],DataDir,PrivDir) ->
+    io:format("Parsing ~p~n", [Filename]),
+    InFile   = filename:join(DataDir, Filename),
+    Parsers  = [{fun epp_dodger:parse_file/1,parse_file},
+		{fun epp_dodger:quick_parse_file/1,quick_parse_file},
+		{fun (File) ->
+			{ok,Dev} = file:open(File,[read]),
+			Res = epp_dodger:parse(Dev),
+			file:close(File),
+			Res
+		 end, parse},
+		{fun (File) ->
+			{ok,Dev} = file:open(File,[read]),
+			Res = epp_dodger:quick_parse(Dev),
+			file:close(File),
+			Res
+		 end, quick_parse}],
+    FsForms  = parse_with(Parsers, InFile),
+    ok = pretty_print_parse_forms(FsForms,PrivDir,Filename),
+    test_epp_dodger(Files,DataDir,PrivDir).
+
+parse_with([],_) -> [];
+parse_with([{Fun,ParserType}|Funs],File) ->
+    {ok, Fs} = Fun(File),
+    [{Fs,ParserType}|parse_with(Funs,File)].
+
+pretty_print_parse_forms([],_,_) -> ok;
+pretty_print_parse_forms([{Fs0,Type}|FsForms],PrivDir,Filename) ->
+    Parser  = atom_to_list(Type),
+    OutFile = filename:join(PrivDir, Parser ++"_" ++ Filename),
+    io:format("Pretty print ~p (~w) to ~p~n", [Filename,Type,OutFile]),
+    Comment = fun (Node,{CntCase,CntTry}=Cnt) ->
+		      case erl_syntax:type(Node) of
+			  case_expr ->
+			      C1    = erl_syntax:comment(2,["Before a case expression"]),
+			      Node1 = erl_syntax:add_precomments([C1],Node),
+			      C2    = erl_syntax:comment(2,["After a case expression"]),
+			      Node2 = erl_syntax:add_postcomments([C2],Node1),
+			      {Node2,{CntCase+1,CntTry}};
+			  try_expr ->
+			      C1    = erl_syntax:comment(2,["Before a try expression"]),
+			      Node1 = erl_syntax:set_precomments(Node,
+						     erl_syntax:get_precomments(Node) ++ [C1]),
+			      C2    = erl_syntax:comment(2,["After a try expression"]),
+			      Node2 = erl_syntax:set_postcomments(Node1,
+						     erl_syntax:get_postcomments(Node1) ++ [C2]),
+			      {Node2,{CntCase,CntTry+1}};
+			  _ ->
+			      {Node,Cnt}
+		      end
+	      end,
+    Fs1 = erl_syntax:form_list(Fs0),
+    {Fs2,{CC,CT}} = erl_syntax_lib:mapfold(Comment,{0,0}, Fs1),
+    io:format("Commented on ~w cases and ~w tries~n", [CC,CT]),
+    PP  = erl_prettypr:format(Fs2),
+    ok  = file:write_file(OutFile,iolist_to_binary(PP)),
+    pretty_print_parse_forms(FsForms,PrivDir,Filename).
+
+
+validate(_,[]) -> ok;
+validate(F,[V|Vs]) ->
+    ok = F(V),
+    validate(F,Vs).
+
+
+validate_abstract_type({Lit,Type}) ->
+    Tree = erl_syntax:abstract(Lit),
+    ok   = validate_special_type(Type,Tree),
+    Type = erl_syntax:type(Tree),
+    true = erl_syntax:is_literal(Tree),
+    ErlT = erl_syntax:revert(Tree),
+    Type = erl_syntax:type(ErlT),
+    ok   = validate_special_type(Type,ErlT),
+    Conc = erl_syntax:concrete(Tree),
+    Lit  = Conc,
+    ok.
+
+validate_erl_parse_type({String,Type,Leaf}) ->
+    ErlT = string_to_expr(String),
+    ok   = validate_special_type(Type,ErlT),
+    Type = erl_syntax:type(ErlT),
+    Leaf = erl_syntax:is_leaf(ErlT),
+    Tree = erl_syntax_lib:map(fun(Node) -> Node end, ErlT),
+    Type = erl_syntax:type(Tree),
+    _    = erl_syntax:meta(Tree),
+    ok   = validate_special_type(Type,Tree),
+    RevT = erl_syntax:revert(Tree),
+    ok   = validate_special_type(Type,RevT),
+    Type = erl_syntax:type(RevT),
+    ok.
+
+validate_special_type(string,Node) ->
+    Val  = erl_syntax:string_value(Node),
+    true = erl_syntax:is_string(Node,Val),
+    _    = erl_syntax:string_literal(Node),
+    ok;
+validate_special_type(variable,Node) ->
+    _ = erl_syntax:variable_literal(Node),
+    ok;
+validate_special_type(fun_expr,Node) ->
+    A = erl_syntax:fun_expr_arity(Node),
+    true = is_integer(A),
+    ok;
+validate_special_type(named_fun_expr,Node) ->
+    A = erl_syntax:named_fun_expr_arity(Node),
+    true = is_integer(A),
+    ok;
+validate_special_type(tuple,Node) ->
+    Size = erl_syntax:tuple_size(Node),
+    true = is_integer(Size),
+    ok;
+validate_special_type(float,Node) ->
+    Str   = erl_syntax:float_literal(Node),
+    Val   = list_to_float(Str),
+    Val   = erl_syntax:float_value(Node),
+    false = erl_syntax:is_proper_list(Node),
+    false = erl_syntax:is_list_skeleton(Node),
+    ok;
+validate_special_type(integer,Node) ->
+    Str   = erl_syntax:integer_literal(Node),
+    Val   = list_to_integer(Str),
+    true  = erl_syntax:is_integer(Node,Val),
+    Val   = erl_syntax:integer_value(Node),
+    false = erl_syntax:is_proper_list(Node),
+    ok;
+validate_special_type(nil,Node) ->
+    true  = erl_syntax:is_proper_list(Node),
+    ok;
+validate_special_type(list,Node) ->
+    true  = erl_syntax:is_list_skeleton(Node),
+    _     = erl_syntax:list_tail(Node),
+    ErrV  = erl_syntax:list_head(Node),
+    false = erl_syntax:is_string(Node,ErrV),
+    Norm  = erl_syntax:normalize_list(Node),
+    list  = erl_syntax:type(Norm),
+    case erl_syntax:is_proper_list(Node) of
+	true ->
+	    true = erl_syntax:is_list_skeleton(Node),
+	    Compact = erl_syntax:compact_list(Node),
+	    list = erl_syntax:type(Compact),
+	    [_|_] = erl_syntax:list_elements(Node),
+	    _  = erl_syntax:list_elements(Node),
+	    N  = erl_syntax:list_length(Node),
+	    true = N > 0,
+	    ok;
+	false ->
+	    ok
+    end;
+validate_special_type(_,_) ->
+    ok.
+
+%%% scan_and_parse
+
+string_to_expr(String) ->
+    io:format("Str: ~p~n", [String]),
+    {ok, Ts, _} = erl_scan:string(String++"."),
+    {ok,[Expr]} = erl_parse:parse_exprs(Ts),
+    Expr.
+
 
 p_run(Test, List) ->
     N = erlang:system_info(schedulers),
@@ -138,4 +426,3 @@ p_run_loop(Test, List, N, Refs0, Errors0) ->
 	    Refs = Refs0 -- [Ref],
 	    p_run_loop(Test, List, N, Refs, Errors)
     end.
-    
diff --git a/lib/syntax_tools/test/syntax_tools_SUITE_data/m1.erl b/lib/syntax_tools/test/syntax_tools_SUITE_data/m1.erl
new file mode 100644
index 0000000000..d0d1911199
--- /dev/null
+++ b/lib/syntax_tools/test/syntax_tools_SUITE_data/m1.erl
@@ -0,0 +1,22 @@
+%%
+%% File:    m1.erl
+%% Author:  Björn-Egil Dahlberg
+%% Created: 2014-10-24
+%%
+
+-module(m1).
+
+-export([foo/0,bar/1,baz/2]).
+
+foo() ->
+    [m2:foo(),
+     m2:bar()].
+
+bar(A) ->
+    [m2:foo(A),
+     m2:bar(A),
+     m2:record_update(3,m2:record())].
+
+baz(A,B) ->
+    [m2:foo(A,B),
+     m2:bar(A,B)].
diff --git a/lib/syntax_tools/test/syntax_tools_SUITE_data/m2.erl b/lib/syntax_tools/test/syntax_tools_SUITE_data/m2.erl
new file mode 100644
index 0000000000..781139317d
--- /dev/null
+++ b/lib/syntax_tools/test/syntax_tools_SUITE_data/m2.erl
@@ -0,0 +1,26 @@
+%%
+%% File:    m2.erl
+%% Author:  Björn-Egil Dahlberg
+%% Created: 2014-10-24
+%%
+
+-module(m2).
+
+
+-export([foo/0,foo/1,foo/2,
+	 bar/0,bar/1,bar/2,
+	 record_update/2, record/0]).
+
+foo() -> ok.
+foo(A) -> [item,A].
+foo(A,B) -> A + B.
+
+bar() -> true.
+bar(A) -> {element,A}.
+bar(A,B) -> A*B.
+
+-record(rec, {a,b}).
+
+record() -> #rec{a=3,b=0}.
+record_update(V,#rec{a=V0}=R) ->
+    R#rec{a=V0+V,b=V0}.
diff --git a/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_SUITE_test_module.erl b/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_SUITE_test_module.erl
new file mode 100644
index 0000000000..07c419b4b7
--- /dev/null
+++ b/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_SUITE_test_module.erl
@@ -0,0 +1,540 @@
+-module(syntax_tools_SUITE_test_module).
+
+-export([foo1/1,foo2/3,start_child/2]).
+
+-export([len/1,equal/2,concat/2,chr/2,rchr/2,str/2,rstr/2,
+	 span/2,cspan/2,substr/2,substr/3,tokens/2,chars/2,chars/3]).
+-export([copies/2,words/1,words/2,strip/1,strip/2,strip/3,
+	 sub_word/2,sub_word/3,left/2,left/3,right/2,right/3,
+	 sub_string/2,sub_string/3,centre/2,centre/3, join/2]).
+-export([to_upper/1, to_lower/1]).
+
+-import(lists,[reverse/1,member/2]).
+
+
+%% @type some_type() = map()
+%% @type some_other_type() = {a, #{ list() => term()}}
+
+-type some_type() :: map().
+-type some_other_type() :: {'a', #{ list() => term()} }.
+
+-spec foo1(Map :: #{ 'a' => integer(), 'b' => term()}) -> term().
+
+%% @doc Gets value from map.
+
+foo1(#{ a:= 1, b := V}) -> V.
+
+%% @spec foo2(some_type(), Type2 :: some_other_type(), map()) -> Value
+%% @doc Gets value from map.
+
+-spec foo2(
+    Type1 :: some_type(),
+    Type2 :: some_other_type(),
+    Map :: #{ get => 'value', 'value' => binary()}) -> binary().
+
+foo2(Type1, {a,#{ "a" := _}}, #{get := value, value := B}) when is_map(Type1) -> B.
+
+%% from supervisor 18.0
+
+-type child()    :: 'undefined' | pid().
+-type child_id() :: term().
+-type mfargs()   :: {M :: module(), F :: atom(), A :: [term()] | undefined}.
+-type modules()  :: [module()] | 'dynamic'.
+-type restart()  :: 'permanent' | 'transient' | 'temporary'.
+-type shutdown() :: 'brutal_kill' | timeout().
+-type worker()   :: 'worker' | 'supervisor'.
+-type sup_ref()  :: (Name :: atom())
+                  | {Name :: atom(), Node :: node()}
+                  | {'global', Name :: atom()}
+                  | {'via', Module :: module(), Name :: any()}
+                  | pid().
+-type child_spec() :: #{name => child_id(),     % mandatory
+			start => mfargs(),      % mandatory
+			restart => restart(),   % optional
+			shutdown => shutdown(), % optional
+			type => worker(),       % optional
+			modules => modules()}   % optional
+                    | {Id :: child_id(),
+                       StartFunc :: mfargs(),
+                       Restart :: restart(),
+                       Shutdown :: shutdown(),
+                       Type :: worker(),
+                       Modules :: modules()}.
+
+-type startchild_err() :: 'already_present'
+			| {'already_started', Child :: child()} | term().
+-type startchild_ret() :: {'ok', Child :: child()}
+                        | {'ok', Child :: child(), Info :: term()}
+			| {'error', startchild_err()}.
+
+
+-spec start_child(SupRef, ChildSpec) -> startchild_ret() when
+      SupRef :: sup_ref(),
+      ChildSpec :: child_spec() | (List :: [term()]).
+start_child(Supervisor, ChildSpec) ->
+    {Supervisor,ChildSpec}.
+
+
+%% From string.erl
+%% Robert's bit
+
+%% len(String)
+%%  Return the length of a string.
+
+-spec len(String) -> Length when
+      String :: string(),
+      Length :: non_neg_integer().
+
+len(S) -> length(S).
+
+%% equal(String1, String2)
+%%  Test if 2 strings are equal.
+
+-spec equal(String1, String2) -> boolean() when
+      String1 :: string(),
+      String2 :: string().
+
+equal(S, S) -> true;
+equal(_, _) -> false.
+
+%% concat(String1, String2)
+%%  Concatenate 2 strings.
+
+-spec concat(String1, String2) -> String3 when
+      String1 :: string(),
+      String2 :: string(),
+      String3 :: string().
+
+concat(S1, S2) -> S1 ++ S2.
+
+%% chr(String, Char)
+%% rchr(String, Char)
+%%  Return the first/last index of the character in a string.
+
+-spec chr(String, Character) -> Index when
+      String :: string(),
+      Character :: char(),
+      Index :: non_neg_integer().
+
+chr(S, C) when is_integer(C) -> chr(S, C, 1).
+
+chr([C|_Cs], C, I) -> I;
+chr([_|Cs], C, I) -> chr(Cs, C, I+1);
+chr([], _C, _I) -> 0.
+
+-spec rchr(String, Character) -> Index when
+      String :: string(),
+      Character :: char(),
+      Index :: non_neg_integer().
+
+rchr(S, C) when is_integer(C) -> rchr(S, C, 1, 0).
+
+rchr([C|Cs], C, I, _L) ->			%Found one, now find next!
+    rchr(Cs, C, I+1, I);
+rchr([_|Cs], C, I, L) ->
+    rchr(Cs, C, I+1, L);
+rchr([], _C, _I, L) -> L.
+
+%% str(String, SubString)
+%% rstr(String, SubString)
+%% index(String, SubString)
+%%  Return the first/last index of the sub-string in a string.
+%%  index/2 is kept for backwards compatibility.
+
+-spec str(String, SubString) -> Index when
+      String :: string(),
+      SubString :: string(),
+      Index :: non_neg_integer().
+
+str(S, Sub) when is_list(Sub) -> str(S, Sub, 1).
+
+str([C|S], [C|Sub], I) ->
+    case prefix(Sub, S) of
+	true -> I;
+	false -> str(S, [C|Sub], I+1)
+    end;
+str([_|S], Sub, I) -> str(S, Sub, I+1);
+str([], _Sub, _I) -> 0.
+
+-spec rstr(String, SubString) -> Index when
+      String :: string(),
+      SubString :: string(),
+      Index :: non_neg_integer().
+
+rstr(S, Sub) when is_list(Sub) -> rstr(S, Sub, 1, 0).
+
+rstr([C|S], [C|Sub], I, L) ->
+    case prefix(Sub, S) of
+	true -> rstr(S, [C|Sub], I+1, I);
+	false -> rstr(S, [C|Sub], I+1, L)
+    end;
+rstr([_|S], Sub, I, L) -> rstr(S, Sub, I+1, L);
+rstr([], _Sub, _I, L) -> L.
+
+prefix([C|Pre], [C|String]) -> prefix(Pre, String);
+prefix([], String) when is_list(String) -> true;
+prefix(Pre, String) when is_list(Pre), is_list(String) -> false.
+
+%% span(String, Chars) -> Length.
+%% cspan(String, Chars) -> Length.
+
+-spec span(String, Chars) -> Length when
+      String :: string(),
+      Chars :: string(),
+      Length :: non_neg_integer().
+
+span(S, Cs) when is_list(Cs) -> span(S, Cs, 0).
+
+span([C|S], Cs, I) ->
+    case member(C, Cs) of
+	true -> span(S, Cs, I+1);
+	false -> I
+    end;
+span([], _Cs, I) -> I.
+
+-spec cspan(String, Chars) -> Length when
+      String :: string(),
+      Chars :: string(),
+      Length :: non_neg_integer().
+
+cspan(S, Cs) when is_list(Cs) -> cspan(S, Cs, 0).
+
+cspan([C|S], Cs, I) ->
+    case member(C, Cs) of
+	true -> I;
+	false -> cspan(S, Cs, I+1)
+    end;
+cspan([], _Cs, I) -> I.
+
+%% substr(String, Start)
+%% substr(String, Start, Length)
+%%  Extract a sub-string from String.
+
+-spec substr(String, Start) -> SubString when
+      String :: string(),
+      SubString :: string(),
+      Start :: pos_integer().
+
+substr(String, 1) when is_list(String) -> 
+    String;
+substr(String, S) when is_integer(S), S > 1 ->
+    substr2(String, S).
+
+-spec substr(String, Start, Length) -> SubString when
+      String :: string(),
+      SubString :: string(),
+      Start :: pos_integer(),
+      Length :: non_neg_integer().
+
+substr(String, S, L) when is_integer(S), S >= 1, is_integer(L), L >= 0 ->
+    substr1(substr2(String, S), L).
+
+substr1([C|String], L) when L > 0 -> [C|substr1(String, L-1)];
+substr1(String, _L) when is_list(String) -> [].	     %Be nice!
+
+substr2(String, 1) when is_list(String) -> String;
+substr2([_|String], S) -> substr2(String, S-1).
+
+%% tokens(String, Seperators).
+%%  Return a list of tokens seperated by characters in Seperators.
+
+-spec tokens(String, SeparatorList) -> Tokens when
+      String :: string(),
+      SeparatorList :: string(),
+      Tokens :: [Token :: nonempty_string()].
+
+tokens(S, Seps) ->
+    tokens1(S, Seps, []).
+
+tokens1([C|S], Seps, Toks) ->
+    case member(C, Seps) of
+	true -> tokens1(S, Seps, Toks);
+	false -> tokens2(S, Seps, Toks, [C])
+    end;
+tokens1([], _Seps, Toks) ->
+    reverse(Toks).
+
+tokens2([C|S], Seps, Toks, Cs) ->
+    case member(C, Seps) of
+	true -> tokens1(S, Seps, [reverse(Cs)|Toks]);
+	false -> tokens2(S, Seps, Toks, [C|Cs])
+    end;
+tokens2([], _Seps, Toks, Cs) ->
+    reverse([reverse(Cs)|Toks]).
+
+-spec chars(Character, Number) -> String when
+      Character :: char(),
+      Number :: non_neg_integer(),
+      String :: string().
+
+chars(C, N) -> chars(C, N, []).
+
+-spec chars(Character, Number, Tail) -> String when
+      Character :: char(),
+      Number :: non_neg_integer(),
+      Tail :: string(),
+      String :: string().
+
+chars(C, N, Tail) when N > 0 ->
+    chars(C, N-1, [C|Tail]);
+chars(C, 0, Tail) when is_integer(C) ->
+    Tail.
+
+%% Torbjörn's bit.
+
+%%% COPIES %%%
+
+-spec copies(String, Number) -> Copies when
+      String :: string(),
+      Copies :: string(),
+      Number :: non_neg_integer().
+
+copies(CharList, Num) when is_list(CharList), is_integer(Num), Num >= 0 ->
+    copies(CharList, Num, []).
+
+copies(_CharList, 0, R) ->
+    R;
+copies(CharList, Num, R) ->
+    copies(CharList, Num-1, CharList++R).
+
+%%% WORDS %%%
+
+-spec words(String) -> Count when
+      String :: string(),
+      Count :: pos_integer().
+
+words(String) -> words(String, $\s).
+
+-spec words(String, Character) -> Count when
+      String :: string(),
+      Character :: char(),
+      Count :: pos_integer().
+
+words(String, Char) when is_integer(Char) ->
+    w_count(strip(String, both, Char), Char, 0).
+
+w_count([], _, Num) -> Num+1;
+w_count([H|T], H, Num) -> w_count(strip(T, left, H), H, Num+1);
+w_count([_H|T], Char, Num) -> w_count(T, Char, Num).
+
+%%% SUB_WORDS %%%
+
+-spec sub_word(String, Number) -> Word when
+      String :: string(),
+      Word :: string(),
+      Number :: integer().
+
+sub_word(String, Index) -> sub_word(String, Index, $\s).
+
+-spec sub_word(String, Number, Character) -> Word when
+      String :: string(),
+      Word :: string(),
+      Number :: integer(),
+      Character :: char().
+
+sub_word(String, Index, Char) when is_integer(Index), is_integer(Char) ->
+    case words(String, Char) of
+	Num when Num < Index ->
+	    [];
+	_Num ->
+	    s_word(strip(String, left, Char), Index, Char, 1, [])
+    end.
+
+s_word([], _, _, _,Res) -> reverse(Res);
+s_word([Char|_],Index,Char,Index,Res) -> reverse(Res);
+s_word([H|T],Index,Char,Index,Res) -> s_word(T,Index,Char,Index,[H|Res]);
+s_word([Char|T],Stop,Char,Index,Res) when Index < Stop -> 
+    s_word(strip(T,left,Char),Stop,Char,Index+1,Res);
+s_word([_|T],Stop,Char,Index,Res) when Index < Stop -> 
+    s_word(T,Stop,Char,Index,Res).
+
+%%% STRIP %%%
+
+-spec strip(string()) -> string().
+
+strip(String) -> strip(String, both).
+
+-spec strip(String, Direction) -> Stripped when
+      String :: string(),
+      Stripped :: string(),
+      Direction :: left | right | both.
+
+strip(String, left) -> strip_left(String, $\s);
+strip(String, right) -> strip_right(String, $\s);
+strip(String, both) ->
+    strip_right(strip_left(String, $\s), $\s).
+
+-spec strip(String, Direction, Character) -> Stripped when
+      String :: string(),
+      Stripped :: string(),
+      Direction :: left | right | both,
+      Character :: char().
+
+strip(String, right, Char) -> strip_right(String, Char);
+strip(String, left, Char) -> strip_left(String, Char);
+strip(String, both, Char) ->
+    strip_right(strip_left(String, Char), Char).
+
+strip_left([Sc|S], Sc) ->
+    strip_left(S, Sc);
+strip_left([_|_]=S, Sc) when is_integer(Sc) -> S;
+strip_left([], Sc) when is_integer(Sc) -> [].
+
+strip_right([Sc|S], Sc) ->
+    case strip_right(S, Sc) of
+	[] -> [];
+	T  -> [Sc|T]
+    end;
+strip_right([C|S], Sc) ->
+    [C|strip_right(S, Sc)];
+strip_right([], Sc) when is_integer(Sc) ->
+    [].
+
+%%% LEFT %%%
+
+-spec left(String, Number) -> Left when
+      String :: string(),
+      Left :: string(),
+      Number :: non_neg_integer().
+
+left(String, Len) when is_integer(Len) -> left(String, Len, $\s).
+
+-spec left(String, Number, Character) -> Left when
+      String :: string(),
+      Left :: string(),
+      Number :: non_neg_integer(),
+      Character :: char().
+
+left(String, Len, Char) when is_integer(Char) ->
+    Slen = length(String),
+    if
+	Slen > Len -> substr(String, 1, Len);
+	Slen < Len -> l_pad(String, Len-Slen, Char);
+	Slen =:= Len -> String
+    end.
+
+l_pad(String, Num, Char) -> String ++ chars(Char, Num).
+
+%%% RIGHT %%%
+
+-spec right(String, Number) -> Right when
+      String :: string(),
+      Right :: string(),
+      Number :: non_neg_integer().
+
+right(String, Len) when is_integer(Len) -> right(String, Len, $\s).
+
+-spec right(String, Number, Character) -> Right when
+      String :: string(),
+      Right :: string(),
+      Number :: non_neg_integer(),
+      Character :: char().
+
+right(String, Len, Char) when is_integer(Char) ->
+    Slen = length(String),
+    if
+	Slen > Len -> substr(String, Slen-Len+1);
+	Slen < Len -> r_pad(String, Len-Slen, Char);
+	Slen =:= Len -> String
+    end.
+
+r_pad(String, Num, Char) -> chars(Char, Num, String).
+
+%%% CENTRE %%%
+
+-spec centre(String, Number) -> Centered when
+      String :: string(),
+      Centered :: string(),
+      Number :: non_neg_integer().
+
+centre(String, Len) when is_integer(Len) -> centre(String, Len, $\s).
+
+-spec centre(String, Number, Character) -> Centered when
+      String :: string(),
+      Centered :: string(),
+      Number :: non_neg_integer(),
+      Character :: char().
+
+centre(String, 0, Char) when is_list(String), is_integer(Char) ->
+    [];                       % Strange cases to centre string
+centre(String, Len, Char) when is_integer(Char) ->
+    Slen = length(String),
+    if
+	Slen > Len -> substr(String, (Slen-Len) div 2 + 1, Len);
+	Slen < Len ->
+	    N = (Len-Slen) div 2,
+	    r_pad(l_pad(String, Len-(Slen+N), Char), N, Char);
+	Slen =:= Len -> String
+    end.
+
+%%% SUB_STRING %%%
+
+-spec sub_string(String, Start) -> SubString when
+      String :: string(),
+      SubString :: string(),
+      Start :: pos_integer().
+
+sub_string(String, Start) -> substr(String, Start).
+
+-spec sub_string(String, Start, Stop) -> SubString when
+      String :: string(),
+      SubString :: string(),
+      Start :: pos_integer(),
+      Stop :: pos_integer().
+
+sub_string(String, Start, Stop) -> substr(String, Start, Stop - Start + 1).
+
+%% ISO/IEC 8859-1 (latin1) letters are converted, others are ignored
+%%
+
+to_lower_char(C) when is_integer(C), $A =< C, C =< $Z ->
+    C + 32;
+to_lower_char(C) when is_integer(C), 16#C0 =< C, C =< 16#D6 ->
+    C + 32;
+to_lower_char(C) when is_integer(C), 16#D8 =< C, C =< 16#DE ->
+    C + 32;
+to_lower_char(C) ->
+    C.
+
+to_upper_char(C) when is_integer(C), $a =< C, C =< $z ->
+    C - 32;
+to_upper_char(C) when is_integer(C), 16#E0 =< C, C =< 16#F6 ->
+    C - 32;
+to_upper_char(C) when is_integer(C), 16#F8 =< C, C =< 16#FE ->
+    C - 32;
+to_upper_char(C) ->
+    C.
+
+-spec to_lower(String) -> Result when
+                  String :: io_lib:latin1_string(),
+                  Result :: io_lib:latin1_string()
+	    ; (Char) -> CharResult when
+                  Char :: char(),
+                  CharResult :: char().
+
+to_lower(S) when is_list(S) ->
+    [to_lower_char(C) || C <- S];
+to_lower(C) when is_integer(C) ->
+    to_lower_char(C).
+
+-spec to_upper(String) -> Result when
+                  String :: io_lib:latin1_string(),
+                  Result :: io_lib:latin1_string()
+	    ; (Char) -> CharResult when
+                  Char :: char(),
+                  CharResult :: char().
+
+to_upper(S) when is_list(S) ->
+    [to_upper_char(C) || C <- S];
+to_upper(C) when is_integer(C) ->
+    to_upper_char(C).
+
+-spec join(StringList, Separator) -> String when
+      StringList :: [string()],
+      Separator :: string(),
+      String :: string().
+
+join([], Sep) when is_list(Sep) ->
+    [];
+join([H|T], Sep) ->
+    H ++ lists:append([Sep ++ X || X <- T]).
diff --git a/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_test.erl b/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_test.erl
new file mode 100644
index 0000000000..dd3f88d7a8
--- /dev/null
+++ b/lib/syntax_tools/test/syntax_tools_SUITE_data/syntax_tools_test.erl
@@ -0,0 +1,115 @@
+%%
+%% File:    syntax_tools_test.erl
+%% Author:  Björn-Egil Dahlberg
+%% Created: 2014-10-23
+%%
+
+-module(syntax_tools_test).
+
+-export([foo1/0,foo2/2,foo3/0,foo4/3,foo5/1]).
+
+-include_lib("kernel/include/file.hrl").
+-record(state, { a, b, c, d}).
+-attribute([foo/0]).
+
+-define(attrib, some_attrib).
+
+-?attrib([foo2/2]).
+
+-define(macro_simple1, ok).
+-define(MACRO_SIMPLE2, (other)).
+-define(macro_simple3, ?MODULE).
+-define(macro_simple4, [?macro_simple3,?MODULE,?MACRO_SIMPLE2]).
+-define(macro_simple5, (process_info)).
+-define(macro_string, "hello world").
+-define(macro_argument1(X), (X + 3)).
+-define(macro_argument2(X,Y), (X + 3 * Y)).
+-define(macro_block(X), begin X end).
+-define(macro_if(X1,X2), if X1 -> X2; true -> none end).
+
+
+-ifdef(macro_def1).
+-define(macro_cond1, yep).
+-else.
+-define(macro_cond1, nope).
+-endif.
+-ifndef(macro_def2).
+-define(macro_cond2, nope).
+-else.
+-define(macro_cond2, yep).
+-endif.
+-undef(macro_def1).
+-undef(macro_def2).
+
+%% basic test
+foo1() ->
+    ok.
+
+%% macro test
+foo2(A,B) ->
+    % string combining ?
+    [?macro_string, ?macro_string
+     ?macro_string,
+     "hello world "
+     "more hello",
+     [?macro_simple1,
+      ?MACRO_SIMPLE2,
+      ?macro_simple3,
+      ?macro_simple4,
+      ?macro_simple5,
+      ?macro_string,
+      ?macro_cond1,
+      ?macro_cond2,
+      ?macro_block(A),
+      ?macro_if(A,B),
+      ?macro_argument1(A),
+      ?macro_argument1(begin A end),
+      ?macro_block(<<"hello">>),
+      ?macro_block("hello"),
+      ?macro_block([$h,$e,$l,$l,$0]),
+      ?macro_argument1(id(<<"hello">>)),
+      ?macro_argument1(if A -> B; true -> 3.14 end),
+      ?macro_argument1(case A of ok -> B; C -> C end),
+      ?macro_argument1(receive M -> M after 100 -> 3 end),
+      ?macro_argument1(try foo5(A) catch C:?macro_simple5 -> {C,B} end),
+      ?macro_argument2(A,B)],
+     A,B,ok].
+
+id(I) -> I.
+%% basic terms
+
+foo3() ->
+    [atom,
+     'some other atom',
+     {tuple,1,2,3},
+     1,2,3,3333,
+     3,3333,2,1,
+     [$a,$b,$c],
+     "hello world",
+     <<"hello world">>,
+     <<1,2,3,4,5:6>>,
+     3.1415,
+     1.03e33].
+
+%% application and records
+
+foo4(A,B,#state{c = C}=S) ->
+    Ls = foo3(),
+    S1 = #state{ a = 1, b = 2 },
+    [foo2(A,Ls),B,C,
+     B(3,C),
+     erlang:process_info(self()),
+     erlang:?macro_simple5(self()),
+     A:?MACRO_SIMPLE2(),
+     A:?macro_simple1(),
+     A:process_info(self()),
+     A:B(3),
+     S#state{ a = 2, b = B, d = S1 }].
+
+foo5(A) ->
+    try foo2(A,A) of
+	R -> R
+    catch
+	error:?macro_simple5 ->
+	    nope
+    end.