Merge branch 'hasse/stdlib/fix_qlc_bug/OTP-14296'

* hasse/stdlib/fix_qlc_bug/OTP-14296:
  stdlib: Fix a test in sofs_SUITE
  debugger: Improve handling of pids, ports, and refs
  stdlib: Improve handling of pids, ports, and refs in qlc
  stdlib: Improve the Erlang shell's handling of references

11 files changed, 468 insertions, 73 deletions

diff --git a/lib/debugger/doc/src/debugger_chapter.xml b/lib/debugger/doc/src/debugger_chapter.xml
index 45dfdb3776..3c37d4b924 100644
--- a/lib/debugger/doc/src/debugger_chapter.xml
+++ b/lib/debugger/doc/src/debugger_chapter.xml
@@ -4,7 +4,7 @@
 <chapter>
 <header>
     <copyright>
-      <year>1997</year><year>2016</year>
+      <year>1997</year><year>2017</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -652,8 +652,10 @@ c_break(Bindings) ->
       <item><p>The Bindings area, displaying all variables bindings. If you
         click a variable name, the value is displayed in the Evaluator area.
 	Double-click a variable name to open a window where
-        the variable value can be edited. Notice however that pid,
-        reference, binary, or port values cannot be edited.</p>
+        the variable value can be edited. Notice however that pid, port,
+	reference, or fun
+        values cannot be edited unless they can be represented in the
+        running system.</p>
       </item>
 
       <item><p>The Trace area, which displays a trace output for the
diff --git a/lib/debugger/src/dbg_icmd.erl b/lib/debugger/src/dbg_icmd.erl
index 57a3719a50..4cd3dce670 100644
--- a/lib/debugger/src/dbg_icmd.erl
+++ b/lib/debugger/src/dbg_icmd.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -382,14 +382,19 @@ eval_restricted({From,_Mod,Cmd,SP}, Bs) ->
     case catch parse_cmd(Cmd, 1) of
 	{'EXIT', _Reason} ->
 	    From ! {self(), {eval_rsp, 'Parse error'}};
-	[{var,_,Var}] ->
+	{[{var,_,Var}], XBs} ->
 	    Bs2 = bindings(Bs, SP),
 	    Res = case get_binding(Var, Bs2) of
 		      {value, Value} -> Value;
-		      unbound -> unbound
+		      unbound ->
+                          case get_binding(Var, XBs) of
+                              {value, _} ->
+                                  'Only possible to inspect variables';
+                              unbound -> unbound
+                          end
 		  end,
 	    From ! {self(), {eval_rsp, Res}};
-	_Forms ->
+	{_Forms, _XBs} ->
 	    Rsp = 'Only possible to inspect variables',
 	    From ! {self(), {eval_rsp, Rsp}}
     end.
@@ -404,22 +409,22 @@ eval_nonrestricted({From, _Mod, Cmd, _SP}, Bs,
 	{'EXIT', _Reason} ->
 	    From ! {self(), {eval_rsp, 'Parse error'}},
 	    Bs;
-	Forms ->
+	{Forms, XBs} ->
 	    mark_running(Line, Le),
+            Bs1 = merge_bindings(Bs, XBs),
 	    {Res, Bs2} =
 		lists:foldl(fun(Expr, {_Res, Bs0}) ->
 				    eval_nonrestricted_1(Expr,Bs0,Ieval)
 			    end,
-			    {null, Bs},
+			    {null, Bs1},
 			    Forms),
 	    mark_break(M, Line, Le),
 	    From ! {self(), {eval_rsp, Res}},
-	    Bs2
+	    remove_binding_structs(Bs2, XBs)
     end.
 
 eval_nonrestricted_1({match,_,{var,_,Var},Expr}, Bs, Ieval) ->
-    {value,Res,Bs2} = 
-	dbg_ieval:eval_expr(Expr, Bs, Ieval#ieval{top=false}),
+    {Res,Bs2} = eval_expr(Expr, Bs, Ieval),
     Bs3 = case lists:keyfind(Var, 1, Bs) of
 	      {Var,_Value} ->
 		  lists:keyreplace(Var, 1, Bs2, {Var,Res});
@@ -433,10 +438,21 @@ eval_nonrestricted_1({var,_,Var}, Bs, _Ieval) ->
 	  end,
     {Res,Bs};
 eval_nonrestricted_1(Expr, Bs, Ieval) ->
-    {value,Res,Bs2} = 
-	dbg_ieval:eval_expr(Expr, Bs, Ieval#ieval{top=false}),
+    eval_expr(Expr, Bs, Ieval).
+
+eval_expr(Expr, Bs, Ieval) ->
+    {value,Res,Bs2} =
+        dbg_ieval:eval_expr(Expr, Bs, Ieval#ieval{top=false}),
     {Res,Bs2}.
 
+%% XBs have unique keys.
+merge_bindings(Bs1, XBs) ->
+    Bs1 ++ erl_eval:bindings(XBs).
+
+remove_binding_structs(Bs1, XBs) ->
+    lists:foldl(fun({N, _V}, Bs) -> lists:keydelete(N, 1, Bs)
+                end, Bs1, erl_eval:bindings(XBs)).
+
 mark_running(LineNo, Le) ->
     put(next_break, running),
     put(user_eval, [{LineNo, Le} | get(user_eval)]),
@@ -450,9 +466,9 @@ mark_break(Cm, LineNo, Le) ->
     dbg_iserver:cast(get(int), {set_status,self(),break,{Cm,LineNo}}).
 
 parse_cmd(Cmd, LineNo) ->
-    {ok,Tokens,_} = erl_scan:string(Cmd, LineNo),
-    {ok,Forms} = erl_parse:parse_exprs(Tokens),
-    Forms.
+    {ok,Tokens,_} = erl_scan:string(Cmd, LineNo, [text]),
+    {ok,Forms,Bs} = lib:extended_parse_exprs(Tokens),
+    {Forms, Bs}.
 
 %%====================================================================
 %% Library functions for attached process handling
diff --git a/lib/debugger/src/dbg_wx_win.erl b/lib/debugger/src/dbg_wx_win.erl
index 2c9d83ea74..f1298154ab 100644
--- a/lib/debugger/src/dbg_wx_win.erl
+++ b/lib/debugger/src/dbg_wx_win.erl
@@ -273,10 +273,9 @@ entry(Parent, Title, Prompt, {Type, Value}) ->
 
 
 verify(Type, Str) ->
-    case erl_scan:string(Str) of
+    case erl_scan:string(Str, 1, [text]) of
 	{ok, Tokens, _EndLine} when Type==term ->
-	    
-	    case erl_parse:parse_term(Tokens++[{dot, erl_anno:new(1)}]) of
+	    case lib:extended_parse_term(Tokens++[{dot, erl_anno:new(1)}]) of
 		{ok, Value} -> {edit, Value};
 		_Error -> 
 		    ignore
diff --git a/lib/stdlib/doc/src/shell.xml b/lib/stdlib/doc/src/shell.xml
index ab62c2fcdd..2593d3690b 100644
--- a/lib/stdlib/doc/src/shell.xml
+++ b/lib/stdlib/doc/src/shell.xml
@@ -569,7 +569,7 @@ Hello Number: 3378
 
     <pre>
 42> <input>E = ets:new(t, []).</input>
-17</pre>
+#Ref&lt;0.1662103692.2407923716.214192></pre>
 
     <p>Command 42 creates an ETS table.</p>
 
@@ -602,7 +602,7 @@ false</pre>
 
     <pre>
 47> <input>E = ets:new(t, []).</input>
-18
+#Ref&lt;0.1662103692.2407923716.214197>
 48> <input>ets:insert({d,1,2}).</input>
 * exception error: undefined function ets:insert/1</pre>
 
@@ -617,10 +617,23 @@ true</pre>
     <p>Command 49 successfully inserts the tuple into the ETS table.</p>
 
     <pre>
-50> <input>halt().</input>
+50> <input>ets:insert(#Ref&lt;0.1662103692.2407923716.214197>, {e,3,4}).</input>
+true</pre>
+
+    <p>Command 50 inserts another tuple into the ETS table. This time
+      the first argument is the table identifier itself. The shell can
+      parse commands with pids (<c>&lt;0.60.0></c>), ports
+      (<c>#Port&lt;0.536></c>), references
+      (<c>#Ref&lt;0.1662103692.2407792644.214210></c>), and external
+      functions (<c>#Fun&lt;a.b.1></c>), but the command fails unless
+      the corresponding pid, port, reference, or function can be created
+      in the running system.</p>
+
+    <pre>
+51> <input>halt().</input>
 strider 2></pre>
 
-    <p>Command 50 exits the Erlang runtime system.</p>
+    <p>Command 51 exits the Erlang runtime system.</p>
   </section>
 
   <section>
diff --git a/lib/stdlib/src/lib.erl b/lib/stdlib/src/lib.erl
index 56654097d9..aa6797bce6 100644
--- a/lib/stdlib/src/lib.erl
+++ b/lib/stdlib/src/lib.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -22,6 +22,9 @@
 -export([flush_receive/0, error_message/2, progname/0, nonl/1, send/2,
 	 sendw/2, eval_str/1]).
 
+-export([extended_parse_exprs/1, extended_parse_term/1,
+         subst_values_for_vars/2]).
+
 -export([format_exception/6, format_exception/7,
          format_stacktrace/4, format_stacktrace/5,
          format_call/4, format_call/5, format_fun/1]).
@@ -127,6 +130,224 @@ all_white([$\t|T]) -> all_white(T);
 all_white([])      -> true;
 all_white(_)       -> false.
 
+%% `Tokens' is assumed to have been scanned with the 'text' option.
+%% The annotations of the returned expressions are locations.
+%%
+%% Can handle pids, ports, references, and external funs ("items").
+%% Known items are represented by variables in the erl_parse tree, and
+%% the items themselves are stored in the returned bindings.
+
+-spec extended_parse_exprs(Tokens) ->
+                {'ok', ExprList, Bindings} | {'error', ErrorInfo} when
+      Tokens :: [erl_scan:token()],
+      ExprList :: [erl_parse:abstract_expr()],
+      Bindings :: erl_eval:binding_struct(),
+      ErrorInfo :: erl_parse:error_info().
+
+extended_parse_exprs(Tokens) ->
+    Ts = tokens_fixup(Tokens),
+    case erl_parse:parse_exprs(Ts) of
+        {ok, Exprs0} ->
+            {Exprs, Bs} = expr_fixup(Exprs0),
+            {ok, reset_expr_anno(Exprs), Bs};
+        _ErrorInfo ->
+            erl_parse:parse_exprs(reset_token_anno(Ts))
+    end.
+
+tokens_fixup([]) -> [];
+tokens_fixup([T|Ts]=Ts0) ->
+    try token_fixup(Ts0) of
+        {NewT, NewTs} ->
+            [NewT|tokens_fixup(NewTs)]
+    catch
+        _:_ ->
+            [T|tokens_fixup(Ts)]
+    end.
+
+token_fixup(Ts) ->
+    {AnnoL, NewTs, FixupTag} = unscannable(Ts),
+    String = lists:append([erl_anno:text(A) || A <- AnnoL]),
+    _ = (fixup_fun(FixupTag))(String),
+    NewAnno = erl_anno:set_text(fixup_text(FixupTag), hd(AnnoL)),
+    {{string, NewAnno, String}, NewTs}.
+
+unscannable([{'#', A1}, {var, A2, 'Fun'}, {'<', A3}, {atom, A4, _},
+             {'.', A5}, {float, A6, _}, {'>', A7}|Ts]) ->
+    {[A1, A2, A3, A4, A5, A6, A7], Ts, function};
+unscannable([{'#', A1}, {var, A2, 'Fun'}, {'<', A3}, {atom, A4, _},
+             {'.', A5}, {atom, A6, _}, {'.', A7}, {integer, A8, _},
+             {'>', A9}|Ts]) ->
+    {[A1, A2, A3, A4, A5, A6, A7, A8, A9], Ts, function};
+unscannable([{'<', A1}, {float, A2, _}, {'.', A3}, {integer, A4, _},
+             {'>', A5}|Ts]) ->
+    {[A1, A2, A3, A4, A5], Ts, pid};
+unscannable([{'#', A1}, {var, A2, 'Port'}, {'<', A3}, {float, A4, _},
+             {'>', A5}|Ts]) ->
+    {[A1, A2, A3, A4, A5], Ts, port};
+unscannable([{'#', A1}, {var, A2, 'Ref'}, {'<', A3}, {float, A4, _},
+             {'.', A5}, {float, A6, _}, {'>', A7}|Ts]) ->
+    {[A1, A2, A3, A4, A5, A6, A7], Ts, reference}.
+
+expr_fixup(Expr0) ->
+    {Expr, Bs, _} = expr_fixup(Expr0, erl_eval:new_bindings(), 1),
+    {Expr, Bs}.
+
+expr_fixup({string,A,S}=T, Bs0, I) ->
+    try string_fixup(A, S) of
+        Value ->
+            Var = new_var(I),
+            Bs = erl_eval:add_binding(Var, Value, Bs0),
+            {{var, A, Var}, Bs, I+1}
+    catch
+        _:_ ->
+            {T, Bs0, I}
+    end;
+expr_fixup(Tuple, Bs0, I0) when is_tuple(Tuple) ->
+    {L, Bs, I} = expr_fixup(tuple_to_list(Tuple), Bs0, I0),
+    {list_to_tuple(L), Bs, I};
+expr_fixup([E0|Es0], Bs0, I0) ->
+    {E, Bs1, I1} = expr_fixup(E0, Bs0, I0),
+    {Es, Bs, I} = expr_fixup(Es0, Bs1, I1),
+    {[E|Es], Bs, I};
+expr_fixup(T, Bs, I) ->
+    {T, Bs, I}.
+
+string_fixup(A, S) ->
+    Text = erl_anno:text(A),
+    FixupTag = fixup_tag(Text, S),
+    (fixup_fun(FixupTag))(S).
+
+new_var(I) ->
+    list_to_atom(lists:concat(['__ExtendedParseExprs_', I, '__'])).
+
+reset_token_anno(Tokens) ->
+    [setelement(2, T, (reset_anno())(element(2, T))) || T <- Tokens].
+
+reset_expr_anno(Exprs) ->
+    [erl_parse:map_anno(reset_anno(), E) || E <- Exprs].
+
+reset_anno() ->
+    fun(A) -> erl_anno:new(erl_anno:location(A)) end.
+
+fixup_fun(function)  -> fun function/1;
+fixup_fun(pid)       -> fun erlang:list_to_pid/1;
+fixup_fun(port)      -> fun erlang:list_to_port/1;
+fixup_fun(reference) -> fun erlang:list_to_ref/1.
+
+function(S) ->
+    %% External function.
+    {ok, [_, _, _,
+          {atom, _, Module}, _,
+          {atom, _, Function}, _,
+          {integer, _, Arity}|_], _} = erl_scan:string(S),
+    erlang:make_fun(Module, Function, Arity).
+
+fixup_text(function)  -> "function";
+fixup_text(pid)       -> "pid";
+fixup_text(port)      -> "port";
+fixup_text(reference) -> "reference".
+
+fixup_tag("function",  "#"++_) -> function;
+fixup_tag("pid",       "<"++_) -> pid;
+fixup_tag("port",      "#"++_) -> port;
+fixup_tag("reference", "#"++_) -> reference.
+
+%%% End of extended_parse_exprs.
+
+%% `Tokens' is assumed to have been scanned with the 'text' option.
+%%
+%% Can handle pids, ports, references, and external funs.
+
+-spec extended_parse_term(Tokens) ->
+                {'ok', Term} | {'error', ErrorInfo} when
+      Tokens :: [erl_scan:token()],
+      Term :: term(),
+      ErrorInfo :: erl_parse:error_info().
+
+extended_parse_term(Tokens) ->
+    case extended_parse_exprs(Tokens) of
+        {ok, [Expr], Bindings} ->
+            try normalise(Expr, Bindings) of
+                Term ->
+                    {ok, Term}
+            catch
+                _:_ ->
+                    Loc = erl_anno:location(element(2, Expr)),
+                    {error,{Loc,?MODULE,"bad term"}}
+            end;
+        {ok, [_,Expr|_], _Bindings} ->
+                Loc = erl_anno:location(element(2, Expr)),
+                {error,{Loc,?MODULE,"bad term"}};
+        {error, _} = Error ->
+            Error
+    end.
+
+%% From erl_parse.
+normalise({var, _, V}, Bs) ->
+    {value, Value} = erl_eval:binding(V, Bs),
+    Value;
+normalise({char,_,C}, _Bs) -> C;
+normalise({integer,_,I}, _Bs) -> I;
+normalise({float,_,F}, _Bs) -> F;
+normalise({atom,_,A}, _Bs) -> A;
+normalise({string,_,S}, _Bs) -> S;
+normalise({nil,_}, _Bs) -> [];
+normalise({bin,_,Fs}, Bs) ->
+    {value, B, _} =
+	eval_bits:expr_grp(Fs, [],
+			   fun(E, _) ->
+				   {value, normalise(E, Bs), []}
+			   end, [], true),
+    B;
+normalise({cons,_,Head,Tail}, Bs) ->
+    [normalise(Head, Bs)|normalise(Tail, Bs)];
+normalise({tuple,_,Args}, Bs) ->
+    list_to_tuple(normalise_list(Args, Bs));
+normalise({map,_,Pairs}, Bs) ->
+    maps:from_list(lists:map(fun
+		%% only allow '=>'
+		({map_field_assoc,_,K,V}) ->
+                                     {normalise(K, Bs),normalise(V, Bs)}
+	    end, Pairs));
+%% Special case for unary +/-.
+normalise({op,_,'+',{char,_,I}}, _Bs) -> I;
+normalise({op,_,'+',{integer,_,I}}, _Bs) -> I;
+normalise({op,_,'+',{float,_,F}}, _Bs) -> F;
+normalise({op,_,'-',{char,_,I}}, _Bs) -> -I;   %Weird, but compatible!
+normalise({op,_,'-',{integer,_,I}}, _Bs) -> -I;
+normalise({op,_,'-',{float,_,F}}, _Bs) -> -F;
+normalise({'fun',_,{function,{atom,_,M},{atom,_,F},{integer,_,A}}}, _Bs) ->
+    %% Since "#Fun<M.F.A>" is recognized, "fun M:F/A" should be too.
+    fun M:F/A.
+
+normalise_list([H|T], Bs) ->
+    [normalise(H, Bs)|normalise_list(T, Bs)];
+normalise_list([], _Bs) ->
+    [].
+
+%% To be used on ExprList and Bindings returned from extended_parse_exprs().
+%% Substitute {value, A, Item} for {var, A, ExtendedParseVar}.
+%% {value, A, Item} is a shell/erl_eval convention, and for example
+%% the linter cannot handle it.
+
+-spec subst_values_for_vars(ExprList, Bindings) -> [term()] when
+      ExprList :: [erl_parse:abstract_expr()],
+      Bindings :: erl_eval:binding_struct().
+
+subst_values_for_vars({var, A, V}=Var, Bs) ->
+    case erl_eval:binding(V, Bs) of
+        {value, Value} ->
+            {value, A, Value};
+        unbound ->
+            Var
+    end;
+subst_values_for_vars(L, Bs) when is_list(L) ->
+    [subst_values_for_vars(E, Bs) || E <- L];
+subst_values_for_vars(T, Bs) when is_tuple(T) ->
+    list_to_tuple(subst_values_for_vars(tuple_to_list(T), Bs));
+subst_values_for_vars(T, _Bs) ->
+    T.
+
 %%% Formatting of exceptions, mfa:s and funs.
 
 %% -> iolist() (no \n at end)
diff --git a/lib/stdlib/src/qlc.erl b/lib/stdlib/src/qlc.erl
index 8c4d835432..20aaa2638c 100644
--- a/lib/stdlib/src/qlc.erl
+++ b/lib/stdlib/src/qlc.erl
@@ -635,14 +635,25 @@ string_to_handle(Str, Options, Bindings) when is_list(Str) ->
         badarg ->
             erlang:error(badarg, [Str, Options, Bindings]);
         [Unique, Cache, MaxLookup, Join, Lookup] ->
-            case erl_scan:string(Str) of
+            case erl_scan:string(Str, 1, [text]) of
                 {ok, Tokens, _} ->
-                    case erl_parse:parse_exprs(Tokens) of
-                        {ok, [Expr]} ->
-                            case qlc_pt:transform_expression(Expr, Bindings) of
+                    ScanRes =
+                        case lib:extended_parse_exprs(Tokens) of
+                            {ok, [Expr0], SBs} ->
+                                {ok, Expr0, SBs};
+                            {ok, _ExprList, _SBs} ->
+                                erlang:error(badarg,
+                                             [Str, Options, Bindings]);
+                            E ->
+                                E
+                        end,
+                    case ScanRes of
+                        {ok, Expr, XBs} ->
+                            Bs1 = merge_binding_structs(Bindings, XBs),
+                            case qlc_pt:transform_expression(Expr, Bs1) of
                                 {ok, {call, _, _QlcQ,  Handle}} ->
                                     {value, QLC_lc, _} = 
-                                        erl_eval:exprs(Handle, Bindings),
+                                        erl_eval:exprs(Handle, Bs1),
                                     O = #qlc_opt{unique = Unique, 
                                                  cache = Cache,
                                                  max_lookup = MaxLookup, 
@@ -652,8 +663,6 @@ string_to_handle(Str, Options, Bindings) when is_list(Str) ->
                                 {not_ok, [{error, Error} | _]} ->
                                     error(Error)
                             end;
-                        {ok, _ExprList} ->
-                            erlang:error(badarg, [Str, Options, Bindings]);
                         {error, ErrorInfo} ->
                             error(ErrorInfo)
                     end;
@@ -770,6 +779,10 @@ all_selections([{I,Cs} | ICs]) ->
 %%% Local functions
 %%%
 
+merge_binding_structs(Bs1, Bs2) ->
+    lists:foldl(fun({N, V}, Bs) -> erl_eval:add_binding(N, V, Bs)
+                end, Bs1, erl_eval:bindings(Bs2)).
+
 aux_name1(Name, N, AllNames) ->
     SN = name_suffix(Name, N),
     case sets:is_element(SN, AllNames) of
@@ -1180,9 +1193,12 @@ abstract1({table, {M, F, As0}}, _NElements, _Depth, Anno)
 abstract1({table, TableDesc}, _NElements, _Depth, _A) ->
     case io_lib:deep_char_list(TableDesc) of
         true ->
-            {ok, Tokens, _} = erl_scan:string(lists:flatten(TableDesc++".")),
-            {ok, [Expr]} = erl_parse:parse_exprs(Tokens),
-            Expr;
+            {ok, Tokens, _} =
+                erl_scan:string(lists:flatten(TableDesc++"."), 1, [text]),
+            {ok, Es, Bs} =
+                lib:extended_parse_exprs(Tokens),
+            [Expr] = lib:subst_values_for_vars(Es, Bs),
+            special(Expr);
         false -> % abstract expression
             TableDesc
     end;
@@ -1210,6 +1226,15 @@ abstract1({list, L}, NElements, Depth, _A) when NElements =:= infinity;
 abstract1({list, L}, NElements, Depth, _A) ->
     abstract_term(depth(lists:sublist(L, NElements), Depth) ++ '...', 1).
 
+special({value, _, Thing}) ->
+    abstract_term(Thing);
+special(Tuple) when is_tuple(Tuple) ->
+    list_to_tuple(special(tuple_to_list(Tuple)));
+special([E|Es]) ->
+    [special(E)|special(Es)];
+special(Expr) ->
+    Expr.
+
 depth(List, infinity) ->
     List;
 depth(List, Depth) ->
diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl
index 961f5f8a30..76a2789406 100644
--- a/lib/stdlib/src/shell.erl
+++ b/lib/stdlib/src/shell.erl
@@ -229,8 +229,9 @@ server_loop(N0, Eval_0, Bs00, RT, Ds00, History0, Results0) ->
     {Eval_1,Bs0,Ds0,Prompt} = prompt(N, Eval_0, Bs00, RT, Ds00),
     {Res,Eval0} = get_command(Prompt, Eval_1, Bs0, RT, Ds0),
     case Res of 
-	{ok,Es0} ->
-            case expand_hist(Es0, N) of
+	{ok,Es0,XBs} ->
+            Es1 = lib:subst_values_for_vars(Es0, XBs),
+            case expand_hist(Es1, N) of
                 {ok,Es} ->
                     {V,Eval,Bs,Ds} = shell_cmd(Es, Eval0, Bs0, RT, Ds0, cmd),
                     {History,Results} = check_and_get_history_and_results(),
@@ -276,10 +277,10 @@ get_command(Prompt, Eval, Bs, RT, Ds) ->
         fun() ->
                 exit(
                   case
-                      io:scan_erl_exprs(group_leader(), Prompt, 1)
+                      io:scan_erl_exprs(group_leader(), Prompt, 1, [text])
                   of
                       {ok,Toks,_EndPos} ->
-                          erl_parse:parse_exprs(Toks);
+                          lib:extended_parse_exprs(Toks);
                       {eof,_EndPos} ->
                           eof;
                       {error,ErrorInfo,_EndPos} ->
diff --git a/lib/stdlib/test/qlc_SUITE.erl b/lib/stdlib/test/qlc_SUITE.erl
index 2b5d52287e..5e9e03e410 100644
--- a/lib/stdlib/test/qlc_SUITE.erl
+++ b/lib/stdlib/test/qlc_SUITE.erl
@@ -1240,6 +1240,17 @@ string_to_handle(Config) when is_list(Config) ->
     {'EXIT', {no_lookup_to_carry_out, _}} =
         (catch qlc:e(qlc:string_to_handle(Q, {lookup,true}, Bs2))),
     ets:delete(Ets),
+
+    %% References can be scanned and parsed.
+    E2 = ets:new(test, [bag]),
+    Ref = make_ref(),
+    true = ets:insert(E2, [{Ref,Ref}]),
+    S2 = "[{Val1} || {Ref1, Val1} <- ets:table("++io_lib:write(E2)++"),"
+         "Ref1 =:= Ref].",
+    Bs = erl_eval:add_binding('Ref', Ref, erl_eval:new_bindings()),
+    [{Ref}] = qlc:e(qlc:string_to_handle(S2, [], Bs)),
+    ets:delete(E2),
+
     ok.
 
 %% table
@@ -4321,7 +4332,18 @@ ets(Config) when is_list(Config) ->
            R = qlc:e(Q),
            ets:delete(E),
            [] = R">>]
-       end
+       end,
+
+       <<"E2 = ets:new(test, [bag]),
+          Ref = make_ref(),
+          true = ets:insert(E2, [{Ref,Ref}]),
+          Q2 = qlc:q([{Val1} ||
+                         {Ref1, Val1} <- ets:table(E2),
+                         Ref1 =:= Ref]),
+          S = qlc:info(Q2),
+          true = is_list(S),
+          [{Ref}] = qlc:e(Q2),
+          ets:delete(E2)">>
 
        ],
     
@@ -7071,7 +7093,7 @@ otp_12946(Config) when is_list(Config) ->
 
 %% Examples from qlc(3).
 manpage(Config) when is_list(Config) ->
-
+    dets:start(),
     ok = compile_gb_table(Config),
 
     Ts = [
@@ -7138,11 +7160,14 @@ manpage(Config) when is_list(Config) ->
               \"               [{X,Z}|{W,Y}] <- V2\n\"
               \"          ])\n\"
               \"end\",
-          Info =
+          Info1 =
              re:replace(qlc:info(Q), 
-                        \"table\\\\(-*[0-9]*\",
+                        \"table\\\\(#Ref<[\\.0-9]*>\",
                         \"table(_\", [{return,list},global]),
-          L = Info,
+          F = fun(C) -> C =/= $\n andalso C =/= $\s end,
+          Info = lists:filter(F, Info1),
+          L1 = lists:filter(F, L),
+          L1 = Info,
           ets:delete(E1),
           ets:delete(E2)">>,
 
@@ -7445,10 +7470,10 @@ etsc(F, Opts, Objs) ->
     V.
 
 join_info(H) ->
-    {qlc, S, Options} = strip_qlc_call(H),
+    {{qlc, S, Options}, Bs} = strip_qlc_call2(H),
     %% "Hide" the call to qlc_pt from the test in run_test().
     LoadedPT = code:is_loaded(qlc_pt),
-    QH = qlc:string_to_handle(S, Options),
+    QH = qlc:string_to_handle(S, Options, Bs),
     _ = [unload_pt() || false <- [LoadedPT]], % doesn't take long...
     case {join_info_count(H), join_info_count(QH)} of
         {N, N} -> 
@@ -7458,30 +7483,34 @@ join_info(H) ->
     end.
 
 strip_qlc_call(H) ->
+    {Expr, _Bs} = strip_qlc_call2(H),
+    Expr.
+
+strip_qlc_call2(H) ->
     S = qlc:info(H, {flat, false}),
-    {ok, Tokens, _EndLine} = erl_scan:string(S++"."),
-    {ok, [Expr]} = erl_parse:parse_exprs(Tokens),
-    case Expr of
-        {call,_,{remote,_,{atom,_,qlc},{atom,_,q}},[LC]} ->
-            {qlc, lists:flatten([erl_pp:expr(LC), "."]), []};
-        {call,_,{remote,_,{atom,_,qlc},{atom,_,q}},[LC, Opts]} ->
-            {qlc, lists:flatten([erl_pp:expr(LC), "."]), 
-             erl_parse:normalise(Opts)};
-        {call,_,{remote,_,{atom,_,ets},{atom,_,match_spec_run}},_} ->
-            {match_spec, Expr};
-        {call,_,{remote,_,{atom,_,M},{atom,_,table}},_} ->
-            {table, M, Expr};
-        _ -> 
-            []
-    end.
+    {ok, Tokens, _EndLine} = erl_scan:string(S++".", 1, [text]),
+    {ok, [Expr], Bs} = lib:extended_parse_exprs(Tokens),
+    {case Expr of
+         {call,_,{remote,_,{atom,_,qlc},{atom,_,q}},[LC]} ->
+             {qlc, lists:flatten([erl_pp:expr(LC), "."]), []};
+         {call,_,{remote,_,{atom,_,qlc},{atom,_,q}},[LC, Opts]} ->
+             {qlc, lists:flatten([erl_pp:expr(LC), "."]),
+              erl_parse:normalise(Opts)};
+         {call,_,{remote,_,{atom,_,ets},{atom,_,match_spec_run}},_} ->
+             {match_spec, Expr};
+         {call,_,{remote,_,{atom,_,M},{atom,_,table}},_} ->
+             {table, M, Expr};
+         _ ->
+             []
+     end, Bs}.
 
 -record(ji, {nmerge = 0, nlookup = 0, nnested_loop = 0, nkeysort = 0}).
 
 %% Counts join options and (all) calls to qlc:keysort().
 join_info_count(H) ->
     S = qlc:info(H, {flat, false}),    
-    {ok, Tokens, _EndLine} = erl_scan:string(S++"."),
-    {ok, [Expr]} = erl_parse:parse_exprs(Tokens),
+    {ok, Tokens, _EndLine} = erl_scan:string(S++".", 1, [text]),
+    {ok, [Expr], _Bs} = lib:extended_parse_exprs(Tokens),
     #ji{nmerge = Nmerge, nlookup = Nlookup, 
         nkeysort = NKeysort, nnested_loop = Nnested_loop} = 
         ji(Expr, #ji{}),
@@ -7524,8 +7553,8 @@ lookup_keys({list,Q,_}, L) ->
 lookup_keys({generate,_,Q}, L) ->
     lookup_keys(Q, L);
 lookup_keys({table,Chars}, L) when is_list(Chars) ->
-    {ok, Tokens, _} = erl_scan:string(lists:flatten(Chars++".")),
-    {ok, [Expr]} = erl_parse:parse_exprs(Tokens),
+    {ok, Tokens, _} = erl_scan:string(lists:flatten(Chars++"."), 1, [text]),
+    {ok, [Expr], _Bs} = lib:extended_parse_exprs(Tokens),
     case Expr of
         {call,_,_,[_fun,AKs]} ->
             case erl_parse:normalise(AKs) of
@@ -7842,7 +7871,7 @@ run_test(Config, Extra, {cres, Body, Opts, ExpectedCompileReturn}) ->
     {module, _} = code:load_abs(AbsFile, Mod),
 
     Ms0 = erlang:process_info(self(),messages),
-    Before = {{get(), ets:all(), Ms0}, pps()},
+    Before = {{get(), lists:sort(ets:all()), Ms0}, pps()},
 
     %% Prepare the check that the qlc module does not call qlc_pt.
     _ = [unload_pt() || {file, Name} <- [code:is_loaded(qlc_pt)], 
@@ -7874,7 +7903,7 @@ run_test(Config, Extra, Body) ->
 
 wait_for_expected(R, {Strict0,PPS0}=Before, SourceFile, Wait) ->
     Ms = erlang:process_info(self(),messages),
-    After = {_,PPS1} = {{get(), ets:all(), Ms}, pps()},
+    After = {_,PPS1} = {{get(), lists:sort(ets:all()), Ms}, pps()},
     case {R, After} of
         {ok, Before} ->
             ok;
diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl
index 56002dda25..99411bc8fd 100644
--- a/lib/stdlib/test/shell_SUITE.erl
+++ b/lib/stdlib/test/shell_SUITE.erl
@@ -31,7 +31,7 @@
 	 progex_lc/1, progex_funs/1,
 	 otp_5990/1, otp_6166/1, otp_6554/1,
 	 otp_7184/1, otp_7232/1, otp_8393/1, otp_10302/1, otp_13719/1,
-         otp_14285/1]).
+         otp_14285/1, otp_14296/1]).
 
 -export([ start_restricted_from_shell/1, 
 	  start_restricted_on_command_line/1,restricted_local/1]).
@@ -92,7 +92,7 @@ groups() ->
        progex_funs]},
      {tickets, [],
       [otp_5990, otp_6166, otp_6554, otp_7184,
-       otp_7232, otp_8393, otp_10302, otp_13719, otp_14285]}].
+       otp_7232, otp_8393, otp_10302, otp_13719, otp_14285, otp_14296]}].
 
 init_per_suite(Config) ->
     Config.
@@ -2841,6 +2841,95 @@ otp_14285(Config) ->
     test_server:stop_node(Node),
     ok.
 
+otp_14296(Config) when is_list(Config) ->
+    fun() ->
+            F = fun() -> a end,
+            LocalFun = term_to_string(F),
+            S = LocalFun ++ ".",
+            "1: syntax error before: Fun" = comm_err(S)
+    end(),
+
+    fun() ->
+            F = fun mod:func/1,
+            ExternalFun = term_to_string(F),
+            S = ExternalFun ++ ".",
+            R = ExternalFun ++ ".\n",
+            R = t(S)
+    end(),
+
+    fun() ->
+            UnknownPid = "<100000.0.0>",
+            S = UnknownPid ++ ".",
+            "1: syntax error before: '<'" = comm_err(S)
+    end(),
+
+    fun() ->
+            KnownPid = term_to_string(self()),
+            S = KnownPid ++ ".",
+            R = KnownPid ++ ".\n",
+            R = t(S)
+    end(),
+
+    fun() ->
+            Port = open_port({spawn, "ls"}, [line]),
+            KnownPort = erlang:port_to_list(Port),
+            S = KnownPort ++ ".",
+            R = KnownPort ++ ".\n",
+            R = t(S)
+    end(),
+
+    fun() ->
+            UnknownPort = "#Port<100000.0>",
+            S = UnknownPort ++ ".",
+            "1: syntax error before: Port" = comm_err(S)
+    end(),
+
+    fun() ->
+            UnknownRef = "#Ref<100000.0.0.0>",
+            S = UnknownRef ++ ".",
+            "1: syntax error before: Ref" = comm_err(S)
+    end(),
+
+    fun() ->
+            KnownRef = term_to_string(make_ref()),
+            S = KnownRef ++ ".",
+            R = KnownRef ++ ".\n",
+            R = t(S)
+    end(),
+
+    %% Test lib:extended_parse_term/1
+    TF = fun(S) ->
+                 {ok, Ts, _} = erl_scan:string(S++".", 1, [text]),
+                 case lib:extended_parse_term(Ts) of
+                     {ok, Term} -> Term;
+                     {error, _}=Error -> Error
+                 end
+         end,
+    Fun = fun m:f/1,
+    Fun = TF(term_to_string(Fun)),
+    Fun = TF("fun m:f/1"),
+    Pid = self(),
+    Pid = TF(term_to_string(Pid)),
+    Ref = make_ref(),
+    Ref = TF(term_to_string(Ref)),
+    Term = {[10, a], {"foo", []}, #{x => <<"bar">>}},
+    Term = TF(lists:flatten(io_lib:format("~p", [Term]))),
+    {$a, F1, "foo"} = TF("{$a, 1.0, \"foo\"}"),
+    true = is_float(F1),
+    3 = TF("+3"),
+    $a = TF("+$a"),
+    true = is_float(TF("+1.0")),
+    true  = -3 =:= TF("-3"),
+    true = -$a =:= TF("-$a"),
+    true = is_float(TF("-1.0")),
+    {error, {_, _, ["syntax error"++_|_]}} = TF("{1"),
+    {error, {_,_,"bad term"}} = TF("fun() -> foo end"),
+    {error, {_,_,"bad term"}} = TF("1, 2"),
+    ok.
+
+term_to_string(T) ->
+    lists:flatten(io_lib:format("~w", [T])).
+
 scan(B) ->
     F = fun(Ts) -> 
                 case erl_parse:parse_term(Ts) of
diff --git a/lib/stdlib/test/sofs_SUITE.erl b/lib/stdlib/test/sofs_SUITE.erl
index f67bf16f0f..39e56c6df6 100644
--- a/lib/stdlib/test/sofs_SUITE.erl
+++ b/lib/stdlib/test/sofs_SUITE.erl
@@ -1783,7 +1783,7 @@ multiple_relative_product(Conf) when is_list(Conf) ->
     ok.
 
 digraph(Conf) when is_list(Conf) ->
-    T0 = ets:all(),
+    T0 = lists:sort(ets:all()),
     E = empty_set(),
     R = relation([{a,b},{b,c},{c,d},{d,a}]),
     F = relation_to_family(R),
@@ -1833,7 +1833,7 @@ digraph(Conf) when is_list(Conf) ->
         true -> ok
     end,
 
-    true = T0 == ets:all(),
+    true = T0 == lists:sort(ets:all()),
     ok.
 
 digraph_fail(ExitReason, Fail) ->
diff --git a/system/doc/efficiency_guide/processes.xml b/system/doc/efficiency_guide/processes.xml
index afa4325d8e..3b64c863ff 100644
--- a/system/doc/efficiency_guide/processes.xml
+++ b/system/doc/efficiency_guide/processes.xml
@@ -4,7 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>2001</year><year>2016</year>
+      <year>2001</year><year>2017</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -222,7 +222,7 @@ kilo_byte(N, Acc) ->
 
       <pre>
 4> <input>T = ets:new(tab, []).</input>
-17
+#Ref&lt;0.1662103692.2407923716.214181>
 5> <input>ets:insert(T, {key,efficiency_guide:kilo_byte()}).</input>
 true
 6> <input>erts_debug:size(element(2, hd(ets:lookup(T, key)))).</input>