Merge pull request #1312 from richcarl/shell-smart-compile

Extend shell c(...) to find and recompile modules

OTP-14190

15 files changed, 563 insertions, 131 deletions

diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl
index 90ef84ca51..06facae5c1 100644
--- a/lib/hipe/main/hipe.erl
+++ b/lib/hipe/main/hipe.erl
@@ -441,7 +441,7 @@ compile(Name, File, Opts0) when is_atom(Name) ->
       ?error_msg("Cannot get Core Erlang code from BEAM binary.",[]),
       ?EXIT({cant_compile_core_from_binary});
     true ->
-      case filename:find_src(filename:rootname(File, ".beam")) of
+      case filelib:find_source(filename:rootname(File,".beam") ++ ".beam") of
 	{error, _} ->
 	  ?error_msg("Cannot find source code for ~p.", [File]),
 	  ?EXIT({cant_find_source_code});
diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml
index c581fa9d1e..b342fff0d3 100644
--- a/lib/kernel/doc/src/kernel_app.xml
+++ b/lib/kernel/doc/src/kernel_app.xml
@@ -433,6 +433,28 @@ MaxT = TickTime + TickTime / 4</code>
           return as soon as possible for <c>application_controller</c>
           to terminate properly.</p>
       </item>
+      <tag><c>source_search_rules = [DirRule] | [SuffixRule] </c></tag>
+      <item>
+	<marker id="source_search_rules"></marker>
+        <p>Where:</p>
+        <list type="bulleted">
+          <item><c>DirRule = {ObjDirSuffix,SrcDirSuffix}</c></item>
+          <item><c>SuffixRule = {ObjSuffix,SrcSuffix,[DirRule]}</c></item>
+          <item><c>ObjDirSuffix = string()</c></item>
+          <item><c>SrcDirSuffix = string()</c></item>
+          <item><c>ObjSuffix = string()</c></item>
+          <item><c>SrcSuffix = string()</c></item>
+        </list>
+        <p>Specifies a list of rules for use by <c>filelib:find_file/2</c> and
+          <c>filelib:find_source/2</c>. If this is set to some other value
+          than the empty list, it replaces the default rules. Rules can be
+          simple pairs of directory suffixes, such as <c>{"ebin",
+          "src"}</c>, which are used by <c>filelib:find_file/2</c>, or
+          triples specifying separate directory suffix rules depending on
+          file name extensions, for example <c>[{".beam", ".erl", [{"ebin",
+          "src"}]}</c>, which are used by <c>filelib:find_source/2</c>. Both
+          kinds of rules can be mixed in the list.</p>
+      </item>
     </taglist>
   </section>
 
diff --git a/lib/stdlib/doc/src/c.xml b/lib/stdlib/doc/src/c.xml
index 55a77d1bc5..7666699183 100644
--- a/lib/stdlib/doc/src/c.xml
+++ b/lib/stdlib/doc/src/c.xml
@@ -52,13 +52,27 @@
     <func>
       <name name="c" arity="1"/>
       <name name="c" arity="2"/>
-      <fsummary>Compile and load code in a file.</fsummary>
+      <name name="c" arity="3"/>
+      <fsummary>Compile and load a file or module.</fsummary>
       <desc>
-        <p>Compiles and then purges and loads the code for a file.
-          <c><anno>Options</anno></c> defaults to <c>[]</c>. Compilation is
-          equivalent to:</p>
-        <code type="none">
-compile:file(<anno>File</anno>, <anno>Options</anno> ++ [report_errors, report_warnings])</code>
+        <p>Compiles and then purges and loads the code for a module.
+          <c><anno>Module</anno></c> can be either a module name or a source
+          file path, with or without <c>.erl</c> extension.
+          <c><anno>Options</anno></c> defaults to <c>[]</c>.</p>
+        <p>If <c><anno>Module</anno></c> is an atom and is not the path of a
+          source file, then the code path is searched to locate the object
+          file for the module and extract its original compiler options and
+          source path. If the source file is not found in the original
+          location, <seealso
+          marker="filelib#find_source/1"><c>filelib:find_source/1</c></seealso>
+          is used to search for it relative to the directory of the object
+          file.</p>
+        <p>The source file is compiled with the the original
+          options appended to the given <c><anno>Options</anno></c>, the
+          output replacing the old object file if and only if compilation
+          succeeds. A function <c><anno>Filter</anno></c> can be specified
+          for removing elements from from the original compiler options
+          before the new options are added.</p>
         <p>Notice that purging the code means that any processes
           lingering in old code for the module are killed without
           warning. For more information, see <c>code/3</c>.</p>
diff --git a/lib/stdlib/doc/src/filelib.xml b/lib/stdlib/doc/src/filelib.xml
index 7c6380ce28..ad73fc254a 100644
--- a/lib/stdlib/doc/src/filelib.xml
+++ b/lib/stdlib/doc/src/filelib.xml
@@ -60,6 +60,12 @@
     <datatype>
       <name name="filename_all"/>
     </datatype>
+    <datatype>
+      <name name="find_file_rule"/>
+    </datatype>
+    <datatype>
+      <name name="find_source_rule"/>
+    </datatype>
   </datatypes>
 
   <funcs>
@@ -226,7 +232,51 @@ filelib:wildcard("lib/**/*.{erl,hrl}")</code>
           directory.</p>
       </desc>
     </func>
+
+    <func>
+      <name name="find_file" arity="2"/>
+      <name name="find_file" arity="3"/>
+      <fsummary>Find a file relative to a given directory.</fsummary>
+      <desc>
+        <p>Looks for a file of the given name by applying suffix rules to
+        the given directory path. For example, a rule <c>{"ebin", "src"}</c>
+        means that if the directory path ends with <c>"ebin"</c>, the
+        corresponding path ending in <c>"src"</c> should be searched.</p>
+        <p>If <c><anno>Rules</anno></c> is left out or is an empty list, the
+        default system rules are used. See also the Kernel application
+        parameter <seealso
+        marker="kernel:kernel_app#source_search_rules"><c>source_search_rules</c></seealso>.</p>
+      </desc>
+    </func>
+    <func>
+      <name name="find_source" arity="1"/>
+      <fsummary>Find the source file for a given object file.</fsummary>
+      <desc>
+        <p>Equivalent to <c>find_source(Base, Dir)</c>, where <c>Dir</c> is
+        <c>filename:dirname(<anno>FilePath</anno>)</c> and <c>Base</c> is
+        <c>filename:basename(<anno>FilePath</anno>)</c>.</p>
+      </desc>
+    </func>
+    <func>
+      <name name="find_source" arity="2"/>
+      <name name="find_source" arity="3"/>
+      <fsummary>Find a source file relative to a given directory.</fsummary>
+      <desc>
+        <p>Applies file extension specific rules to find the source file for
+        a given object file relative to the object directory. For example,
+        for a file with the extension <c>.beam</c>, the default rule is to
+        look for a file with a corresponding extension <c>.erl</c> by
+        replacing the suffix <c>"ebin"</c> of the object directory path with
+        <c>"src"</c>.
+        The file search is done through <seealso
+        marker="#find_file/3"><c>find_file/3</c></seealso>. The directory of
+        the object file is always tried before any other directory specified
+        by the rules.</p>
+        <p>If <c><anno>Rules</anno></c> is left out or is an empty list, the
+        default system rules are used. See also the Kernel application
+        parameter <seealso
+        marker="kernel:kernel_app#source_search_rules"><c>source_search_rules</c></seealso>.</p>
+      </desc>
+    </func>
   </funcs>
 </erlref>
-
-
diff --git a/lib/stdlib/doc/src/filename.xml b/lib/stdlib/doc/src/filename.xml
index 2a413835d0..7acef51ca1 100644
--- a/lib/stdlib/doc/src/filename.xml
+++ b/lib/stdlib/doc/src/filename.xml
@@ -356,10 +356,12 @@ true
         <p>Finds the source filename and compiler options for a module.
           The result can be fed to <seealso marker="compiler:compile#file/2">
           <c>compile:file/2</c></seealso> to compile the file again.</p>
-        <warning><p>It is not recommended to use this function. If possible,
-          use the <seealso marker="beam_lib"><c>beam_lib(3)</c></seealso>
-          module to extract the abstract code format from the Beam file and
-          compile that instead.</p></warning>
+        <warning>
+          <p>This function is deprecated. Use <seealso marker="filelib#find_source/1">
+            <c>filelib:find_source/1</c></seealso> instead for finding source files.</p>
+          <p>If possible, use the <seealso marker="beam_lib"><c>beam_lib(3)</c></seealso>
+          module to extract the compiler options and the abstract code
+          format from the Beam file and compile that instead.</p></warning>
         <p>Argument <c><anno>Beam</anno></c>, which can be a string or an atom,
           specifies either the module name or the path to the source
           code, with or without extension <c>".erl"</c>. In either
diff --git a/lib/stdlib/doc/src/shell.xml b/lib/stdlib/doc/src/shell.xml
index d6e8036d4e..f52bc39deb 100644
--- a/lib/stdlib/doc/src/shell.xml
+++ b/lib/stdlib/doc/src/shell.xml
@@ -165,12 +165,12 @@
       <item>
         <p>Evaluates <c>shell_default:help()</c>.</p>
       </item>
-      <tag><c>c(File)</c></tag>
+      <tag><c>c(Mod)</c></tag>
       <item>
-        <p>Evaluates <c>shell_default:c(File)</c>. This compiles
-          and loads code in <c>File</c> and purges old versions of
-          code, if necessary. Assumes that the file and module names
-          are the same.</p>
+        <p>Evaluates <c>shell_default:c(Mod)</c>. This compiles and
+          loads the module <c>Mod</c> and purges old versions of the
+          code, if necessary. <c>Mod</c> can be either a module name or a
+          a source file path, with or without <c>.erl</c> extension.</p>
       </item>
       <tag><c>catch_exception(Bool)</c></tag>
       <item>
diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl
index d36630214c..d3f9a9c7af 100644
--- a/lib/stdlib/src/c.erl
+++ b/lib/stdlib/src/c.erl
@@ -23,7 +23,7 @@
 
 %% Avoid warning for local function error/2 clashing with autoimported BIF.
 -compile({no_auto_import,[error/2]}).
--export([help/0,lc/1,c/1,c/2,nc/1,nc/2, nl/1,l/1,i/0,i/1,ni/0,
+-export([help/0,lc/1,c/1,c/2,c/3,nc/1,nc/2, nl/1,l/1,i/0,i/1,ni/0,
          y/1, y/2,
 	 lc_batch/0, lc_batch/1,
 	 i/3,pid/3,m/0,m/1,mm/0,lm/0,
@@ -44,7 +44,7 @@
 
 help() ->
     io:put_chars(<<"bt(Pid)    -- stack backtrace for a process\n"
-		   "c(File)    -- compile and load code in <File>\n"
+		   "c(Mod)     -- compile and load module or file <Mod>\n"
 		   "cd(Dir)    -- change working directory\n"
 		   "flush()    -- flush any messages sent to the shell\n"
 		   "help()     -- help info\n"
@@ -72,32 +72,222 @@ help() ->
 		   "xm(M)      -- cross reference check a module\n"
 		   "y(File)    -- generate a Yecc parser\n">>).
 
-%% c(FileName)
-%%  Compile a file/module.
+%% c(Module)
+%%  Compile a module/file.
 
--spec c(File) -> {'ok', Module} | 'error' when
-      File :: file:name(),
-      Module :: module().
+-spec c(Module) -> {'ok', ModuleName} | 'error' when
+      Module :: file:name(),
+      ModuleName :: module().
 
-c(File) -> c(File, []).
+c(Module) -> c(Module, []).
 
--spec c(File, Options) -> {'ok', Module} | 'error' when
-      File :: file:name(),
+-spec c(Module, Options) -> {'ok', ModuleName} | 'error' when
+      Module :: file:name(),
       Options :: [compile:option()],
-      Module :: module().
+      ModuleName :: module().
+
+c(Module, Opts) when is_atom(Module) ->
+    %% either a module name or a source file name (possibly without
+    %% suffix); if such a source file exists, it is used to compile from
+    %% scratch with the given options, otherwise look for an object file
+    Suffix = case filename:extension(Module) of
+                 "" -> src_suffix(Opts);
+                 S -> S
+             end,
+    SrcFile = filename:rootname(Module, Suffix) ++ Suffix,
+    case filelib:is_file(SrcFile) of
+        true ->
+            compile_and_load(SrcFile, Opts);
+        false ->
+            c(Module, Opts, fun (_) -> true end)
+    end;
+c(Module, Opts) ->
+    %% we never interpret a string as a module name, only as a file
+    compile_and_load(Module, Opts).
+
+%% This tries to find an existing object file and use its compile_info and
+%% source path to recompile the module, overwriting the old object file.
+%% The Filter parameter is applied to the old compile options
+
+-spec c(Module, Options, Filter) -> {'ok', ModuleName} | 'error' when
+      Module :: atom(),
+      Options :: [compile:option()],
+      Filter :: fun ((compile:option()) -> boolean()),
+      ModuleName :: module().
+
+c(Module, Options, Filter) when is_atom(Module) ->
+    case find_beam(Module) of
+        BeamFile when is_list(BeamFile) ->
+            c(Module, Options, Filter, BeamFile);
+        Error ->
+            {error, Error}
+    end.
+
+c(Module, Options, Filter, BeamFile) ->
+    case compile_info(Module, BeamFile) of
+        Info when is_list(Info) ->
+            case find_source(BeamFile, Info) of
+                SrcFile when is_list(SrcFile) ->
+                    c(SrcFile, Options, Filter, BeamFile, Info);
+                Error ->
+                    Error
+            end;
+        Error ->
+            Error
+    end.
+
+c(SrcFile, NewOpts, Filter, BeamFile, Info) ->
+    %% Filter old options; also remove options that will be replaced.
+    %% Write new beam over old beam unless other outdir is specified.
+    F = fun (Opt) -> not is_outdir_opt(Opt) andalso Filter(Opt) end,
+    Options = (NewOpts ++ [{outdir,filename:dirname(BeamFile)}]
+               ++ lists:filter(F, old_options(Info))),
+    format("Recompiling ~s\n", [SrcFile]),
+    safe_recompile(SrcFile, Options, BeamFile).
+
+old_options(Info) ->
+    case lists:keyfind(options, 1, Info) of
+        {options, Opts} -> Opts;
+        false -> []
+    end.
+
+%% prefer the source path in the compile info if the file exists,
+%% otherwise do a standard source search relative to the beam file
+find_source(BeamFile, Info) ->
+    case lists:keyfind(source, 1, Info) of
+        {source, SrcFile} ->
+            case filelib:is_file(SrcFile) of
+                true -> SrcFile;
+                false -> find_source(BeamFile)
+            end;
+        _ ->
+            find_source(BeamFile)
+    end.
+
+find_source(BeamFile) ->
+    case filelib:find_source(BeamFile) of
+        {ok, SrcFile} -> SrcFile;
+        _ -> {error, no_source}
+    end.
 
-c(File, Opts0) when is_list(Opts0) ->
-    Opts = [report_errors,report_warnings|Opts0],
+%% find the beam file for a module, preferring the path reported by code:which()
+%% if it still exists, or otherwise by searching the code path
+find_beam(Module) when is_atom(Module) ->
+    case code:which(Module) of
+        Beam when is_list(Beam), Beam =/= "" ->
+            case erlang:module_loaded(Module) of
+                false ->
+                    Beam;  % code:which/1 found this in the path
+                true ->
+                    case filelib:is_file(Beam) of
+                        true -> Beam;
+                        false -> find_beam_1(Module)  % file moved?
+                    end
+            end;
+        Other when Other =:= ""; Other =:= cover_compiled ->
+            %% module is loaded but not compiled directly from source
+            find_beam_1(Module);
+        Error ->
+            Error
+    end.
+
+find_beam_1(Module) ->
+    File = atom_to_list(Module) ++ code:objfile_extension(),
+    case code:where_is_file(File) of
+        Beam when is_list(Beam) ->
+            Beam;
+        Error ->
+            Error
+    end.
+
+%% get the compile_info for a module
+%% -will report the info for the module in memory, if loaded
+%% -will try to find and examine the beam file if not in memory
+%% -will not cause a module to become loaded by accident
+compile_info(Module, Beam) when is_atom(Module) ->
+    case erlang:module_loaded(Module) of
+        true ->
+            %% getting the compile info for a loaded module should normally
+            %% work, but return an empty info list if it fails
+            try erlang:get_module_info(Module, compile)
+            catch _:_ -> []
+            end;
+        false ->
+            case beam_lib:chunks(Beam, [compile_info]) of
+                {ok, {_Module, [{compile_info, Info}]}} ->
+                    Info;
+                Error ->
+                    Error
+            end
+    end.
+
+%% compile module, backing up any existing target file and restoring the
+%% old version if compilation fails (this should only be used when we have
+%% an old beam file that we want to preserve)
+safe_recompile(File, Options, BeamFile) ->
+    %% Note that it's possible that because of options such as 'to_asm',
+    %% the compiler might not actually write a new beam file at all
+    Backup = BeamFile ++ ".bak",
+    case file:rename(BeamFile, Backup) of
+        Status when Status =:= ok; Status =:= {error,enoent} ->
+            case compile_and_load(File, Options) of
+                {ok, _} = Result ->
+                    _ = if Status =:= ok -> file:delete(Backup);
+                           true -> ok
+                        end,
+                    Result;
+                Error ->
+                    _ = if Status =:= ok -> file:rename(Backup, BeamFile);
+                           true -> ok
+                        end,
+                    Error
+            end;
+        Error ->
+            Error
+    end.
+
+%% Compile the file and load the resulting object code (if any).
+%% Automatically ensures that there is an outdir option, by default the
+%% directory of File, and that a 'from' option will be passed to match the
+%% actual source suffix if needed (unless already specified).
+compile_and_load(File, Opts0) when is_list(Opts0) ->
+    Opts = [report_errors, report_warnings
+            | ensure_from(filename:extension(File),
+                          ensure_outdir(filename:dirname(File), Opts0))],
     case compile:file(File, Opts) of
 	{ok,Mod} ->				%Listing file.
-	    machine_load(Mod, File, Opts);
+	    purge_and_load(Mod, File, Opts);
 	{ok,Mod,_Ws} ->				%Warnings maybe turned on.
-	    machine_load(Mod, File, Opts);
+	    purge_and_load(Mod, File, Opts);
 	Other ->				%Errors go here
 	    Other
     end;
-c(File, Opt) -> 
-    c(File, [Opt]).
+compile_and_load(File, Opt) ->
+    compile_and_load(File, [Opt]).
+
+ensure_from(Suffix, Opts0) ->
+    case lists:partition(fun is_from_opt/1, Opts0++from_opt(Suffix)) of
+        {[Opt|_], Opts} -> [Opt | Opts];
+        {[], Opts} -> Opts
+    end.
+
+ensure_outdir(Dir, Opts0) ->
+    {[Opt|_], Opts} = lists:partition(fun is_outdir_opt/1,
+                                      Opts0++[{outdir,Dir}]),
+    [Opt | Opts].
+
+is_outdir_opt({outdir, _}) -> true;
+is_outdir_opt(_) -> false.
+
+is_from_opt(from_core) -> true;
+is_from_opt(from_asm) -> true;
+is_from_opt(from_beam) -> true;
+is_from_opt(_) -> false.
+
+from_opt(".core") -> [from_core];
+from_opt(".S")    -> [from_asm];
+from_opt(".beam") -> [from_beam];
+from_opt(_)       -> [].
 
 %%% Obtain the 'outdir' option from the argument. Return "." if no
 %%% such option was given.
@@ -113,18 +303,29 @@ outdir([Opt|Rest]) ->
 	    outdir(Rest)
     end.
 
+%% mimic how suffix is selected in compile:file().
+src_suffix([from_core|_]) -> ".core";
+src_suffix([from_asm|_])  -> ".S";
+src_suffix([from_beam|_]) -> ".beam";
+src_suffix([_|Opts]) -> src_suffix(Opts);
+src_suffix([]) -> ".erl".
+
 %%% We have compiled File with options Opts. Find out where the
-%%% output file went to, and load it.
-machine_load(Mod, File, Opts) ->
+%%% output file went and load it, purging any old version.
+purge_and_load(Mod, File, Opts) ->
     Dir = outdir(Opts),
-    File2 = filename:join(Dir, filename:basename(File, ".erl")),
+    Base = filename:basename(File, src_suffix(Opts)),
+    OutFile = filename:join(Dir, Base),
     case compile:output_generated(Opts) of
 	true ->
-	    Base = atom_to_list(Mod),
-	    case filename:basename(File, ".erl") of
+	    case atom_to_list(Mod) of
 		Base ->
 		    code:purge(Mod),
-		    check_load(code:load_abs(File2,Mod), Mod);
+                    %% Note that load_abs() adds the object file suffix
+		    case code:load_abs(OutFile, Mod) of
+                        {error, _R}=Error -> Error;
+                        _ -> {ok, Mod}
+                    end;
 		_OtherMod ->
 		    format("** Module name '~p' does not match file name '~tp' **~n",
 			   [Mod,File]),
@@ -135,13 +336,6 @@ machine_load(Mod, File, Opts) ->
 	    ok
     end.
 
-%%% This function previously warned if the loaded module was
-%%% loaded from some other place than current directory.
-%%% Now, loading from other than current directory is supposed to work.
-%%% so this function does nothing special.
-check_load({error, _R} = Error, _) -> Error;
-check_load(_, Mod) -> {ok, Mod}.
-
 %% Compile a list of modules
 %% enables the nice unix shell cmd
 %% erl -s c lc f1 f2 f3 @d c1=v1 @c2 @i IDir @o ODir -s erlang halt
diff --git a/lib/stdlib/src/filelib.erl b/lib/stdlib/src/filelib.erl
index 7029389e2f..daa18da9aa 100644
--- a/lib/stdlib/src/filelib.erl
+++ b/lib/stdlib/src/filelib.erl
@@ -24,6 +24,7 @@
 -export([fold_files/5, last_modified/1, file_size/1, ensure_dir/1]).
 -export([wildcard/3, is_dir/2, is_file/2, is_regular/2]).
 -export([fold_files/6, last_modified/2, file_size/2]).
+-export([find_file/2, find_file/3, find_source/1, find_source/2, find_source/3]).
 
 %% For debugging/testing.
 -export([compile_wildcard/1]).
@@ -517,3 +518,124 @@ eval_list_dir(Dir, erl_prim_loader) ->
     end;
 eval_list_dir(Dir, Mod) ->
     Mod:list_dir(Dir).
+
+%% Getting the rules to use for file search
+
+keep_dir_search_rules(Rules) ->
+    [T || {_,_}=T <- Rules].
+
+keep_suffix_search_rules(Rules) ->
+    [T || {_,_,_}=T <- Rules].
+
+get_search_rules() ->
+    case application:get_env(kernel, source_search_rules) of
+        undefined -> default_search_rules();
+        {ok, []}  -> default_search_rules();
+        {ok, R} when is_list(R) -> R
+    end.
+
+default_search_rules() ->
+    [%% suffix-speficic rules for source search
+     {".beam", ".erl", erl_source_search_rules()},
+     {".erl", ".yrl", []},
+     {"", ".src", erl_source_search_rules()},
+     {".so", ".c", c_source_search_rules()},
+     {".o", ".c", c_source_search_rules()},
+     {"", ".c", c_source_search_rules()},
+     {"", ".in", basic_source_search_rules()},
+     %% plain old directory rules, backwards compatible
+     {"", ""},
+     {"ebin","src"},
+     {"ebin","esrc"}
+    ].
+
+basic_source_search_rules() ->
+    (erl_source_search_rules()
+     ++ c_source_search_rules()).
+
+erl_source_search_rules() ->
+    [{"ebin","src"}, {"ebin","esrc"}].
+
+c_source_search_rules() ->
+    [{"priv","c_src"}, {"priv","src"}, {"bin","c_src"}, {"bin","src"}, {"", "src"}].
+
+%% Looks for a file relative to a given directory
+
+-type find_file_rule() :: {ObjDirSuffix::string(), SrcDirSuffix::string()}.
+
+-spec find_file(filename(), filename()) ->
+        {ok, filename()} | {error, not_found}.
+find_file(Filename, Dir) ->
+    find_file(Filename, Dir, []).
+
+-spec find_file(filename(), filename(), [find_file_rule()]) ->
+        {ok, filename()} | {error, not_found}.
+find_file(Filename, Dir, []) ->
+    find_file(Filename, Dir, get_search_rules());
+find_file(Filename, Dir, Rules) ->
+    try_dir_rules(keep_dir_search_rules(Rules), Filename, Dir).
+
+%% Looks for a source file relative to the object file name and directory
+
+-type find_source_rule() :: {ObjExtension::string(), SrcExtension::string(),
+                             [find_file_rule()]}.
+
+-spec find_source(filename()) ->
+        {ok, filename()} | {error, not_found}.
+find_source(FilePath) ->
+    find_source(filename:basename(FilePath), filename:dirname(FilePath)).
+
+-spec find_source(filename(), filename()) ->
+        {ok, filename()} | {error, not_found}.
+find_source(Filename, Dir) ->
+    find_source(Filename, Dir, []).
+
+-spec find_source(filename(), filename(), [find_source_rule()]) ->
+        {ok, filename()} | {error, not_found}.
+find_source(Filename, Dir, []) ->
+    find_source(Filename, Dir, get_search_rules());
+find_source(Filename, Dir, Rules) ->
+    try_suffix_rules(keep_suffix_search_rules(Rules), Filename, Dir).
+
+try_suffix_rules(Rules, Filename, Dir) ->
+    Ext = filename:extension(Filename),
+    try_suffix_rules(Rules, filename:rootname(Filename, Ext), Dir, Ext).
+
+try_suffix_rules([{Ext,Src,Rules}|Rest], Root, Dir, Ext)
+  when is_list(Src), is_list(Rules) ->
+    case try_dir_rules(add_local_search(Rules), Root ++ Src, Dir) of
+        {ok, File} -> {ok, File};
+        _Other ->
+            try_suffix_rules(Rest, Root, Dir, Ext)
+    end;
+try_suffix_rules([_|Rest], Root, Dir, Ext) ->
+    try_suffix_rules(Rest, Root, Dir, Ext);
+try_suffix_rules([], _Root, _Dir, _Ext) ->
+    {error, not_found}.
+
+%% ensuring we check the directory of the object file before any other directory
+add_local_search(Rules) ->
+    Local = {"",""},
+    [Local] ++ lists:filter(fun (X) -> X =/= Local end, Rules).
+
+try_dir_rules([{From, To}|Rest], Filename, Dir)
+  when is_list(From), is_list(To) ->
+    case try_dir_rule(Dir, Filename, From, To) of
+	{ok, File} -> {ok, File};
+	error      -> try_dir_rules(Rest, Filename, Dir)
+    end;
+try_dir_rules([], _Filename, _Dir) ->
+    {error, not_found}.
+
+try_dir_rule(Dir, Filename, From, To) ->
+    case lists:suffix(From, Dir) of
+	true ->
+	    NewDir = lists:sublist(Dir, 1, length(Dir)-length(From))++To,
+	    Src = filename:join(NewDir, Filename),
+	    case is_regular(Src) of
+		true -> {ok, Src};
+		false -> error
+	    end;
+	false ->
+	    error
+    end.
diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl
index c4586171ca..2a2f25dcd2 100644
--- a/lib/stdlib/src/filename.erl
+++ b/lib/stdlib/src/filename.erl
@@ -19,6 +19,9 @@
 %%
 -module(filename).
 
+-deprecated({find_src,1,next_major_release}).
+-deprecated({find_src,2,next_major_release}).
+
 %% Purpose: Provides generic manipulation of filenames.
 %%
 %% Generally, these functions accept filenames in the native format
@@ -34,8 +37,8 @@
 -export([absname/1, absname/2, absname_join/2, 
 	 basename/1, basename/2, dirname/1,
 	 extension/1, join/1, join/2, pathtype/1,
-	 rootname/1, rootname/2, split/1, nativename/1]).
--export([find_src/1, find_src/2, flatten/1]).
+	 rootname/1, rootname/2, split/1, flatten/1, nativename/1]).
+-export([find_src/1, find_src/2]). % deprecated
 -export([basedir/2, basedir/3]).
 
 %% Undocumented and unsupported exports.
@@ -750,8 +753,12 @@ separators() ->
 	_ -> {false, false}
     end.
 
-
-
+%% NOTE: The find_src/1/2 functions are deprecated; they try to do too much
+%% at once and are not a good fit for this module. Parts of the code have
+%% been moved to filelib:find_file/2 instead. Only this part of this
+%% module is allowed to call the filelib module; such mutual dependency
+%% should otherwise be avoided! This code should eventually be removed.
+%%
 %% find_src(Module) --
 %% find_src(Module, Rules) --
 %%
@@ -793,14 +800,7 @@ separators() ->
               | {'d', atom()},
       ErrorReason :: 'non_existing' | 'preloaded' | 'interpreted'.
 find_src(Mod) ->
-    Default = [{"", ""}, {"ebin", "src"}, {"ebin", "esrc"}],
-    Rules = 
-	case application:get_env(kernel, source_search_rules) of
-	    undefined -> Default;
-	    {ok, []} -> Default;
-	    {ok, R} when is_list(R) -> R
-	end,
-    find_src(Mod, Rules).
+    find_src(Mod, []).
 
 -spec find_src(Beam, Rules) -> {SourceFile, Options}
                              | {error, {ErrorReason, Module}} when
@@ -816,44 +816,47 @@ find_src(Mod) ->
       ErrorReason :: 'non_existing' | 'preloaded' | 'interpreted'.
 find_src(Mod, Rules) when is_atom(Mod) ->
     find_src(atom_to_list(Mod), Rules);
-find_src(File0, Rules) when is_list(File0) ->
-    Mod = list_to_atom(basename(File0, ".erl")),
-    File = rootname(File0, ".erl"),
-    case readable_file(File++".erl") of
-	true  ->
-	    try_file(File, Mod, Rules);
-	false ->
-	    try_file(undefined, Mod, Rules)
-    end.
-
-try_file(File, Mod, Rules) ->
+find_src(ModOrFile, Rules) when is_list(ModOrFile) ->
+    Extension = ".erl",
+    Mod = list_to_atom(basename(ModOrFile, Extension)),
     case code:which(Mod) of
 	Possibly_Rel_Path when is_list(Possibly_Rel_Path) ->
-	    {ok, Cwd} = file:get_cwd(),
-	    Path = join(Cwd, Possibly_Rel_Path),
-	    try_file(File, Path, Mod, Rules);
+            {ok, Cwd} = file:get_cwd(),
+            ObjPath = make_abs_path(Cwd, Possibly_Rel_Path),
+            find_src_1(ModOrFile, ObjPath, Mod, Extension, Rules);
 	Ecode when is_atom(Ecode) -> % Ecode :: ecode()
 	    {error, {Ecode, Mod}}
     end.
 
 %% At this point, the Mod is known to be valid.
 %% If the source name is not known, find it.
-%% Then get the compilation options.
-%% Returns: {SrcFile, Options}
+find_src_1(ModOrFile, ObjPath, Mod, Extension, Rules) ->
+    %% The documentation says this function must return the found path
+    %% without extension in all cases. Also, ModOrFile could be given with
+    %% or without extension. Hence the calls to rootname below.
+    ModOrFileRoot = rootname(ModOrFile, Extension),
+    case filelib:is_regular(ModOrFileRoot++Extension) of
+        true  ->
+            find_src_2(ModOrFileRoot, Mod);
+        false ->
+            SrcName = basename(ObjPath, code:objfile_extension()) ++ Extension,
+            case filelib:find_file(SrcName, dirname(ObjPath), Rules) of
+                {ok, SrcFile} ->
+                    find_src_2(rootname(SrcFile, Extension), Mod);
+                Error ->
+                    Error
+            end
+    end.
 
-try_file(undefined, ObjFilename, Mod, Rules) ->
-    case get_source_file(ObjFilename, Mod, Rules) of
-	{ok, File} -> try_file(File, ObjFilename, Mod, Rules);
-	Error -> Error
-    end;
-try_file(Src, _ObjFilename, Mod, _Rules) ->
+%% Get the compilation options and return {SrcFileRoot, Options}
+find_src_2(SrcRoot, Mod) ->
     List = case Mod:module_info(compile) of
 	       none -> [];
 	       List0 -> List0
 	   end,
     Options = proplists:get_value(options, List, []),
     {ok, Cwd} = file:get_cwd(),
-    AbsPath = make_abs_path(Cwd, Src),
+    AbsPath = make_abs_path(Cwd, SrcRoot),
     {AbsPath, filter_options(dirname(AbsPath), Options, [])}.
 
 %% Filters the options.
@@ -884,42 +887,6 @@ filter_options(Base, [_|Rest], Result) ->
 filter_options(_Base, [], Result) ->
     Result.
 
-%% Gets the source file given path of object code and module name.
-
-get_source_file(Obj, Mod, Rules) ->
-    source_by_rules(dirname(Obj), atom_to_list(Mod), Rules).
-
-source_by_rules(Dir, Base, [{From, To}|Rest]) ->
-    case try_rule(Dir, Base, From, To) of
-	{ok, File} -> {ok, File};
-	error      -> source_by_rules(Dir, Base, Rest)
-    end;
-source_by_rules(_Dir, _Base, []) ->
-    {error, source_file_not_found}.
-
-try_rule(Dir, Base, From, To) ->
-    case lists:suffix(From, Dir) of
-	true -> 
-	    NewDir = lists:sublist(Dir, 1, length(Dir)-length(From))++To,
-	    Src = join(NewDir, Base),
-	    case readable_file(Src++".erl") of
-		true -> {ok, Src};
-		false -> error
-	    end;
-	false ->
-	    error
-    end.
-
-readable_file(File) ->
-    case file:read_file_info(File) of
-	{ok, #file_info{type=regular, access=read}} ->
-	    true;
-	{ok, #file_info{type=regular, access=read_write}} ->
-	    true;
-	_Other ->
-	    false
-    end.
-
 make_abs_path(BasePath, Path) ->
     join(BasePath, Path).
 
diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl
index 5bf77a5160..2a0e3118d0 100644
--- a/lib/stdlib/src/otp_internal.erl
+++ b/lib/stdlib/src/otp_internal.erl
@@ -550,6 +550,13 @@ obsolete_1(overload, _, _) ->
 obsolete_1(rpc, safe_multi_server_call, A) when A =:= 2; A =:= 3 ->
     {removed, {rpc, multi_server_call, A}};
 
+%% Added in OTP 20.
+
+obsolete_1(filename, find_src, 1) ->
+    {deprecated, "deprecated; use filelib:find_source/1 instead"};
+obsolete_1(filename, find_src, 2) ->
+    {deprecated, "deprecated; use filelib:find_source/3 instead"};
+
 %% Removed in OTP 20.
 
 obsolete_1(erlang, hash, 2) ->
diff --git a/lib/stdlib/src/shell_default.erl b/lib/stdlib/src/shell_default.erl
index cd63ab28b5..a0c1d98513 100644
--- a/lib/stdlib/src/shell_default.erl
+++ b/lib/stdlib/src/shell_default.erl
@@ -23,7 +23,7 @@
 
 -module(shell_default).
 
--export([help/0,lc/1,c/1,c/2,nc/1,nl/1,l/1,i/0,pid/3,i/3,m/0,m/1,lm/0,mm/0,
+-export([help/0,lc/1,c/1,c/2,c/3,nc/1,nl/1,l/1,i/0,pid/3,i/3,m/0,m/1,lm/0,mm/0,
          memory/0,memory/1,uptime/0,
 	 erlangrc/1,bi/1, regs/0, flush/0,pwd/0,ls/0,ls/1,cd/1, 
          y/1, y/2,
@@ -72,6 +72,7 @@ bi(I) 		-> c:bi(I).
 bt(Pid)		-> c:bt(Pid).
 c(File) 	-> c:c(File).
 c(File, Opt)    -> c:c(File, Opt).
+c(File, Opt, Filter) -> c:c(File, Opt, Filter).
 cd(D)           -> c:cd(D).
 erlangrc(X) 	-> c:erlangrc(X).
 flush()         -> c:flush().
diff --git a/lib/stdlib/test/filelib_SUITE.erl b/lib/stdlib/test/filelib_SUITE.erl
index 4f8936edbf..87fba815d2 100644
--- a/lib/stdlib/test/filelib_SUITE.erl
+++ b/lib/stdlib/test/filelib_SUITE.erl
@@ -25,7 +25,8 @@
 	 init_per_testcase/2,end_per_testcase/2,
 	 wildcard_one/1,wildcard_two/1,wildcard_errors/1,
 	 fold_files/1,otp_5960/1,ensure_dir_eexist/1,ensure_dir_symlink/1,
-	 wildcard_symlink/1, is_file_symlink/1, file_props_symlink/1]).
+	 wildcard_symlink/1, is_file_symlink/1, file_props_symlink/1,
+         find_source/1]).
 
 -import(lists, [foreach/2]).
 
@@ -45,7 +46,8 @@ suite() ->
 all() -> 
     [wildcard_one, wildcard_two, wildcard_errors,
      fold_files, otp_5960, ensure_dir_eexist, ensure_dir_symlink,
-     wildcard_symlink, is_file_symlink, file_props_symlink].
+     wildcard_symlink, is_file_symlink, file_props_symlink,
+     find_source].
 
 groups() -> 
     [].
@@ -503,3 +505,52 @@ file_props_symlink(Config) ->
 	    FileSize = filelib:file_size(Alias, erl_prim_loader),
 	    FileSize = filelib:file_size(Alias, prim_file)
     end.
+
+find_source(Config) when is_list(Config) ->
+    BeamFile = code:which(lists),
+    BeamName = filename:basename(BeamFile),
+    BeamDir = filename:dirname(BeamFile),
+    SrcName = filename:basename(BeamFile, ".beam") ++ ".erl",
+
+    {ok, BeamFile} = filelib:find_file(BeamName, BeamDir),
+    {ok, BeamFile} = filelib:find_file(BeamName, BeamDir, []),
+    {ok, BeamFile} = filelib:find_file(BeamName, BeamDir, [{"",""},{"ebin","src"}]),
+    {error, not_found} = filelib:find_file(BeamName, BeamDir, [{"ebin","src"}]),
+
+    {ok, SrcFile} = filelib:find_file(SrcName, BeamDir),
+    {ok, SrcFile} = filelib:find_file(SrcName, BeamDir, []),
+    {ok, SrcFile} = filelib:find_file(SrcName, BeamDir, [{"foo","bar"},{"ebin","src"}]),
+    {error, not_found} = filelib:find_file(SrcName, BeamDir, [{"",""}]),
+
+    {ok, SrcFile} = filelib:find_source(BeamFile),
+    {ok, SrcFile} = filelib:find_source(BeamName, BeamDir),
+    {ok, SrcFile} = filelib:find_source(BeamName, BeamDir,
+                                         [{".erl",".yrl",[{"",""}]},
+                                          {".beam",".erl",[{"ebin","src"}]}]),
+    {error, not_found} = filelib:find_source(BeamName, BeamDir,
+                                              [{".erl",".yrl",[{"",""}]}]),
+
+    {ok, ParserErl} = filelib:find_source(code:which(erl_parse)),
+    {ok, ParserYrl} = filelib:find_source(ParserErl),
+    "lry." ++ _ = lists:reverse(ParserYrl),
+    {ok, ParserYrl} = filelib:find_source(ParserErl,
+                                           [{".beam",".erl",[{"ebin","src"}]},
+                                            {".erl",".yrl",[{"",""}]}]),
+
+    %% find_source automatically checks the local directory regardless of rules
+    {ok, ParserYrl} = filelib:find_source(ParserErl),
+    {ok, ParserYrl} = filelib:find_source(ParserErl,
+                                          [{".beam",".erl",[{"ebin","src"}]}]),
+
+    %% find_file does not check the local directory unless in the rules
+    ParserYrlName = filename:basename(ParserYrl),
+    ParserYrlDir = filename:dirname(ParserYrl),
+    {ok, ParserYrl} = filelib:find_file(ParserYrlName, ParserYrlDir,
+                                        [{"",""}]),
+    {error, not_found} = filelib:find_file(ParserYrlName, ParserYrlDir,
+                                           [{"ebin","src"}]),
+
+    %% local directory is in the default list for find_file
+    {ok, ParserYrl} = filelib:find_file(ParserYrlName, ParserYrlDir),
+    {ok, ParserYrl} = filelib:find_file(ParserYrlName, ParserYrlDir, []),
+    ok.
diff --git a/lib/stdlib/test/filename_SUITE.erl b/lib/stdlib/test/filename_SUITE.erl
index b7c4d3a6e5..54066021fb 100644
--- a/lib/stdlib/test/filename_SUITE.erl
+++ b/lib/stdlib/test/filename_SUITE.erl
@@ -421,8 +421,10 @@ t_nativename(Config) when is_list(Config) ->
 find_src(Config) when is_list(Config) ->
     {Source,_} = filename:find_src(file),
     ["file"|_] = lists:reverse(filename:split(Source)),
-    {_,_} = filename:find_src(init, [{".","."}, {"ebin","src"}]),
-    
+    {Source,_} = filename:find_src(file, [{"",""}, {"ebin","src"}]),
+    {Source,_} = filename:find_src(Source),
+    {Source,_} = filename:find_src(Source ++ ".erl"),
+
     %% Try to find the source for a preloaded module.
     {error,{preloaded,init}} = filename:find_src(init),
 
diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl
index 15ccdea284..4864bc3d72 100644
--- a/lib/stdlib/test/shell_SUITE.erl
+++ b/lib/stdlib/test/shell_SUITE.erl
@@ -282,7 +282,7 @@ restricted_local(Config) when is_list(Config) ->
 	comm_err(<<"begin F=fun() -> hello end, foo(F) end.">>),
     "exception error: undefined shell command banan/1" =
 	comm_err(<<"begin F=fun() -> hello end, banan(F) end.">>),
-    "{error,"++_ = t(<<"begin F=fun() -> hello end, c(F) end.">>),
+    "Recompiling "++_ = t(<<"c(shell_SUITE).">>),
     "exception exit: restricted shell does not allow l(" ++ _ =
 	comm_err(<<"begin F=fun() -> hello end, l(F) end.">>),
     "exception error: variable 'F' is unbound" =
diff --git a/lib/syntax_tools/src/igor.erl b/lib/syntax_tools/src/igor.erl
index 72170ec5da..b92cd8d607 100644
--- a/lib/syntax_tools/src/igor.erl
+++ b/lib/syntax_tools/src/igor.erl
@@ -417,7 +417,7 @@ merge_files(Name, Files, Options) ->
 %%
 %%     <dd>Specifies a list of rules for associating object files with
 %%     source files, to be passed to the function
-%%     `filename:find_src/2'. This can be used to change the
+%%     `filelib:find_source/2'. This can be used to change the
 %%     way Igor looks for source files. If this option is not specified,
 %%     the default system rules are used. The first occurrence of this
 %%     option completely overrides any later in the option list.</dd>
@@ -462,7 +462,7 @@ merge_files(Name, Files, Options) ->
 %% @see merge/3
 %% @see merge_files/3
 %% @see merge_sources/3
-%% @see //stdlib/filename:find_src/2
+%% @see //stdlib/filelib:find_source/2
 %% @see epp_dodger
 
 -spec merge_files(atom(), erl_syntax:forms(), [file:filename()], [option()]) ->
@@ -2746,8 +2746,8 @@ read_module(Name, Options) ->
 		    %% It seems that we have no file - go on anyway,
 		    %% just to get a decent error message.
 		    read_module_1(Name, Options);
-		{Name1, _} ->
-		    read_module_1(Name1 ++ ".erl", Options)
+		{ok, Name1} ->
+		    read_module_1(Name1, Options)
 	    end
     end.
 
@@ -2807,9 +2807,9 @@ check_forms([], _) ->
     ok.
 
 find_src(Name, undefined) ->
-    filename:find_src(filename(Name));
+    filelib:find_source(filename(Name));
 find_src(Name, Rules) ->
-    filename:find_src(filename(Name), Rules).
+    filelib:find_source(filename(Name), Rules).
 
 %% file_type(filename()) -> {value, Type} | none