From 4cf08709189ea8b7e2ae20f85c390abd04ae48ae Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Wed, 13 Oct 2010 17:08:32 +0200 Subject: Teach filename to accept raw data and add filename enc option to emu --- lib/kernel/src/file.erl | 4 +- lib/stdlib/src/filename.erl | 335 +++++++++++++++++++++++++------------------- 2 files changed, 192 insertions(+), 147 deletions(-) (limited to 'lib') diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 97d914b043..89432869e9 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -75,7 +75,7 @@ -define(RAM_FILE, ram_file). % Module %% data types --type filename() :: string(). +-type filename() :: string() | binary(). -type file_info() :: #file_info{}. -type fd() :: #file_descriptor{}. -type io_device() :: pid() | fd(). @@ -87,7 +87,7 @@ | 'delayed_write' | {'read_ahead', pos_integer()} | 'read_ahead' | 'compressed' | {'encoding', unicode:encoding()}. --type name() :: string() | atom() | [name()]. +-type name() :: string() | atom() | [name()] | binary(). -type posix() :: 'eacces' | 'eagain' | 'ebadf' | 'ebusy' | 'edquot' | 'eexist' | 'efault' | 'efbig' | 'eintr' | 'einval' | 'eio' | 'eisdir' | 'eloop' | 'emfile' | 'emlink' diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl index 01c06e4596..40df54fe54 100644 --- a/lib/stdlib/src/filename.erl +++ b/lib/stdlib/src/filename.erl @@ -41,6 +41,9 @@ -include_lib("kernel/include/file.hrl"). +-define(IS_DRIVELETTER(Letter),(((Letter >= $A) andalso (Letter =< $Z)) orelse + ((Letter >= $a) andalso (Letter =< $z)))). + %% Converts a relative filename to an absolute filename %% or the filename itself if it already is an absolute filename %% Note that no attempt is made to create the most beatiful @@ -57,12 +60,18 @@ %% (for Unix) : absname("/") -> "/" %% (for WIN32): absname("/") -> "D:/" --spec absname(file:name()) -> string(). + +-spec absname(file:name()) -> file:filename(). absname(Name) -> {ok, Cwd} = file:get_cwd(), absname(Name, Cwd). --spec absname(file:name(), string()) -> string(). +-spec absname(file:name(), file:filename()) -> file:filename(). +absname(Name, AbsBase) when is_binary(Name), is_list(AbsBase) -> + absname(Name,filename_string_to_binary(AbsBase)); +absname(Name, AbsBase) when is_list(Name), is_binary(AbsBase) -> + absname(filename_string_to_binary(Name),AbsBase); + absname(Name, AbsBase) -> case pathtype(Name) of relative -> @@ -77,6 +86,20 @@ absname(Name, AbsBase) -> %% Handles volumerelative names (on Windows only). +absname_vr([<<"/">>|Rest1], [Volume|_], _AbsBase) -> + %% Absolute path on current drive. + join([Volume|Rest1]); +absname_vr([<>|Rest1], [<>|_], AbsBase) -> + %% Relative to current directory on current drive. + absname(join(Rest1), AbsBase); +absname_vr([<>|Name], _, _AbsBase) -> + %% Relative to current directory on another drive. + Dcwd = + case file:get_cwd([X, $:]) of + {ok, Dir} -> filename_string_to_binary(Dir); + {error, _} -> <> + end, + absname(join(Name), Dcwd); absname_vr(["/"|Rest1], [Volume|_], _AbsBase) -> %% Absolute path on current drive. join([Volume|Rest1]); @@ -92,41 +115,13 @@ absname_vr([[X, $:]|Name], _, _AbsBase) -> end, absname(join(Name), Dcwd). -%% Joins a relative filename to an absolute base. For VxWorks the -%% resulting name is fixed to minimize the length by collapsing -%% ".." directories. -%% For other systems this is just a join/2, but assumes that +%% Joins a relative filename to an absolute base. +%% This is just a join/2, but assumes that %% AbsBase must be absolute and Name must be relative. --spec absname_join(string(), file:name()) -> string(). +-spec absname_join(file:filename(), file:name()) -> file:filename(). absname_join(AbsBase, Name) -> - case major_os_type() of - vxworks -> - absname_pretty(AbsBase, split(Name), lists:reverse(split(AbsBase))); - _Else -> - join(AbsBase, flatten(Name)) - end. - -%% Handles absolute filenames for VxWorks - these are 'pretty-printed', -%% since a C function call chdir("/erlang/lib/../bin") really sets -%% cwd to '/erlang/lib/../bin' which also works, but the long term -%% effect is potentially not so good ... -%% -%% absname_pretty("../bin", "/erlang/lib") -> "/erlang/bin" -%% absname_pretty("../../../..", "/erlang") -> "/erlang" - -absname_pretty(Abspath, Relpath, []) -> - %% AbsBase _must_ begin with a vxworks device name - {device, _Rest, Dev} = vxworks_first(Abspath), - absname_pretty(Abspath, Relpath, [lists:reverse(Dev)]); -absname_pretty(_Abspath, [], AbsBase) -> - join(lists:reverse(AbsBase)); -absname_pretty(Abspath, [[$.]|Rest], AbsBase) -> - absname_pretty(Abspath, Rest, AbsBase); -absname_pretty(Abspath, [[$.,$.]|Rest], [_|AbsRest]) -> - absname_pretty(Abspath, Rest, AbsRest); -absname_pretty(Abspath, [First|Rest], AbsBase) -> - absname_pretty(Abspath, Rest, [First|AbsBase]). + join(AbsBase, flatten(Name)). %% Returns the part of the filename after the last directory separator, %% or the filename itself if it has no separators. @@ -136,12 +131,36 @@ absname_pretty(Abspath, [First|Rest], AbsBase) -> %% basename("/usr/foo/") -> "foo" (trailing slashes ignored) %% basename("/") -> [] --spec basename(file:name()) -> string(). +-spec basename(file:name()) -> file:filename(). +basename(Name) when is_binary(Name) -> + case os:type() of + {win32,_} -> + win_basenameb(Name); + _ -> + basenameb(Name,[<<"/">>]) + end; + basename(Name0) -> Name = flatten(Name0), {DirSep2, DrvSep} = separators(), basename1(skip_prefix(Name, DrvSep), [], DirSep2). +win_basenameb(<>) when ?IS_DRIVELETTER(Letter) -> + basenameb(Rest,[<<"/">>,<<"\\">>]); +win_basenameb(O) -> + basenameb(O,[<<"/">>,<<"\\">>]). +basenameb(Bin,Sep) -> + Parts = [ X || X <- binary:split(Bin,Sep,[global]), + X =:= <<>> ], + if + Parts =:= [] -> + []; + true -> + lists:last(Parts) + end. + + + basename1([$/|[]], Tail, DirSep2) -> basename1([], Tail, DirSep2); basename1([$/|Rest], _Tail, DirSep2) -> @@ -155,26 +174,11 @@ basename1([Char|Rest], Tail, DirSep2) when is_integer(Char) -> basename1([], Tail, _DirSep2) -> lists:reverse(Tail). -skip_prefix(Name, false) -> % No prefix for unix, but for VxWorks. - case major_os_type() of - vxworks -> - case vxworks_first(Name) of - {device, Rest, _Device} -> - Rest; - {not_device, _Rest, _First} -> - Name - end; - _Else -> - Name - end; -skip_prefix(Name, DrvSep) -> - skip_prefix1(Name, DrvSep). - -skip_prefix1([L, DrvSep|Name], DrvSep) when is_integer(L) -> +skip_prefix(Name, false) -> + Name; +skip_prefix([L, DrvSep|Name], DrvSep) when ?IS_DRIVELETTER(L) -> Name; -skip_prefix1([L], _) when is_integer(L) -> - [L]; -skip_prefix1(Name, _) -> +skip_prefix(Name, _) -> Name. %% Returns the last component of the filename, with the given @@ -190,7 +194,27 @@ skip_prefix1(Name, _) -> %% rootname(basename("xxx.jam")) -> "xxx" %% rootname(basename("xxx.erl")) -> "xxx" --spec basename(file:name(), file:name()) -> string(). +-spec basename(file:name(), file:name()) -> file:filename(). +basename(Name, Ext) when is_binary(Name), is_list(Ext) -> + basename(Name,filename_string_to_binary(Ext)); +basename(Name, Ext) when is_list(Name), is_binary(Ext) -> + basename(filename_string_to_binary(Name),Ext); +basename(Name, Ext) when is_binary(Name), is_binary(Ext) -> + BName = basename(Name), + LN = byte_size(BName), + LE = byte_size(Ext), + case LN - LE of + Neg when Neg < 0 -> + BName; + Pos -> + case BName of + <> -> + Part; + Other -> + Other + end + end; + basename(Name0, Ext0) -> Name = flatten(Name0), Ext = flatten(Ext0), @@ -216,21 +240,10 @@ basename([], _Ext, Tail, _DrvSep2) -> %% Example: dirname("/usr/src/kalle.erl") -> "/usr/src", %% dirname("kalle.erl") -> "." --spec dirname(file:name()) -> string(). +-spec dirname(file:name()) -> file:filename(). dirname(Name0) -> Name = flatten(Name0), - case os:type() of - vxworks -> - {Devicep, Restname, FirstComp} = vxworks_first(Name), - case Devicep of - device -> - dirname(Restname, FirstComp, [], separators()); - _ -> - dirname(Name, [], [], separators()) - end; - _ -> - dirname(Name, [], [], separators()) - end. + dirname(Name, [], [], separators()). dirname([[_|_]=List|Rest], Dir, File, Seps) -> dirname(List++Rest, Dir, File, Seps); @@ -268,7 +281,7 @@ dirname([], Dir, _, _) -> %% %% On Windows: fn:dirname("\\usr\\src/kalle.erl") -> "/usr/src" --spec extension(file:name()) -> string(). +-spec extension(file:name()) -> file:filename(). extension(Name0) -> Name = flatten(Name0), extension(Name, [], major_os_type()). @@ -281,8 +294,6 @@ extension([$/|Rest], _Result, OsType) -> extension(Rest, [], OsType); extension([$\\|Rest], _Result, win32) -> extension(Rest, [], win32); -extension([$\\|Rest], _Result, vxworks) -> - extension(Rest, [], vxworks); extension([Char|Rest], Result, OsType) when is_integer(Char) -> extension(Rest, [Char|Result], OsType); extension([], Result, _OsType) -> @@ -290,23 +301,36 @@ extension([], Result, _OsType) -> %% Joins a list of filenames with directory separators. --spec join([string()]) -> string(). +-spec join([file:filename()]) -> file:filename(). join([Name1, Name2|Rest]) -> join([join(Name1, Name2)|Rest]); join([Name]) when is_list(Name) -> join1(Name, [], [], major_os_type()); +join([Name]) when is_binary(Name) -> + join1b(Name, <<>>, [], major_os_type()); join([Name]) when is_atom(Name) -> join([atom_to_list(Name)]). %% Joins two filenames with directory separators. --spec join(string(), string()) -> string(). +-spec join(file:filename(), file:filename()) -> file:filename(). join(Name1, Name2) when is_list(Name1), is_list(Name2) -> OsType = major_os_type(), case pathtype(Name2) of relative -> join1(Name1, Name2, [], OsType); _Other -> join1(Name2, [], [], OsType) end; +join(Name1, Name2) when is_binary(Name1), is_list(Name2) -> + join(Name1,filename_string_to_binary(Name2)); +join(Name1, Name2) when is_list(Name1), is_binary(Name2) -> + join(filename_string_to_binary(Name1),Name2); +join(Name1, Name2) when is_binary(Name1), is_binary(Name2) -> + OsType = major_os_type(), + case pathtype(Name2) of + relative -> join1b(Name1, Name2, [], OsType); + _Other -> join1b(Name2, <<>>, [], OsType) + end; + join(Name1, Name2) when is_atom(Name1) -> join(atom_to_list(Name1), Name2); join(Name1, Name2) when is_atom(Name2) -> @@ -321,8 +345,6 @@ when is_integer(UcLetter), UcLetter >= $A, UcLetter =< $Z -> join1(Rest, RelativeName, [$:, UcLetter+$a-$A], win32); join1([$\\|Rest], RelativeName, Result, win32) -> join1([$/|Rest], RelativeName, Result, win32); -join1([$\\|Rest], RelativeName, Result, vxworks) -> - join1([$/|Rest], RelativeName, Result, vxworks); join1([$/|Rest], RelativeName, [$., $/|Result], OsType) -> join1(Rest, RelativeName, [$/|Result], OsType); join1([$/|Rest], RelativeName, [$/|Result], OsType) -> @@ -344,6 +366,26 @@ join1([Char|Rest], RelativeName, Result, OsType) when is_integer(Char) -> join1([Atom|Rest], RelativeName, Result, OsType) when is_atom(Atom) -> join1(atom_to_list(Atom)++Rest, RelativeName, Result, OsType). +join1b(<>, RelativeName, [], win32) +when is_integer(UcLetter), UcLetter >= $A, UcLetter =< $Z -> + join1b(Rest, RelativeName, [$:, UcLetter+$a-$A], win32); +join1b(<<$\\,Rest/binary>>, RelativeName, Result, win32) -> + join1b(<<$/,Rest>>, RelativeName, Result, win32); +join1b(<<$/,Rest/binary>>, RelativeName, [$., $/|Result], OsType) -> + join1b(Rest, RelativeName, [$/|Result], OsType); +join1b(<<$/,Rest/binary>>, RelativeName, [$/|Result], OsType) -> + join1b(Rest, RelativeName, [$/|Result], OsType); +join1b(<<>>, <<>>, Result, OsType) -> + list_to_binary(maybe_remove_dirsep(Result, OsType)); +join1b(<<>>, RelativeName, [$:|Rest], win32) -> + join1b(RelativeName, <<>>, [$:|Rest], win32); +join1b(<<>>, RelativeName, [$/|Result], OsType) -> + join1b(RelativeName, <<>>, [$/|Result], OsType); +join1b(<<>>, RelativeName, Result, OsType) -> + join1b(RelativeName, <<>>, [$/|Result], OsType); +join1b(<>, RelativeName, Result, OsType) when is_integer(Char) -> + join1b(Rest, RelativeName, [Char|Result], OsType). + maybe_remove_dirsep([$/, $:, Letter], win32) -> [Letter, $:, $/]; maybe_remove_dirsep([$/], _) -> @@ -357,7 +399,7 @@ maybe_remove_dirsep(Name, _) -> %% a given base directory, which is is assumed to be normalised %% by a previous call to join/{1,2}. --spec append(string(), file:name()) -> string(). +-spec append(file:filename(), file:name()) -> file:filename(). append(Dir, Name) -> Dir ++ [$/|Name]. @@ -376,19 +418,14 @@ append(Dir, Name) -> -spec pathtype(file:name()) -> 'absolute' | 'relative' | 'volumerelative'. pathtype(Atom) when is_atom(Atom) -> pathtype(atom_to_list(Atom)); -pathtype(Name) when is_list(Name) -> +pathtype(Name) when is_list(Name) or is_binary(Name) -> case os:type() of {unix, _} -> unix_pathtype(Name); - {win32, _} -> win32_pathtype(Name); - vxworks -> case vxworks_first(Name) of - {device, _Rest, _Dev} -> - absolute; - _ -> - relative - end; - {ose,_} -> unix_pathtype(Name) + {win32, _} -> win32_pathtype(Name) end. +unix_pathtype(<<$/,_/binary>>) -> + absolute; unix_pathtype([$/|_]) -> absolute; unix_pathtype([List|Rest]) when is_list(List) -> @@ -404,6 +441,15 @@ win32_pathtype([Atom|Rest]) when is_atom(Atom) -> win32_pathtype(atom_to_list(Atom)++Rest); win32_pathtype([Char, List|Rest]) when is_list(List) -> win32_pathtype([Char|List++Rest]); +win32_pathtype(<<$/, $/, _/binary>>) -> absolute; +win32_pathtype(<<$\\, $/, _/binary>>) -> absolute; +win32_pathtype(<<$/, $\\, _/binary>>) -> absolute; +win32_pathtype(<<$\\, $\\, _/binary>>) -> absolute; +win32_pathtype(<<$/, _/binary>>) -> volumerelative; +win32_pathtype(<<$\\, _/binary>>) -> volumerelative; +win32_pathtype(<<_Letter, $:, $/, _/binary>>) -> absolute; +win32_pathtype(<<_Letter, $:, $\\, _/binary>>) -> absolute; +win32_pathtype(<<_Letter, $:, _/binary>>) -> volumerelative; win32_pathtype([$/, $/|_]) -> absolute; win32_pathtype([$\\, $/|_]) -> absolute; win32_pathtype([$/, $\\|_]) -> absolute; @@ -422,7 +468,7 @@ win32_pathtype(_) -> relative. %% Examples: rootname("/jam.src/kalle") -> "/jam.src/kalle" %% rootname("/jam.src/foo.erl") -> "/jam.src/foo" --spec rootname(file:name()) -> string(). +-spec rootname(file:name()) -> file:filename(). rootname(Name0) -> Name = flatten(Name0), rootname(Name, [], [], major_os_type()). @@ -431,8 +477,6 @@ rootname([$/|Rest], Root, Ext, OsType) -> rootname(Rest, [$/]++Ext++Root, [], OsType); rootname([$\\|Rest], Root, Ext, win32) -> rootname(Rest, [$/]++Ext++Root, [], win32); -rootname([$\\|Rest], Root, Ext, vxworks) -> - rootname(Rest, [$/]++Ext++Root, [], vxworks); rootname([$.|Rest], Root, [], OsType) -> rootname(Rest, Root, ".", OsType); rootname([$.|Rest], Root, Ext, OsType) -> @@ -451,7 +495,7 @@ rootname([], Root, _Ext, _OsType) -> %% Examples: rootname("/jam.src/kalle.jam", ".erl") -> "/jam.src/kalle.jam" %% rootname("/jam.src/foo.erl", ".erl") -> "/jam.src/foo" --spec rootname(file:name(), file:name()) -> string(). +-spec rootname(file:name(), file:name()) -> file:filename(). rootname(Name0, Ext0) -> Name = flatten(Name0), Ext = flatten(Ext0), @@ -471,27 +515,55 @@ rootname2([Char|Rest], Ext, Result) when is_integer(Char) -> %% split("foo/bar") -> ["foo", "bar"] %% split("a:\\msdev\\include") -> ["a:/", "msdev", "include"] --spec split(file:name()) -> [string()]. +-spec split(file:name()) -> [file:filename()]. +split(Name) when is_binary(Name) -> + case os:type() of + {win32, _} -> win32_splitb(Name); + _ -> unix_splitb(Name) + end; + split(Name0) -> Name = flatten(Name0), case os:type() of - {unix, _} -> unix_split(Name); {win32, _} -> win32_split(Name); - vxworks -> vxworks_split(Name); - {ose,_} -> unix_split(Name) + _ -> unix_split(Name) end. -%% If a VxWorks filename starts with '[/\].*[^/\]' '[/\].*:' or '.*:' -%% that part of the filename is considered a device. -%% The rest of the name is interpreted exactly as for win32. -%% XXX - dirty solution to make filename:split([]) return the same thing on -%% VxWorks as on unix and win32. -vxworks_split([]) -> - []; -vxworks_split(L) -> - {_Devicep, Rest, FirstComp} = vxworks_first(L), - split(Rest, [], [lists:reverse(FirstComp)], win32). +unix_splitb(Name) -> + L = binary:split(Name,[<<"/">>],[global]), + LL = case L of + [<<>>|Rest] -> + [<<"/">>|Rest]; + _ -> + L + end, + [ X || X <- LL, X =/= <<>>]. + + +fix_driveletter(Letter0) -> + if + Letter0 >= $A, Letter0 =< $Z -> + Letter0+$a-$A; + true -> + Letter0 + end. +win32_splitb(<>) when (((Slash =:= $\\) orelse (Slash =:= $/)) andalso + ?IS_DRIVELETTER(Letter0)) -> + Letter = fix_driveletter(Letter0), + L = binary:split(Rest,[<<"/">>,<<"\\">>],[global]), + [<> | [ X || X <- L, X =/= <<>> ]]; +win32_splitb(<>) when ?IS_DRIVELETTER(Letter0) -> + Letter = fix_driveletter(Letter0), + L = binary:split(Rest,[<<"/">>,<<"\\">>],[global]), + [<> | [ X || X <- L, X =/= <<>> ]]; +win32_splitb(<>) when ((Slash =:= $\\) orelse (Slash =:= $/)) -> + L = binary:split(Rest,[<<"/">>,<<"\\">>],[global]), + [<<$/>> | [ X || X <- L, X =/= <<>> ]]; +win32_splitb(Name) -> + L = binary:split(Name,[<<"/">>,<<"\\">>],[global]), + [<<$/>> | [ X || X <- L, X =/= <<>> ]]. + unix_split(Name) -> split(Name, [], unix). @@ -502,8 +574,6 @@ win32_split([X, $\\|Rest]) when is_integer(X) -> win32_split([X, $/|Rest]); win32_split([X, Y, $\\|Rest]) when is_integer(X), is_integer(Y) -> win32_split([X, Y, $/|Rest]); -win32_split([$/, $/|Rest]) -> - split(Rest, [], [[$/, $/]]); win32_split([UcLetter, $:|Rest]) when UcLetter >= $A, UcLetter =< $Z -> win32_split([UcLetter+$a-$A, $:|Rest]); win32_split([Letter, $:, $/|Rest]) -> @@ -540,7 +610,7 @@ split([], Comp, Components, OsType) -> %% will be converted to backslashes. On all platforms, the %% name will be normalized as done by join/1. --spec nativename(string()) -> string(). +-spec nativename(file:filename()) -> file:filename(). nativename(Name0) -> Name = join([Name0]), %Normalize. case os:type() of @@ -557,13 +627,12 @@ win32_nativename([]) -> separators() -> case os:type() of - {unix, _} -> {false, false}; {win32, _} -> {$\\, $:}; - vxworks -> {$\\, false}; - {ose,_} -> {false, false} + _ -> {false, false} end. + %% find_src(Module) -- %% find_src(Module, Rules) -- %% @@ -733,45 +802,12 @@ major_os_type() -> OsT -> OsT end. -%% Need to take care of the first pathname component separately -%% due to VxWorks less than good device naming rules. -%% (i.e. this is VxWorks specific ...) -%% The following four all starts with device names -%% elrond:/foo -> elrond: -%% elrond:\\foo.bar -> elrond: -%% /DISK1:foo -> /DISK1: -%% /usr/include -> /usr -%% This one doesn't: -%% foo/bar - -vxworks_first([]) -> - {not_device, [], []}; -vxworks_first([$/|T]) -> - vxworks_first2(device, T, [$/]); -vxworks_first([$\\|T]) -> - vxworks_first2(device, T, [$/]); -vxworks_first([H|T]) when is_list(H) -> - vxworks_first(H++T); -vxworks_first([H|T]) -> - vxworks_first2(not_device, T, [H]). - -vxworks_first2(Devicep, [], FirstComp) -> - {Devicep, [], FirstComp}; -vxworks_first2(Devicep, [$/|T], FirstComp) -> - {Devicep, [$/|T], FirstComp}; -vxworks_first2(Devicep, [$\\|T], FirstComp) -> - {Devicep, [$/|T], FirstComp}; -vxworks_first2(_Devicep, [$:|T], FirstComp)-> - {device, T, [$:|FirstComp]}; -vxworks_first2(Devicep, [H|T], FirstComp) when is_list(H) -> - vxworks_first2(Devicep, H++T, FirstComp); -vxworks_first2(Devicep, [H|T], FirstComp) -> - vxworks_first2(Devicep, T, [H|FirstComp]). - %% flatten(List) %% Flatten a list, also accepting atoms. --spec flatten(file:name()) -> string(). +-spec flatten(file:name()) -> file:filename(). +flatten(Bin) when is_binary(Bin) -> + Bin; flatten(List) -> do_flatten(List, []). @@ -785,3 +821,12 @@ do_flatten([], Tail) -> Tail; do_flatten(Atom, Tail) when is_atom(Atom) -> atom_to_list(Atom) ++ flatten(Tail). + +filename_string_to_binary(List) -> + case unicode:characters_to_binary(List,unicode,file:native_name_encoding()) of + {error,_,_} -> + erlang:error(badarg); + Bin when is_binary(Bin) -> + Bin + end. + -- cgit v1.2.3 From 63eeba2f6829aac2644eaf212ebef9cdf4b59e8d Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Thu, 14 Oct 2010 10:12:57 +0200 Subject: Handle binary file names and conversion of unicode strings --- lib/kernel/src/file.erl | 2 + lib/kernel/src/file_io_server.erl | 4 +- lib/stdlib/src/c.erl | 20 +++++++++- lib/stdlib/src/filename.erl | 79 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 100 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 89432869e9..579b9132c6 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -1033,6 +1033,8 @@ path_open_first([], _Name, _Mode, LastError) -> %% Generates a flat file name from a deep list of atoms and %% characters (integers). +file_name(N) when is_binary(N) -> + N; file_name(N) -> try file_name_1(N) diff --git a/lib/kernel/src/file_io_server.erl b/lib/kernel/src/file_io_server.erl index 39dc32bb79..14da9c1a55 100644 --- a/lib/kernel/src/file_io_server.erl +++ b/lib/kernel/src/file_io_server.erl @@ -44,11 +44,11 @@ format_error(ErrorId) -> erl_posix_msg:message(ErrorId). start(Owner, FileName, ModeList) - when is_pid(Owner), is_list(FileName), is_list(ModeList) -> + when is_pid(Owner), (is_list(FileName) orelse is_binary(FileName)), is_list(ModeList) -> do_start(spawn, Owner, FileName, ModeList). start_link(Owner, FileName, ModeList) - when is_pid(Owner), is_list(FileName), is_list(ModeList) -> + when is_pid(Owner), (is_list(FileName) orelse is_binary(FileName)), is_list(ModeList) -> do_start(spawn_link, Owner, FileName, ModeList). %%%----------------------------------------------------------------- diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index 6d50a575eb..399b91b92f 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -659,7 +659,7 @@ portformat(Name, Id, Cmd) -> pwd() -> case file:get_cwd() of {ok, Str} -> - ok = io:format("~s\n", [Str]); + ok = io:format("~s\n", [fixup_one_bin(Str)]); {error, _} -> ok = io:format("Cannot determine current directory\n") end. @@ -684,11 +684,27 @@ ls() -> ls(Dir) -> case file:list_dir(Dir) of {ok, Entries} -> - ls_print(sort(Entries)); + ls_print(sort(fixup_bin(Entries))); {error,_E} -> format("Invalid directory\n") end. +fixup_one_bin(X) when is_binary(X) -> + L = binary_to_list(X), + [ if + El > 127 -> + $?; + true -> + El + end || El <- L]; +fixup_one_bin(X) -> + X. +fixup_bin([H|T]) -> + [fixup_one_bin(H) | fixup_bin(T)]; +fixup_bin([]) -> + []. + + ls_print([]) -> ok; ls_print(L) -> Width = min([max(lengths(L, [])), 40]) + 5, diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl index 40df54fe54..9ca4b808e1 100644 --- a/lib/stdlib/src/filename.erl +++ b/lib/stdlib/src/filename.erl @@ -228,7 +228,7 @@ basename([$/|[]], Ext, Tail, DrvSep2) -> basename([], Ext, Tail, DrvSep2); basename([$/|Rest], Ext, _Tail, DrvSep2) -> basename(Rest, Ext, [], DrvSep2); -basename([$\\|Rest], Ext, Tail, DirSep2) when is_integer(DirSep2) -> +basename([DirSep2|Rest], Ext, Tail, DirSep2) when is_integer(DirSep2) -> basename([$/|Rest], Ext, Tail, DirSep2); basename([Char|Rest], Ext, Tail, DrvSep2) when is_integer(Char) -> basename(Rest, Ext, [Char|Tail], DrvSep2); @@ -241,6 +241,39 @@ basename([], _Ext, Tail, _DrvSep2) -> %% dirname("kalle.erl") -> "." -spec dirname(file:name()) -> file:filename(). +dirname(Name) when is_binary(Name) -> + {Dsep,Drivesep} = separators(), + SList = case Dsep of + Sep when is_integer(Sep) -> + [ <> ]; + _ -> + [] + end, + {XPart0,Dirs} = case Drivesep of + X when is_integer(X) -> + case Name of + <> when ?IS_DRIVELETTER(DL) -> + {<>,Rest}; + _ -> + {<<>>,Name} + end; + _ -> + {<<>>,Name} + end, + Parts0 = binary:split(Dirs,[<<"/">>|SList],[global]), + %% Fairly short lists of parts, OK to reverse twice... + Parts = case Parts0 of + [] -> []; + _ -> lists:reverse(fstrip(tl(lists:reverse(Parts0)))) + end, + XPart = case {Parts,XPart0} of + {[],<<>>} -> + <<".">>; + _ -> + XPart0 + end, + dirjoin(Parts,XPart,<<"/">>); + dirname(Name0) -> Name = flatten(Name0), dirname(Name, [], [], separators()). @@ -271,6 +304,26 @@ dirname([], [DrvSep,Dl], File, {_,DrvSep}) -> end; dirname([], Dir, _, _) -> lists:reverse(Dir). + +%% Compatibility with lists variant, remove trailing slashes +fstrip([<<>>,X|Y]) -> + fstrip([X|Y]); +fstrip(A) -> + A. + + +dirjoin([<<>>|T],Acc,Sep) -> + dirjoin1(T,<>,Sep); +dirjoin(A,B,C) -> + dirjoin1(A,B,C). + +dirjoin1([],Acc,_) -> + Acc; +dirjoin1([One],Acc,_) -> + <>; +dirjoin1([H|T],Acc,Sep) -> + dirjoin(T,<>,Sep). + %% Given a filename string, returns the file extension, %% including the period. Returns an empty list if there @@ -282,6 +335,30 @@ dirname([], Dir, _, _) -> %% On Windows: fn:dirname("\\usr\\src/kalle.erl") -> "/usr/src" -spec extension(file:name()) -> file:filename(). +extension(Name) when is_binary(Name) -> + {Dsep,_} = separators(), + SList = case Dsep of + Sep when is_integer(Sep) -> + [ <> ]; + _ -> + [] + end, + case binary:matches(Name,[<<".">>]) of + nomatch -> % Bug in binary workaround :( + <<>>; + [] -> + <<>>; + List -> + {Pos,_} = lists:last(List), + <<_:Pos/binary,Part/binary>> = Name, + case binary:match(Part,[<<"/">>|SList]) of + nomatch -> + Part; + _ -> + <<>> + end + end; + extension(Name0) -> Name = flatten(Name0), extension(Name, [], major_os_type()). -- cgit v1.2.3 From 5da29a8c5fe9584e274d2fe9b95dead8334d419a Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Mon, 8 Nov 2010 17:34:41 +0100 Subject: Correct shell utilities to handle unicode and possibly binaries --- lib/stdlib/src/c.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index 399b91b92f..d04d8f191f 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -659,7 +659,7 @@ portformat(Name, Id, Cmd) -> pwd() -> case file:get_cwd() of {ok, Str} -> - ok = io:format("~s\n", [fixup_one_bin(Str)]); + ok = io:format("~ts\n", [fixup_one_bin(Str)]); {error, _} -> ok = io:format("Cannot determine current directory\n") end. @@ -714,7 +714,7 @@ ls_print(X, Width, Len) when Width + Len >= 80 -> io:nl(), ls_print(X, Width, 0); ls_print([H|T], Width, Len) -> - io:format("~-*s",[Width,H]), + io:format("~-*ts",[Width,H]), ls_print(T, Width, Len+Width); ls_print([], _, _) -> io:nl(). -- cgit v1.2.3 From f16693691cbc8d6d4bff89c44cdc2c120ea637c3 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Thu, 18 Nov 2010 16:50:08 +0100 Subject: Add testcases --- lib/kernel/test/Makefile | 1 + lib/kernel/test/file_name_SUITE.erl | 713 ++++++++++++++++++++++++++++++++++++ 2 files changed, 714 insertions(+) create mode 100644 lib/kernel/test/file_name_SUITE.erl (limited to 'lib') diff --git a/lib/kernel/test/Makefile b/lib/kernel/test/Makefile index 293c368e2a..f84b343de8 100644 --- a/lib/kernel/test/Makefile +++ b/lib/kernel/test/Makefile @@ -51,6 +51,7 @@ MODULES= \ error_logger_SUITE \ error_logger_warn_SUITE \ file_SUITE \ + file_name_SUITE \ prim_file_SUITE \ ram_file_SUITE \ gen_tcp_api_SUITE \ diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl new file mode 100644 index 0000000000..ba1a829149 --- /dev/null +++ b/lib/kernel/test/file_name_SUITE.erl @@ -0,0 +1,713 @@ +-module(file_name_SUITE). +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-include("test_server.hrl"). +-include_lib("kernel/include/file.hrl"). + +%% +%% File operations that take filenames as parameters (* not prim_file operation) (** a drive): +%% altname +%% copy (*) +%% del_dir +%% delete +%% get_cwd (**) +%% list_dir +%% make_dir +%% make_link +%% make_symlink +%% open +%% read_file +%% read_file_info +%% read_link +%% read_link_info +%% rename +%% set_cwd +%% write_file +%% write_file_info +%% +%% File operations that opens/uses separate driver port (not connected to file) +%% altname +%% del_dir +%% delete +%% get_cwd +%% list_dir +%% make_dir +%% make_link +%% make_symlink +%% read_file_info +%% read_link +%% read_link_info +%% rename +%% set_cwd +%% write_file_info +%% +%% Operations that use ?FD_DRV in prim_file +%% open +%% read_file +%% write_file +%% +%% +%% Operations that return a filename/path +%% altname +%% get_cwd +%% list_dir +%% read_link + +-export([all/1,init_per_testcase/2, fin_per_testcase/2]). +-export([normal/1,icky/1,very_icky/1]). + + +init_per_testcase(_Func, Config) -> + Dog = test_server:timetrap(test_server:seconds(60)), + [{watchdog,Dog}|Config]. + +fin_per_testcase(_Func, Config) -> + Dog = ?config(watchdog, Config), + test_server:timetrap_cancel(Dog). + + +all(suite) -> + [normal,icky,very_icky]. + +normal(suite) -> + []; +normal(doc) -> + "Check file operations on normal file names regardless of unicode mode"; +normal(Config) when is_list(Config) -> + {ok,Dir} = file:get_cwd(), + try + Priv = ?config(priv_dir, Config), + file:set_cwd(Priv), + put(file_module,prim_file), + ok = check_normal(prim_file), + put(file_module,file), + ok = check_normal(file) + after + file:set_cwd(Dir) + end. + + +icky(suite) -> + []; +icky(doc) -> + "Check file operations on normal file names regardless of unicode mode"; +icky(Config) when is_list(Config) -> + {ok,Dir} = file:get_cwd(), + try + Priv = ?config(priv_dir, Config), + file:set_cwd(Priv), + put(file_module,prim_file), + ok = check_icky(prim_file), + put(file_module,file), + ok = check_icky(file) + after + file:set_cwd(Dir) + end. +very_icky(suite) -> + []; +very_icky(doc) -> + "Check file operations on normal file names regardless of unicode mode"; +very_icky(Config) when is_list(Config) -> + {ok,Dir} = file:get_cwd(), + try + Priv = ?config(priv_dir, Config), + file:set_cwd(Priv), + put(file_module,prim_file), + case check_very_icky(prim_file) of + need_unicode_mode -> + {skipped,"VM needs to be started in Unicode filename mode"}; + ok -> + put(file_module,file), + ok = check_very_icky(file) + end + after + file:set_cwd(Dir) + end. + + +check_normal(Mod) -> + {ok,Dir} = Mod:get_cwd(), + try + ?line make_normal_dir(Mod), + ?line {ok, L0} = Mod:list_dir("."), + ?line L1 = lists:sort(L0), + %erlang:display(L1), + ?line L1 = lists:sort(list(normal_dir())), + ?line {ok,D2} = Mod:get_cwd(), + ?line true = is_list(D2), + ?line case Mod:altname("fil1") of + {error,enotsup} -> + ok; + {ok,LLL} when is_list(LLL) -> + ok + end, + ?line [ true = is_list(El) || El <- L1], + ?line Syms = [ {S,Targ,list_to_binary(get_data(Targ,normal_dir()))} + || {T,S,Targ} <- normal_dir(), T =:= symlink ], + ?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ], + ?line [ {ok, Targ} = Mod:read_link(SymL) || {SymL,Targ,_} <- Syms ], + ?line chk_cre_dir(Mod,[{directory,"temp_dir",normal_dir()}]), + ?line {ok,BeginAt} = Mod:get_cwd(), + ?line true = is_list(BeginAt), + ?line {error,enoent} = Mod:set_cwd("tmp_dir"), + ?line ok = Mod:set_cwd("temp_dir"), + ?line {ok, NowAt} = Mod:get_cwd(), + ?line true = BeginAt =/= NowAt, + ?line ok = Mod:set_cwd(".."), + ?line {ok,BeginAt} = Mod:get_cwd(), + ?line rm_r(Mod,"temp_dir"), + ?line true = is_list(Dir), + ?line [ true = is_list(FN) || FN <- L0 ], + case has_links() of + true -> + ?line ok = Mod:make_link("fil1","nisse"), + ?line {ok, <<"fil1">>} = Mod:read_file("nisse"), + ?line {ok, #file_info{type = regular}} = Mod:read_link_info("nisse"), + ?line ok = Mod:delete("nisse"), + ?line {ok, <<"fil1">>} = Mod:read_file("fil1"), + ?line {error,enoent} = Mod:read_file("nisse"), + ?line {error,enoent} = Mod:read_link_info("nisse"); + false -> + ok + end, + ?line [ begin + ?line {ok, FD} = Mod:open(Name,[read]), + ?line {ok, Content} = Mod:read(FD,1024), + ?line ok = file:close(FD) + end || {regular,Name,Content} <- normal_dir() ], + ?line [ begin + ?line {ok, FD} = Mod:open(Name,[read,binary]), + ?line BC = list_to_binary(Content), + ?line {ok, BC} = Mod:read(FD,1024), + ?line ok = file:close(FD) + end || {regular,Name,Content} <- normal_dir() ], + ?line Mod:rename("fil1","tmp_fil1"), + ?line {ok, <<"fil1">>} = Mod:read_file("tmp_fil1"), + ?line {error,enoent} = Mod:read_file("fil1"), + ?line Mod:rename("tmp_fil1","fil1"), + ?line {ok, <<"fil1">>} = Mod:read_file("fil1"), + ?line {error,enoent} = Mod:read_file("tmp_fil1"), + ?line {ok,FI} = Mod:read_file_info("fil1"), + ?line NewMode = FI#file_info.mode band (bnot 8#333), + ?line NewMode2 = NewMode bor 8#222, + ?line true = NewMode2 =/= NewMode, + ?line ok = Mod:write_file_info("fil1",FI#file_info{mode = NewMode}), + ?line {ok,#file_info{mode = NewMode}} = Mod:read_file_info("fil1"), + ?line ok = Mod:write_file_info("fil1",FI#file_info{mode = NewMode2}), + ?line {ok,#file_info{mode = NewMode2}} = Mod:read_file_info("fil1"), + ok + after + case Mod:read_file_info("fil1") of + {ok,FII} -> + NewModeI = FII#file_info.mode bor 8#777, + Mod:write_file_info("fil1",FII#file_info{mode = NewModeI}); + _ -> + ok + end, + Mod:set_cwd(Dir), + io:format("Wd now: ~s~n",[Dir]) + end. + +check_icky(Mod) -> + {ok,Dir} = Mod:get_cwd(), + try + ?line true=(length("åäö") =:= 3), + ?line UniMode = file:native_name_encoding() =/= latin1, + ?line make_icky_dir(Mod), + ?line {ok, L0} = Mod:list_dir("."), + ?line L1 = lists:sort(L0), + io:format("~p ~p~n",[L1,list(icky_dir())]), + ?line L1 = lists:sort(convlist(list(icky_dir()))), + ?line {ok,D2} = Mod:get_cwd(), + ?line true = is_list(D2), +%% Altname only on windows, and there are no non native filenames there +%% ?line case Mod:altname("fil1") of +%% {error,enotsup} -> +%% ok; +%% {ok,LLL} when is_list(LLL) -> +%% ok +%% end, + ?line [ true = ((is_list(El) or (UniMode and is_binary(El)))) || El <- L1], + ?line Syms = [ {S,conv(Targ),list_to_binary(get_data(Targ,icky_dir()))} + || {T,S,Targ} <- icky_dir(), T =:= symlink ], + ?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ], + ?line [ {ok, Targ} = Mod:read_link(SymL) || {SymL,Targ,_} <- Syms ], + ?line chk_cre_dir(Mod,[{directory,"åäö_dir",icky_dir()}]), + ?line {ok,BeginAt} = Mod:get_cwd(), + ?line true = is_list(BeginAt), + ?line {error,enoent} = Mod:set_cwd("åä_dir"), + ?line ok = Mod:set_cwd("åäö_dir"), + ?line {ok, NowAt} = Mod:get_cwd(), + ?line true = is_list(NowAt), + ?line true = BeginAt =/= NowAt, + ?line ok = Mod:set_cwd(".."), + ?line {ok,BeginAt} = Mod:get_cwd(), + ?line rm_r2(Mod,"åäö_dir"), + {OS,TYPE} = os:type(), + % Check that treat_icky really converts to the same as the OS + case UniMode of + true -> + ?line chk_cre_dir(Mod,[{directory,"åäö_dir",[]}]), + ?line ok = Mod:set_cwd("åäö_dir"), + ?line ok = Mod:write_file(<<"ååå">>,<<"hello">>), + ?line Treated = treat_icky(<<"ååå">>), + ?line {ok,[Treated]} = Mod:list_dir("."), + ?line ok = Mod:delete(<<"ååå">>), + ?line {ok,[]} = Mod:list_dir("."), + ?line ok = Mod:set_cwd(".."), + ?line rm_r2(Mod,"åäö_dir"); + false -> + ok + end, + + ?line chk_cre_dir(Mod,[{directory,treat_icky(<<"åäö_dir">>),icky_dir()}]), + if + UniMode and (OS =/= win32) -> + ?line {error,enoent} = Mod:set_cwd("åäö_dir"); + true -> + ok + end, + ?line ok = Mod:set_cwd(treat_icky(<<"åäö_dir">>)), + ?line {ok, NowAt2} = Mod:get_cwd(), + io:format("~p~n",[NowAt2]), + % Cannot create raw unicode-breaking filenames on windows or macos + ?line true = ((((not UniMode) or (OS =:= win32) or (TYPE=:=darwin)) and is_list(NowAt2)) orelse ((UniMode) and is_binary(NowAt2))), + ?line true = BeginAt =/= NowAt2, + ?line ok = Mod:set_cwd(".."), + ?line {ok,BeginAt} = Mod:get_cwd(), + ?line rm_r2(Mod,conv(treat_icky(<<"åäö_dir">>))), + case has_links() of + true -> + ?line ok = Mod:make_link("fil1","nisseö"), + ?line {ok, <<"fil1">>} = Mod:read_file("nisseö"), + ?line {ok, #file_info{type = regular}} = Mod:read_link_info("nisseö"), + ?line ok = Mod:delete("nisseö"), + ?line ok = Mod:make_link("fil1",treat_icky(<<"nisseö">>)), + ?line {ok, <<"fil1">>} = Mod:read_file(treat_icky(<<"nisseö">>)), + ?line {ok, #file_info{type = regular}} = Mod:read_link_info(treat_icky(<<"nisseö">>)), + ?line ok = Mod:delete(treat_icky(<<"nisseö">>)), + ?line {ok, <<"fil1">>} = Mod:read_file("fil1"), + ?line {error,enoent} = Mod:read_file("nisseö"), + ?line {error,enoent} = Mod:read_link_info("nisseö"), + ?line {error,enoent} = Mod:read_file(treat_icky(<<"nisseö">>)), + ?line {error,enoent} = Mod:read_link_info(treat_icky(<<"nisseö">>)); + false -> + ok + end, + ?line [ begin + ?line {ok, FD} = Mod:open(Name,[read]), + ?line {ok, Content} = Mod:read(FD,1024), + ?line ok = file:close(FD) + end || {regular,Name,Content} <- icky_dir() ], + ?line [ begin + ?line {ok, FD} = Mod:open(Name,[read,binary]), + ?line BC = list_to_binary([Content]), + ?line {ok, BC} = Mod:read(FD,1024), + ?line ok = file:close(FD) + end || {regular,Name,Content} <- icky_dir() ], + ?line Mod:rename("åäö2","åäö_fil1"), + ?line {ok, <<"åäö2">>} = Mod:read_file("åäö_fil1"), + ?line {error,enoent} = Mod:read_file("åäö2"), + ?line Mod:rename("åäö_fil1","åäö2"), + ?line {ok, <<"åäö2">>} = Mod:read_file("åäö2"), + ?line {error,enoent} = Mod:read_file("åäö_fil1"), + + ?line Mod:rename("åäö2",treat_icky(<<"åäö_fil1">>)), + ?line {ok, <<"åäö2">>} = Mod:read_file(treat_icky(<<"åäö_fil1">>)), + if + UniMode and (OS =/= win32) -> + {error,enoent} = Mod:read_file("åäö_fil1"); + true -> + ok + end, + ?line {error,enoent} = Mod:read_file("åäö2"), + ?line Mod:rename(treat_icky(<<"åäö_fil1">>),"åäö2"), + ?line {ok, <<"åäö2">>} = Mod:read_file("åäö2"), + ?line {error,enoent} = Mod:read_file("åäö_fil1"), + ?line {error,enoent} = Mod:read_file(treat_icky(<<"åäö_fil1">>)), + + ?line {ok,FI} = Mod:read_file_info("åäö2"), + ?line NewMode = FI#file_info.mode band (bnot 8#333), + ?line NewMode2 = NewMode bor 8#222, + ?line true = NewMode2 =/= NewMode, + ?line ok = Mod:write_file_info("åäö2",FI#file_info{mode = NewMode}), + ?line {ok,#file_info{mode = NewMode}} = Mod:read_file_info("åäö2"), + ?line ok = Mod:write_file_info("åäö2",FI#file_info{mode = NewMode2}), + ?line {ok,#file_info{mode = NewMode2}} = Mod:read_file_info("åäö2"), + + ?line {ok,FII} = Mod:read_file_info(treat_icky(<<"åäö5">>)), + ?line true = NewMode2 =/= NewMode, + ?line ok = Mod:write_file_info(treat_icky(<<"åäö5">>),FII#file_info{mode = NewMode}), + ?line {ok,#file_info{mode = NewMode}} = Mod:read_file_info(treat_icky(<<"åäö5">>)), + ?line ok = Mod:write_file_info(<<"åäö5">>,FII#file_info{mode = NewMode2}), + ?line {ok,#file_info{mode = NewMode2}} = Mod:read_file_info(treat_icky(<<"åäö5">>)), + ok + after + Mod:set_cwd(Dir), + io:format("Wd now: ~s~n",[Dir]) + end. + +check_very_icky(Mod) -> + {ok,Dir} = Mod:get_cwd(), + try + ?line true=(length("åäö") =:= 3), + ?line UniMode = file:native_name_encoding() =/= latin1, + if + not UniMode -> + throw(need_unicode_mode); + true -> + ok + end, + ?line make_very_icky_dir(Mod), + ?line {ok, L0} = Mod:list_dir("."), + ?line L1 = lists:sort(L0), + ?line L1 = lists:sort(convlist(list(very_icky_dir()))), + ?line {ok,D2} = Mod:get_cwd(), + ?line true = is_list(D2), + ?line [ true = ((is_list(El) or is_binary(El))) || El <- L1], + ?line Syms = [ {S,conv(Targ),list_to_binary(get_data(Targ,very_icky_dir()))} + || {T,S,Targ} <- very_icky_dir(), T =:= symlink ], + ?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ], + ?line [ {ok, Targ} = Mod:read_link(SymL) || {SymL,Targ,_} <- Syms ], + ?line chk_cre_dir(Mod,[{directory,[1088,1079,1091]++"_dir",very_icky_dir()}]), + ?line {ok,BeginAt} = Mod:get_cwd(), + ?line true = is_list(BeginAt), + ?line {error,enoent} = Mod:set_cwd("åä_dir"), + ?line ok = Mod:set_cwd([1088,1079,1091]++"_dir"), + ?line {ok, NowAt} = Mod:get_cwd(), + ?line true = is_list(NowAt), + ?line true = BeginAt =/= NowAt, + ?line ok = Mod:set_cwd(".."), + ?line {ok,BeginAt} = Mod:get_cwd(), + ?line rm_r2(Mod,[1088,1079,1091]++"_dir"), + + case has_links() of + true -> + ?line ok = Mod:make_link("fil1","nisse"++[1088,1079,1091]), + ?line {ok, <<"fil1">>} = + Mod:read_file("nisse"++[1088,1079,1091]), + ?line {ok, #file_info{type = regular}} = + Mod:read_link_info("nisse"++[1088,1079,1091]), + ?line ok = Mod:delete("nisse"++[1088,1079,1091]), + ?line ok = Mod:make_link("fil1",<<"nisseö">>), + ?line {ok, <<"fil1">>} = Mod:read_file(<<"nisseö">>), + ?line {ok, #file_info{type = regular}} = + Mod:read_link_info(<<"nisseö">>), + ?line ok = Mod:delete(<<"nisseö">>), + ?line {ok, <<"fil1">>} = Mod:read_file("fil1"), + ?line {error,enoent} = Mod:read_file("nisse"++[1088,1079,1091]), + ?line {error,enoent} = Mod:read_link_info("nisse"++[1088,1079,1091]), + ?line {error,enoent} = Mod:read_file(<<"nisseö">>), + ?line {error,enoent} = Mod:read_link_info(<<"nisseö">>); + false -> + ok + end, + ?line [ begin + ?line {ok, FD} = Mod:open(Name,[read]), + ?line {ok, Content} = Mod:read(FD,1024), + ?line ok = file:close(FD) + end || {regular,Name,Content} <- very_icky_dir() ], + ?line [ begin + ?line {ok, FD} = Mod:open(Name,[read,binary]), + ?line BC = list_to_binary([Content]), + ?line {ok, BC} = Mod:read(FD,1024), + ?line ok = file:close(FD) + end || {regular,Name,Content} <- very_icky_dir() ], + ?line Mod:rename([956,965,963,954,959,49], + [956,965,963,954,959]++"_fil1"), + ?line {ok, <<"åäö2">>} = Mod:read_file([956,965,963,954,959]++"_fil1"), + ?line {error,enoent} = Mod:read_file([956,965,963,954,959,49]), + ?line Mod:rename([956,965,963,954,959]++"_fil1",[956,965,963,954,959,49]), + ?line {ok, <<"åäö2">>} = Mod:read_file([956,965,963,954,959,49]), + ?line {error,enoent} = Mod:read_file([956,965,963,954,959]++"_fil1"), + + ?line {ok,FI} = Mod:read_file_info([956,965,963,954,959,49]), + ?line NewMode = FI#file_info.mode band (bnot 8#333), + ?line NewMode2 = NewMode bor 8#222, + ?line true = NewMode2 =/= NewMode, + ?line ok = Mod:write_file_info([956,965,963,954,959,49], + FI#file_info{mode = NewMode}), + ?line {ok,#file_info{mode = NewMode}} = + Mod:read_file_info([956,965,963,954,959,49]), + ?line ok = Mod:write_file_info([956,965,963,954,959,49], + FI#file_info{mode = NewMode2}), + ?line {ok,#file_info{mode = NewMode2}} = + Mod:read_file_info([956,965,963,954,959,49]), + ok + catch + throw:need_unicode_mode -> + io:format("Sorry, can only run in unicode mode.~n"), + need_unicode_mode + after + Mod:set_cwd(Dir), + io:format("Wd now: ~s~n",[Dir]) + end. + +%% +%% Utilities +%% + + +rm_rf(Mod,Dir) -> + case Mod:read_link_info(Dir) of + {ok, #file_info{type = directory}} -> + {ok, Content} = Mod:list_dir(Dir), + [ rm_rf(Mod,filename:join(Dir,C)) || C <- Content ], + Mod:del_dir(Dir), + ok; + {ok, #file_info{}} -> + Mod:delete(Dir); + _ -> + ok + end. + +rm_r(Mod,Dir) -> + %erlang:display({rm_r,Dir}), + case Mod:read_link_info(Dir) of + {ok, #file_info{type = directory}} -> + {ok,#file_info{type = directory}} = Mod:read_file_info(Dir), + {ok, Content} = Mod:list_dir(Dir), + [ true = is_list(Part) || Part <- Content ], + [ true = is_list(filename:join(Dir,Part)) || Part <- Content ], + [ rm_r(Mod,filename:join(Dir,C)) || C <- Content ], + ok = Mod:del_dir(Dir), + ok; + {ok, #file_info{type = regular}} -> + {ok,#file_info{type = regular}} = Mod:read_file_info(Dir), + ok = Mod:delete(Dir); + {ok, #file_info{type = symlink}} -> + ok = Mod:delete(Dir) + end. +%% For icky test, allow binaries sometimes +rm_r2(Mod,Dir) -> + %erlang:display({rm_r2,Dir}), + case Mod:read_link_info(Dir) of + {ok, #file_info{type = directory}} -> + {ok,#file_info{type = directory}} = Mod:read_file_info(Dir), + {ok, Content} = Mod:list_dir(Dir), + UniMode = file:native_name_encoding() =/= latin1, + [ true = (is_list(Part) orelse UniMode) || Part <- Content ], + [ true = (is_list(filename:join(Dir,Part)) orelse UniMode) || Part <- Content ], + [ rm_r2(Mod,filename:join(Dir,C)) || C <- Content ], + ok = Mod:del_dir(Dir), + ok; + {ok, #file_info{type = regular}} -> + {ok,#file_info{type = regular}} = Mod:read_file_info(Dir), + ok = Mod:delete(Dir); + {ok, #file_info{type = symlink}} -> + ok = Mod:delete(Dir) + end. +chk_cre_dir(_,[]) -> + ok; +chk_cre_dir(Mod,[{regular,Name,Content}|T]) -> + ok = Mod:write_file(Name,Content), + chk_cre_dir(Mod,T); +chk_cre_dir(Mod,[{link,Name,Target}|T]) -> + ok = Mod:make_link(Target,Name), + chk_cre_dir(Mod,T); +chk_cre_dir(Mod,[{symlink,Name,Target}|T]) -> + ok = Mod:make_symlink(Target,Name), + chk_cre_dir(Mod,T); +chk_cre_dir(Mod,[{directory,Name,Content}|T]) -> + ok = Mod:make_dir(Name), + Content2 = [{Ty,filename:join(Name,N),case Ty of link -> filename:join(Name,C); _ -> C end} || {Ty,N,C} <- Content ], + chk_cre_dir(Mod,Content2), + chk_cre_dir(Mod,T). + +has_links() -> + case os:type() of + {win32,_} -> + case os:version() of + {N,NN,_} when (N > 5) orelse (NN > 1) -> + true; + _ -> + false + end; + _ -> + true + end. + +make_normal_dir(Mod) -> + rm_rf(Mod,"normal_dir"), + Mod:make_dir("normal_dir"), + Mod:set_cwd("normal_dir"), + Mod:write_file("fil1","fil1"), + Mod:write_file("fil2","fil2"), + case has_links() of + true -> + Mod:make_link("fil2","fil3"), + Mod:make_symlink("fil2","fil4"); + _ -> + ok + end, + Mod:make_dir("subdir"), + Mod:write_file(filename:join("subdir","subfil1"),"subfil1"), + ok. + +normal_dir() -> + [{regular,"fil1","fil1"}, + {regular,"fil2","fil2"}] ++ + case has_links() of + true -> + [{regular,"fil3","fil2"}, + {symlink,"fil4","fil2"}]; + false -> + [] + end ++ + [{directory,"subdir", + [{regular,"subfil1","subfil1"}]}]. + +make_icky_dir(Mod) -> + rm_rf(Mod,"icky_dir"), + Icky=icky_dir(), + chk_cre_dir(Mod,[{directory,"icky_dir",linkify([],Icky)}]), + Mod:set_cwd("icky_dir"), + ok. + +linkify(_Passed,[]) -> + []; +linkify(Passed,[{regular,Name,Content}|T]) -> + Regulars = [ {N,C} || {regular,N,C} <- Passed, N =/= Name ], + case lists:keysearch(Content,2,Regulars) of + {value, {Linkto, Content}} -> + [{link,Name,Linkto} | linkify(Passed,T)]; + _ -> + [{regular,Name,Content} | linkify([{regular,Name,Content}|Passed],T)] + end; +linkify(Passed,[{directory, Name, Content}|T]) -> + [{directory,Name, linkify(Content,Content)}|linkify(Passed,T)]; +linkify(Passed,[H|T]) -> + [H|linkify([H|Passed],T)]. + +icky_dir() -> + [{regular,"fil1","fil1"}, + {regular,"åäö2","åäö2"}] ++ + case has_links() of + true -> + [{regular,"åäö3","åäö2"}, + {symlink,"åäö4","åäö2"}]; + false -> + [] + end ++ + [{regular,treat_icky(<<"åäö5">>),"åäö5"}] ++ + case has_links() of + true -> + [{symlink,treat_icky(<<"åäö6">>),treat_icky(<<"åäö5">>)}]; + false -> + [] + end ++ + [{directory,treat_icky(<<"åäösubdir2">>), + [{regular,treat_icky(<<"åäösubfil2">>),"åäösubfil12"}, + {regular,"åäösubfil3","åäösubfil13"}]}, + {directory,"åäösubdir", + [{regular,"åäösubfil1","åäösubfil1"}]}]. + +make_very_icky_dir(Mod) -> + rm_rf(Mod,"very_icky_dir"), + Icky=very_icky_dir(), + chk_cre_dir(Mod,[{directory,"very_icky_dir",linkify([],Icky)}]), + Mod:set_cwd("very_icky_dir"), + ok. + +very_icky_dir() -> + [{regular,"fil1","fil1"}, + {regular,[956,965,963,954,959,49],"åäö2"}] ++ + case has_links() of + true -> + [{regular,[956,965,963,954,959,50],"åäö2"}, + {symlink,[956,965,963,954,959,51],[956,965,963,954,959,49]}]; + false -> + [] + end ++ + [{regular,treat_icky(<<"åäö5">>),"åäö5"}] ++ + case has_links() of + true -> + [{symlink,treat_icky(<<"åäö6">>),treat_icky(<<"åäö5">>)}]; + false -> + [] + end ++ + [{directory,treat_icky(<<"åäösubdir2">>), + [{regular,treat_icky(<<"åäösubfil2">>),"åäösubfil12"}, + {regular,"åäösubfil3","åäösubfil13"}]}, + {directory,[956,965,963,954,959]++"subdir1", + [{regular,[956,965,963,954,959]++"subfil1","åäösubfil1"}]}]. + +%% Some OS'es simply do not allow non UTF8 filenames +treat_icky(Bin) -> + case os:type() of + {unix,darwin} -> + procentify(Bin); + {win32,_} -> + binary_to_list(Bin); + _ -> + Bin + end. + +procentify(<<>>) -> + <<>>; +procentify(<>) when X > 127 -> + T=procentify(Rst), + Y = list_to_binary([$% + | io_lib:format("~2.16B",[X])]), + <>; +procentify(<>) -> + T=procentify(Rst), + <>. + + +list([]) -> + []; +list([{_,Name,_} | T]) -> + [Name | list(T)]. + + +get_data(FN,List) -> + case lists:keysearch(FN,2,List) of + {value,{regular,FN,C}} -> + C; + {value,{symlink,FN,NewFN}} -> + get_data(NewFN,List); + _-> + [] + end. + + +convlist(L) -> + convlist(file:native_name_encoding(),L). +convlist(latin1,[Bin|T]) when is_binary(Bin) -> + %erlang:display('Convert...'), + [binary_to_list(Bin)| convlist(latin1,T)]; +convlist(Any,[H|T]) -> + [H|convlist(Any,T)]; +convlist(_,[]) -> + []. + +conv(L) -> + NoUniMode = file:native_name_encoding() =:= latin1, + if + NoUniMode, is_binary(L) -> + binary_to_list(L); + true -> + L + end. + + + + -- cgit v1.2.3 From 4161de166d46b6c9c25d5ac813c94a025c7a029d Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Mon, 8 Nov 2010 17:25:01 +0100 Subject: Teach file to accept codepoints beyond 255. --- lib/kernel/src/file.erl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 579b9132c6..08825c1c57 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -1037,20 +1037,22 @@ file_name(N) when is_binary(N) -> N; file_name(N) -> try - file_name_1(N) + file_name_1(N,file:native_name_encoding()) catch Reason -> {error, Reason} end. -file_name_1([C|T]) when is_integer(C), C > 0, C =< 255 -> - [C|file_name_1(T)]; -file_name_1([H|T]) -> - file_name_1(H) ++ file_name_1(T); -file_name_1([]) -> +file_name_1([C|T],latin1) when is_integer(C), C < 256-> + [C|file_name_1(T,latin1)]; +file_name_1([C|T],utf8) when is_integer(C) -> + [C|file_name_1(T,utf8)]; +file_name_1([H|T],E) -> + file_name_1(H,E) ++ file_name_1(T,E); +file_name_1([],_) -> []; -file_name_1(N) when is_atom(N) -> +file_name_1(N,_) when is_atom(N) -> atom_to_list(N); -file_name_1(_) -> +file_name_1(_,_) -> throw(badarg). make_binary(Bin) when is_binary(Bin) -> -- cgit v1.2.3 From 9622ab2132e2501ee5769357a914dcc6635e515c Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Tue, 9 Nov 2010 18:08:57 +0100 Subject: Convert filenames read on MacOSX to canonical form --- lib/kernel/test/file_name_SUITE.erl | 980 +++++++++++++++++++++++++++++++++++- 1 file changed, 975 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index ba1a829149..57c4353f36 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -71,7 +71,7 @@ %% read_link -export([all/1,init_per_testcase/2, fin_per_testcase/2]). --export([normal/1,icky/1,very_icky/1]). +-export([normal/1,icky/1,very_icky/1,normalize/1]). init_per_testcase(_Func, Config) -> @@ -84,8 +84,39 @@ fin_per_testcase(_Func, Config) -> all(suite) -> - [normal,icky,very_icky]. + [normal,icky,very_icky,normalize]. +normalize(suite) -> + []; +normalize(doc) -> + ["Check that filename normalization works"]; +normalize(Config) when is_list(Config) -> + random:seed({1290,431421,830412}), + try + ?line UniMode = file:native_name_encoding() =/= latin1, + if + not UniMode -> + throw(need_unicode_mode); + true -> + ok + end, + ?line Pairs = [rand_comp_decomp(200) || _ <- lists:seq(1,1000)], + case os:type() of + {unix,darwin} -> + ?line [ true = (A =:= prim_file:internal_native2name(B)) || + {A,B} <- Pairs ]; + _ -> + ok + end, + ?line [ true = (A =:= prim_file:internal_normalize_utf8(B)) || + {A,B} <- Pairs ] + + catch + throw:need_unicode_mode -> + io:format("Sorry, can only run in unicode mode.~n"), + {skipped,"VM needs to be started in Unicode filename mode"} + end. + normal(suite) -> []; normal(doc) -> @@ -653,7 +684,7 @@ very_icky_dir() -> treat_icky(Bin) -> case os:type() of {unix,darwin} -> - procentify(Bin); + binary_to_list(procentify(Bin)); {win32,_} -> binary_to_list(Bin); _ -> @@ -708,6 +739,945 @@ conv(L) -> L end. - - +rand_comp_decomp(Max) -> + N = random:uniform(Max), + L = [ rand_decomp() || _ <- lists:seq(1,N) ], + LC = [ A || {A,_} <- L], + LD = lists:flatten([B || {_,B} <- L]), + LB = case os:type() of + {win32,_} -> + unicode:characters_to_binary(LD,unicode,{utf16,little}); + _ -> + unicode:characters_to_binary(LD,unicode,utf8) + end, + {LC,LB}. + +rand_decomp() -> + BT = bigtup(), + SZ = tuple_size(BT), + element(random:uniform(SZ),BT). +bigtup() -> + {{192,[65,768]}, + {200,[69,768]}, + {204,[73,768]}, + {210,[79,768]}, + {217,[85,768]}, + {7808,[87,768]}, + {7922,[89,768]}, + {224,[97,768]}, + {232,[101,768]}, + {236,[105,768]}, + {242,[111,768]}, + {249,[117,768]}, + {7809,[119,768]}, + {7923,[121,768]}, + {8173,[168,768]}, + {7846,[65,770,768]}, + {7872,[69,770,768]}, + {7890,[79,770,768]}, + {7847,[97,770,768]}, + {7873,[101,770,768]}, + {7891,[111,770,768]}, + {7700,[69,772,768]}, + {7760,[79,772,768]}, + {7701,[101,772,768]}, + {7761,[111,772,768]}, + {7856,[65,774,768]}, + {7857,[97,774,768]}, + {475,[85,776,768]}, + {476,[117,776,768]}, + {8146,[953,776,768]}, + {8162,[965,776,768]}, + {8074,[913,837,787,768]}, + {8090,[919,837,787,768]}, + {8106,[937,837,787,768]}, + {8066,[945,837,787,768]}, + {8082,[951,837,787,768]}, + {8098,[969,837,787,768]}, + {7946,[913,787,768]}, + {7962,[917,787,768]}, + {7978,[919,787,768]}, + {7994,[921,787,768]}, + {8010,[927,787,768]}, + {8042,[937,787,768]}, + {7938,[945,787,768]}, + {7954,[949,787,768]}, + {7970,[951,787,768]}, + {7986,[953,787,768]}, + {8002,[959,787,768]}, + {8018,[965,787,768]}, + {8034,[969,787,768]}, + {8075,[913,837,788,768]}, + {8091,[919,837,788,768]}, + {8107,[937,837,788,768]}, + {8067,[945,837,788,768]}, + {8083,[951,837,788,768]}, + {8099,[969,837,788,768]}, + {7947,[913,788,768]}, + {7963,[917,788,768]}, + {7979,[919,788,768]}, + {7995,[921,788,768]}, + {8011,[927,788,768]}, + {8027,[933,788,768]}, + {8043,[937,788,768]}, + {7939,[945,788,768]}, + {7955,[949,788,768]}, + {7971,[951,788,768]}, + {7987,[953,788,768]}, + {8003,[959,788,768]}, + {8019,[965,788,768]}, + {8035,[969,788,768]}, + {7900,[79,795,768]}, + {7914,[85,795,768]}, + {7901,[111,795,768]}, + {7915,[117,795,768]}, + {8114,[945,837,768]}, + {8130,[951,837,768]}, + {8178,[969,837,768]}, + {8122,[913,768]}, + {8136,[917,768]}, + {8138,[919,768]}, + {8154,[921,768]}, + {8184,[927,768]}, + {8170,[933,768]}, + {8186,[937,768]}, + {8048,[945,768]}, + {8050,[949,768]}, + {8052,[951,768]}, + {8054,[953,768]}, + {8056,[959,768]}, + {8058,[965,768]}, + {8060,[969,768]}, + {8141,[8127,768]}, + {8157,[8190,768]}, + {193,[65,769]}, + {262,[67,769]}, + {201,[69,769]}, + {500,[71,769]}, + {205,[73,769]}, + {7728,[75,769]}, + {313,[76,769]}, + {7742,[77,769]}, + {323,[78,769]}, + {211,[79,769]}, + {7764,[80,769]}, + {340,[82,769]}, + {346,[83,769]}, + {218,[85,769]}, + {7810,[87,769]}, + {221,[89,769]}, + {377,[90,769]}, + {225,[97,769]}, + {263,[99,769]}, + {233,[101,769]}, + {501,[103,769]}, + {237,[105,769]}, + {7729,[107,769]}, + {314,[108,769]}, + {7743,[109,769]}, + {324,[110,769]}, + {243,[111,769]}, + {7765,[112,769]}, + {341,[114,769]}, + {347,[115,769]}, + {250,[117,769]}, + {7811,[119,769]}, + {253,[121,769]}, + {378,[122,769]}, + {8174,[168,769]}, + {508,[198,769]}, + {510,[216,769]}, + {509,[230,769]}, + {511,[248,769]}, + {7844,[65,770,769]}, + {7870,[69,770,769]}, + {7888,[79,770,769]}, + {7845,[97,770,769]}, + {7871,[101,770,769]}, + {7889,[111,770,769]}, + {7756,[79,771,769]}, + {7800,[85,771,769]}, + {7757,[111,771,769]}, + {7801,[117,771,769]}, + {7702,[69,772,769]}, + {7762,[79,772,769]}, + {7703,[101,772,769]}, + {7763,[111,772,769]}, + {7854,[65,774,769]}, + {7855,[97,774,769]}, + {7726,[73,776,769]}, + {471,[85,776,769]}, + {7727,[105,776,769]}, + {472,[117,776,769]}, + {8147,[953,776,769]}, + {8163,[965,776,769]}, + {506,[65,778,769]}, + {507,[97,778,769]}, + {8076,[913,837,787,769]}, + {8092,[919,837,787,769]}, + {8108,[937,837,787,769]}, + {8068,[945,837,787,769]}, + {8084,[951,837,787,769]}, + {8100,[969,837,787,769]}, + {7948,[913,787,769]}, + {7964,[917,787,769]}, + {7980,[919,787,769]}, + {7996,[921,787,769]}, + {8012,[927,787,769]}, + {8044,[937,787,769]}, + {7940,[945,787,769]}, + {7956,[949,787,769]}, + {7972,[951,787,769]}, + {7988,[953,787,769]}, + {8004,[959,787,769]}, + {8020,[965,787,769]}, + {8036,[969,787,769]}, + {8077,[913,837,788,769]}, + {8093,[919,837,788,769]}, + {8109,[937,837,788,769]}, + {8069,[945,837,788,769]}, + {8085,[951,837,788,769]}, + {8101,[969,837,788,769]}, + {7949,[913,788,769]}, + {7965,[917,788,769]}, + {7981,[919,788,769]}, + {7997,[921,788,769]}, + {8013,[927,788,769]}, + {8029,[933,788,769]}, + {8045,[937,788,769]}, + {7941,[945,788,769]}, + {7957,[949,788,769]}, + {7973,[951,788,769]}, + {7989,[953,788,769]}, + {8005,[959,788,769]}, + {8021,[965,788,769]}, + {8037,[969,788,769]}, + {7898,[79,795,769]}, + {7912,[85,795,769]}, + {7899,[111,795,769]}, + {7913,[117,795,769]}, + {7688,[67,807,769]}, + {7689,[99,807,769]}, + {8116,[945,837,769]}, + {8132,[951,837,769]}, + {8180,[959,837,769]}, + {8123,[913,769]}, + {8137,[917,769]}, + {8139,[919,769]}, + {8155,[921,769]}, + {8185,[927,769]}, + {8171,[933,769]}, + {8187,[937,769]}, + {8049,[945,769]}, + {8051,[949,769]}, + {8053,[951,769]}, + {8055,[953,769]}, + {8057,[959,769]}, + {8059,[965,769]}, + {8061,[969,769]}, + {1027,[1043,769]}, + {1036,[1050,769]}, + {1107,[1075,769]}, + {1116,[1082,769]}, + {8142,[8127,769]}, + {8158,[8190,769]}, + {194,[65,770]}, + {264,[67,770]}, + {202,[69,770]}, + {284,[71,770]}, + {292,[72,770]}, + {206,[73,770]}, + {308,[74,770]}, + {212,[79,770]}, + {348,[83,770]}, + {219,[85,770]}, + {372,[87,770]}, + {374,[89,770]}, + {7824,[90,770]}, + {226,[97,770]}, + {265,[99,770]}, + {234,[101,770]}, + {285,[103,770]}, + {293,[104,770]}, + {238,[105,770]}, + {309,[106,770]}, + {244,[111,770]}, + {349,[115,770]}, + {251,[117,770]}, + {373,[119,770]}, + {375,[121,770]}, + {7825,[122,770]}, + {7852,[65,803,770]}, + {7878,[69,803,770]}, + {7896,[79,803,770]}, + {7853,[97,803,770]}, + {7879,[101,803,770]}, + {7897,[111,803,770]}, + {195,[65,771]}, + {7868,[69,771]}, + {296,[73,771]}, + {209,[78,771]}, + {213,[79,771]}, + {360,[85,771]}, + {7804,[86,771]}, + {7928,[89,771]}, + {227,[97,771]}, + {7869,[101,771]}, + {297,[105,771]}, + {241,[110,771]}, + {245,[111,771]}, + {361,[117,771]}, + {7805,[118,771]}, + {7929,[121,771]}, + {7850,[65,770,771]}, + {7876,[69,770,771]}, + {7894,[79,770,771]}, + {7851,[97,770,771]}, + {7877,[101,770,771]}, + {7895,[111,770,771]}, + {7860,[65,774,771]}, + {7861,[97,774,771]}, + {7904,[79,795,771]}, + {7918,[85,795,771]}, + {7905,[111,795,771]}, + {7919,[117,795,771]}, + {256,[65,772]}, + {274,[69,772]}, + {7712,[71,772]}, + {298,[73,772]}, + {332,[79,772]}, + {362,[85,772]}, + {257,[97,772]}, + {275,[101,772]}, + {7713,[103,772]}, + {299,[105,772]}, + {333,[111,772]}, + {363,[117,772]}, + {482,[198,772]}, + {483,[230,772]}, + {480,[65,775,772]}, + {481,[97,775,772]}, + {478,[65,776,772]}, + {469,[85,776,772]}, + {479,[97,776,772]}, + {470,[117,776,772]}, + {7736,[76,803,772]}, + {7772,[82,803,772]}, + {7737,[108,803,772]}, + {7773,[114,803,772]}, + {492,[79,808,772]}, + {493,[111,808,772]}, + {8121,[913,772]}, + {8153,[921,772]}, + {8169,[933,772]}, + {8113,[945,772]}, + {8145,[953,772]}, + {8161,[965,772]}, + {1250,[1048,772]}, + {1262,[1059,772]}, + {1251,[1080,772]}, + {1263,[1091,772]}, + {258,[65,774]}, + {276,[69,774]}, + {286,[71,774]}, + {300,[73,774]}, + {334,[79,774]}, + {364,[85,774]}, + {259,[97,774]}, + {277,[101,774]}, + {287,[103,774]}, + {301,[105,774]}, + {335,[111,774]}, + {365,[117,774]}, + {7862,[65,803,774]}, + {7863,[97,803,774]}, + {7708,[69,807,774]}, + {7709,[101,807,774]}, + {8120,[913,774]}, + {8152,[921,774]}, + {8168,[933,774]}, + {8112,[945,774]}, + {8144,[953,774]}, + {8160,[965,774]}, + {1232,[1040,774]}, + {1238,[1045,774]}, + {1217,[1046,774]}, + {1049,[1048,774]}, + {1038,[1059,774]}, + {1233,[1072,774]}, + {1239,[1077,774]}, + {1218,[1078,774]}, + {1081,[1080,774]}, + {1118,[1091,774]}, + {7682,[66,775]}, + {266,[67,775]}, + {7690,[68,775]}, + {278,[69,775]}, + {7710,[70,775]}, + {288,[71,775]}, + {7714,[72,775]}, + {304,[73,775]}, + {7744,[77,775]}, + {7748,[78,775]}, + {7766,[80,775]}, + {7768,[82,775]}, + {7776,[83,775]}, + {7786,[84,775]}, + {7814,[87,775]}, + {7818,[88,775]}, + {7822,[89,775]}, + {379,[90,775]}, + {7683,[98,775]}, + {267,[99,775]}, + {7691,[100,775]}, + {279,[101,775]}, + {7711,[102,775]}, + {289,[103,775]}, + {7715,[104,775]}, + {7745,[109,775]}, + {7749,[110,775]}, + {7767,[112,775]}, + {7769,[114,775]}, + {7777,[115,775]}, + {7787,[116,775]}, + {7815,[119,775]}, + {7819,[120,775]}, + {7823,[121,775]}, + {380,[122,775]}, + {7835,[383,775]}, + {7780,[83,769,775]}, + {7781,[115,769,775]}, + {784,[774,775]}, + {7782,[83,780,775]}, + {7783,[115,780,775]}, + {7784,[83,803,775]}, + {7785,[115,803,775]}, + {196,[65,776]}, + {203,[69,776]}, + {7718,[72,776]}, + {207,[73,776]}, + {214,[79,776]}, + {220,[85,776]}, + {7812,[87,776]}, + {7820,[88,776]}, + {376,[89,776]}, + {228,[97,776]}, + {235,[101,776]}, + {7719,[104,776]}, + {239,[105,776]}, + {246,[111,776]}, + {7831,[116,776]}, + {252,[117,776]}, + {7813,[119,776]}, + {7821,[120,776]}, + {255,[121,776]}, + {1242,[399,776]}, + {1258,[415,776]}, + {1243,[601,776]}, + {1259,[629,776]}, + {7758,[79,771,776]}, + {7759,[111,771,776]}, + {7802,[85,772,776]}, + {7803,[117,772,776]}, + {938,[921,776]}, + {939,[933,776]}, + {970,[953,776]}, + {971,[965,776]}, + {980,[978,776]}, + {1031,[1030,776]}, + {1234,[1040,776]}, + {1025,[1045,776]}, + {1244,[1046,776]}, + {1246,[1047,776]}, + {1252,[1048,776]}, + {1254,[1054,776]}, + {1264,[1059,776]}, + {1268,[1063,776]}, + {1272,[1067,776]}, + {1235,[1072,776]}, + {1105,[1077,776]}, + {1245,[1078,776]}, + {1247,[1079,776]}, + {1253,[1080,776]}, + {1255,[1086,776]}, + {1265,[1091,776]}, + {1269,[1095,776]}, + {1273,[1099,776]}, + {1111,[1110,776]}, + {7842,[65,777]}, + {7866,[69,777]}, + {7880,[73,777]}, + {7886,[79,777]}, + {7910,[85,777]}, + {7926,[89,777]}, + {7843,[97,777]}, + {7867,[101,777]}, + {7881,[105,777]}, + {7887,[111,777]}, + {7911,[117,777]}, + {7927,[121,777]}, + {7848,[65,770,777]}, + {7874,[69,770,777]}, + {7892,[79,770,777]}, + {7849,[97,770,777]}, + {7875,[101,770,777]}, + {7893,[111,770,777]}, + {7858,[65,774,777]}, + {7859,[97,774,777]}, + {7902,[79,795,777]}, + {7916,[85,795,777]}, + {7903,[111,795,777]}, + {7917,[117,795,777]}, + {197,[65,778]}, + {366,[85,778]}, + {229,[97,778]}, + {367,[117,778]}, + {7832,[119,778]}, + {7833,[121,778]}, + {336,[79,779]}, + {368,[85,779]}, + {337,[111,779]}, + {369,[117,779]}, + {1266,[1059,779]}, + {1267,[1091,779]}, + {461,[65,780]}, + {268,[67,780]}, + {270,[68,780]}, + {282,[69,780]}, + {486,[71,780]}, + {463,[73,780]}, + {488,[75,780]}, + {317,[76,780]}, + {327,[78,780]}, + {465,[79,780]}, + {344,[82,780]}, + {352,[83,780]}, + {356,[84,780]}, + {467,[85,780]}, + {381,[90,780]}, + {462,[97,780]}, + {269,[99,780]}, + {271,[100,780]}, + {283,[101,780]}, + {487,[103,780]}, + {464,[105,780]}, + {496,[106,780]}, + {489,[107,780]}, + {318,[108,780]}, + {328,[110,780]}, + {466,[111,780]}, + {345,[114,780]}, + {353,[115,780]}, + {357,[116,780]}, + {468,[117,780]}, + {382,[122,780]}, + {494,[439,780]}, + {495,[658,780]}, + {473,[85,776,780]}, + {474,[117,776,780]}, + {901,[168,781]}, + {912,[953,776,781]}, + {944,[965,776,781]}, + {902,[913,781]}, + {904,[917,781]}, + {905,[919,781]}, + {906,[921,781]}, + {908,[927,781]}, + {910,[933,781]}, + {911,[937,781]}, + {940,[945,781]}, + {941,[949,781]}, + {942,[951,781]}, + {943,[953,781]}, + {972,[959,781]}, + {973,[965,781]}, + {974,[969,781]}, + {979,[978,781]}, + {512,[65,783]}, + {516,[69,783]}, + {520,[73,783]}, + {524,[79,783]}, + {528,[82,783]}, + {532,[85,783]}, + {513,[97,783]}, + {517,[101,783]}, + {521,[105,783]}, + {525,[111,783]}, + {529,[114,783]}, + {533,[117,783]}, + {1142,[1140,783]}, + {1143,[1141,783]}, + {514,[65,785]}, + {518,[69,785]}, + {522,[73,785]}, + {526,[79,785]}, + {530,[82,785]}, + {534,[85,785]}, + {515,[97,785]}, + {519,[101,785]}, + {523,[105,785]}, + {527,[111,785]}, + {531,[114,785]}, + {535,[117,785]}, + {8072,[913,837,787]}, + {8088,[919,837,787]}, + {8104,[937,837,787]}, + {8064,[945,837,787]}, + {8080,[951,837,787]}, + {8096,[969,837,787]}, + {7944,[913,787]}, + {7960,[917,787]}, + {7976,[919,787]}, + {7992,[921,787]}, + {8008,[927,787]}, + {8040,[937,787]}, + {7936,[945,787]}, + {7952,[949,787]}, + {7968,[951,787]}, + {7984,[953,787]}, + {8000,[959,787]}, + {8164,[961,787]}, + {8016,[965,787]}, + {8032,[969,787]}, + {8073,[913,837,788]}, + {8089,[919,837,788]}, + {8105,[937,837,788]}, + {8065,[945,837,788]}, + {8081,[951,837,788]}, + {8097,[969,837,788]}, + {7945,[913,788]}, + {7961,[917,788]}, + {7977,[919,788]}, + {7993,[921,788]}, + {8009,[927,788]}, + {8172,[929,788]}, + {8025,[933,788]}, + {8041,[937,788]}, + {7937,[945,788]}, + {7953,[949,788]}, + {7969,[951,788]}, + {7985,[953,788]}, + {8001,[959,788]}, + {8165,[961,788]}, + {8017,[965,788]}, + {8033,[969,788]}, + {416,[79,795]}, + {431,[85,795]}, + {417,[111,795]}, + {432,[117,795]}, + {7840,[65,803]}, + {7684,[66,803]}, + {7692,[68,803]}, + {7864,[69,803]}, + {7716,[72,803]}, + {7882,[73,803]}, + {7730,[75,803]}, + {7734,[76,803]}, + {7746,[77,803]}, + {7750,[78,803]}, + {7884,[79,803]}, + {7770,[82,803]}, + {7778,[83,803]}, + {7788,[84,803]}, + {7908,[85,803]}, + {7806,[86,803]}, + {7816,[87,803]}, + {7924,[89,803]}, + {7826,[90,803]}, + {7841,[97,803]}, + {7685,[98,803]}, + {7693,[100,803]}, + {7865,[101,803]}, + {7717,[104,803]}, + {7883,[105,803]}, + {7731,[107,803]}, + {7735,[108,803]}, + {7747,[109,803]}, + {7751,[110,803]}, + {7885,[111,803]}, + {7771,[114,803]}, + {7779,[115,803]}, + {7789,[116,803]}, + {7909,[117,803]}, + {7807,[118,803]}, + {7817,[119,803]}, + {7925,[121,803]}, + {7827,[122,803]}, + {7906,[79,795,803]}, + {7920,[85,795,803]}, + {7907,[111,795,803]}, + {7921,[117,795,803]}, + {7794,[85,804]}, + {7795,[117,804]}, + {7680,[65,805]}, + {7681,[97,805]}, + {199,[67,807]}, + {7696,[68,807]}, + {290,[71,807]}, + {7720,[72,807]}, + {310,[75,807]}, + {315,[76,807]}, + {325,[78,807]}, + {342,[82,807]}, + {350,[83,807]}, + {354,[84,807]}, + {231,[99,807]}, + {7697,[100,807]}, + {291,[103,807]}, + {7721,[104,807]}, + {311,[107,807]}, + {316,[108,807]}, + {326,[110,807]}, + {343,[114,807]}, + {351,[115,807]}, + {355,[116,807]}, + {260,[65,808]}, + {280,[69,808]}, + {302,[73,808]}, + {490,[79,808]}, + {370,[85,808]}, + {261,[97,808]}, + {281,[101,808]}, + {303,[105,808]}, + {491,[111,808]}, + {371,[117,808]}, + {7698,[68,813]}, + {7704,[69,813]}, + {7740,[76,813]}, + {7754,[78,813]}, + {7792,[84,813]}, + {7798,[85,813]}, + {7699,[100,813]}, + {7705,[101,813]}, + {7741,[108,813]}, + {7755,[110,813]}, + {7793,[116,813]}, + {7799,[117,813]}, + {7722,[72,814]}, + {7723,[104,814]}, + {7706,[69,816]}, + {7724,[73,816]}, + {7796,[85,816]}, + {7707,[101,816]}, + {7725,[105,816]}, + {7797,[117,816]}, + {7686,[66,817]}, + {7694,[68,817]}, + {7732,[75,817]}, + {7738,[76,817]}, + {7752,[78,817]}, + {7774,[82,817]}, + {7790,[84,817]}, + {7828,[90,817]}, + {7687,[98,817]}, + {7695,[100,817]}, + {7830,[104,817]}, + {7733,[107,817]}, + {7739,[108,817]}, + {7753,[110,817]}, + {7775,[114,817]}, + {7791,[116,817]}, + {7829,[122,817]}, + {8129,[168,834]}, + {8151,[953,776,834]}, + {8167,[965,776,834]}, + {8078,[913,837,787,834]}, + {8094,[919,837,787,834]}, + {8110,[937,837,787,834]}, + {8070,[945,837,787,834]}, + {8086,[951,837,787,834]}, + {8102,[969,837,787,834]}, + {7950,[913,787,834]}, + {7982,[919,787,834]}, + {7998,[921,787,834]}, + {8046,[937,787,834]}, + {7942,[945,787,834]}, + {7974,[951,787,834]}, + {7990,[953,787,834]}, + {8022,[965,787,834]}, + {8038,[969,787,834]}, + {8079,[913,837,788,834]}, + {8095,[919,837,788,834]}, + {8111,[937,837,788,834]}, + {8071,[945,837,788,834]}, + {8087,[951,837,788,834]}, + {8103,[969,837,788,834]}, + {7951,[913,788,834]}, + {7983,[919,788,834]}, + {7999,[921,788,834]}, + {8031,[933,788,834]}, + {8047,[937,788,834]}, + {7943,[945,788,834]}, + {7975,[951,788,834]}, + {7991,[953,788,834]}, + {8023,[965,788,834]}, + {8039,[969,788,834]}, + {8119,[945,837,834]}, + {8135,[951,837,834]}, + {8183,[969,837,834]}, + {8118,[945,834]}, + {8134,[951,834]}, + {8150,[953,834]}, + {8166,[965,834]}, + {8182,[969,834]}, + {8143,[8127,834]}, + {8159,[8190,834]}, + {8124,[913,837]}, + {8140,[919,837]}, + {8188,[937,837]}, + {8115,[945,837]}, + {8131,[951,837]}, + {8179,[969,837]}, + {64302,[1488,1463]}, + {64287,[1522,1463]}, + {64303,[1488,1464]}, + {64331,[1493,1465]}, + {64304,[1488,1468]}, + {64305,[1489,1468]}, + {64306,[1490,1468]}, + {64307,[1491,1468]}, + {64308,[1492,1468]}, + {64309,[1493,1468]}, + {64310,[1494,1468]}, + {64312,[1496,1468]}, + {64313,[1497,1468]}, + {64314,[1498,1468]}, + {64315,[1499,1468]}, + {64316,[1500,1468]}, + {64318,[1502,1468]}, + {64320,[1504,1468]}, + {64321,[1505,1468]}, + {64323,[1507,1468]}, + {64324,[1508,1468]}, + {64326,[1510,1468]}, + {64327,[1511,1468]}, + {64328,[1512,1468]}, + {64329,[1513,1468]}, + {64330,[1514,1468]}, + {64332,[1489,1471]}, + {64333,[1499,1471]}, + {64334,[1508,1471]}, + {64300,[1513,1468,1473]}, + {64298,[1513,1473]}, + {64301,[1513,1468,1474]}, + {64299,[1513,1474]}, + {2392,[2325,2364]}, + {2393,[2326,2364]}, + {2394,[2327,2364]}, + {2395,[2332,2364]}, + {2396,[2337,2364]}, + {2397,[2338,2364]}, + {2345,[2344,2364]}, + {2398,[2347,2364]}, + {2399,[2351,2364]}, + {2353,[2352,2364]}, + {2356,[2355,2364]}, + {2524,[2465,2492]}, + {2525,[2466,2492]}, + {2480,[2476,2492]}, + {2527,[2479,2492]}, + {2507,[2503,2494]}, + {2508,[2503,2519]}, + {2649,[2582,2620]}, + {2650,[2583,2620]}, + {2651,[2588,2620]}, + {2652,[2593,2620]}, + {2654,[2603,2620]}, + {2908,[2849,2876]}, + {2909,[2850,2876]}, + {2911,[2863,2876]}, + {2891,[2887,2878]}, + {2888,[2887,2902]}, + {2892,[2887,2903]}, + {3018,[3014,3006]}, + {3019,[3015,3006]}, + {2964,[2962,3031]}, + {3020,[3014,3031]}, + {3144,[3142,3158]}, + {3274,[3270,3266]}, + {3264,[3263,3285]}, + {3275,[3270,3266,3285]}, + {3271,[3270,3285]}, + {3272,[3270,3286]}, + {3402,[3398,3390]}, + {3403,[3399,3390]}, + {3404,[3398,3415]}, + {3635,[3661,3634]}, + {3763,[3789,3762]}, + {3955,[3954,3953]}, + {3957,[3956,3953]}, + {3959,[4018,3968,3953]}, + {3961,[4019,3968,3953]}, + {3958,[4018,3968]}, + {3960,[4019,3968]}, + {3945,[3904,4021]}, + {4025,[3984,4021]}, + {3907,[3906,4023]}, + {3917,[3916,4023]}, + {3922,[3921,4023]}, + {3927,[3926,4023]}, + {3932,[3931,4023]}, + {3987,[3986,4023]}, + {3997,[3996,4023]}, + {4002,[4001,4023]}, + {4007,[4006,4023]}, + {4012,[4011,4023]}, + {12436,[12358,12441]}, + {12364,[12363,12441]}, + {12366,[12365,12441]}, + {12368,[12367,12441]}, + {12370,[12369,12441]}, + {12372,[12371,12441]}, + {12374,[12373,12441]}, + {12376,[12375,12441]}, + {12378,[12377,12441]}, + {12380,[12379,12441]}, + {12382,[12381,12441]}, + {12384,[12383,12441]}, + {12386,[12385,12441]}, + {12389,[12388,12441]}, + {12391,[12390,12441]}, + {12393,[12392,12441]}, + {12400,[12399,12441]}, + {12403,[12402,12441]}, + {12406,[12405,12441]}, + {12409,[12408,12441]}, + {12412,[12411,12441]}, + {12446,[12445,12441]}, + {12532,[12454,12441]}, + {12460,[12459,12441]}, + {12462,[12461,12441]}, + {12464,[12463,12441]}, + {12466,[12465,12441]}, + {12468,[12467,12441]}, + {12470,[12469,12441]}, + {12472,[12471,12441]}, + {12474,[12473,12441]}, + {12476,[12475,12441]}, + {12478,[12477,12441]}, + {12480,[12479,12441]}, + {12482,[12481,12441]}, + {12485,[12484,12441]}, + {12487,[12486,12441]}, + {12489,[12488,12441]}, + {12496,[12495,12441]}, + {12499,[12498,12441]}, + {12502,[12501,12441]}, + {12505,[12504,12441]}, + {12508,[12507,12441]}, + {12535,[12527,12441]}, + {12536,[12528,12441]}, + {12537,[12529,12441]}, + {12538,[12530,12441]}, + {12542,[12541,12441]}, + {12401,[12399,12442]}, + {12404,[12402,12442]}, + {12407,[12405,12442]}, + {12410,[12408,12442]}, + {12413,[12411,12442]}, + {12497,[12495,12442]}, + {12500,[12498,12442]}, + {12503,[12501,12442]}, + {12506,[12504,12442]}, + {12509,[12507,12442]}}. -- cgit v1.2.3 From fed20c731b91f1debb11a809cc5999d8b04dd293 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Mon, 22 Nov 2010 16:24:59 +0100 Subject: Teach spawn_executable about Unicode Also corrected compressed files on Windows --- lib/kernel/test/file_name_SUITE.erl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index 57c4353f36..c96621d036 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -745,12 +745,7 @@ rand_comp_decomp(Max) -> L = [ rand_decomp() || _ <- lists:seq(1,N) ], LC = [ A || {A,_} <- L], LD = lists:flatten([B || {_,B} <- L]), - LB = case os:type() of - {win32,_} -> - unicode:characters_to_binary(LD,unicode,{utf16,little}); - _ -> - unicode:characters_to_binary(LD,unicode,utf8) - end, + LB = unicode:characters_to_binary(LD,unicode,utf8), {LC,LB}. rand_decomp() -> -- cgit v1.2.3 From 0c18c4ba3f561ae6d9ff943863bfc62e5e9099e1 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Fri, 26 Nov 2010 14:13:25 +0100 Subject: Corrected testcases broken by unicode filenames Also corrected type-info for bifs --- lib/hipe/cerl/erl_bif_types.erl | 29 +++++++++++++++++++++++++++++ lib/kernel/src/code.erl | 2 ++ lib/kernel/test/code_SUITE.erl | 18 +++++++++++++++++- lib/kernel/test/os_SUITE.erl | 3 ++- lib/stdlib/test/binary_module_SUITE.erl | 2 +- 5 files changed, 51 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index 696414eb7a..6936e5c7b4 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -2014,6 +2014,18 @@ type(file, set_cwd, 1, Xs) -> end); type(file, write_file, 2, Xs) -> strict(arg_types(file, write_file, 2), Xs, fun (_) -> t_file_return() end); +type(file, native_name_encoding, 0, _) -> + t_file_encoding(); +%%-- prim_file ---------------------------------------------------------------- +type(prim_file, internal_name2native, 1, Xs) -> + strict(arg_types(prim_file, internal_name2native, 1), Xs, + fun (_) -> t_binary() end); +type(prim_file, internal_native2name, 1, Xs) -> + strict(arg_types(prim_file, internal_native2name, 1), Xs, + fun (_) -> t_prim_file_name() end); +type(prim_file, internal_normalize_utf8, 1, Xs) -> + strict(arg_types(prim_file, internal_normalize_utf8, 1), Xs, + fun (_) -> t_binary() end); %%-- gen_tcp ------------------------------------------------------------------ %% NOTE: All type information for this module added to avoid loss of precision type(gen_tcp, accept, 1, Xs) -> @@ -4216,6 +4228,15 @@ arg_types(file, write, 2) -> [t_file_io_device(), t_iodata()]; arg_types(file, write_file, 2) -> [t_file_name(), t_sup(t_binary(), t_list())]; +arg_types(file, native_name_encoding, 0) -> + []; +%%-- prim_file ---------------------------------------------------------------- +arg_types(prim_file, internal_name2native, 1) -> + [t_prim_file_name()]; +arg_types(prim_file, internal_native2name, 1) -> + [t_binary()]; +arg_types(prim_file, internal_normalize_utf8, 1) -> + [t_binary()]; %%------- gen_tcp ------------------------------------------------------------- arg_types(gen_tcp, accept, 1) -> [t_socket()]; @@ -4997,6 +5018,7 @@ t_file_io_device() -> t_file_name() -> t_sup([t_atom(), t_string(), + t_binary(), %% DeepList = [char() | atom() | DeepList] -- approximation below t_list(t_sup([t_atom(), t_string(), t_list()]))]). @@ -5051,6 +5073,10 @@ t_file_posix_error() -> t_file_return() -> t_sup(t_atom('ok'), t_tuple([t_atom('error'), t_file_posix_error()])). +t_prim_file_name() -> + t_sup([t_string(), + t_binary()]). + %% ===================================================================== %% These are used for the built-in functions of 'gen_tcp' %% ===================================================================== @@ -5253,6 +5279,9 @@ t_ML() -> % a binary or a possibly deep list of integers or binaries t_encoding() -> t_atoms(['latin1', 'unicode', 'utf8', 'utf16', 'utf32']). +t_file_encoding() -> + t_atoms(['latin1', 'utf8']). + t_encoding_a2b() -> % for the 2nd arg of atom_to_binary/2 and binary_to_atom/2 t_atoms(['latin1', 'unicode', 'utf8']). diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index ec256d5806..d37b74bc03 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -286,6 +286,8 @@ do_start(Flags) -> ets:module_info(module), os:module_info(module), + binary:module_info(module), + unicode:module_info(module), filename:module_info(module), lists:module_info(module), diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 6f846ebc56..0c7440a13e 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -645,7 +645,7 @@ analyse([], [This={M,F,A}|Path], Visited, ErrCnt0) -> %% These modules should be loaded by code.erl before %% the code_server is started. OK = [erlang, os, prim_file, erl_prim_loader, init, ets, - code_server, lists, lists_sort, filename, packages, + code_server, lists, lists_sort, unicode, binary, filename, packages, gb_sets, gb_trees, hipe_unified_loader, hipe_bifs, prim_zip, zlib], ErrCnt1 = @@ -673,6 +673,22 @@ analyse2(MFA={_,_,_}, Path, Visited0) -> %%%% We need to check these manually... % fun's are ok as long as they are defined locally. +check_funs({'$M_EXPR','$F_EXPR',_}, + [{unicode,characters_to_binary_int,3}, + {unicode,characters_to_binary,3}, + {filename,filename_string_to_binary,1}|_]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',_}, + [{unicode,ml_map,3}, + {unicode,characters_to_binary_int,3}, + {unicode,characters_to_binary,3}, + {filename,filename_string_to_binary,1}|_]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',_}, + [{unicode,do_o_binary2,2}, + {unicode,do_o_binary,2}, + {unicode,o_trans,1}, + {unicode,characters_to_binary_int,3}, + {unicode,characters_to_binary,3}, + {filename,filename_string_to_binary,1}|_]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, [{code_server,load_native_code,4}, {code_server,load_native_code_1,2}, diff --git a/lib/kernel/test/os_SUITE.erl b/lib/kernel/test/os_SUITE.erl index ace9501d18..eacf3c7584 100644 --- a/lib/kernel/test/os_SUITE.erl +++ b/lib/kernel/test/os_SUITE.erl @@ -204,8 +204,9 @@ evil(Config) when is_list(Config) -> evil_loop(Parent, ?EVIL_LOOPS,N) end) end, lists:seq(1, ?EVIL_PROCS)), - Devil = spawn(fun () -> devil(hd(Ps), hd(lists:reverse(Ps))) end), + Devil = spawn_link(fun () -> devil(hd(Ps), hd(lists:reverse(Ps))) end), lists:foreach(fun (P) -> receive {P, done} -> ok end end, Ps), + unlink(Devil), exit(Devil, kill), test_server:timetrap_cancel(Dog), ok. diff --git a/lib/stdlib/test/binary_module_SUITE.erl b/lib/stdlib/test/binary_module_SUITE.erl index 16ed9a2c26..e4cdcf6125 100644 --- a/lib/stdlib/test/binary_module_SUITE.erl +++ b/lib/stdlib/test/binary_module_SUITE.erl @@ -186,7 +186,7 @@ badargs(Config) when is_list(Config) -> binary:match(<<1,2,3>>, {ac,ets:match_spec_compile([{'_',[],['$_']}])}, [{scope,{0,1}}])), - ?line nomatch = + ?line [] = ?MASK_ERROR(binary:matches(<<1,2,3>>,<<1>>,[{scope,{0,0}}])), ?line badarg = ?MASK_ERROR(binary:matches(<<1,2,3>>,{bm,<<>>},[{scope,{0,1}}])), -- cgit v1.2.3 From c9d45aa9f32a76a4b08edd1d6010764d9ab02f9b Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Tue, 30 Nov 2010 15:24:21 +0100 Subject: Treat soft links on Windows correctly in file_name_SUITE --- lib/kernel/test/file_name_SUITE.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index c96621d036..3faae93bea 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -193,7 +193,7 @@ check_normal(Mod) -> ?line Syms = [ {S,Targ,list_to_binary(get_data(Targ,normal_dir()))} || {T,S,Targ} <- normal_dir(), T =:= symlink ], ?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ], - ?line [ {ok, Targ} = Mod:read_link(SymL) || {SymL,Targ,_} <- Syms ], + ?line [ {ok, Targ} = fixlink(Mod:read_link(SymL)) || {SymL,Targ,_} <- Syms ], ?line chk_cre_dir(Mod,[{directory,"temp_dir",normal_dir()}]), ?line {ok,BeginAt} = Mod:get_cwd(), ?line true = is_list(BeginAt), @@ -279,7 +279,7 @@ check_icky(Mod) -> ?line Syms = [ {S,conv(Targ),list_to_binary(get_data(Targ,icky_dir()))} || {T,S,Targ} <- icky_dir(), T =:= symlink ], ?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ], - ?line [ {ok, Targ} = Mod:read_link(SymL) || {SymL,Targ,_} <- Syms ], + ?line [ {ok, Targ} = fixlink(Mod:read_link(SymL)) || {SymL,Targ,_} <- Syms ], ?line chk_cre_dir(Mod,[{directory,"åäö_dir",icky_dir()}]), ?line {ok,BeginAt} = Mod:get_cwd(), ?line true = is_list(BeginAt), @@ -416,7 +416,7 @@ check_very_icky(Mod) -> ?line Syms = [ {S,conv(Targ),list_to_binary(get_data(Targ,very_icky_dir()))} || {T,S,Targ} <- very_icky_dir(), T =:= symlink ], ?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ], - ?line [ {ok, Targ} = Mod:read_link(SymL) || {SymL,Targ,_} <- Syms ], + ?line [ {ok, Targ} = fixlink(Mod:read_link(SymL)) || {SymL,Targ,_} <- Syms ], ?line chk_cre_dir(Mod,[{directory,[1088,1079,1091]++"_dir",very_icky_dir()}]), ?line {ok,BeginAt} = Mod:get_cwd(), ?line true = is_list(BeginAt), @@ -691,6 +691,11 @@ treat_icky(Bin) -> Bin end. +fixlink({ok,Link}) -> + {ok,filename:basename(Link)}; +fixlink(X) -> + X. + procentify(<<>>) -> <<>>; procentify(<>) when X > 127 -> -- cgit v1.2.3 From b21d33041ef30182cbb8e74c0023dc282d069a26 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Tue, 30 Nov 2010 12:31:33 +0100 Subject: Teach filelib to use re in unicode mode when filenames are not raw --- lib/kernel/test/file_name_SUITE.erl | 84 ++++++++++++++++++++++++++----------- lib/stdlib/src/filelib.erl | 29 +++++++------ 2 files changed, 77 insertions(+), 36 deletions(-) (limited to 'lib') diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index 3faae93bea..d91bc32d11 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -140,36 +140,46 @@ icky(suite) -> icky(doc) -> "Check file operations on normal file names regardless of unicode mode"; icky(Config) when is_list(Config) -> - {ok,Dir} = file:get_cwd(), - try - Priv = ?config(priv_dir, Config), - file:set_cwd(Priv), - put(file_module,prim_file), - ok = check_icky(prim_file), - put(file_module,file), - ok = check_icky(file) - after - file:set_cwd(Dir) + case hopeless_darwin() of + true -> + {skipped,"This version of darwin does not support icky names at all."}; + false -> + {ok,Dir} = file:get_cwd(), + try + Priv = ?config(priv_dir, Config), + file:set_cwd(Priv), + put(file_module,prim_file), + ok = check_icky(prim_file), + put(file_module,file), + ok = check_icky(file) + after + file:set_cwd(Dir) + end end. very_icky(suite) -> []; very_icky(doc) -> "Check file operations on normal file names regardless of unicode mode"; very_icky(Config) when is_list(Config) -> - {ok,Dir} = file:get_cwd(), - try - Priv = ?config(priv_dir, Config), - file:set_cwd(Priv), - put(file_module,prim_file), - case check_very_icky(prim_file) of - need_unicode_mode -> - {skipped,"VM needs to be started in Unicode filename mode"}; - ok -> - put(file_module,file), - ok = check_very_icky(file) - end - after - file:set_cwd(Dir) + case hopeless_darwin() of + true -> + {skipped,"This version of darwin does not support icky names at all."}; + false -> + {ok,Dir} = file:get_cwd(), + try + Priv = ?config(priv_dir, Config), + file:set_cwd(Priv), + put(file_module,prim_file), + case check_very_icky(prim_file) of + need_unicode_mode -> + {skipped,"VM needs to be started in Unicode filename mode"}; + ok -> + put(file_module,file), + ok = check_very_icky(file) + end + after + file:set_cwd(Dir) + end end. @@ -481,6 +491,21 @@ check_very_icky(Mod) -> FI#file_info{mode = NewMode2}), ?line {ok,#file_info{mode = NewMode2}} = Mod:read_file_info([956,965,963,954,959,49]), + ?line NumOK0 = case has_links() of + true -> 5; + false -> 3 + end, + ?line NumNOK0 = case has_links() of + true -> 4; + false -> 3 + end, + ?line {NumOK,NumNOK} = case is_binary(treat_icky(<<"foo">>)) of + false -> + {NumOK0+NumNOK0,0}; + true -> + {NumOK0,NumNOK0} + end, + ?line {NumOK,NumNOK} = filelib:fold_files(".",".*",true,fun(_F,{N,M}) when is_list(_F) -> io:format("~ts~n",[_F]),{N+1,M}; (_F,{N,M}) -> io:format("~p~n",[_F]),{N,M+1} end,{0,0}), ok catch throw:need_unicode_mode -> @@ -548,6 +573,7 @@ rm_r2(Mod,Dir) -> chk_cre_dir(_,[]) -> ok; chk_cre_dir(Mod,[{regular,Name,Content}|T]) -> + %io:format("~p~n",[Name]), ok = Mod:write_file(Name,Content), chk_cre_dir(Mod,T); chk_cre_dir(Mod,[{link,Name,Target}|T]) -> @@ -558,7 +584,9 @@ chk_cre_dir(Mod,[{symlink,Name,Target}|T]) -> chk_cre_dir(Mod,T); chk_cre_dir(Mod,[{directory,Name,Content}|T]) -> ok = Mod:make_dir(Name), + %io:format("Content = ~p~n",[Content]), Content2 = [{Ty,filename:join(Name,N),case Ty of link -> filename:join(Name,C); _ -> C end} || {Ty,N,C} <- Content ], + %io:format("Content2 = ~p~n",[Content2]), chk_cre_dir(Mod,Content2), chk_cre_dir(Mod,T). @@ -627,6 +655,14 @@ linkify(Passed,[{directory, Name, Content}|T]) -> linkify(Passed,[H|T]) -> [H|linkify([H|Passed],T)]. +hopeless_darwin() -> + case {os:type(),os:version()} of + {{unix,darwin},{Major,_,_}} when Major < 9 -> + true; + _ -> + false + end. + icky_dir() -> [{regular,"fil1","fil1"}, {regular,"åäö2","åäö2"}] ++ diff --git a/lib/stdlib/src/filelib.erl b/lib/stdlib/src/filelib.erl index d5ddf9ed7e..eaaca750ab 100644 --- a/lib/stdlib/src/filelib.erl +++ b/lib/stdlib/src/filelib.erl @@ -166,36 +166,41 @@ do_is_regular(File, Mod) -> %% If is true all sub-directories to are processed do_fold_files(Dir, RegExp, Recursive, Fun, Acc, Mod) -> - {ok, Re1} = re:compile(RegExp), - do_fold_files1(Dir, Re1, Recursive, Fun, Acc, Mod). + {ok, Re1} = re:compile(RegExp,[unicode]), + do_fold_files1(Dir, Re1, RegExp, Recursive, Fun, Acc, Mod). -do_fold_files1(Dir, RegExp, Recursive, Fun, Acc, Mod) -> +do_fold_files1(Dir, RegExp, OrigRE, Recursive, Fun, Acc, Mod) -> case eval_list_dir(Dir, Mod) of - {ok, Files} -> do_fold_files2(Files, Dir, RegExp, Recursive, Fun, Acc, Mod); + {ok, Files} -> do_fold_files2(Files, Dir, RegExp, OrigRE, + Recursive, Fun, Acc, Mod); {error, _} -> Acc end. -do_fold_files2([], _Dir, _RegExp, _Recursive, _Fun, Acc, _Mod) -> +%% OrigRE is not to be compiled as it's for non conforming filenames, +%% i.e. for filenames that does not comply to the current encoding, which should +%% be very rare. We use it only in those cases and do not want to precompile. +do_fold_files2([], _Dir, _RegExp, _OrigRE, _Recursive, _Fun, Acc, _Mod) -> Acc; -do_fold_files2([File|T], Dir, RegExp, Recursive, Fun, Acc0, Mod) -> +do_fold_files2([File|T], Dir, RegExp, OrigRE, Recursive, Fun, Acc0, Mod) -> FullName = filename:join(Dir, File), case do_is_regular(FullName, Mod) of true -> - case re:run(File, RegExp, [{capture,none}]) of + case re:run(File, if is_binary(File) -> OrigRE; true -> RegExp end, + [{capture,none}]) of match -> Acc = Fun(FullName, Acc0), - do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc, Mod); + do_fold_files2(T, Dir, RegExp, OrigRE, Recursive, Fun, Acc, Mod); nomatch -> - do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc0, Mod) + do_fold_files2(T, Dir, RegExp, OrigRE, Recursive, Fun, Acc0, Mod) end; false -> case Recursive andalso do_is_dir(FullName, Mod) of true -> - Acc1 = do_fold_files1(FullName, RegExp, Recursive, + Acc1 = do_fold_files1(FullName, RegExp, OrigRE, Recursive, Fun, Acc0, Mod), - do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc1, Mod); + do_fold_files2(T, Dir, RegExp, OrigRE, Recursive, Fun, Acc1, Mod); false -> - do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc0, Mod) + do_fold_files2(T, Dir, RegExp, OrigRE, Recursive, Fun, Acc0, Mod) end end. -- cgit v1.2.3 From 9fbce40dcbda8d4e7888507d646bfee22bf7d311 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Wed, 1 Dec 2010 08:45:42 +0100 Subject: Correct testcase regarding windows versions supporting soft links. --- lib/kernel/test/file_name_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index d91bc32d11..02b8d12478 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -594,7 +594,7 @@ has_links() -> case os:type() of {win32,_} -> case os:version() of - {N,NN,_} when (N > 5) orelse (NN > 1) -> + {N,NN,_} when (N > 5) andalso (NN >= 1) -> true; _ -> false -- cgit v1.2.3 From 30ad8cfadba47123920128023cb06af6e0922746 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Wed, 1 Dec 2010 15:11:07 +0100 Subject: Mend on_load_embedded testcase which did not handle windows links --- lib/kernel/test/code_SUITE.erl | 13 +++++++++++-- .../on_load_app-1.0/src/on_load_embedded.erl | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 0c7440a13e..e52f8a0e37 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -1268,7 +1268,8 @@ on_load_embedded_1(Config) -> ?line LibRoot = code:lib_dir(), ?line LinkName = filename:join(LibRoot, "on_load_app-1.0"), ?line OnLoadApp = filename:join(DataDir, "on_load_app-1.0"), - ?line file:delete(LinkName), + ?line del_link(LinkName), + io:format("LinkName :~p, OnLoadApp: ~p~n",[LinkName,OnLoadApp]), case file:make_symlink(OnLoadApp, LinkName) of {error,enotsup} -> throw({skip,"Support for symlinks required"}); @@ -1297,7 +1298,15 @@ on_load_embedded_1(Config) -> %% Clean up. ?line stop_node(Node), - ?line ok = file:delete(LinkName). + ?line ok = del_link(LinkName). + +del_link(LinkName) -> + case file:delete(LinkName) of + {error,eperm} -> + file:del_dir(LinkName); + Other -> + Other + end. create_boot(Config, Options) -> ?line {ok, OldDir} = file:get_cwd(), diff --git a/lib/kernel/test/code_SUITE_data/on_load_app-1.0/src/on_load_embedded.erl b/lib/kernel/test/code_SUITE_data/on_load_app-1.0/src/on_load_embedded.erl index b7fdd4d9ae..646921026d 100644 --- a/lib/kernel/test/code_SUITE_data/on_load_app-1.0/src/on_load_embedded.erl +++ b/lib/kernel/test/code_SUITE_data/on_load_app-1.0/src/on_load_embedded.erl @@ -8,7 +8,7 @@ run_me() -> LibDir = code:lib_dir(on_load_app), PrivDir = code:priv_dir(on_load_app), LibDir = filename:dirname(PrivDir), - ModPath = code:which(?MODULE), + ModPath = filename:join(filename:split(code:which(?MODULE))), LibDir = filename:dirname(filename:dirname(ModPath)), %% Start a process to remember that the on_load was called. -- cgit v1.2.3 From 1ab2f46d91bce11bdc5b7ec65d3b3df46eadb105 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Wed, 1 Dec 2010 17:34:49 +0100 Subject: Make filelib not crash on re codepoints beyond 255 in re when filename is raw --- lib/stdlib/src/filelib.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/stdlib/src/filelib.erl b/lib/stdlib/src/filelib.erl index eaaca750ab..04147d40d1 100644 --- a/lib/stdlib/src/filelib.erl +++ b/lib/stdlib/src/filelib.erl @@ -185,11 +185,14 @@ do_fold_files2([File|T], Dir, RegExp, OrigRE, Recursive, Fun, Acc0, Mod) -> FullName = filename:join(Dir, File), case do_is_regular(FullName, Mod) of true -> - case re:run(File, if is_binary(File) -> OrigRE; true -> RegExp end, - [{capture,none}]) of + case (catch re:run(File, if is_binary(File) -> OrigRE; + true -> RegExp end, + [{capture,none}])) of match -> Acc = Fun(FullName, Acc0), do_fold_files2(T, Dir, RegExp, OrigRE, Recursive, Fun, Acc, Mod); + {'EXIT',_} -> + do_fold_files2(T, Dir, RegExp, OrigRE, Recursive, Fun, Acc0, Mod); nomatch -> do_fold_files2(T, Dir, RegExp, OrigRE, Recursive, Fun, Acc0, Mod) end; -- cgit v1.2.3 From 3e6877b06ae395a9d4310ef664d0360867a47f62 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Wed, 1 Dec 2010 17:35:40 +0100 Subject: Add documentation about raw filenames and Unicode file name translation mode --- lib/kernel/doc/src/file.xml | 72 ++++++++++++++++++++++++++++++++++++++++- lib/stdlib/doc/src/filelib.xml | 27 ++++++++++++++-- lib/stdlib/doc/src/filename.xml | 15 +++++++-- 3 files changed, 107 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 64cdd3a8ea..d3441d3623 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -36,6 +36,61 @@ other Erlang processes to continue executing in parallel with the file operations. See the command line flag +A in erl(1).

+ +

The Erlang VM supports file names in Unicode to a limited + extent. Depending on how the VM is started (with the parameter + +fnu or +fnl), file names given can contain + characters > 255 and the VM system will convert file names + back and forth to the native file name encoding.

+ +

The default behavior for Unicode character translation depends + on to what extent the underlying OS/filesystem enforces consistent + naming. On OSes where all file names are ensured to be in one or + another encoding, Unicode is the default (currently this holds for + Windows and MacOSX). On OSes with completely transparent file + naming (i.e. all Unixes except MacOSX), ISO-latin-1 file naming is + the default. The reason for the ISO-latin-1 default is that + file names are not guaranteed to be possible to interpret according to + the Unicode encoding expected (i.e. UTF-8), and file names that + cannot be decoded will only be accessible by using "raw + file names", in other word file names given as binaries.

+ +

As file names are traditionally not binaries in Erlang, + applications that need to handle raw file names need to be + converted, why the Unicode mode for file names is not default on + systems having completely transparent file naming.

+ + As of R14B01, the most basic file handling modules + (file, prim_file, filelib and + filename) accept raw file names, but the rest of OTP is not + guaranteed to handle them, why Unicode file naming on systems + where it is not default is still considered experimental. + +

Raw file names is a new feature in OTP R14B01, which allows the + user to supply completely uninterpreted file names to the + underlying OS/filesystem. They are supplied as binaries, where it + is up to the user to supply a correct encoding for the + environment. The function file:native_name_encoding() can + be used to check what encoding the VM is working in. If the + function returns latin1 file names are not in any way + converted to Unicode, if it is utf8, raw file names should + be encoded as UTF-8 if they are to follow the convention of the VM + (and usually the convention of the OS as well). Using raw + file names is useful if you have a filesystem with inconsistent + file naming, where some files are named in UTF-8 encoding while + others are not. A file:list_dir on such mixed file name systems + when the VM is in Unicode file name mode might return file names as + raw binaries as they cannot be interpreted as Unicode + file names. Raw file names can also be used to give UTF-8 encoded + file names even though the VM is not started in Unicode file name + translation mode.

+ +

Note that on Windows, file:native_name_encoding() + returns utf8 per default, which is the format for raw + file names even on Windows, although the underlying OS specific + code works in a limited version of little endian UTF16. As far as + the Erlang programmer is concerned, Windows native Unicode format + is UTF-8...

@@ -47,8 +102,14 @@ iodata() = iolist() | binary() io_device() as returned by file:open/2, a process handling IO protocols -name() = string() | atom() | DeepList +name() = string() | atom() | DeepList | RawFilename DeepList = [char() | atom() | DeepList] + RawFilename = binary() + If VM is in unicode filename mode, string() and char() are allowed to be > 255. + RawFilename is a filename not subject to Unicode translation, meaning that it + can contain characters not conforming to the Unicode encoding expected from the + filesystem (i.e. non-UTF-8 characters although the VM is started in Unicode + filename mode). posix() an atom which is named from the POSIX error codes used in @@ -597,6 +658,15 @@ f.txt: {person, "kalle", 25}. + + native_name_encoding() -> latin1 | utf8 + Retunr the VMs configure filename encoding. + +

This function returns the configured default file name encoding to use for raw file names. Generally an application supplying file names raw (as binaries), should obey the character encoding returned by this function.

+

By default, the VM uses ISO-latin-1 file name encoding on filesystems and/or OSes that use completely transparent file naming. This includes all Unix versions except for MacOSX, where the vfs layer enforces UTF-8 file naming. By giving the experimental option +fnu when starting Erlang, UTF-8 translation of file names can be turned on even for those systems. If Unicode file name translation is in effect, the system behaves as usual as long as file names conform to the encoding, but will return file names that are not properly encoded in UTF-8 as raw file names (i.e. binaries).

+

On Windows, this function also returns utf8 by default. The OS uses a pure Unicode naming scheme and file names are always possible to interpret as valid Unicode. The fact that the underlying Windows OS actually encodes file names using little endian UTF-16 can be ignored by the Erlang programmer. Windows and MacOSX are the only operating systems where the VM operates in Unicode file name mode by default.

+ + open(Filename, Modes) -> {ok, IoDevice} | {error, Reason} Open a file diff --git a/lib/stdlib/doc/src/filelib.xml b/lib/stdlib/doc/src/filelib.xml index 4ff3b22f32..969aff4fcb 100644 --- a/lib/stdlib/doc/src/filelib.xml +++ b/lib/stdlib/doc/src/filelib.xml @@ -36,14 +36,23 @@

This module contains utilities on a higher level than the file module.

+

The module supports Unicode file names, so that it will match against regular expressions given in Unicode and that it will find and process raw file names (i.e. files named in a way that does not confirm to the expected encoding).

+

If the VM operates in Unicode file naming mode on a machine with transparent file naming, the fun() provided to fold_files/5 needs to be prepared to handle binary file names.

+

For more information about raw file names, see the file module.
DATA TYPES -filename() = string() | atom() | DeepList -dirname() = filename() -DeepList = [char() | atom() | DeepList] +filename() = = string() | atom() | DeepList | RawFilename + DeepList = [char() | atom() | DeepList] + RawFilename = binary() + If VM is in unicode filename mode, string() and char() are allowed to be > 255. + RawFilename is a filename not subject to Unicode translation, meaning that it + can contain characters not conforming to the Unicode encoding expected from the + filesystem (i.e. non-UTF-8 characters although the VM is started in Unicode + filename mode). +dirname() = filename()
@@ -90,6 +99,18 @@ DeepList = [char() | atom() | DeepList] If Recursive is true all sub-directories to Dir are processed. The regular expression matching is done on just the filename without the directory part.

+ +

If Unicode file name translation is in effect and the file + system is completely transparent, file names that cannot be + interpreted as Unicode may be encountered, in which case the + fun() must be prepared to handle raw file names + (i.e. binaries). If the regular expression contains + codepoints beyond 255, it will not match file names that does + not conform to the expected character encoding (i.e. are not + encoded in valid UTF-8).

+ +

For more information about raw file names, see the + file module.

 diff --git a/lib/stdlib/doc/src/filename.xml b/lib/stdlib/doc/src/filename.xml index fe6c6f898e..cdee6e4a81 100644 --- a/lib/stdlib/doc/src/filename.xml +++ b/lib/stdlib/doc/src/filename.xml @@ -4,7 +4,7 @@
- 19972009 + 19972010 Ericsson AB. All Rights Reserved. @@ -43,13 +43,22 @@ only, even if the arguments contain back slashes. Use join/1 to normalize a file name by removing redundant directory separators.

+

The module supports raw file names in the way that if a binary is present, or the file name cannot be interpreted according to the return value of + file:native_name_encoding/0, a raw file name will also be returned. For example filename:join/1 provided with a path component being a binary (and also not being possible to interpret under the current native file name encoding) will result in a raw file name being returned (the join operation will have been performed of course). For more information about raw file names, see the file module.

DATA TYPES -name() = string() | atom() | DeepList -DeepList = [char() | atom() | DeepList] +name() = string() | atom() | DeepList | RawFilename + DeepList = [char() | atom() | DeepList] + RawFilename = binary() + If VM is in unicode filename mode, string() and char() are allowed to be > 255. + RawFilename is a filename not subject to Unicode translation, meaning that it + can contain characters not conforming to the Unicode encoding expected from the + filesystem (i.e. non-UTF-8 characters although the VM is started in Unicode + filename mode). +
-- cgit v1.2.3 From 47c0ced5a05874c3cbfdad532abbd8900c40e966 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Wed, 1 Dec 2010 17:46:45 +0100 Subject: Correct bug in file_name_SUITE making it fail on Unix instead of Windows7 --- lib/kernel/test/file_name_SUITE.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index 02b8d12478..8829e93473 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -506,6 +506,7 @@ check_very_icky(Mod) -> {NumOK0,NumNOK0} end, ?line {NumOK,NumNOK} = filelib:fold_files(".",".*",true,fun(_F,{N,M}) when is_list(_F) -> io:format("~ts~n",[_F]),{N+1,M}; (_F,{N,M}) -> io:format("~p~n",[_F]),{N,M+1} end,{0,0}), + ?line ok = filelib:fold_files(".",[1076,1089,1072,124,46,42],true,fun(_F,_) -> ok end,false), ok catch throw:need_unicode_mode -> @@ -727,8 +728,14 @@ treat_icky(Bin) -> Bin end. +% Handle windows having absolute soft link targets. fixlink({ok,Link}) -> - {ok,filename:basename(Link)}; + case os:type() of + {win32,_} -> + {ok,filename:basename(Link)}; + _ -> + {ok,Link} + end; fixlink(X) -> X. -- cgit v1.2.3 From 6ba7c90dd6ccd7a6a6c661c51f17c44a124d2182 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Thu, 2 Dec 2010 14:55:03 +0100 Subject: Add section about Unicode file names to stdlib users guide --- lib/stdlib/doc/src/unicode_usage.xml | 44 +++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index f1b0659ea2..df8e6c6b47 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -5,7 +5,7 @@
1999 - 2009 + 2010 Ericsson AB. All Rights Reserved. @@ -168,6 +168,48 @@ Eshell V5.7 (abort with ^G) Unicode characters in allowed and disallowed context
+Unicode file names +

Most modern operating systems support Unicode file names in some way or another. There are several different ways to do this and Erlang by default treats the different approaches differently:

+ +Mandatory Unicode file naming + +

Windows and, for most common uses, MacOSX enforces Unicode support for file names. All files created in the filesystem has names that can consistently be interpreted. In MacOSX, all file names are retrieved in UTF-8 encoding, while Windows have selected an approach where each system call handling file names has a special Unicode aware variant, giving much the same effect. There are no file names on these systems that are not Unicode file names, why the default behavior of the Erlang VM is to work in "Unicode file name translation mode", meaning that a file name can be given as a Unicode list and that will be automatically translated to the proper name encoding for the underlying operating and file system.

+

Doing i.e. a file:list_dir/1 on one of these systems may return Unicode lists with codepoints beyond 255, depending on the content of the actual filesystem.

+

As the feature is fairly new, you may still stumble upon non core applications that cannot handle being provided with file names containing characters with codepoints larger than 255, but the core Erlang system should have no problems with Unicode file names.

+ +Transparent file naming + +

Most Unix operating systems have adopted a simpler approach, namely that Unicode file naming is not enforced, but by convention. Those systems usually use UTF-8 encoding for Unicode file names, but do not enforce it. On such a system, a file name containing characters having codepoints between 128 and 255 may be named either as plain ISO-latin-1 or using UTF-8 encoding. As no consistency is enforced, the Erlang VM can do no consistent translation of all file names. If the VM would automatically select encoding based on heuristics, one could get unexpected behavior on these systems, therefore file names not being encoded in UTF-8 are returned as "raw file names" if Unicode file naming support is turned on.

+

A raw file name is not a list, but a binary. Many non core applications still does not handle file names given as binaries, why such raw names are avoided by default. This means that systems having implemented Unicode file naming through transparent file systems and an UTF-8 convention, does not by default have Unicode file naming turned on. Explicitly turning Unicode file name handling on for these types of systems is considered experimental.

+ + +

The Unicode file naming support was introduced with OTP release R14B01. A VM operating in Unicode file mode can work with files having names in any language or character set (as long as it's supported by the underlying OS and file system). The Unicode character list is used to denote file or directory names and if the file system content is listed, you will also be able to get Unicode lists as return value. The support lies in the kernel and stdlib modules, why most applications (that does not explicitly require the file names to be in the ISO-latin-1 range) will benefit from the Unicode support without change.

+ +

On Operating systems with mandatory Unicode file names, this means that you more easily conform to the file names of other (non Erlang) applications, and you can also process file names that, at least on Windows, where completely inaccessible (due to having names that could not be represented in ISO-latin-1). Also you will avoid creating incomprehensible file names on MacOSX as the vfs layer of the OS will accept all your file names as UTF-8 and will not rewrite them.

+ +

For most systems, turning on Unicode file name translation is no problem even if it uses transparent file naming. Very few systems have mixed file name encodings. A consistent UTF-8 named system will work perfectly in Unicode file name mode. It is still however considered experimental in R14B01. Unicode file name translation is turned on with the +fnu switch to the erl program. If the VM is started in Unicode file name translation mode, file:native_name_encoding/0 will return the atom utf8.

+ +

In Unicode file name mode, file names given to the BIF open_port/2 with the option {spawn_executable,...} are also interpreted as Unicode. So is the parameter list given in the argv option available when using spawn_executable. The UTF-8 translation of arguments can be avoided using binaries, see the discussion about raw file names below.

+ +

It is worth noting that the file encoding options given when opening a file has nothing to do with the file name encoding convention. You can very well open files containing UTF-8 but having file names in ISO-latin-1 or vice versa.

+ +Erlang drivers and NIF shared objects still can not be named with names containing codepoints beyond 127. This is a known limitation to be removed in a future release. Erlang modules however can, but it is definitely not a good idea and is still considered experimental. + +
+Notes about raw file names and automatic file name conversion +

Raw file names is introduced together with Unicode file name support in erts-5.8.2 (OTP R14B01). The reason the "raw file names" is introduced in the system is to be able to consistently represent file names given in different encodings on the same system. Having the VM automatically translate a file name that is not in UTF-8 when to a list of Unicode characters might seem practical, but this would open up for both duplicate file names and other inconsistent behavior. Consider a directory containing a file named "björn" in ISO-latin-1, while the Erlang VM is operating in Unicode file name mode (and therefore expecting UTF-8 file naming). The ISO-latin-1 name is not valid UTF-8 and one could be tempted to think that automatic conversion in for example file:list_dir/1 is a good idea. But what would happen if we later tried to open the file and has the name as a Unicode list (magically converted from the ISO-latin-1 file name)? The VM will convert the file name given to UTF-8, as this is the encoding expected. Effectively this means trying to open the file named <<"björn"/utf8>>. This file does not exist, and even if it existed it would not be the same file as the one that was listed. We could even create two files named "björn", one named in the UTF-8 encoding and one not. If file:list_dir/1 would automatically convert the ISO-latin-1 file name to a list, we would get two identical file names as the result. To avoid this, we need to differentiate between file names being properly encoded according to the Unicode file naming convention (i.e. UTF-8) and file names being invalid under the encoding. This is done by representing invalid encoding as "raw" file names, i.e. as binaries.

+

The core system of Erlang (kernel and stdlib) accept raw file names except for loadable drivers and executables invoked using open_port({spawn, ...} ...). open_port({spawn_executable, ...} ...) however does accept them. As mentioned earlier, the arguments given in the option list to open_port({spawn_executable, ...} ...) undergo the same conversion as the file names, meaning that the executable will be provided with arguments in UTF-8 as well. This translation is avoided consistently with how the file names are treated, by giving the argument as a binary.

+

To force Unicode file name translation mode on systems where this is not the default is considered experimental in OTP R14B01 due to the raw file names possibly being a new experience to the programmer and that the non core applications of OTP are not tested for compliance with raw file names yet. Unicode file name translation is expected to be default in future releases.

+

If working with raw file names, one can still conform to the encoding convention of the Erlang VM by using the file:native_name_encoding/0 function, which returns either the atom latin1 or the atom utf8 depending on the file name translation mode. On Linux, an VM started without explicitly stating the file name translation mode will default to latin1 as the native file name encoding, why file names on the disk encoded as UTF-8 will be returned as a list of the names interpreted as ISO-latin-1. The "UTF-8 list" is not a practical type for displaying or operating on in Erlang, but it is backward compatible and usable in all functions requiring a file name. On Windows and MacOSX, the default behavior is that of file name translation, why the file:native_name_encoding/0 by default returns utf8 on those systems (the fact that Windows actually does not use UTF-8 on the file system level can safely be ignored by the Erlang programmer). The default behavior can be changed using the +fnu or +fnl options to the VM, see the erl command manual page.

+

Even if you are operating without Unicode file naming translation automatically done by the VM, you can access and create files with names in UTF-8 encoding by using raw file names encoded as UTF-8. Enforcing the UTF-8 encoding regardless of the mode the Erlang VM is started in might, in some circumstances be a good idea, as the convention of using UTF-8 file names is spreading.

+
+
+Notes about MacOSX +

MacOSXs vfs layer enforces UTF-8 file names in a quite aggressive way. Older versions did this by simply refusing to create non UTF-8 conforming file names, while newer versions replace offending bytes with the sequence "%HH", where HH is the original character in hexadecimal notation. As Unicode translation is enabled by default on MacOSX, the only way to come up against this is to either start the VM with the +fnl flag or to use a raw file name in latin1 encoding. In that case, the file can not be opened with the same name as the one used to create this. The problem is by design in newer versions of MacOSX.

+

MacOSX also reorganizes the names of files so that the representation of accents etc is denormalized, i.e. the character ö is represented as the codepoints [111,776], where 111 is the character o and 776 is a special accent character. This type of denormalized Unicode is otherwise very seldom used and Erlang normalizes those file names on retrieval, so that denormalized file names is not passed up to the Erlang application. In Erlang the file name "björn" is retrieved as [98,106,246,114,110], not as [98,106,117,776,114,110], even though the file system might think differently.

+
+
+
Unicode-aware modules

Most of the modules in Erlang/OTP are of course Unicode-unaware in the sense that they have no notion of Unicode and really shouldn't have. Typically they handle non-textual or byte-oriented data (like gen_tcp etc).

Modules that actually handle textual data (like io_lib, string etc) are sometimes subject to conversion or extension to be able to handle Unicode characters.

-- cgit v1.2.3 From 6ea8348174c62812057dd552d0890b2d9d4a3c16 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Fri, 3 Dec 2010 09:51:00 +0100 Subject: Add documentation to erlang.xml and slight correction to unicode_usage.xml --- lib/stdlib/doc/src/unicode_usage.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index df8e6c6b47..c02ea3cbcb 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -189,7 +189,7 @@ Eshell V5.7 (abort with ^G)

For most systems, turning on Unicode file name translation is no problem even if it uses transparent file naming. Very few systems have mixed file name encodings. A consistent UTF-8 named system will work perfectly in Unicode file name mode. It is still however considered experimental in R14B01. Unicode file name translation is turned on with the +fnu switch to the erl program. If the VM is started in Unicode file name translation mode, file:native_name_encoding/0 will return the atom utf8.

-

In Unicode file name mode, file names given to the BIF open_port/2 with the option {spawn_executable,...} are also interpreted as Unicode. So is the parameter list given in the argv option available when using spawn_executable. The UTF-8 translation of arguments can be avoided using binaries, see the discussion about raw file names below.

+

In Unicode file name mode, file names given to the BIF open_port/2 with the option {spawn_executable,...} are also interpreted as Unicode. So is the parameter list given in the args option available when using spawn_executable. The UTF-8 translation of arguments can be avoided using binaries, see the discussion about raw file names below.

It is worth noting that the file encoding options given when opening a file has nothing to do with the file name encoding convention. You can very well open files containing UTF-8 but having file names in ISO-latin-1 or vice versa.

-- cgit v1.2.3 From 159d79b6657e5bb4f3accb6a0d74b74f06da4ba2 Mon Sep 17 00:00:00 2001 From: Patrik Nyblom Date: Fri, 3 Dec 2010 13:16:21 +0100 Subject: Test and correct filelib and filename --- lib/kernel/test/file_name_SUITE.erl | 3 + lib/stdlib/src/filelib.erl | 28 +++- lib/stdlib/src/filename.erl | 28 ++-- lib/stdlib/test/filelib_SUITE.erl | 7 +- lib/stdlib/test/filename_SUITE.erl | 315 +++++++++++++++++++++++++++++++++++- 5 files changed, 363 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index 8829e93473..fea4df8539 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -507,6 +507,9 @@ check_very_icky(Mod) -> end, ?line {NumOK,NumNOK} = filelib:fold_files(".",".*",true,fun(_F,{N,M}) when is_list(_F) -> io:format("~ts~n",[_F]),{N+1,M}; (_F,{N,M}) -> io:format("~p~n",[_F]),{N,M+1} end,{0,0}), ?line ok = filelib:fold_files(".",[1076,1089,1072,124,46,42],true,fun(_F,_) -> ok end,false), + ?line SF3 = unicode:characters_to_binary("åäösubfil3",file:native_name_encoding()), + ?line Sorted = lists:sort([SF3,<<"åäösubfil2">>]), + ?line Sorted = lists:sort(filelib:wildcard("*",<<"åäösubdir2">>)), ok catch throw:need_unicode_mode -> diff --git a/lib/stdlib/src/filelib.erl b/lib/stdlib/src/filelib.erl index 04147d40d1..c845b61204 100644 --- a/lib/stdlib/src/filelib.erl +++ b/lib/stdlib/src/filelib.erl @@ -47,14 +47,14 @@ wildcard(Pattern) when is_list(Pattern) -> ?HANDLE_ERROR(do_wildcard(Pattern, file)). -spec wildcard(file:name(), file:name() | atom()) -> [file:filename()]. -wildcard(Pattern, Cwd) when is_list(Pattern), is_list(Cwd) -> +wildcard(Pattern, Cwd) when is_list(Pattern), (is_list(Cwd) or is_binary(Cwd)) -> ?HANDLE_ERROR(do_wildcard(Pattern, Cwd, file)); wildcard(Pattern, Mod) when is_list(Pattern), is_atom(Mod) -> ?HANDLE_ERROR(do_wildcard(Pattern, Mod)). -spec wildcard(file:name(), file:name(), atom()) -> [file:filename()]. wildcard(Pattern, Cwd, Mod) - when is_list(Pattern), is_list(Cwd), is_atom(Mod) -> + when is_list(Pattern), (is_list(Cwd) or is_binary(Cwd)), is_atom(Mod) -> ?HANDLE_ERROR(do_wildcard(Pattern, Cwd, Mod)). -spec is_dir(file:name()) -> boolean(). @@ -118,7 +118,7 @@ do_wildcard_comp({compiled_wildcard,{exists,File}}, Mod) -> do_wildcard_comp({compiled_wildcard,[Base|Rest]}, Mod) -> do_wildcard_1([Base], Rest, Mod). -do_wildcard(Pattern, Cwd, Mod) when is_list(Pattern), is_list(Cwd) -> +do_wildcard(Pattern, Cwd, Mod) when is_list(Pattern), (is_list(Cwd) or is_binary(Cwd)) -> do_wildcard_comp(do_compile_wildcard(Pattern), Cwd, Mod). do_wildcard_comp({compiled_wildcard,{exists,File}}, Cwd, Mod) -> @@ -127,9 +127,18 @@ do_wildcard_comp({compiled_wildcard,{exists,File}}, Cwd, Mod) -> _ -> [] end; do_wildcard_comp({compiled_wildcard,[current|Rest]}, Cwd0, Mod) -> - Cwd = filename:join([Cwd0]), %Slash away redundant slashes. - PrefixLen = length(Cwd)+1, - [lists:nthtail(PrefixLen, N) || N <- do_wildcard_1([Cwd], Rest, Mod)]; + {Cwd,PrefixLen} = case filename:join([Cwd0]) of + Bin when is_binary(Bin) -> {Bin,byte_size(Bin)+1}; + Other -> {Other,length(Other)+1} + end, %Slash away redundant slashes. + [ + if + is_binary(N) -> + <<_:PrefixLen/binary,Res/binary>> = N, + Res; + true -> + lists:nthtail(PrefixLen, N) + end || N <- do_wildcard_1([Cwd], Rest, Mod)]; do_wildcard_comp({compiled_wildcard,[Base|Rest]}, _Cwd, Mod) -> do_wildcard_1([Base], Rest, Mod). @@ -276,6 +285,13 @@ do_wildcard_3(Base, [Pattern|Rest], Result, Mod) -> do_wildcard_3(Base, [], Result, _Mod) -> [Base|Result]. +wildcard_4(Pattern, [File|Rest], Base, Result) when is_binary(File) -> + case wildcard_5(Pattern, binary_to_list(File)) of + true -> + wildcard_4(Pattern, Rest, Base, [join(Base, File)|Result]); + false -> + wildcard_4(Pattern, Rest, Base, Result) + end; wildcard_4(Pattern, [File|Rest], Base, Result) -> case wildcard_5(Pattern, File) of true -> diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl index 9ca4b808e1..e38b8957f2 100644 --- a/lib/stdlib/src/filename.erl +++ b/lib/stdlib/src/filename.erl @@ -151,10 +151,10 @@ win_basenameb(O) -> basenameb(O,[<<"/">>,<<"\\">>]). basenameb(Bin,Sep) -> Parts = [ X || X <- binary:split(Bin,Sep,[global]), - X =:= <<>> ], + X =/= <<>> ], if Parts =:= [] -> - []; + <<>>; true -> lists:last(Parts) end. @@ -201,17 +201,19 @@ basename(Name, Ext) when is_list(Name), is_binary(Ext) -> basename(filename_string_to_binary(Name),Ext); basename(Name, Ext) when is_binary(Name), is_binary(Ext) -> BName = basename(Name), + LAll = byte_size(Name), LN = byte_size(BName), LE = byte_size(Ext), case LN - LE of Neg when Neg < 0 -> BName; Pos -> - case BName of - <> -> + StartLen = LAll - Pos - LE, + case Name of + <<_:StartLen/binary,Part:Pos/binary,Ext/binary>> -> Part; - Other -> - Other + _Other -> + BName end end; @@ -447,7 +449,7 @@ join1b(<>, RelativeName, [], win32) when is_integer(UcLetter), UcLetter >= $A, UcLetter =< $Z -> join1b(Rest, RelativeName, [$:, UcLetter+$a-$A], win32); join1b(<<$\\,Rest/binary>>, RelativeName, Result, win32) -> - join1b(<<$/,Rest>>, RelativeName, Result, win32); + join1b(<<$/,Rest/binary>>, RelativeName, Result, win32); join1b(<<$/,Rest/binary>>, RelativeName, [$., $/|Result], OsType) -> join1b(Rest, RelativeName, [$/|Result], OsType); join1b(<<$/,Rest/binary>>, RelativeName, [$/|Result], OsType) -> @@ -546,6 +548,8 @@ win32_pathtype(_) -> relative. %% rootname("/jam.src/foo.erl") -> "/jam.src/foo" -spec rootname(file:name()) -> file:filename(). +rootname(Name) when is_binary(Name) -> + list_to_binary(rootname(binary_to_list(Name))); % No need to handle unicode, . is < 128 rootname(Name0) -> Name = flatten(Name0), rootname(Name, [], [], major_os_type()). @@ -573,6 +577,12 @@ rootname([], Root, _Ext, _OsType) -> %% rootname("/jam.src/foo.erl", ".erl") -> "/jam.src/foo" -spec rootname(file:name(), file:name()) -> file:filename(). +rootname(Name, Ext) when is_binary(Name), is_binary(Ext) -> + list_to_binary(rootname(binary_to_list(Name),binary_to_list(Ext))); +rootname(Name, Ext) when is_binary(Name) -> + rootname(Name,filename_string_to_binary(Ext)); +rootname(Name, Ext) when is_binary(Ext) -> + rootname(filename_string_to_binary(Name),Ext); rootname(Name0, Ext0) -> Name = flatten(Name0), Ext = flatten(Ext0), @@ -639,7 +649,7 @@ win32_splitb(<>) when ((Slash =:= $\\) orelse (Slash =:= $/)) [<<$/>> | [ X || X <- L, X =/= <<>> ]]; win32_splitb(Name) -> L = binary:split(Name,[<<"/">>,<<"\\">>],[global]), - [<<$/>> | [ X || X <- L, X =/= <<>> ]]. + [ X || X <- L, X =/= <<>> ]. unix_split(Name) -> @@ -900,7 +910,7 @@ do_flatten(Atom, Tail) when is_atom(Atom) -> atom_to_list(Atom) ++ flatten(Tail). filename_string_to_binary(List) -> - case unicode:characters_to_binary(List,unicode,file:native_name_encoding()) of + case unicode:characters_to_binary(flatten(List),unicode,file:native_name_encoding()) of {error,_,_} -> erlang:error(badarg); Bin when is_binary(Bin) -> diff --git a/lib/stdlib/test/filelib_SUITE.erl b/lib/stdlib/test/filelib_SUITE.erl index d54741051f..5a279609c6 100644 --- a/lib/stdlib/test/filelib_SUITE.erl +++ b/lib/stdlib/test/filelib_SUITE.erl @@ -53,8 +53,11 @@ wildcard_one(Config) when is_list(Config) -> wildcard_two(Config) when is_list(Config) -> ?line Dir = filename:join(?config(priv_dir, Config), "wildcard_two"), + ?line DirB = unicode:characters_to_binary(Dir, file:native_name_encoding()), ?line ok = file:make_dir(Dir), - ?line do_wildcard_1(Dir, fun(Wc) -> filelib:wildcard(Wc, Dir) end), + ?line do_wildcard_1(Dir, fun(Wc) -> io:format("~p~n",[{Wc,Dir, X = filelib:wildcard(Wc, Dir)}]),X end), + ?line do_wildcard_1(Dir, fun(Wc) -> io:format("~p~n",[{Wc,DirB, X = filelib:wildcard(Wc, DirB)}]), + [unicode:characters_to_list(Y,file:native_name_encoding()) || Y <- X] end), ?line do_wildcard_1(Dir, fun(Wc) -> filelib:wildcard(Wc, Dir++"/") end), case os:type() of {win32,_} -> @@ -253,5 +256,7 @@ ensure_dir_eexist(Config) when is_list(Config) -> %% There already is a file with the name of the directory %% we want to create. ?line NeedFile = filename:join(Name, "file"), + ?line NeedFileB = filename:join(Name, <<"file">>), ?line {error, eexist} = filelib:ensure_dir(NeedFile), + ?line {error, eexist} = filelib:ensure_dir(NeedFileB), ok. diff --git a/lib/stdlib/test/filename_SUITE.erl b/lib/stdlib/test/filename_SUITE.erl index ab6521f37b..dbce93600b 100644 --- a/lib/stdlib/test/filename_SUITE.erl +++ b/lib/stdlib/test/filename_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,12 +22,19 @@ basename_1/1, basename_2/1, dirname/1, extension/1, join/1, t_nativename/1]). -export([pathtype/1,rootname/1,split/1,find_src/1]). +-export([absname_bin/1, absname_bin_2/1, + basename_bin_1/1, basename_bin_2/1, + dirname_bin/1, extension_bin/1, join_bin/1]). +-export([pathtype_bin/1,rootname_bin/1,split_bin/1]). -include("test_server.hrl"). all(suite) -> [absname, absname_2, basename_1, basename_2, dirname, extension, - join, pathtype, rootname, split, t_nativename, find_src]. + join, pathtype, rootname, split, t_nativename, find_src, + absname_bin, absname_bin_2, basename_bin_1, basename_bin_2, dirname_bin, + extension_bin, + join_bin, pathtype_bin, rootname_bin, split_bin]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -457,3 +464,307 @@ find_src(Config) when is_list(Config) -> %% Try to find the source for a preloaded module. ?line {error,{preloaded,init}} = filename:find_src(init), ok. + +%% +%% +%% With binaries +%% +%% + +absname_bin(Config) when is_list(Config) -> + case os:type() of + {win32, _} -> + ?line [Drive|_] = ?config(priv_dir, Config), + ?line Temp = filename:join([Drive|":/"], "temp"), + ?line case file:make_dir(Temp) of + ok -> ok; + {error,eexist} -> ok + end, + ?line {ok,Cwd} = file:get_cwd(), + ?line ok = file:set_cwd(Temp), + ?line <> = filename:absname(<<"foo">>), + ?line <> = filename:absname(<<"../ebin">>), + ?line <> = filename:absname(<<"/erlang">>), + ?line <> = filename:absname(<<"/erlang/src">>), + ?line <> = filename:absname(<<"\\erlang\\src">>), + ?line <> = filename:absname(<>), + ?line <> = + filename:absname(<>), + ?line <> = + filename:absname(<>), + ?line <<"a:/erlang">> = filename:absname(<<"a:erlang">>), + + ?line file:set_cwd(<>), + ?line <> = filename:absname(<<"foo">>), + ?line <> = filename:absname(<<"../ebin">>), + ?line <> = filename:absname(<<"/erlang">>), + ?line <> = filename:absname(<<"/erlang/src">>), + ?line <> = filename:absname(<<"\\erlang\\\\src">>), + ?line <> = filename:absname(<>), + ?line <> = filename:absname(<>), + ?line <<"a:/erlang">> = filename:absname(<<"a:erlang">>), + + ?line file:set_cwd(Cwd), + ok; + {unix, _} -> + ?line ok = file:set_cwd(<<"/usr">>), + ?line <<"/usr/foo">> = filename:absname(<<"foo">>), + ?line <<"/usr/../ebin">> = filename:absname(<<"../ebin">>), + + ?line file:set_cwd(<<"/">>), + ?line <<"/foo">> = filename:absname(<<"foo">>), + ?line <<"/../ebin">> = filename:absname(<<"../ebin">>), + ?line <<"/erlang">> = filename:absname(<<"/erlang">>), + ?line <<"/erlang/src">> = filename:absname(<<"/erlang/src">>), + ?line <<"/erlang/src">> = filename:absname(<<"/erlang///src">>), + ok + end. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +absname_bin_2(Config) when is_list(Config) -> + case os:type() of + {win32, _} -> + ?line [Drive|_] = ?config(priv_dir, Config), + ?line <> = filename:absname(<<"foo">>, <>), + ?line <> = filename:absname(<<"../ebin">>, + <>), + ?line <> = filename:absname(<<"/erlang">>, <>), + ?line <> = filename:absname(<<"/erlang/src">>, + <>), + ?line <> = filename:absname(<<"\\erlang\\src">>, + <>), + ?line <> = filename:absname(<>, + <>), + ?line <> = filename:absname(<>, + <>), + ?line <> = + filename:absname(<>, <>), + ?line <<"a:/erlang">> = filename:absname(<<"a:erlang">>, <>), + + ?line file:set_cwd(<>), + ?line <> = filename:absname(foo, <>), + ?line <> = filename:absname(<<"foo">>, <>), + ?line <> = filename:absname(<<"../ebin">>, <>), + ?line <> = filename:absname(<<"/erlang">>, <>), + ?line <> = filename:absname(<<"/erlang/src">>, + <>), + ?line <> = filename:absname(<<"\\erlang\\\\src">>, + <>), + ?line <> = filename:absname(<>, + <>), + ?line <> = filename:absname(<>, + <>), + ?line <<"a:/erlang">> = filename:absname(<<"a:erlang">>, <>), + + ok; + {unix, _} -> + ?line <<"/usr/foo">> = filename:absname(<<"foo">>, <<"/usr">>), + ?line <<"/usr/../ebin">> = filename:absname(<<"../ebin">>, <<"/usr">>), + + ?line <<"/foo">> = filename:absname(<<"foo">>, <<"/">>), + ?line <<"/../ebin">> = filename:absname(<<"../ebin">>, <<"/">>), + ?line <<"/erlang">> = filename:absname(<<"/erlang">>, <<"/">>), + ?line <<"/erlang/src">> = filename:absname(<<"/erlang/src">>, <<"/">>), + ?line <<"/erlang/src">> = filename:absname(<<"/erlang///src">>, <<"/">>), + ok + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +basename_bin_1(Config) when is_list(Config) -> + ?line Dog = test_server:timetrap(test_server:seconds(10)), + ?line <<".">> = filename:basename(<<".">>), + ?line <<"foo">> = filename:basename(<<"foo">>), + ?line <<"foo">> = filename:basename(<<"/usr/foo">>), + ?line <<"foo.erl">> = filename:basename(<<"A:usr/foo.erl">>), + ?line case os:type() of + {win32, _} -> + ?line <<"foo">> = filename:basename(<<"A:\\usr\\foo">>), + ?line <<"foo">> = filename:basename(<<"A:foo">>); + {unix, _} -> + ?line <<"strange\\but\\true">> = + filename:basename(<<"strange\\but\\true">>) + end, + ?line test_server:timetrap_cancel(Dog), + ok. + +basename_bin_2(Config) when is_list(Config) -> + ?line Dog = test_server:timetrap(test_server:seconds(10)), + ?line <<".">> = filename:basename(<<".">>, <<".erl">>), + ?line <<"foo">> = filename:basename(<<"foo.erl">>, <<".erl">>), + ?line <<"foo.erl">> = filename:basename(<<"/usr/foo.erl">>, <<".hrl">>), + ?line <<"foo.erl">> = filename:basename(<<"/usr.hrl/foo.erl">>, <<".hrl">>), + ?line <<"foo">> = filename:basename(<<"/usr.hrl/foo">>, <<".hrl">>), + ?line <<"foo">> = filename:basename(<<"usr/foo/">>, <<".erl">>), + ?line <<"foo.erl">> = filename:basename(<<"usr/foo.erl/">>, <<".erl">>), + ?line case os:type() of + {win32, _} -> + ?line <<"foo">> = filename:basename(<<"A:foo">>, <<".erl">>), + ?line <<"foo.erl">> = filename:basename(<<"a:\\usr\\foo.erl">>, + <<".hrl">>), + ?line <<"foo.erl">> = filename:basename(<<"c:\\usr.hrl\\foo.erl">>, + <<".hrl">>), + ?line <<"foo">> = filename:basename(<<"A:\\usr\\foo">>, <<".hrl">>); + {unix, _} -> + ?line <<"strange\\but\\true">> = + filename:basename(<<"strange\\but\\true.erl">>, <<".erl">>), + ?line <<"strange\\but\\true">> = + filename:basename(<<"strange\\but\\true">>, <<".erl">>) + end, + ?line test_server:timetrap_cancel(Dog), + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +dirname_bin(Config) when is_list(Config) -> + case os:type() of + {win32,_} -> + ?line <<"A:/usr">> = filename:dirname(<<"A:/usr/foo.erl">>), + ?line <<"A:usr">> = filename:dirname(<<"A:usr/foo.erl">>), + ?line <<"/usr">> = filename:dirname(<<"\\usr\\foo.erl">>), + ?line <<"/">> = filename:dirname(<<"\\usr">>), + ?line <<"A:">> = filename:dirname(<<"A:">>); + vxworks -> + ?line <<"net:/usr">> = filename:dirname(<<"net:/usr/foo.erl">>), + ?line <<"/disk0:/usr">> = filename:dirname(<<"/disk0:/usr/foo.erl">>), + ?line <<"/usr">> = filename:dirname(<<"\\usr\\foo.erl">>), + ?line <<"/usr">> = filename:dirname(<<"\\usr">>), + ?line <<"net:">> = filename:dirname(<<"net:">>); + _ -> true + end, + ?line <<"usr">> = filename:dirname(<<"usr///foo.erl">>), + ?line <<".">> = filename:dirname(<<"foo.erl">>), + ?line <<".">> = filename:dirname(<<".">>), + case os:type() of + vxworks -> + ?line <<"/">> = filename:dirname(<<"/">>), + ?line <<"/usr">> = filename:dirname(<<"/usr">>); + _ -> + ?line <<"/">> = filename:dirname(<<"/">>), + ?line <<"/">> = filename:dirname(<<"/usr">>) + end, + ok. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +extension_bin(Config) when is_list(Config) -> + ?line <<".erl">> = filename:extension(<<"A:/usr/foo.erl">>), + ?line <<".erl">> = filename:extension(<<"A:/usr/foo.nisse.erl">>), + ?line <<".erl">> = filename:extension(<<"A:/usr.bar/foo.nisse.erl">>), + ?line <<"">> = filename:extension(<<"A:/usr.bar/foo">>), + ?line <<"">> = filename:extension(<<"A:/usr/foo">>), + ?line case os:type() of + {win32, _} -> + ?line <<"">> = filename:extension(<<"A:\\usr\\foo">>), + ?line <<".erl">> = + filename:extension(<<"A:/usr.bar/foo.nisse.erl">>), + ?line <<"">> = filename:extension(<<"A:/usr.bar/foo">>), + ok; + vxworks -> + ?line <<"">> = filename:extension(<<"/disk0:\\usr\\foo">>), + ?line <<".erl">> = + filename:extension(<<"net:/usr.bar/foo.nisse.erl">>), + ?line <<"">> = filename:extension(<<"net:/usr.bar/foo">>), + ok; + _ -> ok + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +join_bin(Config) when is_list(Config) -> + ?line <<"/">> = filename:join([<<"/">>]), + ?line <<"/">> = filename:join([<<"//">>]), + ?line <<"usr/foo.erl">> = filename:join(<<"usr">>,<<"foo.erl">>), + ?line <<"/src/foo.erl">> = filename:join(usr, <<"/src/foo.erl">>), + ?line <<"/src/foo.erl">> = filename:join([<<"/src/">>,'foo.erl']), + ?line <<"/src/foo.erl">> = filename:join(<<"usr">>, ["/sr", 'c/foo.erl']), + ?line <<"/src/foo.erl">> = filename:join(<<"usr">>, <<"/src/foo.erl">>), + + %% Make sure that redundant slashes work too. + ?line <<"a/b/c/d/e/f/g">> = filename:join([<<"a//b/c/////d//e/f/g">>]), + ?line <<"a/b/c/d/e/f/g">> = filename:join([<<"a//b/c/">>, <<"d//e/f/g">>]), + ?line <<"a/b/c/d/e/f/g">> = filename:join([<<"a//b/c">>, <<"d//e/f/g">>]), + ?line <<"/d/e/f/g">> = filename:join([<<"a//b/c">>, <<"/d//e/f/g">>]), + ?line <<"/d/e/f/g">> = filename:join([<<"a//b/c">>, <<"//d//e/f/g">>]), + + ?line <<"foo/bar">> = filename:join([$f,$o,$o,$/,[]], <<"bar">>), + + ?line case os:type() of + {win32, _} -> + ?line <<"d:/">> = filename:join([<<"D:/">>]), + ?line <<"d:/">> = filename:join([<<"D:\\">>]), + ?line <<"d:/abc">> = filename:join([<<"D:/">>, <<"abc">>]), + ?line <<"d:abc">> = filename:join([<<"D:">>, <<"abc">>]), + ?line <<"a/b/c/d/e/f/g">> = + filename:join([<<"a//b\\c//\\/\\d/\\e/f\\g">>]), + ?line <<"a:usr/foo.erl">> = + filename:join([<<"A:">>,<<"usr">>,<<"foo.erl">>]), + ?line <<"/usr/foo.erl">> = + filename:join([<<"A:">>,<<"/usr">>,<<"foo.erl">>]), + ?line <<"c:usr">> = filename:join(<<"A:">>,<<"C:usr">>), + ?line <<"a:usr">> = filename:join(<<"A:">>,<<"usr">>), + ?line <<"c:/usr">> = filename:join(<<"A:">>, <<"C:/usr">>), + ?line <<"c:/usr/foo.erl">> = + filename:join([<<"A:">>,<<"C:/usr">>,<<"foo.erl">>]), + ?line <<"c:usr/foo.erl">> = + filename:join([<<"A:">>,<<"C:usr">>,<<"foo.erl">>]), + ?line <<"d:/foo">> = filename:join([$D, $:, $/, []], <<"foo">>), + ok; + {unix, _} -> + ok + end. + +pathtype_bin(Config) when is_list(Config) -> + ?line relative = filename:pathtype(<<"..">>), + ?line relative = filename:pathtype(<<"foo">>), + ?line relative = filename:pathtype(<<"foo/bar">>), + ?line relative = filename:pathtype('foo/bar'), + case os:type() of + {win32, _} -> + ?line volumerelative = filename:pathtype(<<"/usr/local/bin">>), + ?line volumerelative = filename:pathtype(<<"A:usr/local/bin">>), + ok; + {unix, _} -> + ?line absolute = filename:pathtype(<<"/">>), + ?line absolute = filename:pathtype(<<"/usr/local/bin">>), + ok + end. + +rootname_bin(Config) when is_list(Config) -> + ?line <<"/jam.src/kalle">> = filename:rootname(<<"/jam.src/kalle">>), + ?line <<"/jam.src/foo">> = filename:rootname(<<"/jam.src/foo.erl">>), + ?line <<"/jam.src/foo">> = filename:rootname(<<"/jam.src/foo.erl">>, <<".erl">>), + ?line <<"/jam.src/foo.jam">> = filename:rootname(<<"/jam.src/foo.jam">>, <<".erl">>), + ?line <<"/jam.src/foo.jam">> = filename:rootname(["/jam.sr",'c/foo.j',"am"],<<".erl">>), + ?line <<"/jam.src/foo.jam">> = filename:rootname(["/jam.sr",'c/foo.j'|am],<<".erl">>), + ok. + +split_bin(Config) when is_list(Config) -> + case os:type() of + vxworks -> + ?line [<<"/usr">>,<<"local">>,<<"bin">>] = filename:split(<<"/usr/local/bin">>); + _ -> + ?line [<<"/">>,<<"usr">>,<<"local">>,<<"bin">>] = filename:split(<<"/usr/local/bin">>) + end, + ?line [<<"foo">>,<<"bar">>]= filename:split(<<"foo/bar">>), + ?line [<<"foo">>, <<"bar">>, <<"hello">>]= filename:split(<<"foo////bar//hello">>), + case os:type() of + {win32,_} -> + ?line [<<"a:/">>,<<"msdev">>,<<"include">>] = + filename:split(<<"a:/msdev/include">>), + ?line [<<"a:/">>,<<"msdev">>,<<"include">>] = + filename:split(<<"A:/msdev/include">>), + ?line [<<"msdev">>,<<"include">>] = + filename:split(<<"msdev\\include">>), + ?line [<<"a:/">>,<<"msdev">>,<<"include">>] = + filename:split(<<"a:\\msdev\\include">>), + ?line [<<"a:">>,<<"msdev">>,<<"include">>] = + filename:split(<<"a:msdev\\include">>), + ok; + _ -> + ok + end. + -- cgit v1.2.3