From c48348233c467f7dc96728a31710abbb12cbe1f1 Mon Sep 17 00:00:00 2001 From: Filipe David Manana Date: Tue, 28 Dec 2010 10:57:04 +0000 Subject: Add file:allocate/3 operation This operation allows pre-allocation of space for files. It succeeds only on systems that support such operation. The POSIX standard defines the optional system call posix_fallocate() to implement this feature. However, some systems implement more specific functions to accomplish the same operation. On Linux, if the more specific function fallocate() is implemented, it is used instead of posix_fallocate(), falling back to posix_fallocate() if the fallocate() call failed (it's only supported for the ext4, ocfs2, xfs and btrfs file systems at the moment). On Mac OS X it uses the specific fcntl() operation F_PREALLOCATE, falling back to posix_fallocate() if it's available (at the moment Mac OS X doesn't provide posix_fallocate()). On any other UNIX system, it uses posix_fallocate() if it's available. Any other system not providing this system call or any function to pre-allocate space for files, this operation always fails with the ENOTSUP POSIX error. --- lib/kernel/doc/src/file.xml | 12 ++++++ lib/kernel/src/file.erl | 14 ++++++- lib/kernel/src/file_io_server.erl | 4 ++ lib/kernel/src/ram_file.erl | 7 ++++ lib/kernel/test/file_SUITE.erl | 72 +++++++++++++++++++++++++++++++++++- lib/kernel/test/prim_file_SUITE.erl | 74 ++++++++++++++++++++++++++++++++++++- 6 files changed, 180 insertions(+), 3 deletions(-) (limited to 'lib/kernel') diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 536b98b5f5..e30ade1bd2 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -169,6 +169,18 @@

On some platforms, this function might have no effect.

+ + + Allocate file space + +

allocate/3 can be used to preallocate space for a file.

+

This function only succeeds in platforms that implement this + feature. When it succeeds, space is preallocated for the file but + the file size might not be updated. This behaviour depends on the + preallocation implementation. To guarantee the file size is updated + one must truncate the file to the new size.

+ + Change group of a file diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index de3eaad5a1..16f2dde464 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -38,7 +38,7 @@ %% Specialized -export([ipread_s32bu_p32bu/3]). %% Generic file contents. --export([open/2, close/1, advise/4, +-export([open/2, close/1, advise/4, allocate/3, read/2, write/2, pread/2, pread/3, pwrite/2, pwrite/3, read_line/1, @@ -490,6 +490,18 @@ advise(#file_descriptor{module = Module} = Handle, Offset, Length, Advise) -> advise(_, _, _, _) -> {error, badarg}. +-spec allocate(File, Offset, Length) -> + 'ok' | {'error', posix()} when + File :: io_device(), + Offset :: non_neg_integer(), + Length :: non_neg_integer(). + +allocate(File, Offset, Length) when is_pid(File) -> + R = file_request(File, {allocate, Offset, Length}), + wait_file_reply(File, R); +allocate(#file_descriptor{module = Module} = Handle, Offset, Length) -> + Module:allocate(Handle, Offset, Length). + -spec read(IoDevice, Number) -> {ok, Data} | eof | {error, Reason} when IoDevice :: io_device() | atom(), Number :: non_neg_integer(), diff --git a/lib/kernel/src/file_io_server.erl b/lib/kernel/src/file_io_server.erl index acaffe1e41..fad2ed7fb3 100644 --- a/lib/kernel/src/file_io_server.erl +++ b/lib/kernel/src/file_io_server.erl @@ -211,6 +211,10 @@ file_request({advise,Offset,Length,Advise}, Reply -> {reply,Reply,State} end; +file_request({allocate, Offset, Length}, + #state{handle = Handle} = State) -> + Reply = ?PRIM_FILE:allocate(Handle, Offset, Length), + {reply, Reply, State}; file_request({pread,At,Sz}, #state{handle=Handle,buf=Buf,read_mode=ReadMode}=State) -> case position(Handle, At, Buf) of diff --git a/lib/kernel/src/ram_file.erl b/lib/kernel/src/ram_file.erl index 48ea871433..ca881ff8a4 100644 --- a/lib/kernel/src/ram_file.erl +++ b/lib/kernel/src/ram_file.erl @@ -29,6 +29,7 @@ %% Specialized file operations -export([get_size/1, get_file/1, set_file/2, get_file_close/1]). -export([compress/1, uncompress/1, uuencode/1, uudecode/1, advise/4]). +-export([allocate/3]). -export([open_mode/1]). %% used by ftp-file @@ -72,6 +73,7 @@ -define(RAM_FILE_UUDECODE, 36). -define(RAM_FILE_SIZE, 37). -define(RAM_FILE_ADVISE, 38). +-define(RAM_FILE_ALLOCATE, 39). %% Open modes for RAM_FILE_OPEN -define(RAM_FILE_MODE_READ, 1). @@ -383,6 +385,11 @@ advise(#file_descriptor{module = ?MODULE, data = Port}, Offset, advise(#file_descriptor{}, _Offset, _Length, _Advise) -> {error, enotsup}. +allocate(#file_descriptor{module = ?MODULE, data = Port}, Offset, Length) -> + call_port(Port, <>); +allocate(#file_descriptor{}, _Offset, _Length) -> + {error, enotsup}. + %%%----------------------------------------------------------------- diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 9c507fd437..914f0d6127 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -84,6 +84,8 @@ -export([advise/1]). +-export([allocate/1]). + -export([standard_io/1,mini_server/1]). %% Debug exports @@ -116,7 +118,7 @@ groups() -> {files, [], [{group, open}, {group, pos}, {group, file_info}, {group, consult}, {group, eval}, {group, script}, - truncate, sync, datasync, advise]}, + truncate, sync, datasync, advise, allocate]}, {open, [], [open1, old_modes, new_modes, path_open, close, access, read_write, pread_write, append, open_errors, @@ -1617,6 +1619,74 @@ advise(Config) when is_list(Config) -> ?line test_server:timetrap_cancel(Dog), ok. +allocate(suite) -> []; +allocate(doc) -> "Tests that ?FILE_MODULE:allocate/3 at least doesn't crash."; +allocate(Config) when is_list(Config) -> + ?line Dog = test_server:timetrap(test_server:seconds(5)), + ?line PrivDir = ?config(priv_dir, Config), + ?line Allocate = filename:join(PrivDir, + atom_to_list(?MODULE) + ++"_allocate.fil"), + + Line1 = "Hello\n", + Line2 = "World!\n", + + ?line {ok, Fd} = ?FILE_MODULE:open(Allocate, [write, binary]), + allocate_and_assert(Fd, 1, iolist_size([Line1, Line2])), + ?line ok = io:format(Fd, "~s", [Line1]), + ?line ok = io:format(Fd, "~s", [Line2]), + ?line ok = ?FILE_MODULE:close(Fd), + + ?line {ok, Fd2} = ?FILE_MODULE:open(Allocate, [write, binary]), + allocate_and_assert(Fd2, 1, iolist_size(Line1)), + ?line ok = io:format(Fd2, "~s", [Line1]), + ?line ok = io:format(Fd2, "~s", [Line2]), + ?line ok = ?FILE_MODULE:close(Fd2), + + ?line {ok, Fd3} = ?FILE_MODULE:open(Allocate, [write, binary]), + allocate_and_assert(Fd3, 1, iolist_size(Line1) + 1), + ?line ok = io:format(Fd3, "~s", [Line1]), + ?line ok = io:format(Fd3, "~s", [Line2]), + ?line ok = ?FILE_MODULE:close(Fd3), + + ?line {ok, Fd4} = ?FILE_MODULE:open(Allocate, [write, binary]), + allocate_and_assert(Fd4, 1, 4 * iolist_size([Line1, Line2])), + ?line ok = io:format(Fd4, "~s", [Line1]), + ?line ok = io:format(Fd4, "~s", [Line2]), + ?line ok = ?FILE_MODULE:close(Fd4), + + ?line [] = flush(), + ?line test_server:timetrap_cancel(Dog), + ok. + +allocate_and_assert(Fd, Offset, Length) -> + % Just verify that calls to ?PRIM_FILE:allocate/3 don't crash or have + % any other negative side effect. We can't really asssert against a + % specific return value, because support for file space pre-allocation + % depends on the OS, OS version and underlying filesystem. + % + % The Linux kernel added support for fallocate() in version 2.6.23, + % which currently works only for the ext4, ocfs2, xfs and btrfs file + % systems. posix_fallocate() is available in glibc as of version + % 2.1.94, but it was buggy until glibc version 2.7. + % + % Mac OS X, as of version 10.3, supports the fcntl operation F_PREALLOCATE. + % + % Solaris supports posix_fallocate() but only for the UFS file system + % apparently (not supported for ZFS). + % + % FreeBSD 9.0 is the first FreeBSD release supporting posix_fallocate(). + % + % For Windows there's apparently no way to pre-allocate file space, at + % least with same semantics as posix_fallocate(), fallocate() and + % fcntl F_PREALLOCATE. + Result = ?FILE_MODULE:allocate(Fd, Offset, Length), + case os:type() of + {win32, _} -> + ?line {error, enotsup} = Result; + _ -> + ?line _ = Result + end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/kernel/test/prim_file_SUITE.erl b/lib/kernel/test/prim_file_SUITE.erl index a56746bbc4..4e93a593b3 100644 --- a/lib/kernel/test/prim_file_SUITE.erl +++ b/lib/kernel/test/prim_file_SUITE.erl @@ -57,6 +57,8 @@ %% System probe functions that might be handy to check from the shell -export([unix_free/1]). +-export([allocate/1]). + -include_lib("test_server/include/test_server.hrl"). -include_lib("kernel/include/file.hrl"). @@ -87,7 +89,7 @@ groups() -> cur_dir_1a, cur_dir_1b]}, {files, [], [{group, open}, {group, pos}, {group, file_info}, - truncate, sync, datasync, advise, large_write]}, + truncate, sync, datasync, advise, large_write, allocate]}, {open, [], [open1, modes, close, access, read_write, pread_write, append, exclusive]}, @@ -1359,6 +1361,76 @@ check_large_write(Dog, Fd, _, _, []) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +allocate(suite) -> []; +allocate(doc) -> "Tests that ?PRIM_FILE:allocate/3 at least doesn't crash."; +allocate(Config) when is_list(Config) -> + ?line Dog = test_server:timetrap(test_server:seconds(5)), + ?line PrivDir = ?config(priv_dir, Config), + ?line Allocate = filename:join(PrivDir, + atom_to_list(?MODULE) + ++"_allocate.fil"), + + Line1 = "Hello\n", + Line2 = "World!\n", + + ?line {ok, Fd} = ?PRIM_FILE:open(Allocate, [write, binary]), + allocate_and_assert(Fd, 1, iolist_size([Line1, Line2])), + ?line ok = ?PRIM_FILE:write(Fd, Line1), + ?line ok = ?PRIM_FILE:write(Fd, Line2), + ?line ok = ?PRIM_FILE:close(Fd), + + ?line {ok, Fd2} = ?PRIM_FILE:open(Allocate, [write, binary]), + allocate_and_assert(Fd2, 1, iolist_size(Line1)), + ?line ok = ?PRIM_FILE:write(Fd2, Line1), + ?line ok = ?PRIM_FILE:write(Fd2, Line2), + ?line ok = ?PRIM_FILE:close(Fd2), + + ?line {ok, Fd3} = ?PRIM_FILE:open(Allocate, [write, binary]), + allocate_and_assert(Fd3, 1, iolist_size(Line1) + 1), + ?line ok = ?PRIM_FILE:write(Fd3, Line1), + ?line ok = ?PRIM_FILE:write(Fd3, Line2), + ?line ok = ?PRIM_FILE:close(Fd3), + + ?line {ok, Fd4} = ?PRIM_FILE:open(Allocate, [write, binary]), + allocate_and_assert(Fd4, 1, 4 * iolist_size([Line1, Line2])), + ?line ok = ?PRIM_FILE:write(Fd4, Line1), + ?line ok = ?PRIM_FILE:write(Fd4, Line2), + ?line ok = ?PRIM_FILE:close(Fd4), + + ?line test_server:timetrap_cancel(Dog), + ok. + +allocate_and_assert(Fd, Offset, Length) -> + % Just verify that calls to ?PRIM_FILE:allocate/3 don't crash or have + % any other negative side effect. We can't really asssert against a + % specific return value, because support for file space pre-allocation + % depends on the OS, OS version and underlying filesystem. + % + % The Linux kernel added support for fallocate() in version 2.6.23, + % which currently works only for the ext4, ocfs2, xfs and btrfs file + % systems. posix_fallocate() is available in glibc as of version + % 2.1.94, but it was buggy until glibc version 2.7. + % + % Mac OS X, as of version 10.3, supports the fcntl operation F_PREALLOCATE. + % + % Solaris supports posix_fallocate() but only for the UFS file system + % apparently (not supported for ZFS). + % + % FreeBSD 9.0 is the first FreeBSD release supporting posix_fallocate(). + % + % For Windows there's apparently no way to pre-allocate file space, at + % least with similar API/semantics as posix_fallocate(), fallocate() or + % fcntl F_PREALLOCATE. + Result = ?PRIM_FILE:allocate(Fd, Offset, Length), + case os:type() of + {win32, _} -> + ?line {error, enotsup} = Result; + _ -> + ?line _ = Result + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + delete_a(suite) -> []; delete_a(doc) -> []; delete_a(Config) when is_list(Config) -> -- cgit v1.2.3