From 80b231a0874aa5cd68c3d9f0dc7e13b6736a5dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Wed, 19 May 2010 15:21:15 +0200 Subject: sys.h: Correct the get_int64() macro The get_int64() macro has never been used (it was probably added for completeness at the same time that put_int64() was added), but it was not actually used and therefore it does not actually work. --- erts/emulator/beam/sys.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index a1955235b7..0f20d36167 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1173,14 +1173,14 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); /* Standard set of integer macros .. */ -#define get_int64(s) ((((unsigned char*) (s))[0] << 56) | \ - (((unsigned char*) (s))[1] << 48) | \ - (((unsigned char*) (s))[2] << 40) | \ - (((unsigned char*) (s))[3] << 32) | \ - (((unsigned char*) (s))[4] << 24) | \ - (((unsigned char*) (s))[5] << 16) | \ - (((unsigned char*) (s))[6] << 8) | \ - (((unsigned char*) (s))[7])) +#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \ + (((Uint64)((unsigned char*) (s))[1]) << 48) | \ + (((Uint64)((unsigned char*) (s))[2]) << 40) | \ + (((Uint64)((unsigned char*) (s))[3]) << 32) | \ + (((Uint64)((unsigned char*) (s))[4]) << 24) | \ + (((Uint64)((unsigned char*) (s))[5]) << 16) | \ + (((Uint64)((unsigned char*) (s))[6]) << 8) | \ + (((Uint64)((unsigned char*) (s))[7]))) #define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\ ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\ -- cgit v1.2.3 From 3f53a96a8bd0cd4a18f819b6857e6a764706ede5 Mon Sep 17 00:00:00 2001 From: Filipe David Manana Date: Thu, 22 Apr 2010 23:40:26 +0100 Subject: Add file:datasync/1 for syncing file contents only file:datasync/1 invokes the POSIX system call "int fdatasync(int fd)". This system call is similar to "fsync" but, unlike fsync, it does not update the metadata associated with the file (like the access time for example). It's used by many DBMSs (MySQL and SQLite of example) to increase disk IO performance, as it avoids disk seeks and disk write operations compared to fsync. More details on it at: http://linux.die.net/man/2/fdatasync An example, from the MySQL source: http://bazaar.launchpad.net/~mysql/mysql-server/mysql-5.1-telco-6.1/annotate/head%3A/mysys/my_sync.c#L61 This new function just calls fsync on systems not implementing fdatasync. --- erts/configure.in | 6 ++++++ erts/emulator/drivers/common/efile_drv.c | 23 +++++++++++++++++++++++ erts/emulator/drivers/common/erl_efile.h | 1 + erts/emulator/drivers/common/ram_file_drv.c | 8 ++++++++ erts/emulator/drivers/unix/unix_efile.c | 11 +++++++++++ erts/emulator/drivers/win32/win_efile.c | 9 +++++++++ erts/preloaded/src/prim_file.erl | 6 +++++- 7 files changed, 63 insertions(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/configure.in b/erts/configure.in index a14b10adbf..f11aba1b15 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -1757,6 +1757,12 @@ fi dnl Need by run_erl. AC_CHECK_FUNCS([openpty]) +dnl fdatasync syscall (Unix only) +AC_CHECK_FUNCS([fdatasync]) + +dnl Find which C libraries are required to use fdatasync +AC_SEARCH_LIBS(fdatasync, [rt]) + dnl ---------------------------------------------------------------------- dnl Checks for features/quirks in the system that affects Erlang. dnl ---------------------------------------------------------------------- diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index d2b916000e..b64d27de68 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -53,6 +53,7 @@ #define FILE_IPREAD 27 #define FILE_ALTNAME 28 #define FILE_READ_LINE 29 +#define FILE_FDATASYNC 30 /* Return codes */ @@ -883,6 +884,15 @@ static void invoke_chdir(void *data) invoke_name(data, efile_chdir); } +static void invoke_fdatasync(void *data) +{ + struct t_data *d = (struct t_data *) data; + int fd = (int) d->fd; + + d->again = 0; + d->result_ok = efile_fdatasync(&d->errInfo, fd); +} + static void invoke_fsync(void *data) { struct t_data *d = (struct t_data *) data; @@ -1919,6 +1929,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) case FILE_RMDIR: case FILE_CHDIR: case FILE_DELETE: + case FILE_FDATASYNC: case FILE_FSYNC: case FILE_TRUNCATE: case FILE_LINK: @@ -2209,6 +2220,18 @@ file_output(ErlDrvData e, char* buf, int count) goto done; } + case FILE_FDATASYNC: + { + d = EF_SAFE_ALLOC(sizeof(struct t_data)); + + d->fd = fd; + d->command = command; + d->invoke = invoke_fdatasync; + d->free = free_data; + d->level = 2; + goto done; + } + case FILE_FSYNC: { d = EF_SAFE_ALLOC(sizeof(struct t_data)); diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index 9aa941e550..6821a0e2ee 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -126,6 +126,7 @@ int efile_readdir(Efile_error* errInfo, char* name, int efile_openfile(Efile_error* errInfo, char* name, int flags, int* pfd, Sint64* pSize); void efile_closefile(int fd); +int efile_fdatasync(Efile_error* errInfo, int fd); int efile_fsync(Efile_error* errInfo, int fd); int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, char *name, int info_for_link); diff --git a/erts/emulator/drivers/common/ram_file_drv.c b/erts/emulator/drivers/common/ram_file_drv.c index 4a39a156e6..d4e547ade6 100644 --- a/erts/emulator/drivers/common/ram_file_drv.c +++ b/erts/emulator/drivers/common/ram_file_drv.c @@ -35,6 +35,7 @@ #define RAM_FILE_TRUNCATE 14 #define RAM_FILE_PREAD 17 #define RAM_FILE_PWRITE 18 +#define RAM_FILE_FDATASYNC 19 /* other operations */ #define RAM_FILE_GET 30 @@ -558,6 +559,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count) numeric_reply(f, 0); /* 0 is not used */ break; + case RAM_FILE_FDATASYNC: + if (f->flags == 0) + error_reply(f, EBADF); + else + reply(f, 1, 0); + break; + case RAM_FILE_FSYNC: if (f->flags == 0) error_reply(f, EBADF); diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 1d094ee613..0cece76221 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -773,6 +773,17 @@ efile_closefile(int fd) close(fd); } +int +efile_fdatasync(Efile_error *errInfo, /* Where to return error codes. */ + int fd) /* File descriptor for file to sync data. */ +{ +#ifdef HAVE_FDATASYNC + return check_error(fdatasync(fd), errInfo); +#else + return efile_fsync(errInfo, fd); +#endif +} + int efile_fsync(Efile_error *errInfo, /* Where to return error codes. */ int fd) /* File descriptor for file to sync. */ diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 89aaad31da..cdf3bb424c 100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -763,6 +763,15 @@ int fd; /* File descriptor for file to close. */ CloseHandle((HANDLE) fd); } +int +efile_fdatasync(errInfo, fd) +Efile_error* errInfo; /* Where to return error codes. */ +int fd; /* File descriptor for file to sync. */ +{ + /* Not available in Windows, just call regular fsync */ + return efile_fsync(errInfo, fd); +} + int efile_fsync(errInfo, fd) Efile_error* errInfo; /* Where to return error codes. */ diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 43e6f6cd88..53c741e66e 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -25,7 +25,7 @@ %%% Interface towards a single file's contents. Uses ?FD_DRV. %% Generic file contents operations --export([open/2, close/1, sync/1, position/2, truncate/1, +-export([open/2, close/1, datasync/1, sync/1, position/2, truncate/1, write/2, pwrite/2, pwrite/3, read/2, read_line/1, pread/2, pread/3, copy/3]). %% Specialized file operations @@ -96,6 +96,7 @@ -define(FILE_IPREAD, 27). -define(FILE_ALTNAME, 28). -define(FILE_READ_LINE, 29). +-define(FILE_FDATASYNC, 30). %% Driver responses -define(FILE_RESP_OK, 0). @@ -292,6 +293,9 @@ pwrite(#file_descriptor{module = ?MODULE}, _, _) -> {error, badarg}. +%% Returns {error, Reason} | ok. +datasync(#file_descriptor{module = ?MODULE, data = {Port, _}}) -> + drv_command(Port, [?FILE_FDATASYNC]). %% Returns {error, Reason} | ok. sync(#file_descriptor{module = ?MODULE, data = {Port, _}}) -> -- cgit v1.2.3 From 21a67b797e40df930b83bd407ffc165b3f4b91b2 Mon Sep 17 00:00:00 2001 From: Filipe David Manana Date: Mon, 26 Apr 2010 13:04:40 +0200 Subject: Add file:advise/4 - a wrapper to the POSIX syscall posix_fadvise Useful for informing the Operating System about the access pattern for a file's data, so that it can adapt the caching strategy to maximize disk IO performance. --- erts/configure.in | 1 + erts/emulator/drivers/common/efile_drv.c | 34 ++++++++++++++++++++++++++ erts/emulator/drivers/common/erl_efile.h | 2 ++ erts/emulator/drivers/common/ram_file_drv.c | 9 +++++++ erts/emulator/drivers/unix/unix_efile.c | 11 +++++++++ erts/emulator/drivers/win32/win_efile.c | 9 +++++++ erts/preloaded/src/prim_file.erl | 38 ++++++++++++++++++++++++++++- 7 files changed, 103 insertions(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/configure.in b/erts/configure.in index f11aba1b15..63bf548c89 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -1054,6 +1054,7 @@ fi AC_SUBST(ERTS_BUILD_SMP_EMU) +AC_CHECK_FUNCS([posix_fadvise]) # diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index b64d27de68..60ae4cb108 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -54,6 +54,7 @@ #define FILE_ALTNAME 28 #define FILE_READ_LINE 29 #define FILE_FDATASYNC 30 +#define FILE_FADVISE 31 /* Return codes */ @@ -358,6 +359,11 @@ struct t_data struct t_readdir_buf *first_buf; struct t_readdir_buf *last_buf; } read_dir; + struct { + Sint64 offset; + Sint64 length; + int advise; + } fadvise; } c; char b[1]; }; @@ -1647,6 +1653,18 @@ static void invoke_open(void *data) d->result_ok = status; } +static void invoke_fadvise(void *data) +{ + struct t_data *d = (struct t_data *) data; + int fd = (int) d->fd; + off_t offset = (off_t) d->c.fadvise.offset; + off_t length = (off_t) d->c.fadvise.length; + int advise = (int) d->c.fadvise.advise; + + d->again = 0; + d->result_ok = efile_fadvise(&d->errInfo, fd, offset, length, advise); +} + static void free_readdir(void *data) { struct t_data *d = (struct t_data *) data; @@ -1936,6 +1954,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) case FILE_SYMLINK: case FILE_RENAME: case FILE_WRITE_INFO: + case FILE_FADVISE: reply(desc, d->result_ok, &d->errInfo); free_data(data); break; @@ -2355,6 +2374,21 @@ file_output(ErlDrvData e, char* buf, int count) goto done; } + case FILE_FADVISE: + { + d = EF_SAFE_ALLOC(sizeof(struct t_data)); + + d->fd = fd; + d->command = command; + d->invoke = invoke_fadvise; + d->free = free_data; + d->level = 2; + d->c.fadvise.offset = get_int64((uchar*) buf); + d->c.fadvise.length = get_int64(((uchar*) buf) + sizeof(Sint64)); + d->c.fadvise.advise = get_int32(((uchar*) buf) + 2 * sizeof(Sint64)); + goto done; + } + } /* diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index 6821a0e2ee..61684af6c9 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -151,3 +151,5 @@ int efile_altname(Efile_error* errInfo, char *name, int efile_link(Efile_error* errInfo, char* old, char* new); int efile_symlink(Efile_error* errInfo, char* old, char* new); int efile_may_openfile(Efile_error* errInfo, char *name); +int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length, + int advise); diff --git a/erts/emulator/drivers/common/ram_file_drv.c b/erts/emulator/drivers/common/ram_file_drv.c index d4e547ade6..abedcc933a 100644 --- a/erts/emulator/drivers/common/ram_file_drv.c +++ b/erts/emulator/drivers/common/ram_file_drv.c @@ -46,6 +46,8 @@ #define RAM_FILE_UUENCODE 35 /* uuencode file */ #define RAM_FILE_UUDECODE 36 /* uudecode file */ #define RAM_FILE_SIZE 37 /* get file size */ +#define RAM_FILE_ADVISE 38 /* predeclare the access + * pattern for file data */ /* possible new operations include: DES_ENCRYPT DES_DECRYPT @@ -693,6 +695,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count) case RAM_FILE_UUDECODE: /* uudecode file */ ram_file_uudecode(f); break; + + case RAM_FILE_ADVISE: + if (f->flags == 0) + error_reply(f, EBADF); + else + reply(f, 1, 0); + break; } /* * Ignore anything else -- let the caller hang. diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 0cece76221..ea016526ef 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -1448,3 +1448,14 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) return check_error(symlink(old, new), errInfo); #endif } + +int +efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, + Sint64 length, int advise) +{ +#ifdef HAVE_POSIX_FADVISE + return check_error(posix_fadvise(fd, offset, length, advise), errInfo); +#else + return check_error(0, errInfo); +#endif +} diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index cdf3bb424c..06850b4945 100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1433,3 +1433,12 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) errno = ENOTSUP; return check_error(-1, errInfo); } + +int +efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, + Sint64 length, int advise) +{ + /* posix_fadvise is not available on Windows, do nothing */ + errno = ERROR_SUCCESS; + return check_error(0, errInfo); +} diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 53c741e66e..e9df5752ab 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -25,7 +25,7 @@ %%% Interface towards a single file's contents. Uses ?FD_DRV. %% Generic file contents operations --export([open/2, close/1, datasync/1, sync/1, position/2, truncate/1, +-export([open/2, close/1, datasync/1, sync/1, advise/4, position/2, truncate/1, write/2, pwrite/2, pwrite/3, read/2, read_line/1, pread/2, pread/3, copy/3]). %% Specialized file operations @@ -97,6 +97,7 @@ -define(FILE_ALTNAME, 28). -define(FILE_READ_LINE, 29). -define(FILE_FDATASYNC, 30). +-define(FILE_ADVISE, 31). %% Driver responses -define(FILE_RESP_OK, 0). @@ -131,6 +132,13 @@ %% IPREAD variants -define(IPREAD_S32BU_P32BU, 0). +%% POSIX file advises +-define(POSIX_FADV_NORMAL, 0). +-define(POSIX_FADV_RANDOM, 1). +-define(POSIX_FADV_SEQUENTIAL, 2). +-define(POSIX_FADV_WILLNEED, 3). +-define(POSIX_FADV_DONTNEED, 4). +-define(POSIX_FADV_NOREUSE, 5). %%%----------------------------------------------------------------- @@ -221,7 +229,35 @@ close(#file_descriptor{module = ?MODULE, data = {Port, _}}) -> close(Port) when is_port(Port) -> drv_close(Port). +-define(ADVISE(Offs, Len, Adv), + <>). +%% Returns {error, Reason} | ok. +advise(#file_descriptor{module = ?MODULE, data = {Port, _}}, + Offset, Length, Advise) -> + case Advise of + normal -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_NORMAL), + drv_command(Port, Cmd); + random -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_RANDOM), + drv_command(Port, Cmd); + sequential -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_SEQUENTIAL), + drv_command(Port, Cmd); + will_need -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_WILLNEED), + drv_command(Port, Cmd); + dont_need -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_DONTNEED), + drv_command(Port, Cmd); + no_reuse -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_NOREUSE), + drv_command(Port, Cmd); + _ -> + {error, einval} + end. %% Returns {error, Reason} | ok. write(#file_descriptor{module = ?MODULE, data = {Port, _}}, Bytes) -> -- cgit v1.2.3 From 329aff8ed81e94033c836e58bd294a68a8f7be8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Wed, 19 May 2010 16:31:34 +0200 Subject: Update preloaded modules --- erts/preloaded/ebin/erl_prim_loader.beam | Bin 50816 -> 50708 bytes erts/preloaded/ebin/erlang.beam | Bin 23808 -> 23788 bytes erts/preloaded/ebin/init.beam | Bin 44500 -> 44352 bytes erts/preloaded/ebin/otp_ring0.beam | Bin 1448 -> 1420 bytes erts/preloaded/ebin/prim_file.beam | Bin 29496 -> 30472 bytes erts/preloaded/ebin/prim_inet.beam | Bin 57320 -> 57288 bytes erts/preloaded/ebin/prim_zip.beam | Bin 22600 -> 22432 bytes erts/preloaded/ebin/zlib.beam | Bin 10620 -> 10596 bytes 8 files changed, 0 insertions(+), 0 deletions(-) (limited to 'erts') diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam index afd8a90b3f..fe3cee1c56 100644 Binary files a/erts/preloaded/ebin/erl_prim_loader.beam and b/erts/preloaded/ebin/erl_prim_loader.beam differ diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam index 4ec84948d8..5a4c5e9d1e 100644 Binary files a/erts/preloaded/ebin/erlang.beam and b/erts/preloaded/ebin/erlang.beam differ diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam index c3e746f3ee..cfe2c36cee 100644 Binary files a/erts/preloaded/ebin/init.beam and b/erts/preloaded/ebin/init.beam differ diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam index 4b2d8bb2de..74587de26b 100644 Binary files a/erts/preloaded/ebin/otp_ring0.beam and b/erts/preloaded/ebin/otp_ring0.beam differ diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam index 2916baaa77..c6610b71e6 100644 Binary files a/erts/preloaded/ebin/prim_file.beam and b/erts/preloaded/ebin/prim_file.beam differ diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam index 46912e2bea..8d19923281 100644 Binary files a/erts/preloaded/ebin/prim_inet.beam and b/erts/preloaded/ebin/prim_inet.beam differ diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam index ccf8aff6f6..cd41f36413 100644 Binary files a/erts/preloaded/ebin/prim_zip.beam and b/erts/preloaded/ebin/prim_zip.beam differ diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam index ccd597ba68..ce1163d260 100644 Binary files a/erts/preloaded/ebin/zlib.beam and b/erts/preloaded/ebin/zlib.beam differ -- cgit v1.2.3