diff options
193 files changed, 11858 insertions, 2374 deletions
diff --git a/HOWTO/INSTALL-ANDROID.md b/HOWTO/INSTALL-ANDROID.md new file mode 100644 index 0000000000..31698d4ce3 --- /dev/null +++ b/HOWTO/INSTALL-ANDROID.md @@ -0,0 +1,52 @@ +Cross Compiling Erlang/OTP - ANDROID +==================================== + +Introduction +------------ + +This document describes how to cross compile Erlang OTP to Android/Rasberry Pi platforms. + +### Download and Install Android NDK ### + +https://developer.android.com/tools/sdk/ndk/index.html + +### Define System Variables ### + +export NDK_ROOT=/usr/local/android +export NDK_PLAT=android-9 +export PATH=$NDK_ROOT/toolchains/arm-linux-androideabi-4.8/prebuilt/darwin-x86_64/bin:$PATH + +### Configure OTP ### + +./otp_build configure \ + --xcomp-conf=./xcomp/erl-xcomp-arm-android.conf \ + --without-ssl + +### Compile OTP ### + +make noboot [-j4] + +### Make Release ### + +./otp_build release -a /usr/local/otp_R16B03_arm + +### Target Deployment ### + +Make a tarball out of /usr/local/otp_R16B03_arm and copy it to target device +(e.g. Raspberry Pi). Extract it and install + +./Install /usr/local/otp_R16B03_arm + +Android SDK (adb tool) is used to deploy OTP/Erlang to target device for +evaluation purpose only. + +adb push /usr/local/otp_R16B03_arm /mnt/sdcard/otp_R16B03_arm +adb shell + +### Known Issues ### + + * native inet:gethostbyname/1 return {error, nxdomain} on Raspberry PI. Use dns resolver to by-pass the issue (see http://www.erlang.org/doc/apps/erts/inet_cfg.html) + +### References ### + + The port derives some solutions from https://code.google.com/p/erlang4android/ diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex f5800105d3..b57eef57ab 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex f5800105d3..b57eef57ab 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/configure.in b/configure.in index f6cccb4f4b..be906dcb4e 100644 --- a/configure.in +++ b/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1998-2012. All Rights Reserved. +dnl Copyright Ericsson AB 1998-2014. All Rights Reserved. dnl dnl The contents of this file are subject to the Erlang Public License, dnl Version 1.1, (the "License"); you may not use this file except in @@ -209,7 +209,7 @@ AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support])) AC_ARG_ENABLE(halfword-emulator, AS_HELP_STRING([--enable-halfword-emulator], - [enable halfword emulator (only for 64bit builds)])) + [enable halfword emulator (only for 64bit builds). Note: Halfword emulator is marked as deprecated and scheduled for removal in future major release.])) AC_ARG_ENABLE(smp-support, AS_HELP_STRING([--enable-smp-support], [enable smp support]) diff --git a/erts/configure.in b/erts/configure.in index 074882532f..208c294106 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. -*-m4-*- dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1997-2013. All Rights Reserved. +dnl Copyright Ericsson AB 1997-2014. All Rights Reserved. dnl dnl The contents of this file are subject to the Erlang Public License, dnl Version 1.1, (the "License"); you may not use this file except in @@ -144,7 +144,7 @@ AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support]), AC_ARG_ENABLE(halfword-emulator, AS_HELP_STRING([--enable-halfword-emulator], - [enable halfword emulator (only for 64bit builds)]), + [enable halfword emulator (only for 64bit builds). Note: Halfword emulator is marked as deprecated and scheduled for removal in future major release.]), [ case "$enableval" in no) enable_halfword_emualtor=no ;; *) enable_halfword_emulator=yes ;; @@ -816,6 +816,16 @@ if test "$enable_halfword_emulator" = "yes"; then [Define if building a halfword-heap 64bit emulator]) ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS halfword" AC_MSG_RESULT([yes]) + + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> $ERL_TOP/erts/CONF_INFO <<EOF + + The HALFWORD emulator has been enabled. + This is a DEPRECATED feature scheduled for removal + in a future major release. + +EOF else AC_MSG_ERROR(no; halfword emulator not supported on this architecture) fi @@ -1151,7 +1161,7 @@ fi AC_SUBST(ERTS_BUILD_SMP_EMU) -AC_CHECK_FUNCS([posix_fadvise]) +AC_CHECK_FUNCS([posix_fadvise closefrom]) AC_CHECK_HEADERS([linux/falloc.h]) dnl * Old glibcs have broken fallocate64(). Make sure not to use it. AC_CACHE_CHECK([whether fallocate() works],i_cv_fallocate_works,[ diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 4aa3033f40..9724a1345a 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -536,7 +536,7 @@ <tag><marker id="file_name_encoding"></marker><c><![CDATA[+fnl]]></c></tag> <item> <p>The VM works with file names as if they are encoded using the ISO-latin-1 encoding, disallowing Unicode characters with codepoints beyond 255.</p> - <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names. Note that this value also applies to command-line parameters and environment variables (see <seealso marker="stdlib:unicode_usage#unicode_in_environment_and_parameters">STDLIB User's Guide</seealso>).</p> </item> <tag><c><![CDATA[+fnu[{w|i|e}]]]></c></tag> <item> @@ -555,7 +555,7 @@ encountered. <c>w</c> is the default. Note that <c>file:read_link/1</c> will always return an error if the link points to an invalid file name.</p> - <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names. Note that this value also applies to command-line parameters and environment variables (see <seealso marker="stdlib:unicode_usage#unicode_in_environment_and_parameters">STDLIB User's Guide</seealso>).</p> </item> <tag><c><![CDATA[+fna[{w|i|e}]]]></c></tag> <item> @@ -572,7 +572,7 @@ settings cause the behavior of <c>+fnl</c> to be selected, then <c>w</c>, <c>i</c>, or <c>e</c> will not have any effect.</p> - <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names. Note that this value also applies to command-line parameters and environment variables (see <seealso marker="stdlib:unicode_usage#unicode_in_environment_and_parameters">STDLIB User's Guide</seealso>).</p> </item> <tag><c><![CDATA[+hms Size]]></c></tag> <item> diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index 8da1836da7..ad37813ac0 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -315,10 +315,13 @@ <c>ERL_DRV_EXTENDED_MINOR_VERSION</c> will be incremented when new features are added. The runtime system uses the minor version of the driver to determine what features to use. - The runtime system will refuse to load a driver if the major + The runtime system will normally refuse to load a driver if the major versions differ, or if the major versions are equal and the minor version used by the driver is greater than the one used - by the runtime system.</p> + by the runtime system. Old drivers with lower major versions + will however be allowed after a bump of the major version during + a transition period of two major releases. Such old drivers might + however fail if deprecated features are used.</p> <p>The emulator will refuse to load a driver that does not use the extended driver interface, to allow for 64-bit capable drivers, diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 8b19725c02..6b1f4cccf8 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -316,6 +316,32 @@ ok <p>The library initialization callbacks <c>load</c>, <c>reload</c> and <c>upgrade</c> are all thread-safe even for shared state data.</p> </item> + + <tag><marker id="version_management"/>Version Management</tag> + <item><p> + When a NIF library is built, information about NIF API version + is compiled into the library. When a NIF library is loaded the + runtime system verifies that the library is of a compatible version. + <c>erl_nif.h</c> defines <c>ERL_NIF_MAJOR_VERSION</c>, and + <c>ERL_NIF_MINOR_VERSION</c>. <c>ERL_NIF_MAJOR_VERSION</c> will be + incremented when NIF library incompatible changes are made to the + Erlang runtime system. Normally it will suffice to recompile the NIF + library when the <c>ERL_NIF_MAJOR_VERSION</c> has changed, but it + could, under rare circumstances, mean that NIF libraries have to + be slightly modified. If so, this will of course be documented. + <c>ERL_NIF_MINOR_VERSION</c> will be incremented when + new features are added. The runtime system uses the minor version + to determine what features to use. + </p><p> + The runtime system will normally refuse to load a NIF library if + the major versions differ, or if the major versions are equal and + the minor version used by the NIF library is greater than the one + used by the runtime system. Old NIF libraries with lower major + versions will however be allowed after a bump of the major version + during a transition period of two major releases. Such old NIF + libraries might however fail if deprecated features are used. + </p></item> + <tag>Dirty NIFs</tag> <item><p><marker id="dirty_nifs"/><em>Note that the dirty NIF functionality is experimental</em> and that you have to enable support for dirty diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 2a9a0a5c2f..58e77ed1fa 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -378,7 +378,9 @@ LIBS += -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) endif # erts_internal_r +ifneq ($(TARGET),arm-unknown-linux-androideabi) LIBS += @LIBRT@ +endif LIBS += @LIBCARBON@ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 0cec9ea3ec..9634faff1d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1301,7 +1301,7 @@ void process_main(void) (Eterm)fptr[1], (Uint)fptr[2], NULL, fun_buf); } else { - erts_snprintf(fun_buf, sizeof(fun_buf), + erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), "<unknown/%p>", next); } } diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 3a987e213b..50548850eb 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -48,7 +48,8 @@ copy_object(Eterm obj, Process* to) if (DTRACE_ENABLED(copy_object)) { DTRACE_CHARBUF(proc_name, 64); - erts_snprintf(proc_name, sizeof(proc_name), "%T", to->common.id); + erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), + "%T", to->common.id); DTRACE2(copy_object, proc_name, size); } #endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 6ecf3f0722..ec07ddcd9c 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -851,9 +851,12 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) #ifdef USE_VM_PROBES *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(receiver_name), "%T", remote); + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), + "%T", remote); msize = size_object(message); if (token != NIL && token != am_have_dt_utag) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); @@ -908,9 +911,11 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) #ifdef USE_VM_PROBES *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(receiver_name), + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "{%T,%s}", remote_name, node_name); msize = size_object(message); if (token != NIL && token != am_have_dt_utag) { @@ -971,11 +976,14 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, #ifdef USE_VM_PROBES *node_name = *sender_name = *remote_name = '\0'; if (DTRACE_ENABLED(process_exit_signal_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(remote_name, sizeof(remote_name), + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(remote_name, sizeof(DTRACE_CHARBUF_NAME(remote_name)), "{%T,%s}", remote, node_name); - erts_snprintf(reason_str, sizeof(reason), "%T", reason); + erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), + "%T", reason); if (token != NIL && token != am_have_dt_utag) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); @@ -1797,8 +1805,9 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", cid); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", cid); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", dep->sysname); DTRACE3(dist_port_not_busy, erts_this_node_sysname, port_str, remote_str); @@ -1855,9 +1864,11 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) DTRACE_CHARBUF(remote_str, 64); DTRACE_CHARBUF(pid_str, 16); - erts_snprintf(port_str, sizeof(port_str), "%T", cid); - erts_snprintf(remote_str, sizeof(remote_str), "%T", dep->sysname); - erts_snprintf(pid_str, sizeof(pid_str), "%T", c_p->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", cid); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), + "%T", dep->sysname); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), + "%T", c_p->common.id); DTRACE4(dist_port_busy, erts_this_node_sysname, port_str, remote_str, pid_str); } @@ -1890,8 +1901,9 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE4(dist_output, erts_this_node_sysname, port_str, remote_str, size); @@ -1944,8 +1956,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE4(dist_outputv, erts_this_node_sysname, port_str, remote_str, size); @@ -2280,8 +2293,9 @@ erts_dist_port_not_busy(Port *prt) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE3(dist_port_not_busy, erts_this_node_sysname, port_str, remote_str); @@ -3246,10 +3260,10 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas DTRACE_CHARBUF(type_str, 12); DTRACE_CHARBUF(reason_str, 64); - erts_snprintf(what_str, sizeof(what_str), "%T", what); - erts_snprintf(node_str, sizeof(node_str), "%T", node); - erts_snprintf(type_str, sizeof(type_str), "%T", type); - erts_snprintf(reason_str, sizeof(reason_str), "%T", reason); + erts_snprintf(what_str, sizeof(DTRACE_CHARBUF_NAME(what_str)), "%T", what); + erts_snprintf(node_str, sizeof(DTRACE_CHARBUF_NAME(node_str)), "%T", node); + erts_snprintf(type_str, sizeof(DTRACE_CHARBUF_NAME(type_str)), "%T", type); + erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T", reason); DTRACE5(dist_monitor, erts_this_node_sysname, what_str, node_str, type_str, reason_str); } diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index b3dc327704..decae6b2ca 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -292,7 +292,8 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) if (DTRACE_ENABLED(aio_pool_add)) { DTRACE_CHARBUF(port_str, 16); - erts_snprintf(port_str, sizeof(port_str), "%T", a->port); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", a->port); /* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */ len = -1; DTRACE2(aio_pool_add, port_str, len); @@ -327,7 +328,8 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, if (DTRACE_ENABLED(aio_pool_get)) { DTRACE_CHARBUF(port_str, 16); - erts_snprintf(port_str, sizeof(port_str), "%T", a->port); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", a->port); /* DTRACE TODO: Get the length from erts_thr_q_dequeue() ? */ len = -1; DTRACE2(aio_pool_get, port_str, len); diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 1728b200f7..56cd2ba04f 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1548,8 +1548,10 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) switch (dp->extended_marker) { case ERL_DRV_EXTENDED_MARKER: - if (ERL_DRV_EXTENDED_MAJOR_VERSION != dp->major_version - || ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version) { + if (dp->major_version < ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + || (ERL_DRV_EXTENDED_MAJOR_VERSION < dp->major_version + || (ERL_DRV_EXTENDED_MAJOR_VERSION == dp->major_version + && ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version))) { /* Incompatible driver version */ res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION; goto error; diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 77627a6897..afb33c1cdb 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -882,7 +882,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(p, process_str); - erts_snprintf(port_str, sizeof(port_str), "%T", port->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", port->common.id); DTRACE3(port_open, process_str, name_buf, port_str); } #endif diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 5517c26ba4..3ecb379326 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -136,6 +136,22 @@ typedef struct { #define ERL_DRV_EXTENDED_MINOR_VERSION 0 /* + * The emulator will refuse to load a driver with a major version + * lower than ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load + * may however fail if user have not removed use of deprecated + * symbols. + * + * The ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow + * loading of drivers built at least two major OTP releases + * ago. + * + * Bump of major version to 3 happened in OTP 17. That is, in + * OTP 19 we can increase ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + * to 3. + */ +#define ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2 + +/* * The emulator will refuse to load a driver with different major * version than the one used by the emulator. */ diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 2fff7f9390..fdd2d0c0f6 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -647,22 +647,24 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { *mhp++ = tup; if (is_immed(key)) { - while(n--) { + while (1) { if (*ks == key) { goto found_key; - } else { + } else if (--n) { *mhp++ = *vs++; *thp++ = *ks++; - } + } else + break; } } else { - while(n--) { + while(1) { if (EQ(*ks, key)) { goto found_key; - } else { + } else if (--n) { *mhp++ = *vs++; *thp++ = *ks++; - } + } else + break; } } @@ -676,7 +678,7 @@ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { found_key: /* Copy rest of keys and values */ - if (n) { + if (--n) { sys_memcpy(mhp, vs+1, n*sizeof(Eterm)); sys_memcpy(thp, ks+1, n*sizeof(Eterm)); } diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 6a9030fd99..0eb8117980 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -896,8 +896,10 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send)) { - erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->common.id); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), + "%T", receiver->common.id); } #endif if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 40860e141c..063dba056e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2049,8 +2049,10 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) { ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful"); } - else if (entry->major != ERL_NIF_MAJOR_VERSION - || entry->minor > ERL_NIF_MINOR_VERSION + else if (entry->major < ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + || (ERL_NIF_MAJOR_VERSION < entry->major + || (ERL_NIF_MAJOR_VERSION == entry->major + && ERL_NIF_MINOR_VERSION < entry->minor)) || (entry->major==2 && entry->minor == 5)) { /* experimental maps */ ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index c12ba4d554..5b93c2398e 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -46,6 +46,18 @@ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 6 +/* + * The emulator will refuse to load a nif-lib with a major version + * lower than ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load + * may however fail if user have not removed use of deprecated + * symbols. + * + * The ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow + * loading of nif-libs built at least two major OTP releases + * ago. + */ +#define ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2 + #include <stdlib.h> #ifdef SIZEOF_CHAR diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index fb6048b41f..31d9a1e26e 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -2035,9 +2035,9 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) DTRACE_CHARBUF(pid_str, 16); ErtsProcList* plp2 = plp; - erts_snprintf(port_str, sizeof(port_str), "%T", pp->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", pp->common.id); while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid); DTRACE2(process_port_unblocked, pid_str, port_str); } } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 37e1d07107..8bf0cc9491 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11313,7 +11313,7 @@ send_exit_signal(Process *c_p, /* current process if and only dtrace_pid_str(from, sender_str); dtrace_proc_str(rp, receiver_str); - erts_snprintf(reason_buf, sizeof(reason_buf) - 1, "%T", reason); + erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason); DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf); } #endif diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index cd5060ebb3..edf4a28784 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -914,8 +914,8 @@ int erts_port_handle_xports(Port *prt) (iov)->iov_base = (ptr); \ (iov)->iov_len = (len); \ if (sizeof((iov)->iov_len) < sizeof(len) \ - /* Check if (len) overflowed (iov)->iov_len */ \ - && ((len) >> (sizeof((iov)->iov_len)*CHAR_BIT)) != 0) { \ + /* Check if (len) overflowed (iov)->iov_len */ \ + && (iov)->iov_len != (len)) { \ goto L_overflow; \ } \ *(bv)++ = (bin); \ @@ -2473,7 +2473,7 @@ set_port_connected(int bang_op, DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE); dtrace_pid_str(connect, process_str); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); dtrace_proc_str(rp, newprocess_str); DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str); } @@ -3591,9 +3591,9 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); DTRACE_CHARBUF(rreason_str, 64); - erts_snprintf(from_str, sizeof(from_str), "%T", from); + erts_snprintf(from_str, sizeof(DTRACE_CHARBUF_NAME(from_str)), "%T", from); dtrace_port_str(p, port_str); - erts_snprintf(rreason_str, sizeof(rreason_str), "%T", rreason); + erts_snprintf(rreason_str, sizeof(DTRACE_CHARBUF_NAME(rreason_str)), "%T", rreason); DTRACE4(port_exit, from_str, port_str, p->name, rreason_str); } #endif @@ -4660,7 +4660,7 @@ set_busy_port(ErlDrvPort dprt, int on) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_busy)) { - erts_snprintf(port_str, sizeof(port_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); DTRACE1(port_busy, port_str); } @@ -4673,7 +4673,7 @@ set_busy_port(ErlDrvPort dprt, int on) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_not_busy)) { - erts_snprintf(port_str, sizeof(port_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); DTRACE1(port_not_busy, port_str); } @@ -4725,9 +4725,9 @@ erts_port_resume_procs(Port *prt) DTRACE_CHARBUF(pid_str, 16); ErtsProcList* plp2 = plp; - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid); DTRACE2(process_port_unblocked, pid_str, port_str); } } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index e273056a2b..05f07e57b2 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -154,10 +154,14 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; /* In VC++, noreturn is a declspec that has to be before the types, * but in GNUC it is an att ribute to be placed between return type * and function name, hence __decl_noreturn <types> __noreturn <function name> + * + * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h */ #if __GNUC__ # define __decl_noreturn -# define __noreturn __attribute__((noreturn)) +# ifndef __noreturn +# define __noreturn __attribute__((noreturn)) +# endif #else # if defined(__WIN32__) && defined(_MSC_VER) # define __noreturn diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c index ef539f8f9b..1ef1602ec9 100644 --- a/erts/emulator/drivers/common/gzio.c +++ b/erts/emulator/drivers/common/gzio.c @@ -230,6 +230,7 @@ local ErtsGzFile gz_open (path, mode) errno = 0; #if defined(FILENAMES_16BIT) { + FILE* efile_wfopen(const WCHAR* name, const WCHAR* mode); WCHAR wfmode[80]; int i = 0; int j; @@ -237,7 +238,7 @@ local ErtsGzFile gz_open (path, mode) wfmode[i++] = (WCHAR) fmode[j]; } wfmode[i++] = L'\0'; - s->file = _wfopen((WCHAR *)path, wfmode); + s->file = efile_wfopen((WCHAR *)path, wfmode); if (s->file == NULL) { return s->destroy(s), (ErtsGzFile)Z_NULL; } diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c index 3fe5d282dc..3143e4511d 100644 --- a/erts/emulator/drivers/common/zlib_drv.c +++ b/erts/emulator/drivers/common/zlib_drv.c @@ -21,6 +21,9 @@ * ZLib interface for erlang * */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif #include <stdio.h> #include <zlib.h> #include <errno.h> diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 480ba23239..a321bb9641 100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2013. All Rights Reserved. + * Copyright Ericsson AB 1997-2014. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -29,12 +29,27 @@ #include <wchar.h> #include "erl_efile.h" +#define DBG_TRACE_MASK 0 +/* 1 = file name ops + * 2 = file descr ops + * 4 = errors + * 8 = path name conversion + */ +#if !DBG_TRACE_MASK +# define DBG_TRACE(M,S) +# define DBG_TRACE1(M,FMT,A) +# define DBG_TRACE2(M,FMT,A,B) +#else +# define DBG_TRACE(M,S) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: %s\r\n", __LINE__, (WCHAR*)(S)); }while(0) +# define DBG_TRACE1(M,FMT,A) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: " L##FMT L"\r\n", __LINE__, (A)); }while(0) +# define DBG_TRACE2(M,FMT,A,B) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: " L##FMT L"\r\n", __LINE__, (A), (B)); }while(0) +#endif + /* * Microsoft-specific function to map a WIN32 error code to a Posix errno. */ #define ISSLASH(a) ((a) == L'\\' || (a) == L'/') - #define ISDIR(st) (((st).st_mode&S_IFMT) == S_IFDIR) #define ISREG(st) (((st).st_mode&S_IFMT) == S_IFREG) @@ -69,10 +84,92 @@ static int check_error(int result, Efile_error* errInfo); static int set_error(Efile_error* errInfo); +static int set_os_errno(Efile_error* errInfo, DWORD os_errno); static int is_root_unc_name(const WCHAR *path); static int extract_root(WCHAR *name); static unsigned short dos_to_posix_mode(int attr, const WCHAR *name); + +struct wpath_tmp_buffer { + struct wpath_tmp_buffer* next; + WCHAR buffer[1]; +}; + +typedef struct { + Efile_error* errInfo; + struct wpath_tmp_buffer* buf_list; +}Efile_call_state; + +static void call_state_init(Efile_call_state* state, Efile_error* errInfo) +{ + state->errInfo = errInfo; + state->buf_list = NULL; +} +static WCHAR* wpath_tmp_alloc(Efile_call_state* state, size_t len) +{ + size_t sz = offsetof(struct wpath_tmp_buffer, buffer) + + (len+1)*sizeof(WCHAR); + struct wpath_tmp_buffer* p = driver_alloc(sz); + p->next = state->buf_list; + state->buf_list = p; + return p->buffer; +} +static void call_state_free(Efile_call_state* state) +{ + while(state->buf_list) { + struct wpath_tmp_buffer* next = state->buf_list->next; + driver_free(state->buf_list); + state->buf_list = next; + } +} +static WCHAR* get_cwd_wpath_tmp(Efile_call_state* state) +{ + WCHAR dummy; + DWORD size = GetCurrentDirectoryW(0, &dummy); + WCHAR* ret = NULL; + + if (size) { + ret = wpath_tmp_alloc(state, size); + if (!GetCurrentDirectoryW(size, ret)) { + ret = NULL; + } + } + return ret; +} +static WCHAR* get_full_wpath_tmp(Efile_call_state* state, + const WCHAR* file, + WCHAR** file_part, + DWORD extra) +{ + WCHAR dummy; + DWORD size = GetFullPathNameW(file, 0, &dummy, NULL); + WCHAR* ret = NULL; + + if (size) { + int ok; + ret = wpath_tmp_alloc(state, size + extra); + if (file_part) { + ok = (GetFullPathNameW(file, size, ret, file_part) != 0); + } + else { + ok = (_wfullpath(ret, file, size) != NULL); + } + if (!ok) { + ret = NULL; + } + } + return ret; +} + +static void ensure_wpath_max(Efile_call_state* state, WCHAR** pathp, size_t max); +static int do_rmdir(Efile_call_state*, char* name); +static int do_rename(Efile_call_state*, char* src, char* dst); +static int do_readdir(Efile_call_state*, char* name, EFILE_DIR_HANDLE*, char* buffer, size_t *size); +static int do_fileinfo(Efile_call_state*, Efile_info*, char* orig_name, int info_for_link); +static char* do_readlink(Efile_call_state*, char* name, char* buffer, size_t size); +static int do_altname(Efile_call_state*, char* orig_name, char* buffer, size_t size); + + static int errno_map(DWORD last_error) { switch (last_error) { @@ -154,6 +251,8 @@ static int errno_map(DWORD last_error) { return EAGAIN; case ERROR_CANT_RESOLVE_FILENAME: return EMLINK; + case ERROR_PRIVILEGE_NOT_HELD: + return EPERM; case ERROR_ARENA_TRASHED: case ERROR_INVALID_BLOCK: case ERROR_BAD_ENVIRONMENT: @@ -176,11 +275,23 @@ check_error(int result, Efile_error* errInfo) if (result < 0) { errInfo->posix_errno = errno; errInfo->os_errno = GetLastError(); + DBG_TRACE2(4, "ERROR os_error=%d errno=%d @@@@@@@@@@@@@@@@@@@@@@@@@@@@", + errInfo->os_errno, errInfo->posix_errno); return 0; } return 1; } +static void +save_last_error(Efile_error* errInfo) +{ + errInfo->posix_errno = errno; + errInfo->os_errno = GetLastError(); + DBG_TRACE2(4, "ERROR os_error=%d errno=%d $$$$$$$$$$$$$$$$$$$$$$$$$$$$$", + errInfo->os_errno, errInfo->posix_errno); +} + + /* * Fills the provided error information structure with information * with the error code given by GetLastError() and its corresponding @@ -192,7 +303,18 @@ check_error(int result, Efile_error* errInfo) static int set_error(Efile_error* errInfo) { - errInfo->posix_errno = errno_map(errInfo->os_errno = GetLastError()); + set_os_errno(errInfo, GetLastError()); + return 0; +} + + +static int +set_os_errno(Efile_error* errInfo, DWORD os_errno) +{ + errInfo->os_errno = os_errno; + errInfo->posix_errno = errno_map(os_errno); + DBG_TRACE2(4, "ERROR os_error=%d errno=%d ############################", + errInfo->os_errno, errInfo->posix_errno); return 0; } @@ -226,21 +348,151 @@ win_writev(Efile_error* errInfo, } +/* Check '*pathp' and convert it if needed to something that windows will accept. + * Typically use UNC path with \\?\ prefix if absolute path is longer than 260. + */ +static void ensure_wpath(Efile_call_state* state, WCHAR** pathp) +{ + ensure_wpath_max(state, pathp, MAX_PATH); +} + +static void ensure_wpath_max(Efile_call_state* state, WCHAR** pathp, size_t max) +{ + WCHAR* path = *pathp; + WCHAR* p; + size_t len = wcslen(path); + int unc_fixup = 0; + + if (path[0] == 0) { + DBG_TRACE(8, L"Let empty path pass through"); + return; + } + + DBG_TRACE1(8,"IN: %s", path); + + if (path[1] == L':' && ISSLASH(path[2])) { /* absolute path */ + if (len >= max) { + WCHAR *src, *dst; + + *pathp = wpath_tmp_alloc(state, 4+len+1); + dst = *pathp; + wcscpy(dst, L"\\\\?\\"); + for (src=path,dst+=4; *src; src++) { + if (*src == L'/') { + if (dst[-1] != L'\\') { + *dst++ = L'\\'; + } + /*else ignore redundant slashes */ + } + else + *dst++ = *src; + } + *dst = 0; + unc_fixup = 1; + } + } + else if (!(ISSLASH(path[0]) && ISSLASH(path[1]))) { /* relative path */ + DWORD cwdLen = GetCurrentDirectoryW(0, NULL); + DWORD absLen = cwdLen + 1 + len; + if (absLen >= max) { + WCHAR *fullPath = wpath_tmp_alloc(state, 4+4+absLen); + DWORD fullLen; + + fullLen = GetFullPathNameW(path, 4 + absLen, fullPath+4, NULL); + if (fullLen >= 4+absLen) { + *pathp = path; + DBG_TRACE2(8,"ensure_wpath FAILED absLen=%u %s", (int)absLen, path); + return; + } + /* GetFullPathNameW can return paths longer than MAX_PATH without the \\?\ prefix. + * At least seen on Windows 7. Go figure... + */ + if (fullLen >= max && wcsncmp(fullPath+4, L"\\\\?\\", 4) != 0) { + wcsncpy(fullPath, L"\\\\?\\", 4); + *pathp = fullPath; + } + else { + *pathp = fullPath + 4; + } + } + } + + if (unc_fixup) { + WCHAR* endp; + + p = *pathp; + len = wcslen(p); + endp = p + len; + if (len > 4) { + p += 4; + while (*p) { + if (p[0] == L'\\' && p[1] == L'.') { + if (p[2] == L'\\' || !p[2]) { /* single dot */ + wmemmove(p, p+2, (&endp[1] - &p[2])); + endp -= 2; + } + else if (p[2] == L'.' && (p[3] == L'\\' || !p[3])) { /* double dot */ + WCHAR* r; + for (r=p-1; *r == L'\\'; --r) + /*skip redundant slashes*/; + for (; *r != L'\\'; --r) + /*find start of prev directory*/; + if (r < *pathp + 6) + break; + wmemmove(r, p+3, (&endp[1] - &p[3])); + p = r; + } + else p += 3; + } + else ++p; + } + } + } + DBG_TRACE1(8,"OUT: %s", *pathp); +} int efile_mkdir(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of directory to create. */ { - return check_error(_wmkdir((WCHAR *) name), errInfo); + Efile_call_state state; + WCHAR* wname = (WCHAR*)name; + int ret; + + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ensure_wpath_max(&state, &wname, 248); /* Yes, 248 limit for normal paths */ + + ret = (int) CreateDirectoryW(wname, NULL); + if (!ret) + set_error(errInfo); + + call_state_free(&state); + return ret; } int efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of directory to delete. */ { + Efile_call_state state; + int ret; + + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ret = do_rmdir(&state, name); + call_state_free(&state); + return ret; +} + +static int do_rmdir(Efile_call_state* state, char* name) +{ OSVERSIONINFO os; DWORD attr; WCHAR *wname = (WCHAR *) name; + WCHAR *buffer = NULL; + + ensure_wpath(state, &wname); if (RemoveDirectoryW(wname) != FALSE) { return 1; @@ -270,10 +522,9 @@ efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ if (os.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) { HANDLE handle; WIN32_FIND_DATAW data; - WCHAR buffer[2*MAX_PATH]; - int len; + int len = wcslen(wname); - len = wcslen(wname); + buffer = wpath_tmp_alloc(state, len + 4); wcscpy(buffer, wname); if (buffer[0] && buffer[len-1] != L'\\' && buffer[len-1] != L'/') { wcscat(buffer, L"\\"); @@ -311,16 +562,30 @@ efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ } end: - return check_error(-1, errInfo); + save_last_error(state->errInfo); + return 0; } int efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of file to delete. */ { + Efile_call_state state; + int ret; + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ret = do_delete_file(&state, name); + call_state_free(&state); + return ret; +} + +static int do_delete_file(Efile_call_state* state, char* name) +{ DWORD attr; WCHAR *wname = (WCHAR *) name; + ensure_wpath(state, &wname); + if (DeleteFileW(wname) != FALSE) { return 1; } @@ -359,7 +624,7 @@ efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ errno = EACCES; } - return check_error(-1, errInfo); + return check_error(-1, state->errInfo); } /* @@ -393,14 +658,29 @@ efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ */ int -efile_rename(Efile_error* errInfo, /* Where to return error codes. */ - char* src, /* Original name. */ - char* dst) /* New name. */ +efile_rename(Efile_error* errInfo, char* src, char* dst) +{ + Efile_call_state state; + int ret; + DBG_TRACE(1, src); + call_state_init(&state, errInfo); + ret = do_rename(&state, src, dst); + call_state_free(&state); + return ret; +} + +static int +do_rename(Efile_call_state* state, + char* src, /* Original name. */ + char* dst) /* New name. */ { DWORD srcAttr, dstAttr; WCHAR *wsrc = (WCHAR *) src; WCHAR *wdst = (WCHAR *) dst; - + + ensure_wpath(state, &wsrc); + ensure_wpath(state, &wdst); + if (MoveFileW(wsrc, wdst) != FALSE) { return 1; } @@ -417,23 +697,27 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ if (errno == EBADF) { errno = EACCES; - return check_error(-1, errInfo); + return check_error(-1, state->errInfo); } if (errno == EACCES) { decode: if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) { - WCHAR srcPath[MAX_PATH], dstPath[MAX_PATH]; + WCHAR *srcPath, *dstPath; WCHAR *srcRest, *dstRest; int size; - size = GetFullPathNameW(wsrc, MAX_PATH, srcPath, &srcRest); - if ((size == 0) || (size > MAX_PATH)) { - return check_error(-1, errInfo); + srcPath = get_full_wpath_tmp(state, wsrc, &srcRest, 0); + if (!srcPath) { + save_last_error(state->errInfo); + return 0; } - size = GetFullPathNameW(wdst, MAX_PATH, dstPath, &dstRest); - if ((size == 0) || (size > MAX_PATH)) { - return check_error(-1, errInfo); + + dstPath = get_full_wpath_tmp(state, wdst, &dstRest, 0); + if (!dstPath) { + save_last_error(state->errInfo); + return 0; } + if (srcRest == NULL) { srcRest = srcPath + wcslen(srcPath); } @@ -538,14 +822,16 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ * put temp file back to old name. */ - WCHAR tempName[MAX_PATH]; - int result, size; + WCHAR *tempName; + int result; WCHAR *rest; - size = GetFullPathNameW(wdst, MAX_PATH, tempName, &rest); - if ((size == 0) || (size > MAX_PATH) || (rest == NULL)) { - return check_error(-1, errInfo); + tempName = get_full_wpath_tmp(state, wdst, &rest, 14); + if (!tempName || !rest) { + save_last_error(state->errInfo); + return 0; } + *rest = L'\0'; result = -1; if (GetTempFileNameW(tempName, L"erlr", 0, tempName) != 0) { @@ -578,7 +864,6 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ /* * Decode the EACCES to a more meaningful error. */ - goto decode; } } @@ -586,16 +871,20 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ } } } - return check_error(-1, errInfo); + return check_error(-1, state->errInfo); } int efile_chdir(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of directory to make current. */ -{ - int success = check_error(_wchdir((WCHAR *) name), errInfo); - if (!success && errInfo->posix_errno == EINVAL) - /* POSIXification of errno */ +{
+ /* We don't even try to handle long paths here
+ * as current working directory is always limited to MAX_PATH
+ * even if we use UNC paths and SetCurrentDirectoryW()
+ */
+ int success = check_error(_wchdir((WCHAR *) name), errInfo);
+ if (!success && errInfo->posix_errno == EINVAL)
+ /* POSIXification of errno */
errInfo->posix_errno = ENOENT; return success; } @@ -608,28 +897,45 @@ efile_getdcwd(Efile_error* errInfo, /* Where to return error codes. */ { WCHAR *wbuffer = (WCHAR *) buffer; size_t wbuffer_size = size / 2; - if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL) + DBG_TRACE(1, L"#getdcwd#"); + if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL) { return check_error(-1, errInfo); + } + DBG_TRACE1(8, "getdcwd OS=%s", wbuffer); + if (wcsncmp(wbuffer, L"\\\\?\\", 4) == 0) { + wmemmove(wbuffer, wbuffer+4, wcslen(wbuffer+4)+1); + } for ( ; *wbuffer; wbuffer++) if (*wbuffer == L'\\') *wbuffer = L'/'; + DBG_TRACE1(8, "getdcwd ERLANG=%s", (WCHAR*)buffer); return 1; } int -efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name, /* Name of directory to list */ - EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */ - char* buffer, /* Buffer to put one filename in */ - size_t *size) /* in-out size of buffer/size of filename excluding zero - termination in bytes*/ +efile_readdir(Efile_error* errInfo, char* name, EFILE_DIR_HANDLE* dir_handle, + char* buffer, size_t *size) +{ + Efile_call_state state; + int ret; + DBG_TRACE(dir_handle?2:1, name); + call_state_init(&state, errInfo); + ret = do_readdir(&state, name, dir_handle, buffer, size); + call_state_free(&state); + return ret; +} + +static int do_readdir(Efile_call_state* state, + char* name, /* Name of directory to list */ + EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */ + char* buffer, /* Buffer to put one filename in */ + size_t *size) /* in-out size of buffer/size of filename excluding zero + termination in bytes*/ { HANDLE dir; /* Handle to directory. */ - WCHAR wildcard[MAX_PATH]; /* Wildcard to search for. */ WIN32_FIND_DATAW findData; /* Data found by FindFirstFile() or FindNext(). */ /* Alignment is not honored, this works on x86 because of alignment fixup by processor. Not perfect, but faster than alinging by hand (really) */ - WCHAR *wname = (WCHAR *) name; WCHAR *wbuffer = (WCHAR *) buffer; /* @@ -637,13 +943,15 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ */ if (*dir_handle == NULL) { - int length = wcslen(wname); + WCHAR *wname = (WCHAR *) name; + WCHAR* wildcard; + int length; WCHAR* s; - if (length+3 >= MAX_PATH) { - errno = ENAMETOOLONG; - return check_error(-1, errInfo); - } + ensure_wpath_max(state, &wname, MAX_PATH-2); + length = wcslen(wname); + + wildcard = wpath_tmp_alloc(state, length+3); wcscpy(wildcard, wname); s = wildcard+length-1; @@ -653,8 +961,10 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ *++s = L'\0'; DEBUGF(("Reading %ws\n", wildcard)); dir = FindFirstFileW(wildcard, &findData); - if (dir == INVALID_HANDLE_VALUE) - return set_error(errInfo); + if (dir == INVALID_HANDLE_VALUE) { + set_error(state->errInfo); + return 0; + } *dir_handle = (EFILE_DIR_HANDLE) dir; if (!IS_DOT_OR_DOTDOT(findData.cFileName)) { @@ -664,7 +974,6 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ } } - /* * Retrieve the name of the next file using the directory handle. */ @@ -681,24 +990,36 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ } if (GetLastError() == ERROR_NO_MORE_FILES) { - FindClose(dir); - errInfo->posix_errno = errInfo->os_errno = 0; - return 0; + state->errInfo->posix_errno = state->errInfo->os_errno = 0; + } + else { + set_error(state->errInfo); } - - set_error(errInfo); FindClose(dir); return 0; } } int -efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ - char* name, /* Name of directory to open. */ - int flags, /* Flags to use for opening. */ - int* pfd, /* Where to store the file descriptor. */ - Sint64* pSize) /* Where to store the size of the file. */ +efile_openfile(Efile_error* errInfo, char* name, int flags, int* pfd, Sint64* pSize) { + Efile_call_state state; + int ret; + DBG_TRACE1(1, "openfile(%s)", name); + call_state_init(&state, errInfo); + ret = do_openfile(&state, name, flags, pfd, pSize); + call_state_free(&state); + return ret; +} + +static +int do_openfile(Efile_call_state* state, /* Where to return error codes. */ + char* name, /* Name of directory to open. */ + int flags, /* Flags to use for opening. */ + int* pfd, /* Where to store the file descriptor. */ + Sint64* pSize) /* Where to store the size of the file. */ +{ + Efile_error* errInfo = state->errInfo; BY_HANDLE_FILE_INFORMATION fileInfo; /* File information from a handle. */ HANDLE fd; /* Handle to open file. */ DWORD access; /* Access mode: GENERIC_READ, GENERIC_WRITE. */ @@ -735,6 +1056,7 @@ efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ if (flags & EFILE_MODE_EXCL) { crFlags = CREATE_NEW; } + ensure_wpath(state, &wname); fd = CreateFileW(wname, access, FILE_SHARE_FLAGS, NULL, crFlags, flagsAndAttrs, NULL); @@ -777,34 +1099,56 @@ efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ } int -efile_may_openfile(Efile_error* errInfo, char *name) { +efile_may_openfile(Efile_error* errInfo, char *name) +{ + Efile_call_state state; WCHAR *wname = (WCHAR *) name; DWORD attr; + int ret; + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ensure_wpath(&state, &wname); if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) { errno = ENOENT; - return check_error(-1, errInfo); + ret = check_error(-1, errInfo); } - - if (attr & FILE_ATTRIBUTE_DIRECTORY) { + else if (attr & FILE_ATTRIBUTE_DIRECTORY) { errno = EISDIR; - return check_error(-1, errInfo); + ret = check_error(-1, errInfo); } - return 1; + else ret = 1; + + call_state_free(&state); + return ret; } void efile_closefile(fd) int fd; /* File descriptor for file to close. */ { + DBG_TRACE(2, L""); CloseHandle((HANDLE) fd); } +FILE* efile_wfopen(const WCHAR* name, const WCHAR* mode) +{ + Efile_call_state state; + Efile_error dummy; + FILE* f; + call_state_init(&state, &dummy); + ensure_wpath(&state, (WCHAR**)&name); + f = _wfopen(name, mode); + call_state_free(&state); + return f; +} + int efile_fdatasync(errInfo, fd) Efile_error* errInfo; /* Where to return error codes. */ int fd; /* File descriptor for file to sync. */ { + DBG_TRACE(2, L""); /* Not available in Windows, just call regular fsync */ return efile_fsync(errInfo, fd); } @@ -814,6 +1158,7 @@ efile_fsync(errInfo, fd) Efile_error* errInfo; /* Where to return error codes. */ int fd; /* File descriptor for file to sync. */ { + DBG_TRACE(2, L""); if (!FlushFileBuffers((HANDLE) fd)) { return check_error(-1, errInfo); } @@ -824,64 +1169,87 @@ int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, char* orig_name, int info_for_link) { + Efile_call_state state; + int ret; + DBG_TRACE(1, L""); + call_state_init(&state, errInfo); + ret = do_fileinfo(&state, pInfo, orig_name, info_for_link); + call_state_free(&state); + return ret; +} + +static int +do_fileinfo(Efile_call_state* state, Efile_info* pInfo, + char* orig_name, int info_for_link) +{ + Efile_error* errInfo = state->errInfo; HANDLE findhandle; /* Handle returned by FindFirstFile(). */ WIN32_FIND_DATAW findbuf; /* Data return by FindFirstFile(). */ - WCHAR name[_MAX_PATH]; + WCHAR* name = NULL; + WCHAR* win_path; int name_len; - WCHAR *path; - WCHAR pathbuf[_MAX_PATH]; int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ - WCHAR *worig_name = (WCHAR *) orig_name; + WCHAR *worig_name = (WCHAR *) orig_name; + ensure_wpath(state, &worig_name); /* Don't allow wildcards to be interpreted by system */ - if (wcspbrk(worig_name, L"?*")) { - enoent: - errInfo->posix_errno = ENOENT; - errInfo->os_errno = ERROR_FILE_NOT_FOUND; - return 0; - } /* * Move the name to a buffer and make sure to remove a trailing * slash, because it causes FindFirstFile() to fail on Win95. */ - if ((name_len = wcslen(worig_name)) >= _MAX_PATH) { - goto enoent; - } else { - wcscpy(name, worig_name); - if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != L':') { - name[name_len-1] = L'\0'; - } + name_len = wcslen(worig_name); + + name = wpath_tmp_alloc(state, name_len+1); + wcscpy(name, worig_name); + if (name_len > 2 && ISSLASH(name[name_len-1]) && + name[name_len-2] != L':') { + name[name_len-1] = L'\0'; } - + + win_path = name; + if (wcsncmp(name, L"\\\\?\\", 4) == 0) { + win_path += 4; + } + + if (wcspbrk(win_path, L"?*")) { + enoent: + errInfo->posix_errno = ENOENT; + errInfo->os_errno = ERROR_FILE_NOT_FOUND; + return 0; + } + /* Try to get disk from name. If none, get current disk. */ - if (name[1] != L':') { + if (win_path[1] != L':') { + WCHAR* cwd_path = get_cwd_wpath_tmp(state); drive = 0; - if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) && - pathbuf[1] == L':') { - drive = towlower(pathbuf[0]) - L'a' + 1; + if (cwd_path[1] == L':') { + drive = towlower(cwd_path[0]) - L'a' + 1; } - } else if (*name && name[2] == L'\0') { + } else if (*win_path && win_path[2] == L'\0') { /* * X: and nothing more is an error. */ errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; - } else - drive = towlower(*name) - L'a' + 1; + } else { + drive = towlower(*win_path) - L'a' + 1; + } findhandle = FindFirstFileW(name, &findbuf); if (findhandle == INVALID_HANDLE_VALUE) { + WCHAR* path = NULL; + if (!(wcspbrk(name, L"./\\") && - (path = _wfullpath(pathbuf, name, _MAX_PATH)) && + (path = get_full_wpath_tmp(state, name, NULL, 0)) && /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ ((wcslen(path) == 3) || is_root_unc_name(path)) && (GetDriveTypeW(path) > 1) ) ) { + errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; @@ -908,13 +1276,11 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, /* * given that we know this is a symlink, we should be able to find its target */ - WCHAR target_name[_MAX_PATH]; - if (efile_readlink(errInfo, (char *) name, - (char *) target_name, - _MAX_PATH * sizeof(WCHAR)) == 1) { + WCHAR* target_name = (WCHAR*) do_readlink(state, (char *) name, NULL, 0); + if (target_name) { FindClose(findhandle); - return efile_fileinfo(errInfo, pInfo, - (char *) target_name, info_for_link); + return do_fileinfo(state, pInfo, + (char *) target_name, info_for_link); } } @@ -981,6 +1347,20 @@ efile_write_info(Efile_error* errInfo, Efile_info* pInfo, char* name) { + Efile_call_state state; + int ret; + call_state_init(&state, errInfo); + ret = do_write_info(&state, pInfo, name); + call_state_free(&state); + return ret; +} + +static int +do_write_info(Efile_call_state* state, + Efile_info* pInfo, + char* name) +{ + Efile_error* errInfo = state->errInfo; SYSTEMTIME timebuf; FILETIME ModifyFileTime; FILETIME AccessFileTime; @@ -990,6 +1370,10 @@ efile_write_info(Efile_error* errInfo, DWORD tempAttr; WCHAR *wname = (WCHAR *) name; + DBG_TRACE(1, name); + + ensure_wpath(state, &wname); + /* * Get the attributes for the file. */ @@ -1066,7 +1450,9 @@ char* buf; /* Buffer to write. */ size_t count; /* Number of bytes to write. */ Sint64 offset; /* where to write it */ { - int res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); + int res; + DBG_TRACE(2, L""); + res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); if (res) { return efile_write(errInfo, EFILE_MODE_WRITE, fd, buf, count); } else { @@ -1084,7 +1470,9 @@ char* buf; /* Buffer to read into. */ size_t count; /* Number of bytes to read. */ size_t* pBytesRead; /* Where to return number of bytes read. */ { - int res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); + int res; + DBG_TRACE(2, L""); + res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); if (res) { return efile_read(errInfo, EFILE_MODE_READ, fd, buf, count, pBytesRead); } else { @@ -1106,6 +1494,7 @@ size_t count; /* Number of bytes to write. */ OVERLAPPED overlapped; OVERLAPPED* pOverlapped = NULL; + DBG_TRACE(2, L""); if (flags & EFILE_MODE_APPEND) { memset(&overlapped, 0, sizeof(overlapped)); overlapped.Offset = 0xffffffff; @@ -1135,6 +1524,7 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ OVERLAPPED overlapped; OVERLAPPED* pOverlapped = NULL; + DBG_TRACE(2, L""); ASSERT(iovcnt >= 0); if (flags & EFILE_MODE_APPEND) { @@ -1171,6 +1561,8 @@ size_t count; /* Number of bytes to read. */ size_t* pBytesRead; /* Where to return number of bytes read. */ { DWORD nbytes = 0; + + DBG_TRACE(2, L""); if (!ReadFile((HANDLE) fd, buf, count, &nbytes, NULL)) return set_error(errInfo); @@ -1190,6 +1582,7 @@ Sint64* new_location; /* Resulting new location in file. */ { LARGE_INTEGER off, new_loc; + DBG_TRACE(2, L""); switch (origin) { case EFILE_SEEK_SET: origin = FILE_BEGIN; break; case EFILE_SEEK_CUR: origin = FILE_CURRENT; break; @@ -1221,6 +1614,7 @@ Efile_error* errInfo; /* Where to return error codes. */ int *fd; /* File descriptor for file to truncate. */ int flags; { + DBG_TRACE(2, L""); if (!SetEndOfFile((HANDLE) (*fd))) return set_error(errInfo); return 1; @@ -1373,9 +1767,24 @@ dos_to_posix_mode(int attr, const WCHAR *name) return uxmode; } + int efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) { + Efile_call_state state; + int ret; + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ret = !!do_readlink(&state, name, buffer, size); + call_state_free(&state); + return ret; +} + +/* If buffer==0, return buffer allocated by wpath_tmp_allocate +*/ +static char* +do_readlink(Efile_call_state* state, char* name, char* buffer, size_t size) +{ /* * load dll and see if we have CreateSymbolicLink at runtime: * (Vista only) @@ -1383,6 +1792,9 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) HINSTANCE hModule = NULL; WCHAR *wname = (WCHAR *) name; WCHAR *wbuffer = (WCHAR *) buffer; + DWORD wsize = size / sizeof(WCHAR); + char* ret = NULL; + if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { typedef DWORD (WINAPI * GETFINALPATHNAMEBYHANDLEPTR)( HANDLE hFile, @@ -1393,58 +1805,84 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) GETFINALPATHNAMEBYHANDLEPTR pGetFinalPathNameByHandle = (GETFINALPATHNAMEBYHANDLEPTR)GetProcAddress(hModule, "GetFinalPathNameByHandleW"); - if (pGetFinalPathNameByHandle == NULL) { - FreeLibrary(hModule); - } else { + if (pGetFinalPathNameByHandle != NULL) { + DWORD fileAttributes; + ensure_wpath(state, &wname); /* first check if file is a symlink; {error, einval} otherwise */ - DWORD fileAttributes = GetFileAttributesW(wname); + fileAttributes = GetFileAttributesW(wname); if ((fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - BOOLEAN success = 0; + DWORD success = 0; HANDLE h = CreateFileW(wname, GENERIC_READ, FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); int len; if(h != INVALID_HANDLE_VALUE) { - success = pGetFinalPathNameByHandle(h, wbuffer, size / sizeof(WCHAR),0); - /* GetFinalPathNameByHandle prepends path with "\\?\": */ - len = wcslen(wbuffer); - wmemmove(wbuffer,wbuffer+4,len-3); - if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' && - wbuffer[0] <= L'Z') { - wbuffer[0] = wbuffer[0] + L'a' - L'A'; + if (!wbuffer) { /* dynamic allocation */ + WCHAR dummy; + wsize = pGetFinalPathNameByHandle(h, &dummy, 0, 0); + if (wsize) { + wbuffer = wpath_tmp_alloc(state, wsize); + } } + if (wbuffer + && (success = pGetFinalPathNameByHandle(h, wbuffer, wsize, 0)) + && success < wsize) { + WCHAR* wp; + + /* GetFinalPathNameByHandle prepends path with "\\?\": */ + len = wcslen(wbuffer); + wmemmove(wbuffer,wbuffer+4,len-3); + if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' && + wbuffer[0] <= L'Z') { + wbuffer[0] = wbuffer[0] + L'a' - L'A'; + } - for ( ; *wbuffer; wbuffer++) - if (*wbuffer == L'\\') - *wbuffer = L'/'; + for (wp=wbuffer ; *wp; wp++) + if (*wp == L'\\') + *wp = L'/'; + } CloseHandle(h); - } - FreeLibrary(hModule); + } if (success) { - return 1; + ret = (char*) wbuffer; } else { - return set_error(errInfo); + set_error(state->errInfo); } } else { - FreeLibrary(hModule); errno = EINVAL; - return check_error(-1, errInfo); + save_last_error(state->errInfo); } + goto done; } } errno = ENOTSUP; - return check_error(-1, errInfo); + save_last_error(state->errInfo); + +done: + if (hModule) + FreeLibrary(hModule); + return ret; } int efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) { + Efile_call_state state; + int ret; + DBG_TRACE(1, orig_name); + call_state_init(&state, errInfo); + ret = do_altname(&state, orig_name, buffer, size); + call_state_free(&state); + return ret; +} + +static int +do_altname(Efile_call_state* state, char* orig_name, char* buffer, size_t size) +{ WIN32_FIND_DATAW wfd; HANDLE fh; - WCHAR name[_MAX_PATH+1]; + WCHAR* name; int name_len; - WCHAR* path; - WCHAR pathbuf[_MAX_PATH+1]; /* Unclear weather GetCurrentDirectory will access one char after - _MAX_PATH */ + WCHAR* full_path = NULL; WCHAR *worig_name = (WCHAR *) orig_name; WCHAR *wbuffer = (WCHAR *) buffer; int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ @@ -1453,8 +1891,8 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) if (wcspbrk(worig_name, L"?*")) { enoent: - errInfo->posix_errno = ENOENT; - errInfo->os_errno = ERROR_FILE_NOT_FOUND; + state->errInfo->posix_errno = ENOENT; + state->errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; } @@ -1462,24 +1900,23 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) * Move the name to a buffer and make sure to remove a trailing * slash, because it causes FindFirstFile() to fail on Win95. */ - - if ((name_len = wcslen(worig_name)) >= _MAX_PATH) { - goto enoent; - } else { - wcscpy(name, worig_name); - if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != L':') { - name[name_len-1] = L'\0'; - } + ensure_wpath(state, &worig_name); + name_len = wcslen(worig_name); + + name = wpath_tmp_alloc(state, name_len + 1); + wcscpy(name, worig_name); + if (name_len > 2 && ISSLASH(name[name_len-1]) && + name[name_len-2] != L':') { + name[name_len-1] = L'\0'; } /* Try to get disk from name. If none, get current disk. */ if (name[1] != L':') { + WCHAR* cwd_path = get_cwd_wpath_tmp(state); drive = 0; - if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) && - pathbuf[1] == L':') { - drive = towlower(pathbuf[0]) - L'a' + 1; + if (cwd_path[1] == L':') { + drive = towlower(cwd_path[0]) - L'a' + 1; } } else if (*name && name[2] == L'\0') { /* @@ -1491,13 +1928,15 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) } fh = FindFirstFileW(name,&wfd); if (fh == INVALID_HANDLE_VALUE) { + DWORD fff_error = GetLastError(); if (!(wcspbrk(name, L"./\\") && - (path = _wfullpath(pathbuf, name, _MAX_PATH)) && + (full_path = get_full_wpath_tmp(state, name, NULL, 0)) && /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ - ((wcslen(path) == 3) || is_root_unc_name(path)) && - (GetDriveTypeW(path) > 1) ) ) { - errno = errno_map(GetLastError()); - return check_error(-1, errInfo); + ((wcslen(full_path) == 3) || is_root_unc_name(full_path)) && + (GetDriveTypeW(full_path) > 1) ) ) { + + set_os_errno(state->errInfo, fff_error); + return 0; } /* * Root directories (such as C:\ or \\server\share\ are fabricated. @@ -1518,17 +1957,37 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) int efile_link(Efile_error* errInfo, char* old, char* new) { + Efile_call_state state; WCHAR *wold = (WCHAR *) old; WCHAR *wnew = (WCHAR *) new; + int ret; + DBG_TRACE(1, old); + call_state_init(&state, errInfo); + ensure_wpath(&state, &wold); + ensure_wpath(&state, &wnew); if(!CreateHardLinkW(wnew, wold, NULL)) { - return set_error(errInfo); + ret = set_error(errInfo); } - return 1; + else ret =1; + call_state_free(&state); + return ret; } int efile_symlink(Efile_error* errInfo, char* old, char* new) { + Efile_call_state state; + int ret; + DBG_TRACE2(1, "symlink(%s <- %s)", old, new); + call_state_init(&state, errInfo); + ret = do_symlink(&state, old, new); + call_state_free(&state); + return ret; +} + +static int +do_symlink(Efile_call_state* state, char* old, char* new) +{ /* * Load dll and see if we have CreateSymbolicLink at runtime: * (Vista only) @@ -1536,6 +1995,8 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) HINSTANCE hModule = NULL; WCHAR *wold = (WCHAR *) old; WCHAR *wnew = (WCHAR *) new; + + DBG_TRACE(1, old); if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { typedef BOOLEAN (WINAPI * CREATESYMBOLICLINKFUNCPTR) ( LPCWSTR lpSymlinkFileName, @@ -1547,6 +2008,9 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) "CreateSymbolicLinkW"); /* A for MBCS, W for UNICODE... char* above implies 'W'! */ if (pCreateSymbolicLink != NULL) { + ensure_wpath(state, &wold); + ensure_wpath(state, &wnew); + { DWORD attr = GetFileAttributesW(wold); int flag = (attr != INVALID_FILE_ATTRIBUTES && attr & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0; @@ -1557,19 +2021,21 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) if (success) { return 1; } else { - return set_error(errInfo); + return set_error(state->errInfo); } + } } else FreeLibrary(hModule); } errno = ENOTSUP; - return check_error(-1, errInfo); + return check_error(-1, state->errInfo); } int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length, int advise) { + DBG_TRACE(2, L""); /* posix_fadvise is not available on Windows, do nothing */ errno = ERROR_SUCCESS; return check_error(0, errInfo); @@ -1578,6 +2044,7 @@ efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, int efile_fallocate(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length) { + DBG_TRACE(2, L""); /* No file preallocation method available in Windows. */ errno = errno_map(ERROR_NOT_SUPPORTED); SetLastError(ERROR_NOT_SUPPORTED); diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index 7637049bc3..054911e822 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -182,3 +182,10 @@ BIF_RETTYPE hipe_bifs_debug_native_called_2(BIF_ALIST_2) BIF_RET(am_ok); } +/* Stub-BIF for LLVM: + * Reloads BP, SP (in llvm unwind label) */ + +BIF_RETTYPE hipe_bifs_llvm_fix_pinned_regs_0(BIF_ALIST_0) +{ + BIF_RET(am_ok); +} diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab index 45a395bf57..1b659cfa90 100644 --- a/erts/emulator/hipe/hipe_bif2.tab +++ b/erts/emulator/hipe/hipe_bif2.tab @@ -30,3 +30,4 @@ bif hipe_bifs:in_native/0 bif hipe_bifs:modeswitch_debug_on/0 bif hipe_bifs:modeswitch_debug_off/0 bif hipe_bifs:debug_native_called/2 +bif hipe_bifs:llvm_fix_pinned_regs/0 diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 245841a768..1db673e7f3 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -527,7 +527,8 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, /* fast track to stop_select callback */ stop_select_fn = prt->drv_ptr->stop_select; #ifdef USE_VM_PROBES - strncpy(name, prt->drv_ptr->name, sizeof(name)-1); + strncpy(name, prt->drv_ptr->name, + sizeof(DTRACE_CHARBUF_NAME(name))-1); name[sizeof(name)-1] = '\0'; #endif ret = 0; diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 7c6e4a2f37..94eb6b1547 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -54,6 +54,17 @@ void sys_sigrelease(int sig) #endif /* !SIG_SIGNAL */ #endif /* !SIG_SIGSET */ +#if defined(__ANDROID__) +int __system_properties_fd(void); +#endif /* __ANDROID__ */ + +#if defined(__ANDROID__) +#define SHELL "/system/bin/sh" +#else +#define SHELL "/bin/sh" +#endif /* __ANDROID__ */ + + int main(int argc, char *argv[]) { @@ -89,8 +100,23 @@ main(int argc, char *argv[]) if (sscanf(argv[CS_ARGV_FD_CR_IX], "%d:%d", &from, &to) != 2) return 1; + +#if defined(__ANDROID__) + for (i = from; i <= to; i++) { + if (i!=__system_properties_fd) + (void) close(i); + } +#else for (i = from; i <= to; i++) (void) close(i); +#endif /* __ANDROID__ */ + +#if defined(HAVE_CLOSEFROM) + closefrom(from); +#else + for (i = from; i <= to; i++) + (void) close(i); +#endif if (!(argv[CS_ARGV_WD_IX][0] == '.' && argv[CS_ARGV_WD_IX][1] == '\0') && chdir(argv[CS_ARGV_WD_IX]) < 0) @@ -116,7 +142,25 @@ main(int argc, char *argv[]) execv(argv[CS_ARGV_NO_OF_ARGS],&(argv[CS_ARGV_NO_OF_ARGS + 1])); } } else { - execl("/bin/sh", "sh", "-c", argv[CS_ARGV_CMD_IX], (char *) NULL); + execl(SHELL, "sh", "-c", argv[CS_ARGV_CMD_IX], (char *) NULL); } return 1; } + + + +#if defined(__ANDROID__) +int __system_properties_fd(void) +{ + int s, fd; + char *env; + + env = getenv("ANDROID_PROPERTY_WORKSPACE"); + if (!env) { + return -1; + } + fd = atoi(env); + return fd; +} +#endif /* __ANDROID__ */ + diff --git a/erts/emulator/sys/unix/erl_unix_sys_ddll.c b/erts/emulator/sys/unix/erl_unix_sys_ddll.c index 8760b58839..2659d623c7 100644 --- a/erts/emulator/sys/unix/erl_unix_sys_ddll.c +++ b/erts/emulator/sys/unix/erl_unix_sys_ddll.c @@ -123,6 +123,7 @@ int erts_sys_ddll_open(const char *full_name, void **handle, ErtsSysDdllError* e int erts_sys_ddll_open_noext(char *dlname, void **handle, ErtsSysDdllError* err) { +#if defined(HAVE_DLOPEN) int ret = ERL_DE_NO_ERROR; char *str; dlerror(); @@ -148,6 +149,9 @@ int erts_sys_ddll_open_noext(char *dlname, void **handle, ErtsSysDdllError* err) ret = ERL_DE_DYNAMIC_ERROR_OFFSET - find_errcode(str, err); } return ret; +#else + return ERL_DE_ERROR_NO_DDLL_FUNCTIONALITY; +#endif } /* diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 865cb50a56..c3d7440409 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -149,6 +149,13 @@ extern void erl_crash_dump(char* file, int line, char* fmt, ...); #define DIR_SEPARATOR_CHAR '/' +#if defined(__ANDROID__) +#define SHELL "/system/bin/sh" +#else +#define SHELL "/bin/sh" +#endif /* __ANDROID__ */ + + #if defined(DEBUG) #define ERL_BUILD_TYPE_MARKER ".debug" #elif defined(PURIFY) @@ -1596,7 +1603,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op } } } else { - execle("/bin/sh", "sh", "-c", cmd_line, (char *) NULL, new_environ); + execle(SHELL, "sh", "-c", cmd_line, (char *) NULL, new_environ); } child_error: _exit(1); @@ -1717,7 +1724,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op fcntl(i, F_SETFD, 1); qnx_spawn_options.flags = _SPAWN_SETSID; - if ((pid = spawnl(P_NOWAIT, "/bin/sh", "/bin/sh", "-c", cmd_line, + if ((pid = spawnl(P_NOWAIT, SHELL, SHELL, "-c", cmd_line, (char *) 0)) < 0) { erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); reset_qnx_spawn(); diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h index 8015e8f378..a78dbf64af 100644 --- a/erts/emulator/sys/win32/erl_win_sys.h +++ b/erts/emulator/sys/win32/erl_win_sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2012. All Rights Reserved. + * Copyright Ericsson AB 1997-2014. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -60,16 +60,18 @@ #include <windows.h> #undef WIN32_LEAN_AND_MEAN -/* - * Define MAXPATHLEN in terms of MAXPATH if available. - */ - -#ifndef MAXPATH -#define MAXPATH MAX_PATH -#endif /* MAXPATH */ #ifndef MAXPATHLEN -#define MAXPATHLEN MAXPATH +#define MAXPATHLEN 4096 +/* + erts-6.0 (OTP 17.0): + We now accept windows paths longer than 260 (MAX_PATH) by conversion to + UNC path format. In order to also return long paths from the driver we + increased MAXPATHLEN from 260 to larger (but arbitrary) value 4096. + It would of course be nicer to instead dynamically allocate large enough + tmp buffers when efile_drv needs to return really long paths, and do that + for unix as well. + */ #endif /* MAXPATHLEN */ /* diff --git a/erts/emulator/test/driver_SUITE_data/smaller_major_vsn_drv.c b/erts/emulator/test/driver_SUITE_data/smaller_major_vsn_drv.c index a1299fe807..6b9d4745ba 100644 --- a/erts/emulator/test/driver_SUITE_data/smaller_major_vsn_drv.c +++ b/erts/emulator/test/driver_SUITE_data/smaller_major_vsn_drv.c @@ -20,12 +20,12 @@ * Author: Rickard Green * * Description: Implementation of a driver with a smaller major - * driver version than the current system. + * driver version than allowed on load. */ #define VSN_MISMATCH_DRV_NAME_STR "smaller_major_vsn_drv" #define VSN_MISMATCH_DRV_NAME smaller_major_vsn_drv -#define VSN_MISMATCH_DRV_MAJOR_VSN_DIFF (-1) +#define VSN_MISMATCH_DRV_MAJOR_VSN_DIFF (ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD - ERL_DRV_EXTENDED_MAJOR_VERSION - 1) #define VSN_MISMATCH_DRV_MINOR_VSN_DIFF 0 #include "vsn_mismatch_drv_impl.c" diff --git a/erts/epmd/src/epmd.c b/erts/epmd/src/epmd.c index 5d5c3a1c3c..1678d537d1 100644 --- a/erts/epmd/src/epmd.c +++ b/erts/epmd/src/epmd.c @@ -52,7 +52,7 @@ static int epmd_main(int, char **, int); int epmd_dbg(int level,int port) /* Utility to debug epmd... */ { - char* argv[MAX_DEBUG+2]; + char* argv[MAX_DEBUG+4]; char ibuff[100]; int argc = 0; diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in index 5c2cd8aded..cfd36af962 100644 --- a/erts/etc/common/Makefile.in +++ b/erts/etc/common/Makefile.in @@ -66,7 +66,9 @@ LIBS = @LIBS@ LDFLAGS = @LDFLAGS@ # For clock_gettime in heart +ifneq ($(TARGET),arm-unknown-linux-androideabi) RTLIBS = @LIBRT@ +endif ifeq ($(TARGET),win32) ifeq ($(TYPE),debug) diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c index 7f27b5f29c..79784c5b84 100644 --- a/erts/lib_src/pthread/ethread.c +++ b/erts/lib_src/pthread/ethread.c @@ -541,7 +541,11 @@ int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset) return EINVAL; } #endif - return pthread_sigmask(how, set, oset); +#if defined(__ANDROID__) + return sigprocmask(how, set, oset); +#else + return pthread_sigmask(how, set, oset); +#endif } int ethr_sigwait(const sigset_t *set, int *sig) diff --git a/erts/start_scripts/no_dot_erlang.rel.src b/erts/start_scripts/no_dot_erlang.rel.src index 03b64ebf1a..6208572c00 100644 --- a/erts/start_scripts/no_dot_erlang.rel.src +++ b/erts/start_scripts/no_dot_erlang.rel.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,6 +16,6 @@ %% %% %CopyrightEnd% %% -{release, {"OTP APN 181 01","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, +{release, {"Erlang/OTP","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, [{kernel,"%KERNEL_VSN%"}, {stdlib,"%STDLIB_VSN%"}]}. diff --git a/erts/start_scripts/start_all_example.rel.src b/erts/start_scripts/start_all_example.rel.src index 581eb2eb0b..2a1cabe7bb 100644 --- a/erts/start_scripts/start_all_example.rel.src +++ b/erts/start_scripts/start_all_example.rel.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,7 +16,7 @@ %% %% %CopyrightEnd% %% -{release, {"OTP APN 181 01","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, +{release, {"Erlang/OTP","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, [{kernel,"%KERNEL_VSN%"}, {stdlib,"%STDLIB_VSN%"}, {sasl, "%SASL_VSN%"}, diff --git a/erts/start_scripts/start_clean.rel.src b/erts/start_scripts/start_clean.rel.src index d2df422c51..e229721e36 100644 --- a/erts/start_scripts/start_clean.rel.src +++ b/erts/start_scripts/start_clean.rel.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,6 +16,6 @@ %% %% %CopyrightEnd% %% -{release, {"OTP APN 181 01","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, +{release, {"Erlang/OTP","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, [{kernel,"%KERNEL_VSN%"}, {stdlib,"%STDLIB_VSN%"}]}. diff --git a/erts/start_scripts/start_sasl.rel.src b/erts/start_scripts/start_sasl.rel.src index e521e8df91..e68a34af76 100644 --- a/erts/start_scripts/start_sasl.rel.src +++ b/erts/start_scripts/start_sasl.rel.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,7 +16,7 @@ %% %% %CopyrightEnd% %% -{release, {"OTP APN 181 01","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, +{release, {"Erlang/OTP","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, [{kernel,"%KERNEL_VSN%"}, {stdlib,"%STDLIB_VSN%"}, {sasl, "%SASL_VSN%"}]}. diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl index 4707e517b4..44b050e59d 100644 --- a/lib/asn1/src/asn1ct_gen.erl +++ b/lib/asn1/src/asn1ct_gen.erl @@ -1125,7 +1125,22 @@ pgen_info() -> open_hrl(OutFile,Module) -> File = lists:concat([OutFile,".hrl"]), _ = open_output_file(File), - gen_hrlhead(Module). + gen_hrlhead(Module), + Protector = hrl_protector(OutFile), + emit(["-ifndef(",Protector,").\n", + "-define(",Protector,", true).\n" + "\n"]). + +hrl_protector(OutFile) -> + BaseName = filename:basename(OutFile), + P = "_" ++ string:to_upper(BaseName) ++ "_HRL_", + [if + $A =< C, C =< $Z -> C; + $a =< C, C =< $a -> C; + $0 =< C, C =< $9 -> C; + true -> $_ + end || C <- P]. + %% EMIT functions ************************ %% *************************************** @@ -1232,6 +1247,8 @@ pgen_hrl(Erules,Module,TypeOrVal,Options,_Indent) -> 0 -> 0; Y -> + Protector = hrl_protector(get(outfile)), + emit(["-endif. %% ",Protector,"\n"]), close_output_file(), asn1ct:verbose("--~p--~n", [{generated,lists:concat([get(outfile),".hrl"])}], diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl index d733df27dc..cff02a46d9 100644 --- a/lib/common_test/src/ct_conn_log_h.erl +++ b/lib/common_test/src/ct_conn_log_h.erl @@ -204,13 +204,8 @@ pretty_head({{{Y,Mo,D},{H,Mi,S}},MicroS},ConnMod,Text0) -> micro2milli(MicroS)]). pretty_title(#conn_log{client=Client}=Info) -> - case actionstr(Info) of - {no_server,Action} -> - io_lib:format("= Client ~w ~s ",[Client,Action]); - Action -> - io_lib:format("= Client ~w ~s ~ts ",[Client,Action, - serverstr(Info)]) - end. + io_lib:format("= Client ~w ~s ~ts ", + [Client,actionstr(Info),serverstr(Info)]). actionstr(#conn_log{action=send}) -> "----->"; actionstr(#conn_log{action=cmd}) -> "----->"; diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 63bfea68c4..7d577462b0 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -249,8 +249,8 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) -> end end. -ct_suite_init(Suite, Func, PostInitHook, Config) when is_list(Config) -> - case ct_hooks:init_tc(Suite, Func, Config) of +ct_suite_init(Suite, FuncSpec, PostInitHook, Config) when is_list(Config) -> + case ct_hooks:init_tc(Suite, FuncSpec, Config) of NewConfig when is_list(NewConfig) -> PostInitHookResult = do_post_init_hook(PostInitHook, NewConfig), {ok, [PostInitHookResult ++ NewConfig]}; @@ -660,10 +660,7 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> ct_util:delete_testdata(comment), ct_util:delete_suite_data(last_saved_config), - FuncSpec = case group_or_func(Func,Args) of - {_,_GroupName,_} = Group -> Group; - _ -> Func - end, + FuncSpec = group_or_func(Func,Args), {Result1,FinalNotify} = case ct_hooks:end_tc( diff --git a/lib/common_test/src/ct_gen_conn.erl b/lib/common_test/src/ct_gen_conn.erl index 239f5b5f25..56082086f6 100644 --- a/lib/common_test/src/ct_gen_conn.erl +++ b/lib/common_test/src/ct_gen_conn.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2013. All Rights Reserved. +%% Copyright Ericsson AB 2003-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -307,7 +307,8 @@ call(Pid, Msg, Timeout) -> end. return({To,Ref},Result) -> - To ! {Ref, Result}. + To ! {Ref, Result}, + ok. init_gen(Parent,Opts) -> process_flag(trap_exit,true), diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index 2e667030a9..df4c98d9d1 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -64,11 +64,16 @@ terminate(Hooks) -> %% @doc Called as each test case is started. This includes all configuration %% tests. --spec init_tc(Mod :: atom(), Func :: atom(), Args :: list()) -> +-spec init_tc(Mod :: atom(), + FuncSpec :: atom() | + {ConfigFunc :: init_per_group | end_per_group, + GroupName :: atom(), + Properties :: list()}, + Args :: list()) -> NewConfig :: proplists:proplist() | - {skip, Reason :: term()} | - {auto_skip, Reason :: term()} | - {fail, Reason :: term()}. + {skip, Reason :: term()} | + {auto_skip, Reason :: term()} | + {fail, Reason :: term()}. init_tc(Mod, init_per_suite, Config) -> Info = try proplists:get_value(ct_hooks, Mod:suite(),[]) of @@ -82,8 +87,8 @@ init_tc(Mod, init_per_suite, Config) -> call(fun call_generic/3, Config ++ Info, [pre_init_per_suite, Mod]); init_tc(Mod, end_per_suite, Config) -> call(fun call_generic/3, Config, [pre_end_per_suite, Mod]); -init_tc(Mod, {init_per_group, GroupName, Opts}, Config) -> - maybe_start_locker(Mod, GroupName, Opts), +init_tc(Mod, {init_per_group, GroupName, Properties}, Config) -> + maybe_start_locker(Mod, GroupName, Properties), call(fun call_generic/3, Config, [pre_init_per_group, GroupName]); init_tc(_Mod, {end_per_group, GroupName, _}, Config) -> call(fun call_generic/3, Config, [pre_end_per_group, GroupName]); @@ -93,15 +98,18 @@ init_tc(_Mod, TC, Config) -> %% @doc Called as each test case is completed. This includes all configuration %% tests. -spec end_tc(Mod :: atom(), - Func :: atom(), + FuncSpec :: atom() | + {ConfigFunc :: init_per_group | end_per_group, + GroupName :: atom(), + Properties :: list()}, Args :: list(), Result :: term(), - Resturn :: term()) -> + Return :: term()) -> NewConfig :: proplists:proplist() | - {skip, Reason :: term()} | - {auto_skip, Reason :: term()} | - {fail, Reason :: term()} | - ok | '$ct_no_change'. + {skip, Reason :: term()} | + {auto_skip, Reason :: term()} | + {fail, Reason :: term()} | + ok | '$ct_no_change'. end_tc(Mod, init_per_suite, Config, _Result, Return) -> call(fun call_generic/3, Return, [post_init_per_suite, Mod, Config], @@ -112,10 +120,10 @@ end_tc(Mod, end_per_suite, Config, Result, _Return) -> end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config], '$ct_no_change'); -end_tc(Mod, {end_per_group, GroupName, Opts}, Config, Result, _Return) -> +end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) -> Res = call(fun call_generic/3, Result, [post_end_per_group, GroupName, Config], '$ct_no_change'), - maybe_stop_locker(Mod, GroupName,Opts), + maybe_stop_locker(Mod, GroupName, Properties), Res; end_tc(_Mod, TC, Config, Result, _Return) -> call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 35920ec1dc..6fc840745d 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -1,7 +1,7 @@ %%---------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2013. All Rights Reserved. +%% Copyright Ericsson AB 2012-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1334,7 +1334,7 @@ handle_data(NewData,#state{connection=Connection,buff=Buff} = State) -> %% first answer P=#pending{tref=TRef,caller=Caller} = lists:last(Pending), - timer:cancel(TRef), + _ = timer:cancel(TRef), Reason1 = {failed_to_parse_received_data,Reason}, ct_gen_conn:return(Caller,{error,Reason1}), lists:delete(P,Pending) @@ -1454,7 +1454,7 @@ decode({Tag,Attrs,_}=E, #state{connection=Connection,pending=Pending}=State) -> {noreply,State#state{hello_status = {error,Reason}}} end; #pending{tref=TRef,caller=Caller} -> - timer:cancel(TRef), + _ = timer:cancel(TRef), case decode_hello(E) of {ok,SessionId,Capabilities} -> ct_gen_conn:return(Caller,ok), @@ -1482,7 +1482,7 @@ decode({Tag,Attrs,_}=E, #state{connection=Connection,pending=Pending}=State) -> case [P || P = #pending{msg_id=undefined,op=undefined} <- Pending] of [#pending{tref=TRef, caller=Caller}] -> - timer:cancel(TRef), + _ = timer:cancel(TRef), ct_gen_conn:return(Caller,E), {noreply,State#state{pending=[]}}; _ -> @@ -1504,7 +1504,7 @@ get_msg_id(Attrs) -> decode_rpc_reply(MsgId,{_,Attrs,Content0}=E,#state{pending=Pending} = State) -> case lists:keytake(MsgId,#pending.msg_id,Pending) of {value, #pending{tref=TRef,op=Op,caller=Caller}, Pending1} -> - timer:cancel(TRef), + _ = timer:cancel(TRef), Content = forward_xmlns_attr(Attrs,Content0), {CallerReply,{ServerReply,State2}} = do_decode_rpc_reply(Op,Content,State#state{pending=Pending1}), @@ -1519,7 +1519,7 @@ decode_rpc_reply(MsgId,{_,Attrs,Content0}=E,#state{pending=Pending} = State) -> msg_id=undefined, op=undefined, caller=Caller}] -> - timer:cancel(TRef), + _ = timer:cancel(TRef), ct_gen_conn:return(Caller,E), {noreply,State#state{pending=[]}}; _ -> @@ -1862,10 +1862,7 @@ ssh_open(#options{host=Host,timeout=Timeout,port=Port,ssh=SshOpts,name=Name}) -> end; {error, Reason} -> ssh:close(CM), - {error,{ssh,could_not_open_channel,Reason}}; - Other -> - %% Bug in ssh?? got {closed,0} here once... - {error,{ssh,unexpected_from_session_channel,Other}} + {error,{ssh,could_not_open_channel,Reason}} end; {error,Reason} -> {error,{ssh,could_not_connect_to_server,Reason}} diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index 8c3ce03732..0a067b3a08 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -413,9 +413,11 @@ expect(Connection,Patterns) -> %%% Prompt = string() %%% Tag = term() %%% Opts = [Opt] -%%% Opt = {timeout,Timeout} | repeat | {repeat,N} | sequence | -%%% {halt,HaltPatterns} | ignore_prompt | no_prompt_check -%%% Timeout = integer() +%%% Opt = {idle_timeout,IdleTimeout} | {total_timeout,TotalTimeout} | +%%% repeat | {repeat,N} | sequence | {halt,HaltPatterns} | +%%% ignore_prompt | no_prompt_check +%%% IdleTimeout = infinity | integer() +%%% TotalTimeout = infinity | integer() %%% N = integer() %%% HaltPatterns = Patterns %%% MatchList = [Match] @@ -441,11 +443,16 @@ expect(Connection,Patterns) -> %%% will also include the matched <code>Tag</code>. Else, only %%% <code>RxMatch</code> is returned.</p> %%% -%%% <p>The <code>timeout</code> option indicates that the function +%%% <p>The <code>idle_timeout</code> option indicates that the function %%% shall return if the telnet client is idle (i.e. if no data is -%%% received) for more than <code>Timeout</code> milliseconds. Default +%%% received) for more than <code>IdleTimeout</code> milliseconds. Default %%% timeout is 10 seconds.</p> %%% +%%% <p>The <code>total_timeout</code> option sets a time limit for +%%% the complete expect operation. After <code>TotalTimeout</code> +%%% milliseconds, <code>{error,timeout}</code> is returned. The default +%%% value is <code>infinity</code> (i.e. no time limit).</p> +%%% %%% <p>The function will always return when a prompt is found, unless %%% any of the <code>ignore_prompt</code> or %%% <code>no_prompt_check</code> options are used, in which case it @@ -578,14 +585,14 @@ handle_msg({cmd,Cmd,Timeout},State) -> State#state.buffer, prompt, State#state.prx, - [{timeout,2000}]); + [{idle_timeout,2000}]); {ip,false} -> silent_teln_expect(State#state.name, State#state.teln_pid, State#state.buffer, prompt, State#state.prx, - [{timeout,200}]); + [{idle_timeout,200}]); {ip,true} -> ok end, @@ -619,14 +626,14 @@ handle_msg({send,Cmd},State) -> State#state.buffer, prompt, State#state.prx, - [{timeout,2000}]); + [{idle_timeout,2000}]); {ip,false} -> silent_teln_expect(State#state.name, State#state.teln_pid, State#state.buffer, prompt, State#state.prx, - [{timeout,200}]); + [{idle_timeout,200}]); {ip,true} -> ok end, @@ -880,7 +887,8 @@ teln_get_all_data(Pid,Prx,Data,Acc,LastLine) -> %% Expect options record -record(eo,{teln_pid, prx, - timeout, + idle_timeout, + total_timeout, haltpatterns=[], seq=false, repeat=false, @@ -922,11 +930,12 @@ teln_expect(Name,Pid,Data,Pattern0,Prx,Opts) -> Seq = get_seq(Opts), Pattern = convert_pattern(Pattern0,Seq), - Timeout = get_timeout(Opts), + {IdleTimeout,TotalTimeout} = get_timeouts(Opts), EO = #eo{teln_pid=Pid, prx=Prx, - timeout=Timeout, + idle_timeout=IdleTimeout, + total_timeout=TotalTimeout, seq=Seq, haltpatterns=HaltPatterns, prompt_check=PromptCheck}, @@ -965,11 +974,22 @@ rm_dupl([P|Ps],Acc) -> rm_dupl([],Acc) -> lists:reverse(Acc). -get_timeout(Opts) -> - case lists:keysearch(timeout,1,Opts) of - {value,{timeout,T}} -> T; - false -> ?DEFAULT_TIMEOUT - end. +get_timeouts(Opts) -> + {case lists:keysearch(idle_timeout,1,Opts) of + {value,{_,T}} -> + T; + false -> + %% this check is for backwards compatibility (pre CT v1.8) + case lists:keysearch(timeout,1,Opts) of + {value,{_,T}} -> T; + false -> ?DEFAULT_TIMEOUT + end + end, + case lists:keysearch(total_timeout,1,Opts) of + {value,{_,T}} -> T; + false -> infinity + end}. + get_repeat(Opts) -> case lists:keysearch(repeat,1,Opts) of {value,{repeat,N}} when is_integer(N) -> @@ -1011,7 +1031,8 @@ repeat_expect(Name,Pid,Data,Pattern,Acc,EO) -> {error,Reason} end. -teln_expect1(Name,Pid,Data,Pattern,Acc,EO) -> +teln_expect1(Name,Pid,Data,Pattern,Acc,EO=#eo{idle_timeout=IdleTO, + total_timeout=TotalTO}) -> ExpectFun = case EO#eo.seq of true -> fun() -> seq_expect(Name,Pid,Data,Pattern,Acc,EO) @@ -1028,17 +1049,26 @@ teln_expect1(Name,Pid,Data,Pattern,Acc,EO) -> NotFinished -> %% Get more data Fun = fun() -> get_data1(EO#eo.teln_pid) end, - case timer:tc(ct_gen_conn, do_within_time, [Fun, EO#eo.timeout]) of + case timer:tc(ct_gen_conn, do_within_time, [Fun, IdleTO]) of {_,{error,Reason}} -> %% A timeout will occur when the telnet connection - %% is idle for EO#eo.timeout milliseconds. + %% is idle for EO#eo.idle_timeout milliseconds. {error,Reason}; + {_,{ok,Data1}} when TotalTO == infinity -> + case NotFinished of + {nomatch,Rest} -> + %% One expect + teln_expect1(Name,Pid,Rest++Data1,Pattern,[],EO); + {continue,Patterns1,Acc1,Rest} -> + %% Sequence + teln_expect1(Name,Pid,Rest++Data1,Patterns1,Acc1,EO) + end; {Elapsed,{ok,Data1}} -> - TVal = trunc(EO#eo.timeout - (Elapsed/1000)), + TVal = trunc(TotalTO - (Elapsed/1000)), if TVal =< 0 -> {error,timeout}; true -> - EO1 = EO#eo{timeout = TVal}, + EO1 = EO#eo{total_timeout = TVal}, case NotFinished of {nomatch,Rest} -> %% One expect diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl index a731c8054c..0e6c877c5d 100644 --- a/lib/common_test/src/cth_conn_log.erl +++ b/lib/common_test/src/cth_conn_log.erl @@ -100,7 +100,6 @@ get_log_opts(Opts) -> Hosts = proplists:get_value(hosts,Opts,[]), {LogType,Hosts}. - pre_init_per_testcase(TestCase,Config,CthState) -> Logs = lists:map( diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index 7ed2018bdf..bb12171ea7 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -79,6 +79,10 @@ init(Path, Opts) -> url_base = proplists:get_value(url_base,Opts), timer = now() }. +pre_init_per_suite(Suite,SkipOrFail,State) when is_tuple(SkipOrFail) -> + {SkipOrFail, init_tc(State#state{curr_suite = Suite, + curr_suite_ts = now()}, + SkipOrFail) }; pre_init_per_suite(Suite,Config,#state{ test_cases = [] } = State) -> TcLog = proplists:get_value(tc_logfile,Config), CurrLogDir = filename:dirname(TcLog), diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index 7408cbe376..e90513f888 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -81,7 +81,8 @@ end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -suite() -> [{ct_hooks,[ts_install_cth]}]. +suite() -> [{timetrap,{seconds,60}}, + {ct_hooks,[ts_install_cth]}]. all() -> [ct_master_test]. diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl index 394d64c2ed..0ee0525216 100644 --- a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl @@ -20,7 +20,9 @@ all() -> expect_repeat, expect_sequence, expect_error_prompt, - expect_error_timeout, + expect_error_timeout1, + expect_error_timeout2, + expect_error_timeout3, no_prompt_check, no_prompt_check_repeat, no_prompt_check_sequence, @@ -87,13 +89,34 @@ expect_error_prompt(_) -> %% Check that expect returns after idle timeout, and even if the %% expected pattern is received - as long as not newline or prompt is %% received it will not match. -expect_error_timeout(_) -> +expect_error_timeout1(_) -> {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_no_prompt xxx"), {error,timeout} = ct_telnet:expect(Handle, ["xxx"], [{timeout,1000}]), ok = ct_telnet:close(Handle), ok. +expect_error_timeout2(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "echo_no_prompt xxx"), + {error,timeout} = ct_telnet:expect(Handle, ["xxx"], [{idle_timeout,1000}, + {total_timeout,infinity}]), + ok = ct_telnet:close(Handle), + ok. + +%% Check that if server loops and pattern not matching, the operation +%% can be aborted +expect_error_timeout3(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "echo_loop 5000 xxx"), + {error,timeout} = ct_telnet:expect(Handle, ["yyy"], + [{idle_timeout,infinity}, + {total_timeout,3000}]), + ok = ct_telnet:send(Handle, "echo ayt"), + {ok,["ayt"]} = ct_telnet:expect(Handle, ["ayt"]), + ok = ct_telnet:close(Handle), + ok. + %% expect with ignore_prompt option should not return even if a prompt %% is found. The pattern after the prompt (here "> ") can be matched. ignore_prompt(_) -> diff --git a/lib/common_test/test/telnet_server.erl b/lib/common_test/test/telnet_server.erl index 3b0c3cbcb5..ae56787819 100644 --- a/lib/common_test/test/telnet_server.erl +++ b/lib/common_test/test/telnet_server.erl @@ -211,6 +211,11 @@ do_handle_data("echo_ml_no_prompt " ++ Data,State) -> ReturnData = string:join(Lines,"\n"), send(ReturnData,State), {ok,State}; +do_handle_data("echo_loop " ++ Data,State) -> + [TStr|Lines] = string:tokens(Data," "), + ReturnData = string:join(Lines,"\n"), + send_loop(list_to_integer(TStr),ReturnData,State), + {ok,State}; do_handle_data("disconnect_after " ++WaitStr,State) -> Wait = list_to_integer(string:strip(WaitStr,right,$\n)), dbg("Server will close connection in ~w ms...", [Wait]), @@ -251,6 +256,20 @@ send(Data,State) -> throw({error,send,Error}) end. +send_loop(T,Data,State) -> + dbg("Server sending ~p in loop for ~w ms...~n",[Data,T]), + send_loop(now(),T,Data,State). + +send_loop(T0,T,Data,State) -> + ElapsedMS = trunc(timer:now_diff(now(),T0)/1000), + if ElapsedMS >= T -> + ok; + true -> + send(Data,State), + timer:sleep(500), + send_loop(T0,T,Data,State) + end. + get_line([$\r,$\n|Rest],Acc) -> {lists:reverse(Acc),Rest}; get_line([$\r,0|Rest],Acc) -> diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index 1b7b0226cc..cec94a49fd 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -447,7 +447,6 @@ message_to_string({opaque_size, [SizeType, Size]}) -> message_to_string({opaque_call, [M, F, Args, Culprit, OpaqueType]}) -> io_lib:format("The call ~s:~s~s breaks the opaqueness of the term ~s :: ~s\n", [M, F, Args, Culprit, OpaqueType]); - %%----- Warnings for concurrency errors -------------------- message_to_string({race_condition, [M, F, Args, Reason]}) -> io_lib:format("The call ~w:~w~s ~s\n", [M, F, Args, Reason]); @@ -564,4 +563,4 @@ form_position_string(ArgNs) -> ordinal(1) -> "1st"; ordinal(2) -> "2nd"; ordinal(3) -> "3rd"; -ordinal(N) when is_integer(N) -> io_lib:format("~wth",[N]). +ordinal(N) when is_integer(N) -> io_lib:format("~wth", [N]). diff --git a/lib/dialyzer/src/dialyzer.hrl b/lib/dialyzer/src/dialyzer.hrl index 6cb4af6a46..9a25f86512 100644 --- a/lib/dialyzer/src/dialyzer.hrl +++ b/lib/dialyzer/src/dialyzer.hrl @@ -89,12 +89,6 @@ -type dial_error() :: any(). %% XXX: underspecified %%-------------------------------------------------------------------- -%% THIS TYPE SHOULD ONE DAY DISAPPEAR -- IT DOES NOT BELONG HERE -%%-------------------------------------------------------------------- - --type ordset(T) :: [T] . %% XXX: temporarily - -%%-------------------------------------------------------------------- %% Basic types used either in the record definitions below or in other %% parts of the application %%-------------------------------------------------------------------- @@ -144,7 +138,7 @@ init_plts = [] :: [file:filename()], include_dirs = [] :: [file:filename()], output_plt = none :: 'none' | file:filename(), - legal_warnings = ordsets:new() :: ordset(dial_warn_tag()), + legal_warnings = ordsets:new() :: ordsets:ordset(dial_warn_tag()), report_mode = normal :: rep_mode(), erlang_mode = false :: boolean(), use_contracts = true :: boolean(), @@ -168,4 +162,4 @@ dialyzer_timing:end_stamp(Server), Var end). --define(timing(Server, Msg, Expr),?timing(Server, Msg, _T, Expr)). +-define(timing(Server, Msg, Expr), ?timing(Server, Msg, _T, Expr)). diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 46eaeaa303..283031eb9a 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -20,6 +20,8 @@ -module(dialyzer_contracts). +-compile(export_all). + -export([check_contract/2, check_contracts/4, contracts_without_fun/3, @@ -439,7 +441,8 @@ contract_from_form([], _RecDict, _FileLine, TypeAcc, FormAcc) -> {lists:reverse(TypeAcc), lists:reverse(FormAcc)}. process_constraints(Constrs, RecDict, ExpTypes, AllRecords) -> - Init = initialize_constraints(Constrs, RecDict, ExpTypes, AllRecords), + Init0 = initialize_constraints(Constrs, RecDict, ExpTypes, AllRecords), + Init = remove_cycles(Init0), constraints_fixpoint(Init, RecDict, ExpTypes, AllRecords). initialize_constraints(Constrs, RecDict, ExpTypes, AllRecords) -> @@ -479,12 +482,9 @@ constraints_fixpoint(OldVarDict, Constrs, RecDict, ExpTypes, AllRecords) -> constraints_fixpoint(NewVarDict, Constrs, RecDict, ExpTypes, AllRecords) end. --define(TYPE_LIMIT, 4). - final_form(Form, RecDict, ExpTypes, AllRecords, VarDict) -> T1 = erl_types:t_from_form(Form, RecDict, VarDict), - T2 = erl_types:t_solve_remote(T1, ExpTypes, AllRecords), - erl_types:t_limit(T2, ?TYPE_LIMIT). + erl_types:t_solve_remote(T1, ExpTypes, AllRecords). constraints_to_dict(Constrs, RecDict, ExpTypes, AllRecords, VarDict) -> Subtypes = @@ -499,6 +499,74 @@ constraints_to_subs([C|Rest], RecDict, ExpTypes, AllRecords, VarDict, Acc) -> NewAcc = [{subtype, T1, T2}|Acc], constraints_to_subs(Rest, RecDict, ExpTypes, AllRecords, VarDict, NewAcc). +%% Replaces variables with '_' when necessary to break up cycles among +%% the constraints. + +remove_cycles(Constrs0) -> + Uses = find_uses(Constrs0), + G = digraph:new(), + Vs0 = [V || {V, _} <- Uses] ++ [V || {_, V} <- Uses], + Vs = lists:usort(Vs0), + lists:foreach(fun(V) -> _ = digraph:add_vertex(G, V) end, Vs), + lists:foreach(fun({From, To}) -> + _ = digraph:add_edge(G, {From, To}, From, To, []) + end, Uses), + ok = remove_cycles(G, Vs), + ToRemove = ordsets:subtract(ordsets:from_list(Uses), + ordsets:from_list(digraph:edges(G))), + Constrs = remove_uses(ToRemove, Constrs0), + digraph:delete(G), + Constrs. + +find_uses([{Var, Form}|Constrs]) -> + UsedVars = form_vars(Form, []), + VarName = erl_types:t_var_name(Var), + [{VarName, UsedVar} || UsedVar <- UsedVars] ++ find_uses(Constrs); +find_uses([]) -> + []. + +form_vars({var, _, '_'}, Vs) -> Vs; +form_vars({var, _, V}, Vs) -> [V|Vs]; +form_vars(T, Vs) when is_tuple(T) -> + form_vars(tuple_to_list(T), Vs); +form_vars([E|Es], Vs) -> + form_vars(Es, form_vars(E, Vs)); +form_vars(_, Vs) -> Vs. + +remove_cycles(G, Vs) -> + NumberOfEdges = digraph:no_edges(G), + lists:foreach(fun(V) -> + case digraph:get_cycle(G, V) of + false -> true; + [V] -> digraph:del_edge(G, {V, V}); + [V, V1|_] -> digraph:del_edge(G, {V, V1}) + end + end, Vs), + case digraph:no_edges(G) =:= NumberOfEdges of + true -> ok; + false -> remove_cycles(G, Vs) + end. + +remove_uses([], Constrs) -> Constrs; +remove_uses([{Var, Use}|ToRemove], Constrs0) -> + Constrs = remove_uses(Var, Use, Constrs0), + remove_uses(ToRemove, Constrs). + +remove_uses(_Var, _Use, []) -> []; +remove_uses(Var, Use, [Constr|Constrs]) -> + {V, Form} = Constr, + case erl_types:t_var_name(V) =:= Var of + true -> [{V, remove_use(Form, Use)}|Constrs]; + false -> [Constr|remove_uses(Var, Use, Constrs)] + end. + +remove_use({var, L, V}, V) -> {var, L, '_'}; +remove_use(T, V) when is_tuple(T) -> + list_to_tuple(remove_use(tuple_to_list(T), V)); +remove_use([E|Es], V) -> + [remove_use(E, V)|remove_use(Es, V)]; +remove_use(T, _V) -> T. + %% Gets the most general domain of a list of domains of all %% the overloaded contracts diff --git a/lib/dialyzer/src/dialyzer_dep.erl b/lib/dialyzer/src/dialyzer_dep.erl index f1ac41ff04..e3ece144c9 100644 --- a/lib/dialyzer/src/dialyzer_dep.erl +++ b/lib/dialyzer/src/dialyzer_dep.erl @@ -55,11 +55,11 @@ %% %% Letrecs = a dict mapping var labels to their recursive definition. %% top-level letrecs are not included as they are handled -%% separatedly. +%% separately. %% -spec analyze(cerl:c_module()) -> - {dict:dict(), ordset('external' | label()), dict:dict(), dict:dict()}. + {dict:dict(), ordsets:ordset('external' | label()), dict:dict(), dict:dict()}. analyze(Tree) -> %% io:format("Handling ~w\n", [cerl:atom_val(cerl:module_name(Tree))]), diff --git a/lib/dialyzer/src/dialyzer_gui_wx.erl b/lib/dialyzer/src/dialyzer_gui_wx.erl index 08f31c1e13..7070fa240d 100644 --- a/lib/dialyzer/src/dialyzer_gui_wx.erl +++ b/lib/dialyzer/src/dialyzer_gui_wx.erl @@ -61,7 +61,7 @@ init_plt :: dialyzer_plt:plt(), dir_entry :: wx:wx_object(), file_box :: wx:wx_object(), - files_to_analyze :: ordset(string()), + files_to_analyze :: ordsets:ordset(string()), gui :: wx:wx_object(), log :: wx:wx_object(), menu :: menu(), diff --git a/lib/dialyzer/src/dialyzer_races.erl b/lib/dialyzer/src/dialyzer_races.erl index 48fcde8014..b1f849b16f 100644 --- a/lib/dialyzer/src/dialyzer_races.erl +++ b/lib/dialyzer/src/dialyzer_races.erl @@ -85,6 +85,12 @@ -type race_tag() :: 'whereis_register' | 'whereis_unregister' | 'ets_lookup_insert' | 'mnesia_dirty_read_write'. +%% The following type is similar to the dial_warning() type but has a +%% tag which is local to this module and is not propagated to outside +-type dial_race_warning() :: {race_warn_tag(), file_line(), {atom(), [term()]}}. +-type race_warn_tag() :: ?WARN_WHEREIS_REGISTER | ?WARN_WHEREIS_UNREGISTER + | ?WARN_ETS_LOOKUP_INSERT | ?WARN_MNESIA_DIRTY_READ_WRITE. + -record(beg_clause, {arg :: var_to_map1(), pats :: var_to_map1(), guard :: cerl:cerl()}). @@ -103,7 +109,7 @@ args :: args(), arg_types :: [erl_types:erl_type()], vars :: [core_vars()], - state :: _, %% XXX: recursive + state :: dialyzer_dataflow:state(), file_line :: file_line(), var_map :: dict:dict()}). -record(fun_call, {caller :: dialyzer_callgraph:mfa_or_funlbl(), @@ -141,7 +147,7 @@ race_tags = [] :: [#race_fun{}], %% true for fun types and warning mode race_analysis = false :: boolean(), - race_warnings = [] :: [dial_warning()]}). + race_warnings = [] :: [dial_race_warning()]}). %%% =========================================================================== %%% @@ -1763,7 +1769,7 @@ ets_list_args(MaybeList) -> catch _:_ -> [?no_label] end; false -> [ets_tuple_args(MaybeList)] - end. + end. ets_list_argtypes(ListStr) -> ListStr1 = string:strip(ListStr, left, $[), diff --git a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes index bfa33cd296..fbdd182358 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes +++ b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes @@ -1,27 +1,28 @@ contracts_with_subtypes.erl:106: The call contracts_with_subtypes:rec_arg({'a','b'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) contracts_with_subtypes.erl:107: The call contracts_with_subtypes:rec_arg({'b','a'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:108: The call contracts_with_subtypes:rec_arg({'a',{'b','a'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) contracts_with_subtypes.erl:109: The call contracts_with_subtypes:rec_arg({'b',{'a','b'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:110: The call contracts_with_subtypes:rec_arg({'a',{'b',{'a','b'}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:111: The call contracts_with_subtypes:rec_arg({'b',{'a',{'b','a'}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:142: The pattern 1 can never match the type string() -contracts_with_subtypes.erl:145: The pattern 'alpha' can never match the type {'ok',_} | {'ok',_,string()} -contracts_with_subtypes.erl:147: The pattern 42 can never match the type {'ok',_} | {'ok',_,string()} -contracts_with_subtypes.erl:163: The pattern 'alpha' can never match the type {'ok',_} -contracts_with_subtypes.erl:165: The pattern 42 can never match the type {'ok',_} -contracts_with_subtypes.erl:183: The pattern 'alpha' can never match the type {'ok',_} -contracts_with_subtypes.erl:185: The pattern 42 can never match the type {'ok',_} -contracts_with_subtypes.erl:202: The pattern 1 can never match the type string() -contracts_with_subtypes.erl:205: The pattern {'ok', _} can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:206: The pattern 'alpha' can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:207: The pattern {'ok', 42} can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:208: The pattern 42 can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:234: Function flat_ets_new_t/0 has no local return -contracts_with_subtypes.erl:235: The call contracts_with_subtypes:flat_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,'set' | 'ordered_set' | 'bag' | 'duplicate_bag' | 'public' | 'protected' | 'private' | 'named_table' | {'keypos',integer()} | {'heir',pid(),term()} | {'heir','none'} | {'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed') +contracts_with_subtypes.erl:135: The call contracts_with_subtypes:rec2({'a','b'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:136: The call contracts_with_subtypes:rec2({'b','a'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:137: The call contracts_with_subtypes:rec2({'a',{'b','a'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:138: The call contracts_with_subtypes:rec2({'b',{'a','b'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:171: The pattern 1 can never match the type string() +contracts_with_subtypes.erl:174: The pattern 'alpha' can never match the type {'ok',_} | {'ok',_,string()} +contracts_with_subtypes.erl:176: The pattern 42 can never match the type {'ok',_} | {'ok',_,string()} +contracts_with_subtypes.erl:192: The pattern 'alpha' can never match the type {'ok',_} +contracts_with_subtypes.erl:194: The pattern 42 can never match the type {'ok',_} +contracts_with_subtypes.erl:212: The pattern 'alpha' can never match the type {'ok',_} +contracts_with_subtypes.erl:214: The pattern 42 can never match the type {'ok',_} +contracts_with_subtypes.erl:231: The pattern 1 can never match the type string() +contracts_with_subtypes.erl:234: The pattern {'ok', _} can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:235: The pattern 'alpha' can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:236: The pattern {'ok', 42} can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:237: The pattern 42 can never match the type {'ok',_,string()} contracts_with_subtypes.erl:23: Invalid type specification for function contracts_with_subtypes:extract2/0. The success typing is () -> 'something' -contracts_with_subtypes.erl:261: Function factored_ets_new_t/0 has no local return -contracts_with_subtypes.erl:262: The call contracts_with_subtypes:factored_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,Type | Access | 'named_table' | {'keypos',Pos} | {'heir',Pid::pid(),HeirData} | {'heir','none'} | Tweaks), is_subtype(Type,type()), is_subtype(Access,access()), is_subtype(Tweaks,{'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed'), is_subtype(Pos,pos_integer()), is_subtype(HeirData,term()) +contracts_with_subtypes.erl:263: Function flat_ets_new_t/0 has no local return +contracts_with_subtypes.erl:264: The call contracts_with_subtypes:flat_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,'set' | 'ordered_set' | 'bag' | 'duplicate_bag' | 'public' | 'protected' | 'private' | 'named_table' | {'keypos',integer()} | {'heir',pid(),term()} | {'heir','none'} | {'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed') +contracts_with_subtypes.erl:290: Function factored_ets_new_t/0 has no local return +contracts_with_subtypes.erl:291: The call contracts_with_subtypes:factored_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,Type | Access | 'named_table' | {'keypos',Pos} | {'heir',Pid::pid(),HeirData} | {'heir','none'} | Tweaks), is_subtype(Type,type()), is_subtype(Access,access()), is_subtype(Tweaks,{'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed'), is_subtype(Pos,pos_integer()), is_subtype(HeirData,term()) contracts_with_subtypes.erl:77: The call contracts_with_subtypes:foo1(5) breaks the contract (Arg1) -> Res when is_subtype(Arg1,atom()), is_subtype(Res,atom()) contracts_with_subtypes.erl:78: The call contracts_with_subtypes:foo2(5) breaks the contract (Arg1) -> Res when is_subtype(Arg1,Arg2), is_subtype(Arg2,atom()), is_subtype(Res,atom()) contracts_with_subtypes.erl:79: The call contracts_with_subtypes:foo3(5) breaks the contract (Arg1) -> Res when is_subtype(Arg2,atom()), is_subtype(Arg1,Arg2), is_subtype(Res,atom()) diff --git a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 new file mode 100644 index 0000000000..9f5433a13d --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 @@ -0,0 +1,3 @@ + +contracts_with_subtypes2.erl:18: Function t/0 has no local return +contracts_with_subtypes2.erl:19: The call contracts_with_subtypes2:t({'a',{'b',{'c',{'d',{'e',{'g',3}}}}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A}), is_subtype(A,{'b',B}), is_subtype(B,{'c',C}), is_subtype(C,{'d',D}), is_subtype(D,{'e',E}), is_subtype(E,{'f',_}) diff --git a/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 b/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 new file mode 100644 index 0000000000..4565112ea0 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/invalid_spec_2 @@ -0,0 +1,2 @@ + +scala_user.erl:5: Invalid type specification for function scala_user:is_list/2. The success typing is (maybe_improper_list() | tuple(),_) -> boolean() diff --git a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl index d72138d509..d7dfd9752e 100644 --- a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl +++ b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl @@ -103,15 +103,44 @@ c(babb) -> rec_arg({b, {a, {b, b}}}); c(ababb) -> rec_arg({a, {b, {a, {b, b}}}}); c(babaa) -> rec_arg({b, {a, {b, {a, a}}}}). -w(ab) -> rec_arg({a, b}); -w(ba) -> rec_arg({b, a}); -w(aba) -> rec_arg({a, {b, a}}); -w(bab) -> rec_arg({b, {a, b}}); -w(abab) -> rec_arg({a, {b, {a, b}}}); -w(baba) -> rec_arg({b, {a, {b, a}}}); +w(ab) -> rec_arg({a, b}); % breaks the contract +w(ba) -> rec_arg({b, a}); % breaks the contract +w(aba) -> rec_arg({a, {b, a}}); % no longer breaks the contract +w(bab) -> rec_arg({b, {a, b}}); % breaks the contract +w(abab) -> rec_arg({a, {b, {a, b}}}); % no longer breaks the contract +w(baba) -> rec_arg({b, {a, {b, a}}}); % no longer breaks the contract w(ababa) -> rec_arg({a, {b, {a, {b, a}}}}); w(babab) -> rec_arg({b, {a, {b, {a, b}}}}). +%% For comparison: the same thing with types + +-type ab() :: {a, a()} | {b, b()}. +-type a() :: a | {b, b()}. +-type b() :: b | {a, a()}. + +-spec rec2(Arg) -> ok when + Arg :: ab(). + +rec2(X) -> get(X). + +d(aa) -> rec2({a, a}); +d(bb) -> rec2({b, b}); +d(abb) -> rec2({a, {b, b}}); +d(baa) -> rec2({b, {a, a}}); +d(abaa) -> rec2({a, {b, {a, a}}}); +d(babb) -> rec2({b, {a, {b, b}}}); +d(ababb) -> rec2({a, {b, {a, {b, b}}}}); +d(babaa) -> rec2({b, {a, {b, {a, a}}}}). + +q(ab) -> rec2({a, b}); % breaks the contract +q(ba) -> rec2({b, a}); % breaks the contract +q(aba) -> rec2({a, {b, a}}); % breaks the contract +q(bab) -> rec2({b, {a, b}}); % breaks the contract +q(abab) -> rec2({a, {b, {a, b}}}); +q(baba) -> rec2({b, {a, {b, a}}}); +q(ababa) -> rec2({a, {b, {a, {b, a}}}}); +q(babab) -> rec2({b, {a, {b, {a, b}}}}). + %=============================================================================== -type dublo(X) :: {X, X}. @@ -143,7 +172,7 @@ st(X) when is_atom(X) -> _Other -> ok end; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. @@ -161,7 +190,7 @@ dt(X) when is_atom(X) -> err2 -> ok; {ok, X} -> ok; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. @@ -181,7 +210,7 @@ dt2(X) when is_atom(X) -> err2 -> ok; {ok, X} -> ok; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. diff --git a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl new file mode 100644 index 0000000000..d2f945b284 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl @@ -0,0 +1,40 @@ +-module(contracts_with_subtypes2). + +-compile(export_all). + +-behaviour(supervisor). + +-spec t(Arg) -> ok when + Arg :: {a, A}, + A :: {b, B}, + B :: {c, C}, + C :: {d, D}, + D :: {e, E}, + E :: {f, _}. + +t(X) -> + get(X). + +t() -> + t({a, {b, {c, {d, {e, {g, 3}}}}}}). % breaks the contract + +%% This one should possibly result in warnings about unused variables. +-spec l() -> ok when + X :: Y, + Y :: X. + +l() -> + ok. + +%% This is the example from seq12547 (ticket OTP-11798). +%% There used to be a warning. + +-spec init(term()) -> Result when + Result :: {ok, {{supervisor:strategy(), + non_neg_integer(), + pos_integer()}, + [supervisor:child_spec()]}} + | ignore. + +init(_) -> + foo:bar(). diff --git a/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_data.erl b/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_data.erl new file mode 100644 index 0000000000..c26787fe24 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_data.erl @@ -0,0 +1,5 @@ +-module(scala_data). + +-export_type([data/0]). + +-opaque data() :: {'data', term()}. diff --git a/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_user.erl b/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_user.erl new file mode 100644 index 0000000000..4e981f3b74 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/invalid_spec_2/scala_user.erl @@ -0,0 +1,8 @@ +-module(scala_user). + +-export([is_list/2]). + +-spec is_list(atom(), scala_data:data()) -> boolean(). + +is_list( List,Data) when is_list(List) -> true; +is_list(Tuple,Data) when is_tuple(Tuple) -> false. diff --git a/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl b/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl new file mode 100644 index 0000000000..70059f73b6 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl @@ -0,0 +1,12 @@ +-module(maps_redef). + +-export([t/0]). + +%% OK in Erlang/OTP 17, at least. + +-type map() :: atom(). % redefine built-in type + +-spec t() -> map(). + +t() -> + a. % OK diff --git a/lib/eldap/src/Makefile b/lib/eldap/src/Makefile index ebb7967e11..2e1110ec2c 100644 --- a/lib/eldap/src/Makefile +++ b/lib/eldap/src/Makefile @@ -88,7 +88,7 @@ $(TARGET_FILES): $(HRL_FILES) # Special Build Targets # ---------------------------------------------------- $(ASN1_HRL): ../asn1/$(ASN1_FILES) - $(asn_verbose)$(ERLC) -o $(EBIN) -bber $(ERL_COMPILE_FLAGS) ../asn1/ELDAPv3.asn1 + $(asn_verbose)$(ERLC) -o $(EBIN) +legacy_erlang_types -bber $(ERL_COMPILE_FLAGS) ../asn1/ELDAPv3.asn1 # ---------------------------------------------------- # Release Target diff --git a/lib/eldap/test/eldap_misc_SUITE.erl b/lib/eldap/test/eldap_misc_SUITE.erl new file mode 100644 index 0000000000..ca810ee33c --- /dev/null +++ b/lib/eldap/test/eldap_misc_SUITE.erl @@ -0,0 +1,51 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012-2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(eldap_misc_SUITE). + +-compile(export_all). %% Use this only in test suites... + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eldap/include/eldap.hrl"). +-include_lib("eldap/ebin/ELDAPv3.hrl"). + +all() -> + [ + encode, + decode + ]. + + +encode(_Config) -> + {ok,Bin} = 'ELDAPv3':encode('AddRequest', #'AddRequest'{entry="hejHopp" ,attributes=[]} ), + Expected = <<104,11,4,7,104,101,106,72,111,112,112,48,0>>, + Expected = Bin. + +decode(_Config) -> + {ok,Res} = 'ELDAPv3':decode('AddRequest', <<104,11,4,7,104,101,106,72,111,112,112,48,0>>), + ct:log("Res = ~p", [Res]), + Expected = #'AddRequest'{entry = "hejHopp",attributes = []}, + case Res of + Expected -> ok; + #'AddRequest'{entry= <<"hejHopp">>, attributes=[]} -> + {fail, "decoded to (correct) binary!!"}; + _ -> + {fail, "Bad decode"} + end. + diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index 3175d1bdfd..2e8418d61e 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -1166,7 +1166,11 @@ static unsigned int gen_challenge(void) uname(&s.name); s.cpu = clock(); s.pid = getpid(); +#ifndef __ANDROID__ s.hid = gethostid(); +#else + s.hid = 0; +#endif s.uid = getuid(); s.gid = getgid(); diff --git a/lib/erl_interface/src/connect/ei_resolve.c b/lib/erl_interface/src/connect/ei_resolve.c index 74dcba61a7..cffcac801c 100644 --- a/lib/erl_interface/src/connect/ei_resolve.c +++ b/lib/erl_interface/src/connect/ei_resolve.c @@ -642,7 +642,7 @@ struct hostent *ei_gethostbyname_r(const char *name, #ifndef HAVE_GETHOSTBYNAME_R return my_gethostbyname_r(name,hostp,buffer,buflen,h_errnop); #else -#if (defined(__GLIBC__) || (__FreeBSD_version >= 602000) || defined(__DragonFly__)) +#if (defined(__GLIBC__) || (__FreeBSD_version >= 602000) || defined(__DragonFly__) || defined(__ANDROID__)) struct hostent *result; gethostbyname_r(name, hostp, buffer, buflen, &result, h_errnop); diff --git a/lib/hipe/Makefile b/lib/hipe/Makefile index a9e24f4d17..46cbc33ae2 100644 --- a/lib/hipe/Makefile +++ b/lib/hipe/Makefile @@ -22,7 +22,7 @@ include $(ERL_TOP)/make/target.mk include $(ERL_TOP)/make/$(TARGET)/otp.mk ifdef HIPE_ENABLED -HIPE_SUBDIRS = regalloc sparc ppc x86 amd64 arm opt tools +HIPE_SUBDIRS = regalloc sparc ppc x86 amd64 arm opt tools llvm else HIPE_SUBDIRS = endif diff --git a/lib/hipe/arm/hipe_arm_assemble.erl b/lib/hipe/arm/hipe_arm_assemble.erl index 2af786994e..e9de96a927 100644 --- a/lib/hipe/arm/hipe_arm_assemble.erl +++ b/lib/hipe/arm/hipe_arm_assemble.erl @@ -44,8 +44,8 @@ assemble(CompiledCode, Closures, Exports, Options) -> print("Total num bytes=~w\n", [CodeSize], Options), %% SC = hipe_pack_constants:slim_constmap(ConstMap), - DataRelocs = mk_data_relocs(RefsFromConsts, LabelMap), - SSE = slim_sorted_exportmap(ExportMap,Closures,Exports), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap,Closures,Exports), SlimRefs = hipe_pack_constants:slim_refs(AccRefs), Bin = term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, ConstAlign, ConstSize, @@ -320,7 +320,7 @@ do_pseudo_li(I, MFA, ConstMap, Address, PrevImms, PendImms) -> Atom when is_atom(Atom) -> {load_atom, Atom}; {Label,constant} -> - ConstNo = find_const({MFA,Label}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,Label}, ConstMap), {load_address, {constant,ConstNo}}; {Label,closure} -> {load_address, {closure,Label}}; @@ -518,37 +518,6 @@ fix_pc_refs(I, InsnAddress, FunAddress, LabelMap) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -mk_data_relocs(RefsFromConsts, LabelMap) -> - lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). - -mk_data_relocs([{MFA,Labels} | Rest], LabelMap, Acc) -> - Map = [case Label of - {L,Pos} -> - Offset = find({MFA,L}, LabelMap), - {Pos,Offset}; - {sorted,Base,OrderedLabels} -> - {sorted, Base, [begin - Offset = find({MFA,L}, LabelMap), - {Order, Offset} - end - || {L,Order} <- OrderedLabels]} - end - || Label <- Labels], - %% msg("Map: ~w Map\n",[Map]), - mk_data_relocs(Rest, LabelMap, [Map,Acc]); -mk_data_relocs([],_,Acc) -> Acc. - -find({_MFA,_L} = MFAL, LabelMap) -> - gb_trees:get(MFAL, LabelMap). - -slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> - IsClosure = lists:member({M,F,A}, Closures), - IsExported = is_exported(F, A, Exports), - [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; -slim_sorted_exportmap([],_,_) -> []. - -is_exported(F, A, Exports) -> lists:member({F,A}, Exports). - %%% %%% Assembly listing support (pp_asm option). %%% @@ -594,17 +563,6 @@ fill_spaces(N) when N > 0 -> fill_spaces(0) -> []. -%%% -%%% Lookup a constant in a ConstMap. -%%% - -find_const({MFA,Label},[{pcm_entry,MFA,Label,ConstNo,_,_,_}|_]) -> - ConstNo; -find_const(N,[_|R]) -> - find_const(N,R); -find_const(C,[]) -> - ?EXIT({constant_not_found,C}). - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index aa69b57fa2..28281a2fac 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -502,9 +502,9 @@ t_contains_opaque(?int_range(_From, _To), _Opaques) -> false; t_contains_opaque(?int_set(_Set), _Opaques) -> false; t_contains_opaque(?list(Type, Tail, _), Opaques) -> t_contains_opaque(Type, Opaques) orelse t_contains_opaque(Tail, Opaques); -t_contains_opaque(?map(Pairs), Opaques) -> - list_contains_opaque([V||{_,V}<-Pairs], Opaques) orelse - list_contains_opaque([K||{K,_}<-Pairs], Opaques); +t_contains_opaque(?map(_) = Map, Opaques) -> + list_contains_opaque(map_values(Map), Opaques) orelse + list_contains_opaque(map_keys(Map), Opaques); t_contains_opaque(?matchstate(_P, _Slots), _Opaques) -> false; t_contains_opaque(?nil, _Opaques) -> false; t_contains_opaque(?number(_Set, _Tag), _Opaques) -> false; @@ -618,7 +618,7 @@ t_decorate_with_opaque(T1, T2, Opaques) -> end end. -decorate(?none=Type, _, _Opaques) -> Type; +decorate(Type, ?none, _Opaques) -> Type; decorate(?function(Domain, Range), ?function(D, R), Opaques) -> ?function(decorate(Domain, D, Opaques), decorate(Range, R, Opaques)); decorate(?list(Types, Tail, Size), ?list(Ts, Tl, _Sz), Opaques) -> @@ -684,6 +684,7 @@ union_decorate(U1, U2, Opaques) -> List = [A,B,F,I,L,N,T,M,Map], DecList = [Dec || E <- List, + not t_is_none(E), not t_is_none(Dec = decorate(E, Opaque, Opaques))], t_sup([Union|DecList]). @@ -2093,6 +2094,8 @@ t_has_var(?tuple(Elements, _, _)) -> t_has_var_list(Elements); t_has_var(?tuple_set(_) = T) -> t_has_var_list(t_tuple_subtypes(T)); +t_has_var(?map(_)= Map) -> + t_has_var_list(map_keys(Map)) orelse t_has_var_list(map_values(Map)); t_has_var(?opaque(Set)) -> %% Assume variables in 'args' are also present i 'struct' t_has_var_list([O#opaque.struct || O <- set_to_list(Set)]); @@ -2120,21 +2123,28 @@ t_collect_vars(?function(Domain, Range), Acc) -> t_collect_vars(?list(Contents, Termination, _), Acc) -> ordsets:union(t_collect_vars(Contents, Acc), t_collect_vars(Termination, [])); t_collect_vars(?product(Types), Acc) -> - lists:foldl(fun(T, TmpAcc) -> t_collect_vars(T, TmpAcc) end, Acc, Types); + t_collect_vars_list(Types, Acc); t_collect_vars(?tuple(?any, ?any, ?any), Acc) -> Acc; t_collect_vars(?tuple(Types, _, _), Acc) -> - lists:foldl(fun(T, TmpAcc) -> t_collect_vars(T, TmpAcc) end, Acc, Types); + t_collect_vars_list(Types, Acc); t_collect_vars(?tuple_set(_) = TS, Acc) -> - lists:foldl(fun(T, TmpAcc) -> t_collect_vars(T, TmpAcc) end, Acc, - t_tuple_subtypes(TS)); + t_collect_vars_list(t_tuple_subtypes(TS), Acc); +t_collect_vars(?map(_) = Map, Acc0) -> + Acc = t_collect_vars_list(map_keys(Map), Acc0), + t_collect_vars_list(map_values(Map), Acc); t_collect_vars(?opaque(Set), Acc) -> %% Assume variables in 'args' are also present i 'struct' - lists:foldl(fun(T, TmpAcc) -> t_collect_vars(T, TmpAcc) end, Acc, - [O#opaque.struct || O <- set_to_list(Set)]); + t_collect_vars_list([O#opaque.struct || O <- set_to_list(Set)], Acc); +t_collect_vars(?union(List), Acc) -> + t_collect_vars_list(List, Acc); t_collect_vars(_, Acc) -> Acc. +t_collect_vars_list([T|Ts], Acc0) -> + Acc = t_collect_vars(T, Acc0), + t_collect_vars_list(Ts, Acc); +t_collect_vars_list([], Acc) -> Acc. %%============================================================================= %% @@ -3081,6 +3091,9 @@ t_subst_dict(?tuple(Elements, _Arity, _Tag), Dict) -> t_tuple([t_subst_dict(E, Dict) || E <- Elements]); t_subst_dict(?tuple_set(_) = TS, Dict) -> t_sup([t_subst_dict(T, Dict) || T <- t_tuple_subtypes(TS)]); +t_subst_dict(?map(Pairs), Dict) -> + ?map([{t_subst_dict(K, Dict), t_subst_dict(V, Dict)} || + {K, V} <- Pairs]); t_subst_dict(?opaque(Es), Dict) -> List = [Opaque#opaque{args = [t_subst_dict(Arg, Dict) || Arg <- Args], struct = t_subst_dict(S, Dict)} || @@ -3130,6 +3143,9 @@ t_subst_aux(?tuple(Elements, _Arity, _Tag), VarMap) -> t_tuple([t_subst_aux(E, VarMap) || E <- Elements]); t_subst_aux(?tuple_set(_) = TS, VarMap) -> t_sup([t_subst_aux(T, VarMap) || T <- t_tuple_subtypes(TS)]); +t_subst_aux(?map(Pairs), VarMap) -> + ?map([{t_subst_aux(K, VarMap), t_subst_aux(V, VarMap)} || + {K, V} <- Pairs]); t_subst_aux(?opaque(Es), VarMap) -> List = [Opaque#opaque{args = [t_subst_aux(Arg, VarMap) || Arg <- Args], struct = t_subst_aux(S, VarMap)} || @@ -3705,7 +3721,7 @@ t_unopaque(T) -> t_unopaque(?opaque(_) = T, Opaques) -> case Opaques =:= 'universe' orelse is_opaque_type(T, Opaques) of true -> t_unopaque(t_opaque_structure(T), Opaques); - false -> T % XXX: needs revision for parametric opaque data types + false -> T end; t_unopaque(?list(ElemT, Termination, Sz), Opaques) -> ?list(t_unopaque(ElemT, Opaques), t_unopaque(Termination, Opaques), Sz); @@ -3725,11 +3741,12 @@ t_unopaque(?union([A,B,F,I,L,N,T,M,O,R,Map]), Opaques) -> UL = t_unopaque(L, Opaques), UT = t_unopaque(T, Opaques), UF = t_unopaque(F, Opaques), + UMap = t_unopaque(Map, Opaques), {OF,UO} = case t_unopaque(O, Opaques) of ?opaque(_) = O1 -> {O1, []}; Type -> {?none, [Type]} end, - t_sup([?union([A,B,UF,I,UL,N,UT,M,OF,R,Map])|UO]); + t_sup([?union([A,B,UF,I,UL,N,UT,M,OF,R,UMap])|UO]); t_unopaque(T, _) -> T. @@ -4236,8 +4253,8 @@ t_from_form({type, _L, list, []}, _TypeNames, _RecDict, _VarDict) -> t_from_form({type, _L, list, [Type]}, TypeNames, RecDict, VarDict) -> {T, R} = t_from_form(Type, TypeNames, RecDict, VarDict), {t_list(T), R}; -t_from_form({type, _L, map, _}, _TypeNames, _RecDict, _VarDict) -> - {t_map([]), []}; +t_from_form({type, _L, map, _}, TypeNames, RecDict, VarDict) -> + builtin_type(map, t_map([]), TypeNames, RecDict, VarDict); t_from_form({type, _L, mfa, []}, _TypeNames, _RecDict, _VarDict) -> {t_mfa(), []}; t_from_form({type, _L, module, []}, _TypeNames, _RecDict, _VarDict) -> @@ -4700,6 +4717,12 @@ is_same_type_name2(gb_trees, gb_tree, [], gb_trees, tree, [_, _]) -> true; is_same_type_name2(gb_trees, tree, [_, _], gb_trees, gb_tree, []) -> true; is_same_type_name2(_, _, _, _, _, _) -> false. +map_keys(?map(Pairs)) -> + [K || {K, _} <- Pairs]. + +map_values(?map(Pairs)) -> + [V || {_, V} <- Pairs]. + %% ----------------------------------- %% Set %% diff --git a/lib/hipe/llvm/Makefile b/lib/hipe/llvm/Makefile new file mode 100644 index 0000000000..92f378924a --- /dev/null +++ b/lib/hipe/llvm/Makefile @@ -0,0 +1,109 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2001-2014. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +ifndef EBIN +EBIN = ../ebin +endif + +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- +include ../vsn.mk +VSN=$(HIPE_VSN) + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- +RELSYSDIR = $(RELEASE_PATH)/lib/hipe-$(VSN) + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- +ifdef HIPE_ENABLED +HIPE_MODULES = hipe_rtl_to_llvm \ + hipe_llvm \ + elf_format \ + hipe_llvm_main \ + hipe_llvm_merge \ + hipe_llvm_liveness +else +HIPE_MODULES = +endif + +MODULES = $(HIPE_MODULES) + +HRL_FILES= elf_format.hrl elf32_format.hrl elf64_format.hrl \ + hipe_llvm_arch.hrl +ERL_FILES= $(MODULES:%=%.erl) +TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) + +# APP_FILE= +# App_SRC= $(APP_FILE).src +# APP_TARGET= $(EBIN)/$(APP_FILE) +# +# APPUP_FILE= +# APPUP_SRC= $(APPUP_FILE).src +# APPUP_TARGET= $(EBIN)/$(APPUP_FILE) + +# ---------------------------------------------------- +# FLAGS: Please keep +inline below +# ---------------------------------------------------- + +include ../native.mk + +ERL_COMPILE_FLAGS += +inline #+warn_missing_spec + +# if in 32 bit backend define BIT32 symbol +ARCH = $(shell echo $(TARGET) | sed 's/^\(x86_64\)-.*/64bit/') +ifneq ($(ARCH), 64bit) +ERL_COMPILE_FLAGS += -DBIT32 +endif + +# ---------------------------------------------------- +# Targets +# ---------------------------------------------------- + +debug opt: $(TARGET_FILES) + +docs: + +clean: + rm -f $(TARGET_FILES) + rm -f core erl_crash.dump + +# ---------------------------------------------------- +# Special Build Targets +# ---------------------------------------------------- + + +# ---------------------------------------------------- +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +release_spec: opt + $(INSTALL_DIR) $(RELSYSDIR)/llvm + $(INSTALL_DATA) $(ERL_FILES) $(HRL_FILES) $(RELSYSDIR)/llvm + $(INSTALL_DIR) $(RELSYSDIR)/ebin + $(INSTALL_DATA) $(TARGET_FILES) $(RELSYSDIR)/ebin + +release_docs_spec: diff --git a/lib/hipe/llvm/elf32_format.hrl b/lib/hipe/llvm/elf32_format.hrl new file mode 100644 index 0000000000..af1d95bf5b --- /dev/null +++ b/lib/hipe/llvm/elf32_format.hrl @@ -0,0 +1,59 @@ +%% -*- erlang-indent-level: 2 -*- + +%%% @copyright 2011-2014 Yiannis Tsiouris <[email protected]>, +%%% Chris Stavrakakis <[email protected]> +%%% @author Yiannis Tsiouris <[email protected]> +%%% [http://www.softlab.ntua.gr/~gtsiour/] + +%%% @doc This header file contains very very useful macros for handling +%%% various segments of an ELF-32 formated object file, such as sizes, +%%% offsets and predefined constants. For further information about +%%% each field take a quick look at +%%% "[http://www.sco.com/developers/gabi/latest/contents.html]" +%%% that contain the current HP/Intel definition of the ELF object +%%% file format. + +%%------------------------------------------------------------------------------ +%% ELF-32 Data Types (in bytes) +%%------------------------------------------------------------------------------ +-define(ELF_ADDR_SIZE, 4). +-define(ELF_OFF_SIZE, 4). +-define(ELF_HALF_SIZE, 2). +-define(ELF_WORD_SIZE, 4). +-define(ELF_SWORD_SIZE, 4). +-define(ELF_XWORD_SIZE, ?ELF_WORD_SIZE). % for compatibility +-define(ELF_SXWORD_SIZE, ?ELF_WORD_SIZE). +-define(ELF_UNSIGNED_CHAR_SIZE, 1). + +%%------------------------------------------------------------------------------ +%% ELF-32 Symbol Table Entries +%%------------------------------------------------------------------------------ +%% Precomputed offset for Symbol Table entries in SymTab binary (needed because +%% of the different offsets in 32 and 64 bit formats). +-define(ST_NAME_OFFSET, 0). +-define(ST_VALUE_OFFSET, (?ST_NAME_OFFSET + ?ST_NAME_SIZE) ). +-define(ST_SIZE_OFFSET, (?ST_VALUE_OFFSET + ?ST_VALUE_SIZE) ). +-define(ST_INFO_OFFSET, (?ST_SIZE_OFFSET + ?ST_SIZE_SIZE) ). +-define(ST_OTHER_OFFSET, (?ST_INFO_OFFSET + ?ST_INFO_SIZE) ). +-define(ST_SHNDX_OFFSET, (?ST_OTHER_OFFSET + ?ST_OTHER_SIZE) ). + +%%------------------------------------------------------------------------------ +%% ELF-64 Relocation Entries +%%------------------------------------------------------------------------------ +%% Useful macros to extract information from r_info field +-define(ELF_R_SYM(I), (I bsr 8) ). +-define(ELF_R_TYPE(I), (I band 16#ff) ). +-define(ELF_R_INFO(S, T), ((S bsl 8) + (T band 16#ff)) ). + +%%------------------------------------------------------------------------------ +%% ELF-64 Program Header Table +%%------------------------------------------------------------------------------ +%% Offsets of various fields in a Program Header Table entry binary. +-define(P_TYPE_OFFSET, 0). +-define(P_OFFSET_OFFSET, (?P_FLAGS_OFFSET + ?P_FLAGS_SIZE) ). +-define(P_VADDR_OFFSET, (?P_OFFSET_OFFSET + ?P_OFFSET_SIZE) ). +-define(P_PADDR_OFFSET, (?P_VADDR_OFFSET + ?P_VADDR_SIZE) ). +-define(P_FILESZ_OFFSET, (?P_PVADDR_OFFSET + ?P_PVADDR_SIZE) ). +-define(P_MEMSZ_OFFSET, (?P_FILESZ_OFFSET + ?P_FILESZ_SIZE) ). +-define(P_FLAGS_OFFSET, (?P_TYPE_OFFSET + ?P_TYPE_SIZE) ). +-define(P_ALIGN_OFFSET, (?P_MEMSZ_OFFSET + ?P_MEMSZ_SIZE) ). diff --git a/lib/hipe/llvm/elf64_format.hrl b/lib/hipe/llvm/elf64_format.hrl new file mode 100644 index 0000000000..794746ffdc --- /dev/null +++ b/lib/hipe/llvm/elf64_format.hrl @@ -0,0 +1,58 @@ +%% -*- erlang-indent-level: 2 -*- + +%%% @copyright 2011-2014 Yiannis Tsiouris <[email protected]>, +%%% Chris Stavrakakis <[email protected]> +%%% @author Yiannis Tsiouris <[email protected]> +%%% [http://www.softlab.ntua.gr/~gtsiour/] + +%%% @doc This header file contains very very useful macros for handling +%%% various segments of an ELF-64 formated object file, such as sizes, +%%% offsets and predefined constants. For further information about +%%% each field take a quick look at +%%% "[http://downloads.openwatcom.org/ftp/devel/docs/elf-64-gen.pdf]" +%%% that contain the current HP/Intel definition of the ELF object +%%% file format. + +%%------------------------------------------------------------------------------ +%% ELF-64 Data Types (in bytes) +%%------------------------------------------------------------------------------ +-define(ELF_ADDR_SIZE, 8). +-define(ELF_OFF_SIZE, 8). +-define(ELF_HALF_SIZE, 2). +-define(ELF_WORD_SIZE, 4). +-define(ELF_SWORD_SIZE, 4). +-define(ELF_XWORD_SIZE, 8). +-define(ELF_SXWORD_SIZE, 8). +-define(ELF_UNSIGNED_CHAR_SIZE, 1). + +%%------------------------------------------------------------------------------ +%% ELF-64 Symbol Table Entries +%%------------------------------------------------------------------------------ +%% Precomputed offset for Symbol Table entries in SymTab binary +-define(ST_NAME_OFFSET, 0). +-define(ST_INFO_OFFSET, (?ST_NAME_OFFSET + ?ST_NAME_SIZE) ). +-define(ST_OTHER_OFFSET, (?ST_INFO_OFFSET + ?ST_INFO_SIZE) ). +-define(ST_SHNDX_OFFSET, (?ST_OTHER_OFFSET + ?ST_OTHER_SIZE) ). +-define(ST_VALUE_OFFSET, (?ST_SHNDX_OFFSET + ?ST_SHNDX_SIZE) ). +-define(ST_SIZE_OFFSET, (?ST_VALUE_OFFSET + ?ST_VALUE_SIZE) ). + +%%------------------------------------------------------------------------------ +%% ELF-64 Relocation Entries +%%------------------------------------------------------------------------------ +%% Useful macros to extract information from r_info field +-define(ELF_R_SYM(I), (I bsr 32) ). +-define(ELF_R_TYPE(I), (I band 16#ffffffff) ). +-define(ELF_R_INFO(S, T), ((S bsl 32) + (T band 16#ffffffff)) ). + +%%------------------------------------------------------------------------------ +%% ELF-64 Program Header Table +%%------------------------------------------------------------------------------ +%% Offsets of various fields in a Program Header Table entry binary. +-define(P_TYPE_OFFSET, 0). +-define(P_FLAGS_OFFSET, (?P_TYPE_OFFSET + ?P_TYPE_SIZE) ). +-define(P_OFFSET_OFFSET, (?P_FLAGS_OFFSET + ?P_FLAGS_SIZE) ). +-define(P_VADDR_OFFSET, (?P_OFFSET_OFFSET + ?P_OFFSET_SIZE) ). +-define(P_PADDR_OFFSET, (?P_VADDR_OFFSET + ?P_VADDR_SIZE) ). +-define(P_FILESZ_OFFSET, (?P_PVADDR_OFFSET + ?P_PVADDR_SIZE) ). +-define(P_MEMSZ_OFFSET, (?P_FILESZ_OFFSET + ?P_FILESZ_SIZE) ). +-define(P_ALIGN_OFFSET, (?P_MEMSZ_OFFSET + ?P_MEMSZ_SIZE) ). diff --git a/lib/hipe/llvm/elf_format.erl b/lib/hipe/llvm/elf_format.erl new file mode 100644 index 0000000000..260da9b5e6 --- /dev/null +++ b/lib/hipe/llvm/elf_format.erl @@ -0,0 +1,790 @@ +%% -*- erlang-indent-level: 2 -*- + +%%% @copyright 2011-2014 Yiannis Tsiouris <[email protected]>, +%%% Chris Stavrakakis <[email protected]>, +%%% Kostis Sagonas <[email protected]> +%%% @author Yiannis Tsiouris <[email protected]> +%%% [http://www.softlab.ntua.gr/~gtsiour/] + +%%% @doc This module contains functions for extracting various pieces of +%%% information from an ELF formated Object file. To fully understand +%%% the ELF format and the use of these functions please read +%%% "[http://www.linuxjournal.com/article/1060?page=0,0]" carefully. + +-module(elf_format). + +-export([get_tab_entries/1, + %% Relocations + get_rodata_relocs/1, + get_text_relocs/1, + extract_rela/2, + get_rela_addends/1, + %% Note + extract_note/2, + %% Executable code + extract_text/1, + %% GCC Exception Table + get_exn_handlers/1, + %% Misc. + set_architecture_flag/1, + is64bit/0 + ]). + +-include("elf_format.hrl"). + +%%------------------------------------------------------------------------------ +%% Types +%%------------------------------------------------------------------------------ + +-type elf() :: binary(). + +-type lp() :: non_neg_integer(). % landing pad +-type num() :: non_neg_integer(). +-type index() :: non_neg_integer(). +-type offset() :: non_neg_integer(). +-type size() :: non_neg_integer(). +-type start() :: non_neg_integer(). + +-type info() :: index(). +-type nameoff() :: offset(). +-type valueoff() :: offset(). + +-type name() :: string(). +-type name_size() :: {name(), size()}. +-type name_sizes() :: [name_size()]. + +%%------------------------------------------------------------------------------ +%% Abstract Data Types and Accessors for ELF Structures. +%%------------------------------------------------------------------------------ + +%% File header +-record(elf_ehdr, {ident, % ELF identification + type, % Object file type + machine, % Machine Type + version, % Object file version + entry, % Entry point address + phoff, % Program header offset + shoff :: offset(), % Section header offset + flags, % Processor-specific flags + ehsize :: size(), % ELF header size + phentsize :: size(), % Size of program header entry + phnum :: num(), % Number of program header entries + shentsize :: size(), % Size of section header entry + shnum :: num(), % Number of section header entries + shstrndx :: index() % Section name string table index + }). +-type elf_ehdr() :: #elf_ehdr{}. + +-record(elf_ehdr_ident, {class, % File class + data, % Data encoding + version, % File version + osabi, % OS/ABI identification + abiversion, % ABI version + pad, % Start of padding bytes + nident % Size of e_ident[] + }). +%% -type elf_ehdr_ident() :: #elf_ehdr_ident{}. + +%% Section header entries +-record(elf_shdr, {name, % Section name + type, % Section type + flags, % Section attributes + addr, % Virtual address in memory + offset :: offset(), % Offset in file + size :: size(), % Size of section + link, % Link to other section + info, % Miscellaneous information + addralign, % Address align boundary + entsize % Size of entries, if section has table + }). +%% -type elf_shdr() :: #elf_shdr{}. + +%% Symbol table entries +-record(elf_sym, {name :: nameoff(), % Symbol name + info, % Type and Binding attributes + other, % Reserved + shndx, % Section table index + value :: valueoff(), % Symbol value + size :: size() % Size of object + }). +-type elf_sym() :: #elf_sym{}. + +%% Relocations +-record(elf_rel, {r_offset :: offset(), % Address of reference + r_info :: info() % Symbol index and type of relocation + }). +-type elf_rel() :: #elf_rel{}. + +-record(elf_rela, {r_offset :: offset(), % Address of reference + r_info :: info(), % Symbol index and type of relocation + r_addend :: offset() % Constant part of expression + }). +-type elf_rela() :: #elf_rela{}. + +%% %% Program header table +%% -record(elf_phdr, {type, % Type of segment +%% flags, % Segment attributes +%% offset, % Offset in file +%% vaddr, % Virtual address in memory +%% paddr, % Reserved +%% filesz, % Size of segment in file +%% memsz, % Size of segment in memory +%% align % Alignment of segment +%% }). + +%% %% GCC exception table +%% -record(elf_gccexntab, {lpbenc, % Landing pad base encoding +%% lpbase, % Landing pad base +%% ttenc, % Type table encoding +%% ttoff, % Type table offset +%% csenc, % Call-site table encoding +%% cstabsize, % Call-site table size +%% cstab :: cstab() % Call-site table +%% }). +%% -type elf_gccexntab() :: #elf_gccexntab{}. + +-record(elf_gccexntab_callsite, {start :: start(), % Call-site start + size :: size(), % Call-site size + lp :: lp(), % Call-site landing pad + % (exception handler) + onaction % On action (e.g. cleanup) + }). +%% -type elf_gccexntab_callsite() :: #elf_gccexntab_callsite{}. + +%%------------------------------------------------------------------------------ +%% Accessor Functions +%%------------------------------------------------------------------------------ + +%% File header +%% -spec mk_ehdr(...) -> elf_ehrd(). +mk_ehdr(Ident, Type, Machine, Version, Entry, Phoff, Shoff, Flags, Ehsize, + Phentsize, Phnum, Shentsize, Shnum, Shstrndx) -> + #elf_ehdr{ident = Ident, type = Type, machine = Machine, version = Version, + entry = Entry, phoff = Phoff, shoff = Shoff, flags = Flags, + ehsize = Ehsize, phentsize = Phentsize, phnum = Phnum, + shentsize = Shentsize, shnum = Shnum, shstrndx = Shstrndx}. + +%% -spec ehdr_shoff(elf_ehdr()) -> offset(). +%% ehdr_shoff(#elf_ehdr{shoff = Offset}) -> Offset. +%% +%% -spec ehdr_shentsize(elf_ehdr()) -> size(). +%% ehdr_shentsize(#elf_ehdr{shentsize = Size}) -> Size. +%% +%% -spec ehdr_shnum(elf_ehdr()) -> num(). +%% ehdr_shnum(#elf_ehdr{shnum = Num}) -> Num. +%% +%% -spec ehdr_shstrndx(elf_ehdr()) -> index(). +%% ehdr_shstrndx(#elf_ehdr{shstrndx = Index}) -> Index. + + +%%-spec mk_ehdr_ident(...) -> elf_ehdr_ident(). +mk_ehdr_ident(Class, Data, Version, OsABI, AbiVersion, Pad, Nident) -> + #elf_ehdr_ident{class = Class, data = Data, version = Version, osabi = OsABI, + abiversion = AbiVersion, pad = Pad, nident = Nident}. + +%%%------------------------- +%%% Section header entries +%%%------------------------- +mk_shdr(Name, Type, Flags, Addr, Offset, Size, Link, Info, AddrAlign, EntSize) -> + #elf_shdr{name = Name, type = Type, flags = Flags, addr = Addr, + offset = Offset, size = Size, link = Link, info = Info, + addralign = AddrAlign, entsize = EntSize}. + +%% -spec shdr_offset(elf_shdr()) -> offset(). +%% shdr_offset(#elf_shdr{offset = Offset}) -> Offset. +%% +%% -spec shdr_size(elf_shdr()) -> size(). +%% shdr_size(#elf_shdr{size = Size}) -> Size. + +%%%------------------------- +%%% Symbol Table Entries +%%%------------------------- +mk_sym(Name, Info, Other, Shndx, Value, Size) -> + #elf_sym{name = Name, info = Info, other = Other, + shndx = Shndx, value = Value, size = Size}. + +-spec sym_name(elf_sym()) -> nameoff(). +sym_name(#elf_sym{name = Name}) -> Name. + +%% -spec sym_value(elf_sym()) -> valueoff(). +%% sym_value(#elf_sym{value = Value}) -> Value. +%% +%% -spec sym_size(elf_sym()) -> size(). +%% sym_size(#elf_sym{size = Size}) -> Size. + +%%%------------------------- +%%% Relocations +%%%------------------------- +-spec mk_rel(offset(), info()) -> elf_rel(). +mk_rel(Offset, Info) -> + #elf_rel{r_offset = Offset, r_info = Info}. + +%% The following two functions capitalize on the fact that the two kinds of +%% relocation records (for 32- and 64-bit architectures have similar structure. + +-spec r_offset(elf_rel() | elf_rela()) -> offset(). +r_offset(#elf_rel{r_offset = Offset}) -> Offset; +r_offset(#elf_rela{r_offset = Offset}) -> Offset. + +-spec r_info(elf_rel() | elf_rela()) -> info(). +r_info(#elf_rel{r_info = Info}) -> Info; +r_info(#elf_rela{r_info = Info}) -> Info. + +-spec mk_rela(offset(), info(), offset()) -> elf_rela(). +mk_rela(Offset, Info, Addend) -> + #elf_rela{r_offset = Offset, r_info = Info, r_addend = Addend}. + +-spec rela_addend(elf_rela()) -> offset(). +rela_addend(#elf_rela{r_addend = Addend}) -> Addend. + +%% %%%------------------------- +%% %%% GCC exception table +%% %%%------------------------- +%% -type cstab() :: [elf_gccexntab_callsite()]. +%% +%% mk_gccexntab(LPbenc, LPbase, TTenc, TToff, CSenc, CStabsize, CStab) -> +%% #elf_gccexntab{lpbenc = LPbenc, lpbase = LPbase, ttenc = TTenc, +%% ttoff = TToff, csenc = CSenc, cstabsize = CStabsize, +%% cstab = CStab}. +%% +%% -spec gccexntab_cstab(elf_gccexntab()) -> cstab(). +%% gccexntab_cstab(#elf_gccexntab{cstab = CSTab}) -> CSTab. + +mk_gccexntab_callsite(Start, Size, LP, Action) -> + #elf_gccexntab_callsite{start = Start, size=Size, lp=LP, onaction=Action}. + +%% -spec gccexntab_callsite_start(elf_gccexntab_callsite()) -> start(). +%% gccexntab_callsite_start(#elf_gccexntab_callsite{start = Start}) -> Start. +%% +%% -spec gccexntab_callsite_size(elf_gccexntab_callsite()) -> size(). +%% gccexntab_callsite_size(#elf_gccexntab_callsite{size = Size}) -> Size. +%% +%% -spec gccexntab_callsite_lp(elf_gccexntab_callsite()) -> lp(). +%% gccexntab_callsite_lp(#elf_gccexntab_callsite{lp = LP}) -> LP. + +%%------------------------------------------------------------------------------ +%% Functions to manipulate the ELF File Header +%%------------------------------------------------------------------------------ + +%% @doc Extracts the File Header from an ELF formatted object file. Also sets +%% the ELF class variable in the process dictionary (used by many functions +%% in this and hipe_llvm_main modules). +-spec extract_header(elf()) -> elf_ehdr(). +extract_header(Elf) -> + Ehdr_bin = get_binary_segment(Elf, 0, ?ELF_EHDR_SIZE), + << %% Structural pattern matching on fields. + Ident_bin:?E_IDENT_SIZE/binary, + Type:?bits(?E_TYPE_SIZE)/integer-little, + Machine:?bits(?E_MACHINE_SIZE)/integer-little, + Version:?bits(?E_VERSION_SIZE)/integer-little, + Entry:?bits(?E_ENTRY_SIZE)/integer-little, + Phoff:?bits(?E_PHOFF_SIZE)/integer-little, + Shoff:?bits(?E_SHOFF_SIZE)/integer-little, + Flags:?bits(?E_FLAGS_SIZE)/integer-little, + Ehsize:?bits(?E_EHSIZE_SIZE)/integer-little, + Phentsize:?bits(?E_PHENTSIZE_SIZE)/integer-little, + Phnum:?bits(?E_PHNUM_SIZE)/integer-little, + Shentsize:?bits(?E_SHENTSIZE_SIZE)/integer-little, + Shnum:?bits(?E_SHENTSIZE_SIZE)/integer-little, + Shstrndx:?bits(?E_SHSTRNDX_SIZE)/integer-little + >> = Ehdr_bin, + <<16#7f, $E, $L, $F, Class, Data, Version, Osabi, Abiversion, + Pad:6/binary, Nident + >> = Ident_bin, + Ident = mk_ehdr_ident(Class, Data, Version, Osabi, + Abiversion, Pad, Nident), + mk_ehdr(Ident, Type, Machine, Version, Entry, Phoff, Shoff, Flags, + Ehsize, Phentsize, Phnum, Shentsize, Shnum, Shstrndx). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Section Header Entries +%%------------------------------------------------------------------------------ + +%% @doc Extracts the Section Header Table from an ELF formated Object File. +extract_shdrtab(Elf) -> + %% Extract File Header to get info about Section Header Offset (in bytes), + %% Entry Size (in bytes) and Number of entries + #elf_ehdr{shoff = ShOff, shentsize = ShEntsize, shnum = ShNum} = + extract_header(Elf), + %% Get actual Section header table (binary) + ShdrBin = get_binary_segment(Elf, ShOff, ShNum * ShEntsize), + get_shdrtab_entries(ShdrBin, []). + +get_shdrtab_entries(<<>>, Acc) -> + lists:reverse(Acc); +get_shdrtab_entries(ShdrBin, Acc) -> + <<%% Structural pattern matching on fields. + Name:?bits(?SH_NAME_SIZE)/integer-little, + Type:?bits(?SH_TYPE_SIZE)/integer-little, + Flags:?bits(?SH_FLAGS_SIZE)/integer-little, + Addr:?bits(?SH_ADDR_SIZE)/integer-little, + Offset:?bits(?SH_OFFSET_SIZE)/integer-little, + Size:?bits(?SH_SIZE_SIZE)/integer-little, + Link:?bits(?SH_LINK_SIZE)/integer-little, + Info:?bits(?SH_INFO_SIZE)/integer-little, + Addralign:?bits(?SH_ADDRALIGN_SIZE)/integer-little, + Entsize:?bits(?SH_ENTSIZE_SIZE)/integer-little, + MoreShdrE/binary + >> = ShdrBin, + ShdrE = mk_shdr(Name, Type, Flags, Addr, Offset, + Size, Link, Info, Addralign, Entsize), + get_shdrtab_entries(MoreShdrE, [ShdrE | Acc]). + +%% @doc Extracts a specific Entry of a Section Header Table. This function +%% takes as argument the Section Header Table (`SHdrTab') and the entry's +%% serial number (`EntryNum') and returns the entry (`shdr'). +get_shdrtab_entry(SHdrTab, EntryNum) -> + lists:nth(EntryNum + 1, SHdrTab). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Section Header String Table +%%------------------------------------------------------------------------------ + +%% @doc Extracts the Section Header String Table. This section is not a known +%% ELF Object File section. It is just a "hidden" table storing the +%% names of all sections that exist in current object file. +-spec extract_shstrtab(elf()) -> [name()]. +extract_shstrtab(Elf) -> + %% Extract Section Name String Table Index + #elf_ehdr{shstrndx = ShStrNdx} = extract_header(Elf), + ShHdrTab = extract_shdrtab(Elf), + %% Extract Section header entry and get actual Section-header String Table + #elf_shdr{offset = ShStrOffset, size = ShStrSize} = + get_shdrtab_entry(ShHdrTab, ShStrNdx), + case get_binary_segment(Elf, ShStrOffset, ShStrSize) of + <<>> -> %% Segment empty + []; + ShStrTab -> %% Convert to string table + [Name || {Name, _Size} <- get_names(ShStrTab)] + end. + +%%------------------------------------------------------------------------------ + +-spec get_tab_entries(elf()) -> [{name(), valueoff(), size()}]. +get_tab_entries(Elf) -> + SymTab = extract_symtab(Elf), + Ts = [{Name, Value, Size div ?ELF_XWORD_SIZE} + || #elf_sym{name = Name, value = Value, size = Size} <- SymTab, + Name =/= 0], + {NameIndices, ValueOffs, Sizes} = lists:unzip3(Ts), + %% Find the names of the symbols. + %% Get string table entries ([{Name, Offset in strtab section}]). Keep only + %% relevant entries: + StrTab = extract_strtab(Elf), + Relevant = [get_strtab_entry(StrTab, Off) || Off <- NameIndices], + %% Zip back to {Name, ValueOff, Size} + lists:zip3(Relevant, ValueOffs, Sizes). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Symbol Table +%%------------------------------------------------------------------------------ + +%% @doc Function that extracts Symbol Table from an ELF Object file. +extract_symtab(Elf) -> + Symtab_bin = extract_segment_by_name(Elf, ?SYMTAB), + get_symtab_entries(Symtab_bin, []). + +get_symtab_entries(<<>>, Acc) -> + lists:reverse(Acc); +get_symtab_entries(Symtab_bin, Acc) -> + <<SymE_bin:?ELF_SYM_SIZE/binary, MoreSymE/binary>> = Symtab_bin, + case is64bit() of + true -> + <<%% Structural pattern matching on fields. + Name:?bits(?ST_NAME_SIZE)/integer-little, + Info:?bits(?ST_INFO_SIZE)/integer-little, + Other:?bits(?ST_OTHER_SIZE)/integer-little, + Shndx:?bits(?ST_SHNDX_SIZE)/integer-little, + Value:?bits(?ST_VALUE_SIZE)/integer-little, + Size:?bits(?ST_SIZE_SIZE)/integer-little + >> = SymE_bin; + false -> + << %% Same fields in different order: + Name:?bits(?ST_NAME_SIZE)/integer-little, + Value:?bits(?ST_VALUE_SIZE)/integer-little, + Size:?bits(?ST_SIZE_SIZE)/integer-little, + Info:?bits(?ST_INFO_SIZE)/integer-little, + Other:?bits(?ST_OTHER_SIZE)/integer-little, + Shndx:?bits(?ST_SHNDX_SIZE)/integer-little + >> = SymE_bin + end, + SymE = mk_sym(Name, Info, Other, Shndx, Value, Size), + get_symtab_entries(MoreSymE, [SymE | Acc]). + +%% @doc Extracts a specific entry from the Symbol Table (as binary). +%% This function takes as arguments the Symbol Table (`SymTab') +%% and the entry's serial number and returns that entry (`sym'). +get_symtab_entry(SymTab, EntryNum) -> + lists:nth(EntryNum + 1, SymTab). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate String Table +%%------------------------------------------------------------------------------ + +%% @doc Extracts String Table from an ELF formated Object File. +-spec extract_strtab(elf()) -> [{string(), offset()}]. +extract_strtab(Elf) -> + Strtab_bin = extract_segment_by_name(Elf, ?STRTAB), + NamesSizes = get_names(Strtab_bin), + make_offsets(NamesSizes). + +%% @doc Returns the name of the symbol at the given offset. The string table +%% contains entries of the form {Name, Offset}. If no such offset exists +%% returns the empty string (`""'). +%% XXX: There might be a bug here because of the "compact" saving the ELF +%% format uses: e.g. only stores ".rela.text" for ".rela.text" and ".text". +get_strtab_entry(Strtab, Offset) -> + case lists:keyfind(Offset, 2, Strtab) of + {Name, Offset} -> Name; + false -> "" + end. + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Relocations +%%------------------------------------------------------------------------------ + +%% @doc This function gets as argument an ELF binary file and returns a list +%% with all .rela.rodata labels (i.e. constants and literals in code) +%% or an empty list if no ".rela.rodata" section exists in code. +-spec get_rodata_relocs(elf()) -> [offset()]. +get_rodata_relocs(Elf) -> + case is64bit() of + true -> + %% Only care about the addends (== offsets): + get_rela_addends(extract_rela(Elf, ?RODATA)); + false -> + %% Find offsets hardcoded in ".rodata" entry + %%XXX: Treat all 0s as padding and skip them! + [SkipPadding || SkipPadding <- extract_rodata(Elf), SkipPadding =/= 0] + end. + +-spec get_rela_addends([elf_rela()]) -> [offset()]. +get_rela_addends(RelaEntries) -> + [rela_addend(E) || E <- RelaEntries]. + +%% @doc Extract a list of the form `[{SymbolName, Offset}]' with all relocatable +%% symbols and their offsets in the code from the ".text" section. +-spec get_text_relocs(elf()) -> [{name(), offset()}]. +get_text_relocs(Elf) -> + %% Only care about the symbol table index and the offset: + NameOffsetTemp = [{?ELF_R_SYM(r_info(E)), r_offset(E)} + || E <- extract_rela(Elf, ?TEXT)], + {NameIndices, ActualOffsets} = lists:unzip(NameOffsetTemp), + %% Find the names of the symbols: + %% + %% Get those symbol table entries that are related to Text relocs: + Symtab = extract_symtab(Elf), + SymtabEs = [get_symtab_entry(Symtab, Index) || Index <- NameIndices], + %XXX: not zero-indexed! + %% Symbol table entries contain the offset of the name of the symbol in + %% String Table: + SymtabEs2 = [sym_name(E) || E <- SymtabEs], %XXX: Do we need to sort SymtabE? + %% Get string table entries ([{Name, Offset in strtab section}]). Keep only + %% relevant entries: + Strtab = extract_strtab(Elf), + Relevant = [get_strtab_entry(Strtab, Off) || Off <- SymtabEs2], + %% Zip back with actual offsets: + lists:zip(Relevant, ActualOffsets). + +%% @doc Extract the Relocations segment for section `Name' (that is passed +%% as second argument) from an ELF formated Object file binary. +-spec extract_rela(elf(), name()) -> [elf_rel() | elf_rela()]. +extract_rela(Elf, Name) -> + SegName = + case is64bit() of + true -> ?RELA(Name); % ELF-64 uses ".rela" + false -> ?REL(Name) % ...while ELF-32 uses ".rel" + end, + Rela_bin = extract_segment_by_name(Elf, SegName), + get_rela_entries(Rela_bin, []). + +get_rela_entries(<<>>, Acc) -> + lists:reverse(Acc); +get_rela_entries(Bin, Acc) -> + E = case is64bit() of + true -> + <<%% Structural pattern matching on fields of a Rela Entry. + Offset:?bits(?R_OFFSET_SIZE)/integer-little, + Info:?bits(?R_INFO_SIZE)/integer-little, + Addend:?bits(?R_ADDEND_SIZE)/integer-little, + Rest/binary + >> = Bin, + mk_rela(Offset, Info, Addend); + false -> + <<%% Structural pattern matching on fields of a Rel Entry. + Offset:?bits(?R_OFFSET_SIZE)/integer-little, + Info:?bits(?R_INFO_SIZE)/integer-little, + Rest/binary + >> = Bin, + mk_rel(Offset, Info) + end, + get_rela_entries(Rest, [E | Acc]). + +%% %% @doc Extract the `EntryNum' (serial number) Relocation Entry. +%% get_rela_entry(Rela, EntryNum) -> +%% lists:nth(EntryNum + 1, Rela). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Executable Code segment +%%------------------------------------------------------------------------------ + +%% @doc This function gets as arguments an ELF formated binary file and +%% returns the Executable Code (".text" segment) or an empty binary if it +%% is not found. +-spec extract_text(elf()) -> binary(). +extract_text(Elf) -> + extract_segment_by_name(Elf, ?TEXT). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Note Section +%%------------------------------------------------------------------------------ + +%% @doc Extract specific Note Section from an ELF Object file. The function +%% takes as first argument the object file (`Elf') and the `Name' of the +%% wanted Note Section (<b>without</b> the ".note" prefix!). It returns +%% the specified binary segment or an empty binary if no such section +%% exists. +-spec extract_note(elf(), string()) -> binary(). +extract_note(Elf, Name) -> + extract_segment_by_name(Elf, ?NOTE(Name)). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate GCC Exception Table segment +%%------------------------------------------------------------------------------ + +%% A description for the C++ exception table formats can be found at Exception +%% Handling Tables (http://www.codesourcery.com/cxx-abi/exceptions.pdf). + +%% A list with `{Start, End, HandlerOffset}' for all call sites in the code +-spec get_exn_handlers(elf()) -> [{start(), start(), lp()}]. +get_exn_handlers(Elf) -> + CallSites = extract_gccexntab_callsites(Elf), + [{Start, Start + Size, LP} + || #elf_gccexntab_callsite{start = Start, size = Size, lp = LP} <- CallSites]. + +%% @doc This function gets as argument an ELF binary file and returns +%% the table (list) of call sites which is stored in GCC +%% Exception Table (".gcc_except_table") section. +%% It returns an empty list if the Exception Table is not found. +%% XXX: Assumes there is *no* Action Record Table. +extract_gccexntab_callsites(Elf) -> + case extract_segment_by_name(Elf, ?GCC_EXN_TAB) of + <<>> -> + []; + ExnTab -> + %% First byte of LSDA is Landing Pad base encoding. + <<LBenc:8, More/binary>> = ExnTab, + %% Second byte is the Landing Pad base (if its encoding is not + %% DW_EH_PE_omit) (optional). + {_LPBase, LSDACont} = + case LBenc =:= ?DW_EH_PE_omit of + true -> % No landing pad base byte. (-1 denotes that) + {-1, More}; + false -> % Landing pad base. + <<Base:8, More2/binary>> = More, + {Base, More2} + end, + %% Next byte of LSDA is the encoding of the Type Table. + <<TTenc:8, More3/binary>> = LSDACont, + %% Next byte is the Types Table offset encoded in U-LEB128 (optional). + {_TTOff, LSDACont2} = + case TTenc =:= ?DW_EH_PE_omit of + true -> % There is no Types Table pointer. (-1 denotes that) + {-1, More3}; + false -> % The byte offset from this field to the start of the Types + % Table used for exception matching. + leb128_decode(More3) + end, + %% Next byte of LSDA is the encoding of the fields in the Call-site Table. + <<_CSenc:8, More4/binary>> = LSDACont2, + %% Sixth byte is the size (in bytes) of the Call-site Table encoded in + %% U-LEB128. + {_CSTabSize, CSTab} = leb128_decode(More4), + %% Extract all call site information + get_gccexntab_callsites(CSTab, []) + end. + +get_gccexntab_callsites(<<>>, Acc) -> + lists:reverse(Acc); +get_gccexntab_callsites(CSTab, Acc) -> + %% We are only interested in the Landing Pad of every entry. + <<Start:32/integer-little, Size:32/integer-little, + LP:32/integer-little, OnAction:8, More/binary + >> = CSTab, + GccCS = mk_gccexntab_callsite(Start, Size, LP, OnAction), + get_gccexntab_callsites(More, [GccCS | Acc]). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Read-only Data (.rodata) +%%------------------------------------------------------------------------------ +extract_rodata(Elf) -> + Rodata_bin = extract_segment_by_name(Elf, ?RODATA), + get_rodata_entries(Rodata_bin, []). + +get_rodata_entries(<<>>, Acc) -> + lists:reverse(Acc); +get_rodata_entries(Rodata_bin, Acc) -> + <<Num:?bits(?ELF_ADDR_SIZE)/integer-little, More/binary>> = Rodata_bin, + get_rodata_entries(More, [Num | Acc]). + +%%------------------------------------------------------------------------------ +%% Helper functions +%%------------------------------------------------------------------------------ + +%% @doc Returns the binary segment starting at `Offset' with length `Size' +%% (bytes) from a binary file. If `Offset' is bigger than the byte size of +%% the binary, an empty binary (`<<>>') is returned. +-spec get_binary_segment(binary(), offset(), size()) -> binary(). +get_binary_segment(Bin, Offset, _Size) when Offset > byte_size(Bin) -> + <<>>; +get_binary_segment(Bin, Offset, Size) -> + <<_Hdr:Offset/binary, BinSeg:Size/binary, _More/binary>> = Bin, + BinSeg. + +%% @doc This function gets as arguments an ELF formated binary object and +%% a string with the segments' name and returns the specified segment or +%% an empty binary (`<<>>') if there exists no segment with that name. +%% There are handy macros defined in elf_format.hrl for all Standard +%% Section Names. +-spec extract_segment_by_name(elf(), string()) -> binary(). +extract_segment_by_name(Elf, SectionName) -> + %% Extract Section Header Table and Section Header String Table from binary + SHdrTable = extract_shdrtab(Elf), + Names = extract_shstrtab(Elf), + %% Zip to a list of (Name,ShdrE) + [_Zero | ShdrEs] = lists:keysort(2, SHdrTable), % Skip first entry (zeros). + L = lists:zip(Names, ShdrEs), + %% Find Section Header Table entry by name + case lists:keyfind(SectionName, 1, L) of + {SectionName, ShdrE} -> %% Note: Same name. + #elf_shdr{offset = Offset, size = Size} = ShdrE, + get_binary_segment(Elf, Offset, Size); + false -> %% Not found. + <<>> + end. + +%% @doc Extracts a list of strings with (zero-separated) names from a binary. +%% Returns tuples of `{Name, Size}'. +%% XXX: Skip trailing 0. +-spec get_names(<<_:8,_:_*8>>) -> name_sizes(). +get_names(<<0, Bin/binary>>) -> + NamesSizes = get_names(Bin, []), + fix_names(NamesSizes, []). + +get_names(<<>>, Acc) -> + lists:reverse(Acc); +get_names(Bin, Acc) -> + {Name, MoreNames} = bin_get_string(Bin), + get_names(MoreNames, [{Name, length(Name)} | Acc]). + +%% @doc Fix names: +%% e.g. If ".rela.text" exists, ".text" does not. Same goes for +%% ".rel.text". In that way, the Section Header String Table is more +%% compact. Add ".text" just *before* the corresponding rela-field, +%% etc. +-spec fix_names(name_sizes(), name_sizes()) -> name_sizes(). +fix_names([], Acc) -> + lists:reverse(Acc); +fix_names([{Name, Size}=T | Names], Acc) -> + case is64bit() of + true -> + case string:str(Name, ".rela") =:= 1 of + true -> %% Name starts with ".rela": + Section = string:substr(Name, 6), + fix_names(Names, [{Section, Size - 5} + | [T | Acc]]); % XXX: Is order ok? (".text" + % always before ".rela.text") + false -> %% Name does not start with ".rela": + fix_names(Names, [T | Acc]) + end; + false -> + case string:str(Name, ".rel") =:= 1 of + true -> %% Name starts with ".rel": + Section = string:substr(Name, 5), + fix_names(Names, [{Section, Size - 4} + | [T | Acc]]); % XXX: Is order ok? (".text" + % always before ".rela.text") + false -> %% Name does not start with ".rel": + fix_names(Names, [T | Acc]) + end + end. + + +%% @doc A function that byte-reverses a binary. This might be needed because of +%% little (fucking!) endianess. +-spec bin_reverse(binary()) -> binary(). +bin_reverse(Bin) when is_binary(Bin) -> + bin_reverse(Bin, <<>>). + +-spec bin_reverse(binary(), binary()) -> binary(). +bin_reverse(<<>>, Acc) -> + Acc; +bin_reverse(<<Head, More/binary>>, Acc) -> + bin_reverse(More, <<Head, Acc/binary>>). + +%% @doc A function that extracts a null-terminated string from a binary. It +%% returns the found string along with the rest of the binary. +-spec bin_get_string(binary()) -> {string(), binary()}. +bin_get_string(Bin) -> + bin_get_string(Bin, <<>>). + +bin_get_string(<<>>, BinAcc) -> + Bin = bin_reverse(BinAcc), % little endian! + {binary_to_list(Bin), <<>>}; +bin_get_string(<<0, MoreBin/binary>>, BinAcc) -> + Bin = bin_reverse(BinAcc), % little endian! + {binary_to_list(Bin), MoreBin}; +bin_get_string(<<Letter, Tail/binary>>, BinAcc) -> + bin_get_string(Tail, <<Letter, BinAcc/binary>>). + +%% @doc +make_offsets(NamesSizes) -> + {Names, Sizes} = lists:unzip(NamesSizes), + Offsets = make_offsets_from_sizes(Sizes, 1, []), + lists:zip(Names, Offsets). + +make_offsets_from_sizes([], _, Acc) -> + lists:reverse(Acc); +make_offsets_from_sizes([Size | Sizes], Cur, Acc) -> + make_offsets_from_sizes(Sizes, Size+Cur+1, [Cur | Acc]). % For the "."! + +%% @doc Little-Endian Base 128 (LEB128) Decoder +%% This function extracts the <b>first</b> LEB128-encoded integer in a +%% binary and returns that integer along with the remaining binary. This is +%% done because a LEB128 number has variable bit-size and that is a way of +%% extracting only one number in a binary and continuing parsing the binary +%% for other kind of data (e.g. different encoding). +%% FIXME: Only decodes unsigned data! +-spec leb128_decode(binary()) -> {integer(), binary()}. +leb128_decode(LebNum) -> + leb128_decode(LebNum, 0, <<>>). + +-spec leb128_decode(binary(), integer(), binary()) -> {integer(), binary()}. +leb128_decode(LebNum, NoOfBits, Acc) -> + <<Sentinel:1/bits, NextBundle:7/bits, MoreLebNums/bits>> = LebNum, + case Sentinel of + <<1:1>> -> % more bytes to follow + leb128_decode(MoreLebNums, NoOfBits+7, <<NextBundle:7/bits, Acc/bits>>); + <<0:1>> -> % byte bundle stop + Size = NoOfBits+7, + <<Num:Size/integer>> = <<NextBundle:7/bits, Acc/bits>>, + {Num, MoreLebNums} + end. + +%% @doc Extract ELF Class from ELF header and export symbol to process +%% dictionary. +-spec set_architecture_flag(elf()) -> 'ok'. +set_architecture_flag(Elf) -> + %% Extract information about ELF Class from ELF Header + <<16#7f, $E, $L, $F, EI_Class, _MoreHeader/binary>> + = get_binary_segment(Elf, 0, ?ELF_EHDR_SIZE), + put(elf_class, EI_Class), + ok. + +%% @doc Read from object file header if the file class is ELF32 or ELF64. +-spec is64bit() -> boolean(). +is64bit() -> + case get(elf_class) of + ?ELFCLASS64 -> true; + ?ELFCLASS32 -> false + end. diff --git a/lib/hipe/llvm/elf_format.hrl b/lib/hipe/llvm/elf_format.hrl new file mode 100644 index 0000000000..78592e6e2a --- /dev/null +++ b/lib/hipe/llvm/elf_format.hrl @@ -0,0 +1,488 @@ +%% -*- erlang-indent-level: 2 -*- + +%%% @copyright 2011-2014 Yiannis Tsiouris <[email protected]>, +%%% Chris Stavrakakis <[email protected]> +%%% @author Yiannis Tsiouris <[email protected]> +%%% [http://www.softlab.ntua.gr/~gtsiour/] + +%%------------------------------------------------------------------------------ +%% +%% ELF Header File +%% +%%------------------------------------------------------------------------------ + +-ifdef(BIT32). +-include("elf32_format.hrl"). % ELF32-specific definitions. +-else. +-include("elf64_format.hrl"). % ELF64-specific definitions. +-endif. + +%%------------------------------------------------------------------------------ +%% ELF Data Types (in bytes) +%%------------------------------------------------------------------------------ +%%XXX: Included in either elf32_format or elf64_format. + +%%------------------------------------------------------------------------------ +%% ELF File Header +%%------------------------------------------------------------------------------ +-define(ELF_EHDR_SIZE, (?E_IDENT_SIZE + ?E_TYPE_SIZE + ?E_MACHINE_SIZE + +?E_VERSION_SIZE + ?E_ENTRY_SIZE + ?E_PHOFF_SIZE + +?E_SHOFF_SIZE + ?E_FLAGS_SIZE + ?E_EHSIZE_SIZE + +?E_PHENTSIZE_SIZE + ?E_PHNUM_SIZE + ?E_SHENTSIZE_SIZE + +?E_SHNUM_SIZE + ?E_SHSTRNDX_SIZE) ). + +-define(E_IDENT_SIZE, (16 * ?ELF_UNSIGNED_CHAR_SIZE) ). +-define(E_TYPE_SIZE, ?ELF_HALF_SIZE). +-define(E_MACHINE_SIZE, ?ELF_HALF_SIZE). +-define(E_VERSION_SIZE, ?ELF_WORD_SIZE). +-define(E_ENTRY_SIZE, ?ELF_ADDR_SIZE). +-define(E_PHOFF_SIZE, ?ELF_OFF_SIZE). +-define(E_SHOFF_SIZE, ?ELF_OFF_SIZE). +-define(E_FLAGS_SIZE, ?ELF_WORD_SIZE). +-define(E_EHSIZE_SIZE, ?ELF_HALF_SIZE). +-define(E_PHENTSIZE_SIZE, ?ELF_HALF_SIZE). +-define(E_PHNUM_SIZE, ?ELF_HALF_SIZE). +-define(E_SHENTSIZE_SIZE, ?ELF_HALF_SIZE). +-define(E_SHNUM_SIZE, ?ELF_HALF_SIZE). +-define(E_SHSTRNDX_SIZE, ?ELF_HALF_SIZE). + +%% Useful arithmetics for computing byte offsets for various File Header +%% entries from a File Header (erlang) binary +-define(E_IDENT_OFFSET, 0). +-define(E_TYPE_OFFSET, (?E_IDENT_OFFSET + ?E_IDENT_SIZE) ). +-define(E_MACHINE_OFFSET, (?E_TYPE_OFFSET + ?E_TYPE_SIZE) ). +-define(E_VERSION_OFFSET, (?E_MACHINE_OFFSET + ?E_MACHINE_SIZE) ). +-define(E_ENTRY_OFFSET, (?E_VERSION_OFFSET + ?E_VERSION_SIZE) ). +-define(E_PHOFF_OFFSET, (?E_ENTRY_OFFSET + ?E_ENTRY_SIZE) ). +-define(E_SHOFF_OFFSET, (?E_PHOFF_OFFSET + ?E_PHOFF_SIZE) ). +-define(E_FLAGS_OFFSET, (?E_SHOFF_OFFSET + ?E_SHOFF_SIZE) ). +-define(E_EHSIZE_OFFSET, (?E_FLAGS_OFFSET + ?E_FLAGS_SIZE) ). +-define(E_PHENTSIZE_OFFSET, (?E_EHSIZE_OFFSET + ?E_EHSIZE_SIZE) ). +-define(E_PHNUM_OFFSET, (?E_PHENTSIZE_OFFSET + ?E_PHENTSIZE_SIZE) ). +-define(E_SHENTSIZE_OFFSET, (?E_PHNUM_OFFSET + ?E_PHNUM_SIZE) ). +-define(E_SHNUM_OFFSET, (?E_SHENTSIZE_OFFSET + ?E_SHENTSIZE_SIZE) ). +-define(E_SHSTRNDX_OFFSET, (?E_SHNUM_OFFSET + ?E_SHNUM_SIZE) ). + +%% Name aliases of File Header fields information used in get_header_field +%% function of elf64_format module. +-define(E_IDENT, {?E_IDENT_OFFSET, ?E_IDENT_SIZE}). +-define(E_TYPE, {?E_TYPE_OFFSET, ?E_TYPE_SIZE}). +-define(E_MACHINE, {?E_MACHINE_OFFSET, ?E_MACHINE_SIZE}). +-define(E_VERSION, {?E_VERSION_OFFSET, ?E_VERSION_SIZE}) +-define(E_ENTRY, {?E_ENTRY_OFFSET, ?E_ENTRY_SIZE}). +-define(E_PHOFF, {?E_PHOFF_OFFSET, ?E_PHOFF_SIZE}). +-define(E_SHOFF, {?E_SHOFF_OFFSET, ?E_SHOFF_SIZE}). +-define(E_FLAGS, {?E_FLAGS_OFFSET, ?E_FLAGS_SIZE}). +-define(E_EHSIZE, {?E_EHSIZE_OFFSET, ?E_EHSIZE_SIZE}). +-define(E_PHENTSIZE, {?E_PHENTSIZE_OFFSET, ?E_PHENTSIZE_SIZE}). +-define(E_PHNUM, {?E_PHNUM_OFFSET, ?E_PHNUM_SIZE}). +-define(E_SHENTSIZE, {?E_SHENTSIZE_OFFSET, ?E_SHENTSIZE_SIZE}). +-define(E_SHNUM, {?E_SHNUM_OFFSET, ?E_SHNUM_SIZE}). +-define(E_SHSTRNDX, {?E_SHSTRNDX_OFFSET, ?E_SHSTRNDX_SIZE}). + +%% ELF Identification (e_ident) +-define(EI_MAG0, 0). +-define(EI_MAG1, 1). +-define(EI_MAG2, 2). +-define(EI_MAG3, 3). +-define(EI_CLASS, 4). +-define(EI_DATA, 5). +-define(EI_VERSION, 6). +-define(EI_OSABI, 7). +-define(EI_ABIVERSION, 8). +-define(EI_PAD, 9). +-define(EI_NIDENT, 16). + +%% Object File Classes (e_ident[EI_CLASS]) +-define(ELFCLASSNONE, 0). +-define(ELFCLASS32, 1). +-define(ELFCLASS64, 2). + +%% Data Encodings (e_ident[EI_DATA]) +-define(ELFDATA2LSB, 1). +-define(ELFDATA2MSB, 2). + +%% Operating System and ABI Identifiers (e_ident[EI_OSABI]) +-define(ELFOSABI_SYSV, 0). +-define(ELFOSABI_HPUX, 1). +-define(ELFOSABI_STANDALONE, 255). + +%% Object File Types (e_type) +-define(ET_NONE, 0). +-define(ET_REL, 1). +-define(ET_EXEC, 2). +-define(ET_DYN, 3). +-define(ET_CORE, 4). +-define(ET_LOOS, 16#FE00). +-define(ET_HIOS, 16#FEFF). +-define(ET_LOPROC, 16#FF00). +-define(ET_HIPROC, 16#FFFF). + +%%------------------------------------------------------------------------------ +%% ELF Section Header +%%------------------------------------------------------------------------------ +-define(ELF_SHDRENTRY_SIZE, (?SH_NAME_SIZE + ?SH_TYPE_SIZE + ?SH_FLAGS_SIZE + +?SH_ADDR_SIZE + ?SH_OFFSET_SIZE + ?SH_SIZE_SIZE + +?SH_LINK_SIZE + ?SH_INFO_SIZE + +?SH_ADDRALIGN_SIZE + ?SH_ENTSIZE_SIZE) ). + +-define(SH_NAME_SIZE, ?ELF_WORD_SIZE). +-define(SH_TYPE_SIZE, ?ELF_WORD_SIZE). +-define(SH_FLAGS_SIZE, ?ELF_XWORD_SIZE). +-define(SH_ADDR_SIZE, ?ELF_ADDR_SIZE). +-define(SH_OFFSET_SIZE, ?ELF_OFF_SIZE). +-define(SH_SIZE_SIZE, ?ELF_XWORD_SIZE). +-define(SH_LINK_SIZE, ?ELF_WORD_SIZE). +-define(SH_INFO_SIZE, ?ELF_WORD_SIZE). +-define(SH_ADDRALIGN_SIZE, ?ELF_XWORD_SIZE). +-define(SH_ENTSIZE_SIZE, ?ELF_XWORD_SIZE). + +%% Useful arithmetics for computing byte offsets for various fields from a +%% Section Header Entry (erlang) binary +-define(SH_NAME_OFFSET, 0). +-define(SH_TYPE_OFFSET, (?SH_NAME_OFFSET + ?SH_NAME_SIZE) ). +-define(SH_FLAGS_OFFSET, (?SH_TYPE_OFFSET + ?SH_TYPE_SIZE) ). +-define(SH_ADDR_OFFSET, (?SH_FLAGS_OFFSET + ?SH_FLAGS_SIZE) ). +-define(SH_OFFSET_OFFSET, (?SH_ADDR_OFFSET + ?SH_ADDR_SIZE) ). +-define(SH_SIZE_OFFSET, (?SH_OFFSET_OFFSET + ?SH_OFFSET_SIZE) ). +-define(SH_LINK_OFFSET, (?SH_SIZE_OFFSET + ?SH_SIZE_SIZE) ). +-define(SH_INFO_OFFSET, (?SH_LINK_OFFSET + ?SH_LINK_SIZE) ). +-define(SH_ADDRALIGN_OFFSET, (?SH_INFO_OFFSET + ?SH_INFO_SIZE) ). +-define(SH_ENTSIZE_OFFSET, (?SH_ADDRALIGN_OFFSET + ?SH_ADDRALIGN_SIZE) ). + +%% Name aliases of Section Header Table entry information used in +%% get_shdrtab_entry function of elf64_format module. +-define(SH_NAME, {?SH_NAME_OFFSET, ?SH_NAME_SIZE}). +-define(SH_TYPE, {?SH_TYPE_OFFSET, ?SH_TYPE_SIZE}). +-define(SH_FLAGS, {?SH_FLAGS_OFFSET, ?SH_FLAGS_SIZE}). +-define(SH_ADDR, {?SH_ADDR_OFFSET, ?SH_ADDR_SIZE}). +-define(SH_OFFSET, {?SH_OFFSET_OFFSET, ?SH_OFFSET_SIZE}). +-define(SH_SIZE, {?SH_SIZE_OFFSET, ?SH_SIZE_SIZE}). +-define(SH_LINK, {?SH_LINK_OFFSET, ?SH_LINK_SIZE}). +-define(SH_INFO, {?SH_INFO_OFFSET, ?SH_INFO_SIZE}). +-define(SH_ADDRALIGN, {?SH_ADDRALIGN_OFFSET, ?SH_ADDRALIGN_SIZE}). +-define(SH_ENTSIZE, {?SH_ENTSIZE_OFFSET, ?SH_ENTSIZE_SIZE}). + +%% Section Indices +-define(SHN_UNDEF, 0). +-define(SHN_LOPROC, 16#FF00). +-define(SHN_HIPROC, 16#FF1F). +-define(SHN_LOOS, 16#FF20). +-define(SHN_HIOS, 16#FF3F). +-define(SHN_ABS, 16#FFF1). +-define(SHN_COMMON, 16#FFF2). + +%% Section Types (sh_type) +-define(SHT_NULL, 0). +-define(SHT_PROGBITS, 1). +-define(SHT_SYMTAB, 2). +-define(SHT_STRTAB, 3). +-define(SHT_RELA, 4). +-define(SHT_HASH, 5). +-define(SHT_DYNAMIC, 6). +-define(SHT_NOTE, 7). +-define(SHT_NOBITS, 8). +-define(SHT_REL, 9). +-define(SHT_SHLIB, 10). +-define(SHT_DYNSYM, 11). +-define(SHT_LOOS, 16#60000000). +-define(SHT_HIOS, 16#6FFFFFFF). +-define(SHT_LOPROC, 16#70000000). +-define(SHT_HIPROC, 16#7FFFFFFF). + +%% Section Attributes (sh_flags) +-define(SHF_WRITE, 16#1). +-define(SHF_ALLOC, 16#2). +-define(SHF_EXECINSTR, 16#4). +-define(SHF_MASKOS, 16#0F000000). +-define(SHF_MASKPROC, 16#F0000000). + +%% +%% Standard Section names for Code and Data +%% +-define(BSS, ".bss"). +-define(DATA, ".data"). +-define(INTERP, ".interp"). +-define(RODATA, ".rodata"). +-define(TEXT, ".text"). +%% Other Standard Section names +-define(COMMENT, ".comment"). +-define(DYNAMIC, ".dynamic"). +-define(DYNSTR, ".dynstr"). +-define(GOT, ".got"). +-define(HASH, ".hash"). +-define(NOTE(Name), (".note" ++ Name)). +-define(PLT, ".plt"). +-define(REL(Name), (".rel" ++ Name) ). +-define(RELA(Name), (".rela" ++ Name) ). +-define(SHSTRTAB, ".shstrtab"). +-define(STRTAB, ".strtab"). +-define(SYMTAB, ".symtab"). +-define(GCC_EXN_TAB, ".gcc_except_table"). + +%%------------------------------------------------------------------------------ +%% ELF Symbol Table Entries +%%------------------------------------------------------------------------------ +-define(ELF_SYM_SIZE, (?ST_NAME_SIZE + ?ST_INFO_SIZE + ?ST_OTHER_SIZE + +?ST_SHNDX_SIZE + ?ST_VALUE_SIZE + ?ST_SIZE_SIZE) ). + +-define(ST_NAME_SIZE, ?ELF_WORD_SIZE). +-define(ST_INFO_SIZE, ?ELF_UNSIGNED_CHAR_SIZE). +-define(ST_OTHER_SIZE, ?ELF_UNSIGNED_CHAR_SIZE). +-define(ST_SHNDX_SIZE, ?ELF_HALF_SIZE). +-define(ST_VALUE_SIZE, ?ELF_ADDR_SIZE). +-define(ST_SIZE_SIZE, ?ELF_XWORD_SIZE). + +%% Precomputed offset for Symbol Table entries in SymTab binary +%%XXX: Included in either elf32_format or elf64_format. + +%% Name aliases for Symbol Table entry information +-define(ST_NAME, {?ST_NAME_OFFSET, ?ST_NAME_SIZE}). +-define(ST_INFO, {?ST_INFO_OFFSET, ?ST_INFO_SIZE}). +-define(ST_OTHER, {?ST_OTHER_OFFSET, ?ST_OTHER_SIZE}). +-define(ST_SHNDX, {?ST_SHNDX_OFFSET, ?ST_SHNDX_SIZE}). +-define(ST_VALUE, {?ST_VALUE_OFFSET, ?ST_VALUE_SIZE}). +-define(ST_SIZE, {?ST_SIZE_OFFSET, ?ST_SIZE_SIZE}). + +%% Macros to extract information from st_type +-define(ELF_ST_BIND(I), (I bsr 4) ). +-define(ELF_ST_TYPE(I), (I band 16#f) ). +-define(ELF_ST_INFO(B,T), (B bsl 4 + T band 16#f) ). + +%% Symbol Bindings +-define(STB_LOCAL, 0). +-define(STB_GLOBAL, 1). +-define(STB_WEAK, 2). +-define(STB_LOOS, 10). +-define(STB_HIOS, 12). +-define(STB_LOPROC, 13). +-define(STB_HIPROC, 15). + +%% Symbol Types +-define(STT_NOTYPE, 0). +-define(STT_OBJECT, 1). +-define(STT_FUNC, 2). +-define(STT_SECTION, 3). +-define(STT_FILE, 4). +-define(STT_LOOS, 10). +-define(STT_HIOS, 12). +-define(STT_LOPROC, 13). +-define(STT_HIPROC, 15). + +%%------------------------------------------------------------------------------ +%% ELF Relocation Entries +%%------------------------------------------------------------------------------ +-define(ELF_REL_SIZE, (?R_OFFSET_SIZE + ?R_INFO_SIZE) ). +-define(ELF_RELA_SIZE, (?R_OFFSET_SIZE + ?R_INFO_SIZE + ?R_ADDEND_SIZE) ). + +-define(R_OFFSET_SIZE, ?ELF_ADDR_SIZE). +-define(R_INFO_SIZE, ?ELF_XWORD_SIZE). +-define(R_ADDEND_SIZE, ?ELF_SXWORD_SIZE). + +%% Arithmetics for computing byte offsets in a Relocation entry binary +-define(R_OFFSET_OFFSET, 0). +-define(R_INFO_OFFSET, (?R_OFFSET_OFFSET + ?R_OFFSET_SIZE) ). +-define(R_ADDEND_OFFSET, (?R_INFO_OFFSET + ?R_INFO_SIZE) ). + +%% Name aliases for Relocation field information +-define(R_OFFSET, {?R_OFFSET_OFFSET, ?R_OFFSET_SIZE}). +-define(R_INFO, {?R_INFO_OFFSET, ?R_INFO_SIZE}). +-define(R_ADDEND, {?R_ADDEND_OFFSET, ?R_ADDEND_SIZE}). + +%% Useful macros to extract information from r_info field +%%XXX: Included in either elf32_format or elf64_format. + +%%------------------------------------------------------------------------------ +%% ELF Program Header Table +%%------------------------------------------------------------------------------ +-define(ELF_PHDR_SIZE, (?P_TYPE_SIZE + ?P_FLAGS_SIZE + ?P_OFFSET_SIZE + +?P_VADDR_SIZE + ?P_PADDR_SIZE + ?P_FILESZ_SIZE + +?P_MEMSZ_SIZE + ?P_ALIGN_SIZE) ). + +-define(P_TYPE_SIZE, ?ELF_WORD_SIZE). +-define(P_FLAGS_SIZE, ?ELF_WORD_SIZE). +-define(P_OFFSET_SIZE, ?ELF_OFF_SIZE). +-define(P_VADDR_SIZE, ?ELF_ADDR_SIZE). +-define(P_PADDR_SIZE, ?ELF_ADDR_SIZE). +-define(P_FILESZ_SIZE, ?ELF_XWORD_SIZE). +-define(P_MEMSZ_SIZE, ?ELF_XWORD_SIZE). +-define(P_ALIGN_SIZE, ?ELF_XWORD_SIZE). + +%% Offsets of various fields in a Program Header Table entry binary. +%%XXX: Included in either elf32_format or elf64_format. + +%% Name aliases for each Program Header Table entry field information. +-define(P_TYPE, {?P_TYPE_OFFSET, ?P_TYPE_SIZE} ). +-define(P_FLAGS, {?P_FLAGS_OFFSET, ?P_FLAGS_SIZE} ). +-define(P_OFFSET, {?P_OFFSET_OFFSET, ?P_OFFSET_SIZE} ). +-define(P_VADDR, {?P_VADDR_OFFSET, ?P_VADDR_SIZE} ). +-define(P_PADDR, {?P_PADDR_OFFSET, ?P_PADDR_SIZE} ). +-define(P_FILESZ, {?P_FILESZ_OFFSET, ?P_FILESZ_SIZE} ). +-define(P_MEMSZ, {?P_MEMSZ_OFFSET, ?P_MEMSZ_SIZE} ). +-define(P_ALIGN, {?P_ALIGN_OFFSET, ?P_ALIGN_SIZE} ). + +%% Segment Types (p_type) +-define(PT_NULL, 0). +-define(PT_LOAD, 1). +-define(PT_DYNAMIC, 2). +-define(PT_INTERP, 3). +-define(PT_NOTE, 4). +-define(PT_SHLIB, 5). +-define(PT_PHDR, 6). +-define(PT_LOOS, 16#60000000). +-define(PT_HIOS, 16#6FFFFFFF). +-define(PT_LOPROC, 16#70000000). +-define(PT_HIPROC, 16#7FFFFFFF). + +%% Segment Attributes (p_flags) +-define(PF_X, 16#1). +-define(PF_W, 16#2). +-define(PF_R, 16#4). +-define(PF_MASKOS, 16#00FF0000). +-define(PF_MASKPROC, 16#FF000000). + +%%------------------------------------------------------------------------------ +%% ELF Dynamic Table +%%------------------------------------------------------------------------------ +-define(ELF_DYN_SIZE, (?D_TAG_SIZE + ?D_VAL_PTR_SIZE) ). + +-define(D_TAG_SIZE, ?ELF_SXWORD_SIZE). +-define(D_VAL_PTR_SIZE, ?ELF_ADDR_SIZE). + +%% Offsets of each field in Dynamic Table entry in binary +-define(D_TAG_OFFSET, 0). +-define(D_VAL_PTR_OFFSET, (?D_TAG_OFFSET + ?D_TAG_SIZE)). + +%% Name aliases for each field of a Dynamic Table entry information +-define(D_TAG, {?D_TAG_OFFSET, ?D_TAG_SIZE} ). +-define(D_VAL_PTR, {?D_VAL_PTR_OFFSET, ?D_VAL_PTR_SIZE} ). + +%% Dynamic Table Entries +-define(DT_NULL, 0). +-define(DT_NEEDED, 1). +-define(DT_PLTRELSZ, 2). +-define(DT_PLTGOT, 3). +-define(DT_HASH, 4). +-define(DT_STRTAB, 5). +-define(DT_SYMTAB, 6). +-define(DT_RELA, 7). +-define(DT_RELASZ, 8). +-define(DT_RELAENT, 9). +-define(DT_STRSZ, 10). +-define(DT_SYMENT, 11). +-define(DT_INIT, 12). +-define(DT_FINI, 13). +-define(DT_SONAME, 14). +-define(DT_RPATH, 15). +-define(DT_SYMBOLIC, 16). +-define(DT_REL, 17). +-define(DT_RELSZ, 18). +-define(DT_RELENT, 19). +-define(DT_PLTREL, 20). +-define(DT_DEBUG, 21). +-define(DT_TEXTREL, 22). +-define(DT_JMPREL, 23). +-define(DT_BIND_NOW, 24). +-define(DT_INIT_ARRAY, 25). +-define(DT_FINI_ARRAY, 26). +-define(DT_INIT_ARRAYSZ, 27). +-define(DT_FINI_ARRAYSZ, 28). +-define(DT_LOOS, 16#60000000). +-define(DT_HIOS, 16#6FFFFFFF). +-define(DT_LOPROC, 16#700000000). +-define(DT_HIPROC, 16#7FFFFFFFF). + +%%------------------------------------------------------------------------------ +%% ELF GCC Exception Table +%%------------------------------------------------------------------------------ + +%% The DWARF Exception Header Encoding is used to describe the type of data used +%% in the .eh_frame_hdr (and .gcc_except_table) section. The upper 4 bits +%% indicate how the value is to be applied. The lower 4 bits indicate the format +%% of the data. + +%% DWARF Exception Header value format +-define(DW_EH_PE_omit, 16#ff). % No value is present. +-define(DW_EH_PE_uleb128, 16#01). % Unsigned value encoded using LEB128. +-define(DW_EH_PE_udata2, 16#02). % A 2 bytes unsigned value. +-define(DW_EH_PE_udata4, 16#03). % A 4 bytes unsigned value. +-define(DW_EH_PE_udata8, 16#04). % An 8 bytes unsigned value. +-define(DW_EH_PE_sleb128, 16#09). % Signed value encoded using LEB128. +-define(DW_EH_PE_sdata2, 16#0a). % A 2 bytes signed value. +-define(DW_EH_PE_sdata4, 16#0b). % A 4 bytes signed value. +-define(DW_EH_PE_sdata8, 16#0c). % An 8 bytes signed value. + +%% DWARF Exception Header application +-define(DW_EH_PE_absptr, 16#00). % Value is used with no modification. +-define(DW_EH_PE_pcrel, 16#10). % Value is relative to the current PC. +-define(DW_EH_PE_datarel, 16#30). % Value is relative to the beginning of the + % section. + +%%------------------------------------------------------------------------------ +%% ELF Read-only data (constants, literlas etc.) +%%------------------------------------------------------------------------------ +-define(RO_ENTRY_SIZE, 8). + +%%------------------------------------------------------------------------------ +%% Custom Note section: ".note.gc" for Erlang GC +%%------------------------------------------------------------------------------ + +%% The structure of this section is the following: +%% +%% .short <n> # number of safe points in code +%% +%% .long .L<label1> # safe point address | +%% .long .L<label2> # safe point address |-> safe point addrs +%% ..... | +%% .long .L<label3> # safe point address | +%% +%% .short <n> # stack frame size (in words) |-> fixed-size part +%% .short <n> # stack arity | +%% .short <n> # number of live roots that follow | +%% +%% .short <n> # live root's stack index | +%% ..... |-> live root indices +%% .short <n> # >> | + +%% The name of the custom Note Section +-define(NOTE_ERLGC_NAME, ".gc"). + +%% The first word of a Note Section for Erlang GC (".note.gc") is always the +%% number of safepoints in code. +-define(SP_COUNT, {?SP_COUNT_OFFSET, ?SP_COUNT_SIZE}). +-define(SP_COUNT_SIZE, ?ELF_HALF_SIZE). +-define(SP_COUNT_OFFSET, 0). %(always the first entry in sdesc) + +%% The fixed-size part of a safe point (SP) entry consists of 4 words: the SP +%% address (offset in code), the stack frame size of the function (where the SP +%% is located), the stack arity of the function (the registered values are *not* +%% counted), the number of live roots in the specific SP. +-define(SP_FIXED, {?SP_FIXED_OFF, ?SP_FIXED_SIZE}). +-define(SP_FIXED_OFF, 0). +%%XXX: Exclude SP_ADDR_SIZE from SP_FIXED_SIZE in lew of new GC layout +-define(SP_FIXED_SIZE, (?SP_STKFRAME_SIZE + ?SP_STKARITY_SIZE + + ?SP_LIVEROOTCNT_SIZE)). + +-define(SP_ADDR_SIZE, ?ELF_WORD_SIZE). +-define(SP_STKFRAME_SIZE, ?ELF_HALF_SIZE). +-define(SP_STKARITY_SIZE, ?ELF_HALF_SIZE). +-define(SP_LIVEROOTCNT_SIZE, ?ELF_HALF_SIZE). + +%%XXX: SP_STKFRAME is the first piece of information in the new GC layout +-define(SP_STKFRAME_OFFSET, 0). +-define(SP_STKARITY_OFFSET, (?SP_STKFRAME_OFFSET + ?SP_STKFRAME_SIZE) ). +-define(SP_LIVEROOTCNT_OFFSET, (?SP_STKARITY_OFFSET + ?SP_STKARITY_SIZE) ). + +%% Name aliases for safepoint fields. +-define(SP_STKFRAME, {?SP_STKFRAME_OFFSET, ?SP_STKFRAME_SIZE}). +-define(SP_STKARITY, {?SP_STKARITY_OFFSET, ?SP_STKARITY_SIZE}). +-define(SP_LIVEROOTCNT, {?SP_LIVEROOTCNT_OFFSET, ?SP_LIVEROOTCNT_SIZE}). + +%% After the fixed-size part a variable-size part exists. This part holds the +%% stack frame index of every live root in the specific SP. +-define(LR_STKINDEX_SIZE, ?ELF_HALF_SIZE). + +%%------------------------------------------------------------------------------ +%% Misc. +%%------------------------------------------------------------------------------ +-define(bits(Bytes), ((Bytes) bsl 3)). diff --git a/lib/hipe/llvm/hipe_llvm.erl b/lib/hipe/llvm/hipe_llvm.erl new file mode 100644 index 0000000000..5e33731a2b --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm.erl @@ -0,0 +1,1131 @@ +%% -*- erlang-indent-level: 2 -*- + +-module(hipe_llvm). + +-export([ + mk_ret/1, + ret_ret_list/1, + + mk_br/1, + br_dst/1, + + mk_br_cond/3, + mk_br_cond/4, + br_cond_cond/1, + br_cond_true_label/1, + br_cond_false_label/1, + br_cond_meta/1, + + mk_indirectbr/3, + indirectbr_type/1, + indirectbr_address/1, + indirectbr_label_list/1, + + mk_switch/4, + switch_type/1, + switch_value/1, + switch_default_label/1, + switch_value_label_list/1, + + mk_invoke/9, + invoke_dst/1, + invoke_cconv/1, + invoke_ret_attrs/1, + invoke_type/1, + invoke_fnptrval/1, + invoke_arglist/1, + invoke_fn_attrs/1, + invoke_to_label/1, + invoke_unwind_label/1, + + mk_operation/6, + operation_dst/1, + operation_op/1, + operation_type/1, + operation_src1/1, + operation_src2/1, + operation_options/1, + + mk_extractvalue/5, + extractvalue_dst/1, + extractvalue_type/1, + extractvalue_val/1, + extractvalue_idx/1, + extractvalue_idxs/1, + + mk_insertvalue/7, + insertvalue_dst/1, + insertvalue_val_type/1, + insertvalue_val/1, + insertvalue_elem_type/1, + insertvalue_elem/1, + insertvalue_idx/1, + insertvalue_idxs/1, + + mk_alloca/4, + alloca_dst/1, + alloca_type/1, + alloca_num/1, + alloca_align/1, + + mk_load/6, + load_dst/1, + load_p_type/1, + load_pointer/1, + load_alignment/1, + load_nontemporal/1, + load_volatile/1, + + mk_store/7, + store_type/1, + store_value/1, + store_p_type/1, + store_pointer/1, + store_alignment/1, + store_nontemporal/1, + store_volatile/1, + + mk_getelementptr/5, + getelementptr_dst/1, + getelementptr_p_type/1, + getelementptr_value/1, + getelementptr_typed_idxs/1, + getelementptr_inbounds/1, + + mk_conversion/5, + conversion_dst/1, + conversion_op/1, + conversion_src_type/1, + conversion_src/1, + conversion_dst_type/1, + + mk_sitofp/4, + sitofp_dst/1, + sitofp_src_type/1, + sitofp_src/1, + sitofp_dst_type/1, + + mk_ptrtoint/4, + ptrtoint_dst/1, + ptrtoint_src_type/1, + ptrtoint_src/1, + ptrtoint_dst_type/1, + + mk_inttoptr/4, + inttoptr_dst/1, + inttoptr_src_type/1, + inttoptr_src/1, + inttoptr_dst_type/1, + + mk_icmp/5, + icmp_dst/1, + icmp_cond/1, + icmp_type/1, + icmp_src1/1, + icmp_src2/1, + + mk_fcmp/5, + fcmp_dst/1, + fcmp_cond/1, + fcmp_type/1, + fcmp_src1/1, + fcmp_src2/1, + + mk_phi/3, + phi_dst/1, + phi_type/1, + phi_value_label_list/1, + + mk_select/6, + select_dst/1, + select_cond/1, + select_typ1/1, + select_val1/1, + select_typ2/1, + select_val2/1, + + mk_call/8, + call_dst/1, + call_is_tail/1, + call_cconv/1, + call_ret_attrs/1, + call_type/1, + call_fnptrval/1, + call_arglist/1, + call_fn_attrs/1, + + mk_fun_def/10, + fun_def_linkage/1, + fun_def_visibility/1, + fun_def_cconv/1, + fun_def_ret_attrs/1, + fun_def_type/1, + fun_def_name/1, + fun_def_arglist/1, + fun_def_fn_attrs/1, + fun_def_align/1, + fun_def_body/1, + + mk_fun_decl/8, + fun_decl_linkage/1, + fun_decl_visibility/1, + fun_decl_cconv/1, + fun_decl_ret_attrs/1, + fun_decl_type/1, + fun_decl_name/1, + fun_decl_arglist/1, + fun_decl_align/1, + + mk_landingpad/0, + + mk_comment/1, + comment_text/1, + + mk_label/1, + label_label/1, + is_label/1, + + mk_const_decl/4, + const_decl_dst/1, + const_decl_decl_type/1, + const_decl_type/1, + const_decl_value/1, + + mk_asm/1, + asm_instruction/1, + + mk_adj_stack/3, + adj_stack_offset/1, + adj_stack_register/1, + adj_stack_type/1, + + mk_branch_meta/3, + branch_meta_id/1, + branch_meta_true_weight/1, + branch_meta_false_weight/1 + ]). + +-export([ + mk_void/0, + + mk_label_type/0, + + mk_int/1, + int_width/1, + + mk_double/0, + + mk_pointer/1, + pointer_type/1, + + mk_array/2, + array_size/1, + array_type/1, + + mk_vector/2, + vector_size/1, + vector_type/1, + + mk_struct/1, + struct_type_list/1, + + mk_fun/2, + function_ret_type/1, + function_arg_type_list/1 + ]). + +-export([pp_ins_list/2, pp_ins/2]). + + +%%----------------------------------------------------------------------------- +%% Abstract Data Types for LLVM Assembly. +%%----------------------------------------------------------------------------- + +%% Terminator Instructions +-record(llvm_ret, {ret_list=[]}). +-type llvm_ret() :: #llvm_ret{}. + +-record(llvm_br, {dst}). +-type llvm_br() :: #llvm_br{}. + +-record(llvm_br_cond, {'cond', true_label, false_label, meta=[]}). +-type llvm_br_cond() :: #llvm_br_cond{}. + +-record(llvm_indirectbr, {type, address, label_list}). +-type llvm_indirectbr() :: #llvm_indirectbr{}. + +-record(llvm_switch, {type, value, default_label, value_label_list=[]}). +-type llvm_switch() :: #llvm_switch{}. + +-record(llvm_invoke, {dst, cconv=[], ret_attrs=[], type, fnptrval, arglist=[], + fn_attrs=[], to_label, unwind_label}). +-type llvm_invoke() :: #llvm_invoke{}. + +%% Binary Operations +-record(llvm_operation, {dst, op, type, src1, src2, options=[]}). +-type llvm_operation() :: #llvm_operation{}. + +%% Aggregate Operations +-record(llvm_extractvalue, {dst, type, val, idx, idxs=[]}). +-type llvm_extractvalue() :: #llvm_extractvalue{}. + +-record(llvm_insertvalue, {dst, val_type, val, elem_type, elem, idx, idxs=[]}). +-type llvm_insertvalue() :: #llvm_insertvalue{}. + +%% Memory Access and Addressing Operations +-record(llvm_alloca, {dst, type, num=[], align=[]}). +-type llvm_alloca() :: #llvm_alloca{}. + +-record(llvm_load, {dst, p_type, pointer, alignment=[], nontemporal=[], + volatile=false}). +-type llvm_load() :: #llvm_load{}. + +-record(llvm_store, {type, value, p_type, pointer, alignment=[], + nontemporal=[], volatile=false}). +-type llvm_store() :: #llvm_store{}. + +-record(llvm_getelementptr, {dst, p_type, value, typed_idxs, inbounds}). +-type llvm_getelementptr() :: #llvm_getelementptr{}. + +%% Conversion Operations +-record(llvm_conversion, {dst, op, src_type, src, dst_type}). +-type llvm_conversion() :: #llvm_conversion{}. + +-record(llvm_sitofp, {dst, src_type, src, dst_type}). +-type llvm_sitofp() :: #llvm_sitofp{}. + +-record(llvm_ptrtoint, {dst, src_type, src, dst_type}). +-type llvm_ptrtoint() :: #llvm_ptrtoint{}. + +-record(llvm_inttoptr, {dst, src_type, src, dst_type}). +-type llvm_inttoptr() :: #llvm_inttoptr{}. + +%% Other Operations +-record(llvm_icmp, {dst, 'cond', type, src1, src2}). +-type llvm_icmp() :: #llvm_icmp{}. + +-record(llvm_fcmp, {dst, 'cond', type, src1, src2}). +-type llvm_fcmp() :: #llvm_fcmp{}. + +-record(llvm_phi, {dst, type, value_label_list}). +-type llvm_phi() :: #llvm_phi{}. + +-record(llvm_select, {dst, 'cond', typ1, val1, typ2, val2}). +-type llvm_select() :: #llvm_select{}. + +-record(llvm_call, {dst=[], is_tail = false, cconv = [], ret_attrs = [], type, + fnptrval, arglist = [], fn_attrs = []}). +-type llvm_call() :: #llvm_call{}. + +-record(llvm_fun_def, {linkage=[], visibility=[], cconv=[], ret_attrs=[], + type, 'name', arglist=[], fn_attrs=[], align=[], body=[]}). +-type llvm_fun_def() :: #llvm_fun_def{}. + +-record(llvm_fun_decl, {linkage=[], visibility=[], cconv=[], ret_attrs=[], + type, 'name', arglist=[], align=[]}). +-type llvm_fun_decl() :: #llvm_fun_decl{}. + +-record(llvm_landingpad, {}). +-type llvm_landingpad() :: #llvm_landingpad{}. + +-record(llvm_comment, {text}). +-type llvm_comment() :: #llvm_comment{}. + +-record(llvm_label, {label}). +-type llvm_label() :: #llvm_label{}. + +-record(llvm_const_decl, {dst, decl_type, type, value}). +-type llvm_const_decl() :: #llvm_const_decl{}. + +-record(llvm_asm, {instruction}). +-type llvm_asm() :: #llvm_asm{}. + +-record(llvm_adj_stack, {offset, 'register', type}). +-type llvm_adj_stack() :: #llvm_adj_stack{}. + +-record(llvm_branch_meta, {id, true_weight, false_weight}). +-type llvm_branch_meta() :: #llvm_branch_meta{}. + +%% A type for any LLVM instruction +-type llvm_instr() :: llvm_ret() | llvm_br() | llvm_br_cond() + | llvm_indirectbr() | llvm_switch() | llvm_invoke() + | llvm_operation() | llvm_extractvalue() + | llvm_insertvalue() | llvm_alloca() | llvm_load() + | llvm_store() | llvm_getelementptr() | llvm_conversion() + | llvm_sitofp() | llvm_ptrtoint() | llvm_inttoptr() + | llvm_icmp() | llvm_fcmp() | llvm_phi() | llvm_select() + | llvm_call() | llvm_fun_def() | llvm_fun_decl() + | llvm_landingpad() | llvm_comment() | llvm_label() + | llvm_const_decl() | llvm_asm() | llvm_adj_stack() + | llvm_branch_meta(). + +%% Types +-record(llvm_void, {}). +%-type llvm_void() :: #llvm_void{}. + +-record(llvm_label_type, {}). +%-type llvm_label_type() :: #llvm_label_type{}. + +-record(llvm_int, {width}). +%-type llvm_int() :: #llvm_int{}. + +-record(llvm_float, {}). +%-type llvm_float() :: #llvm_float{}. + +-record(llvm_double, {}). +%-type llvm_double() :: #llvm_double{}. + +-record(llvm_fp80, {}). +%-type llvm_fp80() :: #llvm_fp80{}. + +-record(llvm_fp128, {}). +%-type llvm_fp128() :: #llvm_fp128{}. + +-record(llvm_ppc_fp128, {}). +%-type llvm_ppc_fp128() :: #llvm_ppc_fp128{}. + +-record(llvm_pointer, {type}). +%-type llvm_pointer() :: #llvm_pointer{}. + +-record(llvm_vector, {'size', type}). +%-type llvm_vector() :: #llvm_vector{}. + +-record(llvm_struct, {type_list}). +%-type llvm_struct() :: #llvm_struct{}. + +-record(llvm_array, {'size', type}). +%-type llvm_array() :: #llvm_array{}. + +-record(llvm_fun, {ret_type, arg_type_list}). +%-type llvm_fun() :: #llvm_fun{}. + +%%----------------------------------------------------------------------------- +%% Accessor Functions +%%----------------------------------------------------------------------------- + +%% ret +mk_ret(Ret_list) -> #llvm_ret{ret_list=Ret_list}. +ret_ret_list(#llvm_ret{ret_list=Ret_list}) -> Ret_list. + +%% br +mk_br(Dst) -> #llvm_br{dst=Dst}. +br_dst(#llvm_br{dst=Dst}) -> Dst. + +%% br_cond +mk_br_cond(Cond, True_label, False_label) -> + #llvm_br_cond{'cond'=Cond, true_label=True_label, false_label=False_label}. +mk_br_cond(Cond, True_label, False_label, Metadata) -> + #llvm_br_cond{'cond'=Cond, true_label=True_label, false_label=False_label, + meta=Metadata}. +br_cond_cond(#llvm_br_cond{'cond'=Cond}) -> Cond. +br_cond_true_label(#llvm_br_cond{true_label=True_label}) -> True_label. +br_cond_false_label(#llvm_br_cond{false_label=False_label}) -> + False_label. +br_cond_meta(#llvm_br_cond{meta=Metadata}) -> Metadata. + +%% indirectbr +mk_indirectbr(Type, Address, Label_list) -> #llvm_indirectbr{type=Type, address=Address, label_list=Label_list}. +indirectbr_type(#llvm_indirectbr{type=Type}) -> Type. +indirectbr_address(#llvm_indirectbr{address=Address}) -> Address. +indirectbr_label_list(#llvm_indirectbr{label_list=Label_list}) -> Label_list. + +%% invoke +mk_invoke(Dst, Cconv, Ret_attrs, Type, Fnptrval, Arglist, Fn_attrs, To_label, Unwind_label) -> + #llvm_invoke{dst=Dst, cconv=Cconv, ret_attrs=Ret_attrs, type=Type, + fnptrval=Fnptrval, arglist=Arglist, fn_attrs=Fn_attrs, to_label=To_label, + unwind_label=Unwind_label}. +invoke_dst(#llvm_invoke{dst=Dst}) -> Dst. +invoke_cconv(#llvm_invoke{cconv=Cconv}) -> Cconv. +invoke_ret_attrs(#llvm_invoke{ret_attrs=Ret_attrs}) -> Ret_attrs. +invoke_type(#llvm_invoke{type=Type}) -> Type. +invoke_fnptrval(#llvm_invoke{fnptrval=Fnptrval}) -> Fnptrval. +invoke_arglist(#llvm_invoke{arglist=Arglist}) -> Arglist. +invoke_fn_attrs(#llvm_invoke{fn_attrs=Fn_attrs}) -> Fn_attrs. +invoke_to_label(#llvm_invoke{to_label=To_label}) -> To_label. +invoke_unwind_label(#llvm_invoke{unwind_label=Unwind_label}) -> Unwind_label. + +%% switch +mk_switch(Type, Value, Default_label, Value_label_list) -> + #llvm_switch{type=Type, value=Value, default_label=Default_label, + value_label_list=Value_label_list}. +switch_type(#llvm_switch{type=Type}) -> Type. +switch_value(#llvm_switch{value=Value}) -> Value. +switch_default_label(#llvm_switch{default_label=Default_label}) -> + Default_label. +switch_value_label_list(#llvm_switch{value_label_list=Value_label_list}) -> + Value_label_list. + +%% operation +mk_operation(Dst, Op, Type, Src1, Src2, Options) -> + #llvm_operation{dst=Dst, op=Op, type=Type, src1=Src1, src2=Src2, + options=Options}. +operation_dst(#llvm_operation{dst=Dst}) -> Dst. +operation_op(#llvm_operation{op=Op}) -> Op. +operation_type(#llvm_operation{type=Type}) -> Type. +operation_src1(#llvm_operation{src1=Src1}) -> Src1. +operation_src2(#llvm_operation{src2=Src2}) -> Src2. +operation_options(#llvm_operation{options=Options}) -> Options. + +%% extractvalue +mk_extractvalue(Dst, Type, Val, Idx, Idxs) -> + #llvm_extractvalue{dst=Dst,type=Type,val=Val,idx=Idx,idxs=Idxs}. +extractvalue_dst(#llvm_extractvalue{dst=Dst}) -> Dst. +extractvalue_type(#llvm_extractvalue{type=Type}) -> Type. +extractvalue_val(#llvm_extractvalue{val=Val}) -> Val. +extractvalue_idx(#llvm_extractvalue{idx=Idx}) -> Idx. +extractvalue_idxs(#llvm_extractvalue{idxs=Idxs}) -> Idxs. + +%% insertvalue +mk_insertvalue(Dst, Val_type, Val, Elem_type, Elem, Idx, Idxs) -> + #llvm_insertvalue{dst=Dst, val_type=Val_type, val=Val, elem_type=Elem_type, + elem=Elem, idx=Idx, idxs=Idxs}. +insertvalue_dst(#llvm_insertvalue{dst=Dst}) -> Dst. +insertvalue_val_type(#llvm_insertvalue{val_type=Val_type}) -> Val_type. +insertvalue_val(#llvm_insertvalue{val=Val}) -> Val. +insertvalue_elem_type(#llvm_insertvalue{elem_type=Elem_type}) -> Elem_type. +insertvalue_elem(#llvm_insertvalue{elem=Elem}) -> Elem. +insertvalue_idx(#llvm_insertvalue{idx=Idx}) -> Idx. +insertvalue_idxs(#llvm_insertvalue{idxs=Idxs}) -> Idxs. + +%% alloca +mk_alloca(Dst, Type, Num, Align) -> + #llvm_alloca{dst=Dst, type=Type, num=Num, align=Align}. +alloca_dst(#llvm_alloca{dst=Dst}) -> Dst. +alloca_type(#llvm_alloca{type=Type}) -> Type. +alloca_num(#llvm_alloca{num=Num}) -> Num. +alloca_align(#llvm_alloca{align=Align}) -> Align. + +%% load +mk_load(Dst, Type, Pointer, Alignment, Nontemporal, Volatile) -> + #llvm_load{dst=Dst, p_type=Type, pointer=Pointer, alignment=Alignment, + nontemporal=Nontemporal, volatile=Volatile}. +load_dst(#llvm_load{dst=Dst}) -> Dst. +load_p_type(#llvm_load{p_type=Type}) -> Type. +load_pointer(#llvm_load{pointer=Pointer}) -> Pointer. +load_alignment(#llvm_load{alignment=Alignment}) -> Alignment. +load_nontemporal(#llvm_load{nontemporal=Nontemporal}) -> Nontemporal. +load_volatile(#llvm_load{volatile=Volatile}) -> Volatile. + +%% store +mk_store(Type, Value, P_Type, Pointer, Alignment, Nontemporal, Volatile) -> + #llvm_store{type=Type, value=Value, p_type=P_Type, pointer=Pointer, alignment=Alignment, + nontemporal=Nontemporal, volatile=Volatile}. +store_type(#llvm_store{type=Type}) -> Type. +store_value(#llvm_store{value=Value}) -> Value. +store_p_type(#llvm_store{p_type=P_Type}) -> P_Type. +store_pointer(#llvm_store{pointer=Pointer}) -> Pointer. +store_alignment(#llvm_store{alignment=Alignment}) -> Alignment. +store_nontemporal(#llvm_store{nontemporal=Nontemporal}) -> Nontemporal. +store_volatile(#llvm_store{volatile=Volatile}) -> Volatile. + +%% getelementptr +mk_getelementptr(Dst, P_Type, Value, Typed_Idxs, Inbounds) -> + #llvm_getelementptr{dst=Dst,p_type=P_Type, value=Value, + typed_idxs=Typed_Idxs, inbounds=Inbounds}. +getelementptr_dst(#llvm_getelementptr{dst=Dst}) -> Dst. +getelementptr_p_type(#llvm_getelementptr{p_type=P_Type}) -> P_Type. +getelementptr_value(#llvm_getelementptr{value=Value}) -> Value. +getelementptr_typed_idxs(#llvm_getelementptr{typed_idxs=Typed_Idxs}) -> Typed_Idxs. +getelementptr_inbounds(#llvm_getelementptr{inbounds=Inbounds}) -> Inbounds. + +%% conversion +mk_conversion(Dst, Op, Src_type, Src, Dst_type) -> + #llvm_conversion{dst=Dst, op=Op, src_type=Src_type, src=Src, dst_type=Dst_type}. +conversion_dst(#llvm_conversion{dst=Dst}) -> Dst. +conversion_op(#llvm_conversion{op=Op}) -> Op. +conversion_src_type(#llvm_conversion{src_type=Src_type}) -> Src_type. +conversion_src(#llvm_conversion{src=Src}) -> Src. +conversion_dst_type(#llvm_conversion{dst_type=Dst_type}) -> Dst_type. + +%% sitofp +mk_sitofp(Dst, Src_type, Src, Dst_type) -> + #llvm_sitofp{dst=Dst, src_type=Src_type, src=Src, dst_type=Dst_type}. +sitofp_dst(#llvm_sitofp{dst=Dst}) -> Dst. +sitofp_src_type(#llvm_sitofp{src_type=Src_type}) -> Src_type. +sitofp_src(#llvm_sitofp{src=Src}) -> Src. +sitofp_dst_type(#llvm_sitofp{dst_type=Dst_type}) -> Dst_type. + +%% ptrtoint +mk_ptrtoint(Dst, Src_Type, Src, Dst_Type) -> + #llvm_ptrtoint{dst=Dst, src_type=Src_Type, src=Src, dst_type=Dst_Type}. +ptrtoint_dst(#llvm_ptrtoint{dst=Dst}) -> Dst. +ptrtoint_src_type(#llvm_ptrtoint{src_type=Src_Type}) -> Src_Type. +ptrtoint_src(#llvm_ptrtoint{src=Src}) -> Src. +ptrtoint_dst_type(#llvm_ptrtoint{dst_type=Dst_Type}) -> Dst_Type . + +%% inttoptr +mk_inttoptr(Dst, Src_Type, Src, Dst_Type) -> + #llvm_inttoptr{dst=Dst, src_type=Src_Type, src=Src, dst_type=Dst_Type}. +inttoptr_dst(#llvm_inttoptr{dst=Dst}) -> Dst. +inttoptr_src_type(#llvm_inttoptr{src_type=Src_Type}) -> Src_Type. +inttoptr_src(#llvm_inttoptr{src=Src}) -> Src. +inttoptr_dst_type(#llvm_inttoptr{dst_type=Dst_Type}) -> Dst_Type . + +%% icmp +mk_icmp(Dst, Cond, Type, Src1, Src2) -> + #llvm_icmp{dst=Dst,'cond'=Cond,type=Type,src1=Src1,src2=Src2}. +icmp_dst(#llvm_icmp{dst=Dst}) -> Dst. +icmp_cond(#llvm_icmp{'cond'=Cond}) -> Cond. +icmp_type(#llvm_icmp{type=Type}) -> Type. +icmp_src1(#llvm_icmp{src1=Src1}) -> Src1. +icmp_src2(#llvm_icmp{src2=Src2}) -> Src2. + +%% fcmp +mk_fcmp(Dst, Cond, Type, Src1, Src2) -> + #llvm_fcmp{dst=Dst,'cond'=Cond,type=Type,src1=Src1,src2=Src2}. +fcmp_dst(#llvm_fcmp{dst=Dst}) -> Dst. +fcmp_cond(#llvm_fcmp{'cond'=Cond}) -> Cond. +fcmp_type(#llvm_fcmp{type=Type}) -> Type. +fcmp_src1(#llvm_fcmp{src1=Src1}) -> Src1. +fcmp_src2(#llvm_fcmp{src2=Src2}) -> Src2. + +%% phi +mk_phi(Dst, Type, Value_label_list) -> + #llvm_phi{dst=Dst, type=Type,value_label_list=Value_label_list}. +phi_dst(#llvm_phi{dst=Dst}) -> Dst. +phi_type(#llvm_phi{type=Type}) -> Type. +phi_value_label_list(#llvm_phi{value_label_list=Value_label_list}) -> + Value_label_list. + +%% select +mk_select(Dst, Cond, Typ1, Val1, Typ2, Val2) -> + #llvm_select{dst=Dst, 'cond'=Cond, typ1=Typ1, val1=Val1, typ2=Typ2, val2=Val2}. +select_dst(#llvm_select{dst=Dst}) -> Dst. +select_cond(#llvm_select{'cond'=Cond}) -> Cond. +select_typ1(#llvm_select{typ1=Typ1}) -> Typ1. +select_val1(#llvm_select{val1=Val1}) -> Val1. +select_typ2(#llvm_select{typ2=Typ2}) -> Typ2. +select_val2(#llvm_select{val2=Val2}) -> Val2. + +%% call +mk_call(Dst, Is_tail, Cconv, Ret_attrs, Type, Fnptrval, Arglist, Fn_attrs) -> + #llvm_call{dst=Dst, is_tail=Is_tail, cconv=Cconv, ret_attrs=Ret_attrs, + type=Type, fnptrval=Fnptrval, arglist=Arglist, fn_attrs=Fn_attrs}. +call_dst(#llvm_call{dst=Dst}) -> Dst. +call_is_tail(#llvm_call{is_tail=Is_tail}) -> Is_tail. +call_cconv(#llvm_call{cconv=Cconv}) -> Cconv. +call_ret_attrs(#llvm_call{ret_attrs=Ret_attrs}) -> Ret_attrs. +call_type(#llvm_call{type=Type}) -> Type. +call_fnptrval(#llvm_call{fnptrval=Fnptrval}) -> Fnptrval. +call_arglist(#llvm_call{arglist=Arglist}) -> Arglist. +call_fn_attrs(#llvm_call{fn_attrs=Fn_attrs}) -> Fn_attrs. + +%% fun_def +mk_fun_def(Linkage, Visibility, Cconv, Ret_attrs, Type, Name, Arglist, + Fn_attrs, Align, Body) -> + #llvm_fun_def{ + linkage=Linkage, + visibility=Visibility, + cconv=Cconv, + ret_attrs=Ret_attrs, + type=Type, + 'name'=Name, + arglist=Arglist, + fn_attrs=Fn_attrs, + align=Align, + body=Body + }. + +fun_def_linkage(#llvm_fun_def{linkage=Linkage}) -> Linkage. +fun_def_visibility(#llvm_fun_def{visibility=Visibility}) -> Visibility. +fun_def_cconv(#llvm_fun_def{cconv=Cconv}) -> Cconv . +fun_def_ret_attrs(#llvm_fun_def{ret_attrs=Ret_attrs}) -> Ret_attrs. +fun_def_type(#llvm_fun_def{type=Type}) -> Type. +fun_def_name(#llvm_fun_def{'name'=Name}) -> Name. +fun_def_arglist(#llvm_fun_def{arglist=Arglist}) -> Arglist. +fun_def_fn_attrs(#llvm_fun_def{fn_attrs=Fn_attrs}) -> Fn_attrs. +fun_def_align(#llvm_fun_def{align=Align}) -> Align. +fun_def_body(#llvm_fun_def{body=Body}) -> Body. + +%% fun_decl +mk_fun_decl(Linkage, Visibility, Cconv, Ret_attrs, Type, Name, Arglist, Align)-> + #llvm_fun_decl{ + linkage=Linkage, + visibility=Visibility, + cconv=Cconv, + ret_attrs=Ret_attrs, + type=Type, + 'name'=Name, + arglist=Arglist, + align=Align + }. + +fun_decl_linkage(#llvm_fun_decl{linkage=Linkage}) -> Linkage. +fun_decl_visibility(#llvm_fun_decl{visibility=Visibility}) -> Visibility. +fun_decl_cconv(#llvm_fun_decl{cconv=Cconv}) -> Cconv . +fun_decl_ret_attrs(#llvm_fun_decl{ret_attrs=Ret_attrs}) -> Ret_attrs. +fun_decl_type(#llvm_fun_decl{type=Type}) -> Type. +fun_decl_name(#llvm_fun_decl{'name'=Name}) -> Name. +fun_decl_arglist(#llvm_fun_decl{arglist=Arglist}) -> Arglist. +fun_decl_align(#llvm_fun_decl{align=Align}) -> Align. + +%% landingpad +mk_landingpad() -> #llvm_landingpad{}. + +%% comment +mk_comment(Text) -> #llvm_comment{text=Text}. +comment_text(#llvm_comment{text=Text}) -> Text. + +%% label +mk_label(Label) -> #llvm_label{label=Label}. +label_label(#llvm_label{label=Label}) -> Label. + +-spec is_label(llvm_instr()) -> boolean(). +is_label(#llvm_label{}) -> true; +is_label(#llvm_ret{}) -> false; +is_label(#llvm_br{}) -> false; +is_label(#llvm_br_cond{}) -> false; +is_label(#llvm_indirectbr{}) -> false; +is_label(#llvm_switch{}) -> false; +is_label(#llvm_invoke{}) -> false; +is_label(#llvm_operation{}) -> false; +is_label(#llvm_extractvalue{}) -> false; +is_label(#llvm_insertvalue{}) -> false; +is_label(#llvm_alloca{}) -> false; +is_label(#llvm_load{}) -> false; +is_label(#llvm_store{}) -> false; +is_label(#llvm_getelementptr{}) -> false; +is_label(#llvm_conversion{}) -> false; +is_label(#llvm_sitofp{}) -> false; +is_label(#llvm_ptrtoint{}) -> false; +is_label(#llvm_inttoptr{}) -> false; +is_label(#llvm_icmp{}) -> false; +is_label(#llvm_fcmp{}) -> false; +is_label(#llvm_phi{}) -> false; +is_label(#llvm_select{}) -> false; +is_label(#llvm_call{}) -> false; +is_label(#llvm_fun_def{}) -> false; +is_label(#llvm_fun_decl{}) -> false; +is_label(#llvm_landingpad{}) -> false; +is_label(#llvm_comment{}) -> false; +is_label(#llvm_const_decl{}) -> false; +is_label(#llvm_asm{}) -> false; +is_label(#llvm_adj_stack{}) -> false; +is_label(#llvm_branch_meta{}) -> false. + +%% const_decl +mk_const_decl(Dst, Decl_type, Type, Value) -> + #llvm_const_decl{dst=Dst, decl_type=Decl_type, type=Type, value=Value}. +const_decl_dst(#llvm_const_decl{dst=Dst}) -> Dst. +const_decl_decl_type(#llvm_const_decl{decl_type=Decl_type}) -> Decl_type. +const_decl_type(#llvm_const_decl{type=Type}) -> Type. +const_decl_value(#llvm_const_decl{value=Value}) -> Value. + +%% asm +mk_asm(Instruction) -> #llvm_asm{instruction=Instruction}. +asm_instruction(#llvm_asm{instruction=Instruction}) -> Instruction. + +%% adj_stack +mk_adj_stack(Offset, Register, Type) -> + #llvm_adj_stack{offset=Offset, 'register'=Register, type=Type}. +adj_stack_offset(#llvm_adj_stack{offset=Offset}) -> Offset. +adj_stack_register(#llvm_adj_stack{'register'=Register}) -> Register. +adj_stack_type(#llvm_adj_stack{type=Type}) -> Type. + +%% branch meta-data +mk_branch_meta(Id, True_weight, False_weight) -> + #llvm_branch_meta{id=Id, true_weight=True_weight, false_weight=False_weight}. +branch_meta_id(#llvm_branch_meta{id=Id}) -> Id. +branch_meta_true_weight(#llvm_branch_meta{true_weight=True_weight}) -> + True_weight. +branch_meta_false_weight(#llvm_branch_meta{false_weight=False_weight}) -> + False_weight. + +%% types +mk_void() -> #llvm_void{}. + +mk_label_type() -> #llvm_label_type{}. + +mk_int(Width) -> #llvm_int{width=Width}. +int_width(#llvm_int{width=Width}) -> Width. + +mk_double() -> #llvm_double{}. + +mk_pointer(Type) -> #llvm_pointer{type=Type}. +pointer_type(#llvm_pointer{type=Type}) -> Type. + +mk_array(Size, Type) -> #llvm_array{'size'=Size, type=Type}. +array_size(#llvm_array{'size'=Size}) -> Size. +array_type(#llvm_array{type=Type}) -> Type. + +mk_vector(Size, Type) -> #llvm_vector{'size'=Size, type=Type}. +vector_size(#llvm_vector{'size'=Size}) -> Size. +vector_type(#llvm_vector{type=Type}) -> Type. + +mk_struct(Type_list) -> #llvm_struct{type_list=Type_list}. +struct_type_list(#llvm_struct{type_list=Type_list}) -> Type_list. + +mk_fun(Ret_type, Arg_type_list) -> + #llvm_fun{ret_type=Ret_type, arg_type_list=Arg_type_list}. +function_ret_type(#llvm_fun{ret_type=Ret_type}) -> Ret_type. +function_arg_type_list(#llvm_fun{arg_type_list=Arg_type_list}) -> + Arg_type_list. + +%%---------------------------------------------------------------------------- +%% Pretty-printer Functions +%%---------------------------------------------------------------------------- + +%% @doc Pretty-print a list of LLVM instructions to a Device. +pp_ins_list(_Dev, []) -> ok; +pp_ins_list(Dev, [I|Is]) -> + pp_ins(Dev, I), + pp_ins_list(Dev, Is). + +pp_ins(Dev, I) -> + case indent(I) of + true -> write(Dev, " "); + false -> ok + end, + case I of + #llvm_ret{} -> + write(Dev, "ret "), + case ret_ret_list(I) of + [] -> write(Dev, "void"); + List -> pp_args(Dev, List) + end, + write(Dev, "\n"); + #llvm_br{} -> + write(Dev, ["br label ", br_dst(I), "\n"]); + #llvm_switch{} -> + write(Dev, "switch "), + pp_type(Dev, switch_type(I)), + write(Dev, [" ", switch_value(I), ", label ", switch_default_label(I), + " \n [\n"]), + pp_switch_value_label_list(Dev, switch_type(I), + switch_value_label_list(I)), + write(Dev, " ]\n"); + #llvm_invoke{} -> + write(Dev, [invoke_dst(I), " = invoke ", invoke_cconv(I), " "]), + pp_options(Dev, invoke_ret_attrs(I)), + pp_type(Dev, invoke_type(I)), + write(Dev, [" ", invoke_fnptrval(I), "("]), + pp_args(Dev, invoke_arglist(I)), + write(Dev, ") "), + pp_options(Dev, invoke_fn_attrs(I)), + write(Dev, [" to label ", invoke_to_label(I)," unwind label ", + invoke_unwind_label(I), " \n"]); + #llvm_br_cond{} -> + write(Dev, ["br i1 ", br_cond_cond(I), ", label ", br_cond_true_label(I), + ", label ", br_cond_false_label(I)]), + case br_cond_meta(I) of + [] -> ok; + Metadata -> + write(Dev, [", !prof !", Metadata]) + end, + write(Dev, "\n"); + #llvm_indirectbr{} -> + write(Dev, "indirectbr "), + pp_type(Dev, indirectbr_type(I)), + write(Dev, [" ", indirectbr_address(I), ", [ "]), + pp_args(Dev, indirectbr_label_list(I)), + write(Dev, " ]\n"); + #llvm_operation{} -> + write(Dev, [operation_dst(I), " = ", atom_to_list(operation_op(I)), " "]), + case op_has_options(operation_op(I)) of + true -> pp_options(Dev, operation_options(I)); + false -> ok + end, + pp_type(Dev, operation_type(I)), + write(Dev, [" ", operation_src1(I), ", ", operation_src2(I), "\n"]); + #llvm_extractvalue{} -> + write(Dev, [extractvalue_dst(I), " = extractvalue "]), + pp_type(Dev, extractvalue_type(I)), + %% TODO Print idxs + write(Dev, [" ", extractvalue_val(I), ", ", extractvalue_idx(I), "\n"]); + #llvm_insertvalue{} -> + write(Dev, [insertvalue_dst(I), " = insertvalue "]), + pp_type(Dev, insertvalue_val_type(I)), + write(Dev, [" ", insertvalue_val(I), ", "]), + pp_type(Dev, insertvalue_elem_type(I)), + %%TODO Print idxs + write(Dev, [" ", insertvalue_elem(I), ", ", insertvalue_idx(I), "\n"]); + #llvm_alloca{} -> + write(Dev, [alloca_dst(I), " = alloca "]), + pp_type(Dev, alloca_type(I)), + case alloca_num(I) of + [] -> ok; + Num -> + write(Dev, ", "), + pp_type(Dev, alloca_type(I)), + write(Dev, [" ", Num, " "]) + end, + case alloca_align(I) of + [] -> ok; + Align -> write(Dev, [",align ", Align]) + end, + write(Dev, "\n"); + #llvm_load{} -> + write(Dev, [load_dst(I), " = "]), + write(Dev, "load "), + case load_volatile(I) of + true -> write(Dev, "volatile "); + false -> ok + end, + pp_type(Dev, load_p_type(I)), + write(Dev, [" ", load_pointer(I), " "]), + case load_alignment(I) of + [] -> ok; + Al -> write(Dev, [", align ", Al, " "]) + end, + case load_nontemporal(I) of + [] -> ok; + In -> write(Dev, [", !nontemporal !", In]) + end, + write(Dev, "\n"); + #llvm_store{} -> + write(Dev, "store "), + case store_volatile(I) of + true -> write(Dev, "volatile "); + false -> ok + end, + pp_type(Dev, store_type(I)), + write(Dev, [" ", store_value(I), ", "]), + pp_type(Dev, store_p_type(I)), + write(Dev, [" ", store_pointer(I), " "]), + case store_alignment(I) of + [] -> ok; + Al -> write(Dev, [", align ", Al, " "]) + end, + case store_nontemporal(I) of + [] -> ok; + In -> write(Dev, [", !nontemporal !", In]) + end, + write(Dev, "\n"); + #llvm_getelementptr{} -> + write(Dev, [getelementptr_dst(I), " = getelementptr "]), + case getelementptr_inbounds(I) of + true -> write(Dev, "inbounds "); + false -> ok + end, + pp_type(Dev, getelementptr_p_type(I)), + write(Dev, [" ", getelementptr_value(I)]), + pp_typed_idxs(Dev, getelementptr_typed_idxs(I)), + write(Dev, "\n"); + #llvm_conversion{} -> + write(Dev, [conversion_dst(I), " = ", atom_to_list(conversion_op(I)), " "]), + pp_type(Dev, conversion_src_type(I)), + write(Dev, [" ", conversion_src(I), " to "]), + pp_type(Dev, conversion_dst_type(I)), + write(Dev, "\n"); + #llvm_icmp{} -> + write(Dev, [icmp_dst(I), " = icmp ", atom_to_list(icmp_cond(I)), " "]), + pp_type(Dev, icmp_type(I)), + write(Dev, [" ", icmp_src1(I), ", ", icmp_src2(I), "\n"]); + #llvm_fcmp{} -> + write(Dev, [fcmp_dst(I), " = fcmp ", atom_to_list(fcmp_cond(I)), " "]), + pp_type(Dev, fcmp_type(I)), + write(Dev, [" ", fcmp_src1(I), ", ", fcmp_src2(I), "\n"]); + #llvm_phi{} -> + write(Dev, [phi_dst(I), " = phi "]), + pp_type(Dev, phi_type(I)), + pp_phi_value_labels(Dev, phi_value_label_list(I)), + write(Dev, "\n"); + #llvm_select{} -> + write(Dev, [select_dst(I), " = select i1 ", select_cond(I), ", "]), + pp_type(Dev, select_typ1(I)), + write(Dev, [" ", select_val1(I), ", "]), + pp_type(Dev, select_typ2(I)), + write(Dev, [" ", select_val2(I), "\n"]); + #llvm_call{} -> + case call_dst(I) of + [] -> ok; + Dst -> write(Dev, [Dst, " = "]) + end, + case call_is_tail(I) of + true -> write(Dev, "tail "); + false -> ok + end, + write(Dev, ["call ", call_cconv(I), " "]), + pp_options(Dev, call_ret_attrs(I)), + pp_type(Dev, call_type(I)), + write(Dev, [" ", call_fnptrval(I), "("]), + pp_args(Dev, call_arglist(I)), + write(Dev, ") "), + pp_options(Dev, call_fn_attrs(I)), + write(Dev, "\n"); + #llvm_fun_def{} -> + write(Dev, "define "), + pp_options(Dev, fun_def_linkage(I)), + pp_options(Dev, fun_def_visibility(I)), + case fun_def_cconv(I) of + [] -> ok; + Cc -> write(Dev, [Cc, " "]) + end, + pp_options(Dev, fun_def_ret_attrs(I)), + write(Dev, " "), + pp_type(Dev, fun_def_type(I)), + write(Dev, [" @", fun_def_name(I), "("]), + pp_args(Dev, fun_def_arglist(I)), + write(Dev, ") "), + pp_options(Dev, fun_def_fn_attrs(I)), + case fun_def_align(I) of + [] -> ok; + N -> write(Dev, ["align ", N]) + end, + write(Dev, "{\n"), + pp_ins_list(Dev, fun_def_body(I)), + write(Dev, "}\n"); + #llvm_fun_decl{} -> + write(Dev, "declare "), + pp_options(Dev, fun_decl_linkage(I)), + pp_options(Dev, fun_decl_visibility(I)), + case fun_decl_cconv(I) of + [] -> ok; + Cc -> write(Dev, [Cc, " "]) + end, + pp_options(Dev, fun_decl_ret_attrs(I)), + pp_type(Dev, fun_decl_type(I)), + write(Dev, [" ", fun_decl_name(I), "("]), + pp_type_list(Dev, fun_decl_arglist(I)), + write(Dev, ") "), + case fun_decl_align(I) of + [] -> ok; + N -> write(Dev, ["align ", N]) + end, + write(Dev, "\n"); + #llvm_comment{} -> + write(Dev, ["; ", atom_to_list(comment_text(I)), "\n"]); + #llvm_label{} -> + write(Dev, [label_label(I), ":\n"]); + #llvm_const_decl{} -> + write(Dev, [const_decl_dst(I), " = ", const_decl_decl_type(I), " "]), + pp_type(Dev, const_decl_type(I)), + write(Dev, [" ", const_decl_value(I), "\n"]); + #llvm_landingpad{} -> + write(Dev, "landingpad { i8*, i32 } personality i32 (i32, i64, i8*,i8*)* + @__gcc_personality_v0 cleanup\n"); + #llvm_asm{} -> + write(Dev, [asm_instruction(I), "\n"]); + #llvm_adj_stack{} -> + write(Dev, ["call void asm sideeffect \"sub $0, ", + adj_stack_register(I), "\", \"r\"("]), + pp_type(Dev, adj_stack_type(I)), + write(Dev, [" ", adj_stack_offset(I),")\n"]); + #llvm_branch_meta{} -> + write(Dev, ["!", branch_meta_id(I), " = metadata !{metadata !\"branch_weights\", + i32 ", branch_meta_true_weight(I), ", i32 ", + branch_meta_false_weight(I), "}\n"]); + Other -> + exit({?MODULE, pp_ins, {"Unknown LLVM instruction", Other}}) + end. + +%% @doc Pretty-print a list of types +pp_type_list(_Dev, []) -> ok; +pp_type_list(Dev, [T]) -> + pp_type(Dev, T); +pp_type_list(Dev, [T|Ts]) -> + pp_type(Dev, T), + write(Dev, ", "), + pp_type_list(Dev, Ts). + +pp_type(Dev, Type) -> + case Type of + #llvm_void{} -> + write(Dev, "void"); + #llvm_label_type{} -> + write(Dev, "label"); + %% Integer + #llvm_int{} -> + write(Dev, ["i", integer_to_list(int_width(Type))]); + %% Float + #llvm_float{} -> + write(Dev, "float"); + #llvm_double{} -> + write(Dev, "double"); + #llvm_fp80{} -> + write(Dev, "x86_fp80"); + #llvm_fp128{} -> + write(Dev, "fp128"); + #llvm_ppc_fp128{} -> + write(Dev, "ppc_fp128"); + %% Pointer + #llvm_pointer{} -> + pp_type(Dev, pointer_type(Type)), + write(Dev, "*"); + %% Function + #llvm_fun{} -> + pp_type(Dev, function_ret_type(Type)), + write(Dev, " ("), + pp_type_list(Dev, function_arg_type_list(Type)), + write(Dev, ")"); + %% Aggregate + #llvm_array{} -> + write(Dev, ["[", integer_to_list(array_size(Type)), " x "]), + pp_type(Dev, array_type(Type)), + write(Dev, "]"); + #llvm_struct{} -> + write(Dev, "{"), + pp_type_list(Dev, struct_type_list(Type)), + write(Dev, "}"); + #llvm_vector{} -> + write(Dev, ["{", integer_to_list(vector_size(Type)), " x "]), + pp_type(Dev, vector_type(Type)), + write(Dev, "}") + end. + +%% @doc Pretty-print a list of typed arguments +pp_args(_Dev, []) -> ok; +pp_args(Dev, [{Type, Arg} | []]) -> + pp_type(Dev, Type), + write(Dev, [" ", Arg]); +pp_args(Dev, [{Type, Arg} | Args]) -> + pp_type(Dev, Type), + write(Dev, [" ", Arg, ", "]), + pp_args(Dev, Args). + +%% @doc Pretty-print a list of options +pp_options(_Dev, []) -> ok; +pp_options(Dev, [O|Os]) -> + write(Dev, [atom_to_list(O), " "]), + pp_options(Dev, Os). + +%% @doc Pretty-print a list of phi value-labels +pp_phi_value_labels(_Dev, []) -> ok; +pp_phi_value_labels(Dev, [{Value, Label}|[]]) -> + write(Dev, ["[ ", Value, ", ", Label, " ]"]); +pp_phi_value_labels(Dev,[{Value, Label}|VL]) -> + write(Dev, ["[ ", Value, ", ", Label, " ], "]), + pp_phi_value_labels(Dev, VL). + +%% @doc Pretty-print a list of typed indexes +pp_typed_idxs(_Dev, []) -> ok; +pp_typed_idxs(Dev, [{Type, Id} | Tids]) -> + write(Dev, ", "), + pp_type(Dev, Type), + write(Dev, [" ", Id]), + pp_typed_idxs(Dev, Tids). + +%% @doc Pretty-print a switch label list +pp_switch_value_label_list(_Dev, _Type, []) -> ok; +pp_switch_value_label_list(Dev, Type, [{Value, Label} | VLs]) -> + write(Dev, " "), + pp_type(Dev, Type), + write(Dev, [" ", Value, ", label ", Label, "\n"]), + pp_switch_value_label_list(Dev, Type, VLs). + +%%---------------------------------------------------------------------------- +%% Auxiliary Functions +%%---------------------------------------------------------------------------- + +%% @doc Returns if an instruction needs to be intended +indent(I) -> + case I of + #llvm_label{} -> false; + #llvm_fun_def{} -> false; + #llvm_fun_decl{} -> false; + #llvm_const_decl{} -> false; + #llvm_branch_meta{} -> false; + _ -> true + end. + +op_has_options(Op) -> + case Op of + 'and' -> false; + 'or' -> false; + 'xor' -> false; + _ -> true + end. + +%% @doc Abstracts actual writing to file operations +write(Dev, Msg) -> + ok = file:write(Dev, Msg). diff --git a/lib/hipe/llvm/hipe_llvm_arch.hrl b/lib/hipe/llvm/hipe_llvm_arch.hrl new file mode 100644 index 0000000000..689a5a52ea --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm_arch.hrl @@ -0,0 +1,11 @@ +-ifdef(BIT32). +-define(NR_PINNED_REGS, 2). +-define(NR_ARG_REGS, 3). +-define(ARCH_REGISTERS, hipe_x86_registers). +-define(FLOAT_OFFSET, 2). +-else. +-define(NR_PINNED_REGS, 2). +-define(NR_ARG_REGS, 4). +-define(ARCH_REGISTERS, hipe_amd64_registers). +-define(FLOAT_OFFSET, 6). +-endif. diff --git a/lib/hipe/llvm/hipe_llvm_liveness.erl b/lib/hipe/llvm/hipe_llvm_liveness.erl new file mode 100644 index 0000000000..d1c90ed4c9 --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm_liveness.erl @@ -0,0 +1,112 @@ +-module(hipe_llvm_liveness). + +-export([analyze/1]). + +%% @doc Find gc roots and explicitly mark when they go out of scope, based +%% on the liveness analyzis performed by the hipe_rtl_liveness:analyze/1. +analyze(RtlCfg) -> + Liveness = hipe_rtl_liveness:analyze(RtlCfg), + Roots = find_roots(RtlCfg, Liveness), + %% erlang:display(Roots), + NewRtlCfg = mark_dead_roots(RtlCfg, Liveness, Roots), + {NewRtlCfg, Roots}. + +%% @doc Determine which are the GC Roots.Possible roots are all +%% RTL variables (rtl_var). However, since safe points are function calls, we +%% consider as possible GC roots only RTL variables that are live around +%% function calls. +find_roots(Cfg, Liveness) -> + Labels = hipe_rtl_cfg:postorder(Cfg), + Roots = find_roots_bb(Labels, Cfg, Liveness, []), + lists:usort(lists:flatten(Roots)). + +find_roots_bb([], _Cfg, _Liveness, RootAcc) -> + RootAcc; +find_roots_bb([L|Ls], Cfg, Liveness, RootAcc) -> + Block = hipe_rtl_cfg:bb(Cfg, L), + BlockCode = hipe_bb:code(Block), + LiveIn = ordsets:from_list(strip(hipe_rtl_liveness:livein(Liveness, L))), + LiveOut = ordsets:from_list(strip(hipe_rtl_liveness:liveout(Liveness, L))), + Roots = do_find_roots_bb(BlockCode, L, LiveOut, LiveIn, []), + find_roots_bb(Ls, Cfg, Liveness, Roots++RootAcc). + +%% For each call inside a BB the GC roots are those RTL variables that +%% are live before and after the call. +%% --> Live Before Call: These are the RTL variables that belong to the +%% LiveIn list or are initialized inside the BB before the call +%% --> Live After Call: These are the RTL variables that belong to the +%% LiveOut list or are used after the call inside the BB (they die +%% inside the BB and so do not belong to the LiveOut list) +do_find_roots_bb([], _Label, _LiveOut, _LiveBefore, RootAcc) -> + RootAcc; +do_find_roots_bb([I|Is], L, LiveOut, LiveBefore, RootAcc) -> + case hipe_rtl:is_call(I) of + true -> + %% Used inside the BB after the call + UsedAfterCall_ = strip(lists:flatten([hipe_rtl:uses(V) || V <- Is])), + UsedAfterCall = ordsets:from_list(UsedAfterCall_), + LiveAfter = ordsets:union(UsedAfterCall, LiveOut), + %% The Actual Roots + Roots = ordsets:intersection(LiveBefore, LiveAfter), + %% The result of the instruction + Defines = ordsets:from_list(strip(hipe_rtl:defines(I))), + LiveBefore1 = ordsets:union(LiveBefore, Defines), + do_find_roots_bb(Is, L, LiveOut, LiveBefore1, [Roots|RootAcc]); + false -> + %% The result of the instruction + Defines = ordsets:from_list(strip(hipe_rtl:defines(I))), + LiveBefore1 = ordsets:union(LiveBefore, Defines), + do_find_roots_bb(Is, L, LiveOut, LiveBefore1, RootAcc) + end. + +%% @doc This function is responsible for marking when GC Roots, which can be +%% only RTL variables go out of scope (dead). This pass is needed for the LLVM +%% back end because the LLVM framework forces us to explicit mark when gc roots +%% are no longer live. +mark_dead_roots(CFG, Liveness, Roots) -> + Labels = hipe_rtl_cfg:postorder(CFG), + mark_dead_bb(Labels, CFG, Liveness, Roots). + +mark_dead_bb([], Cfg, _Liveness, _Roots) -> + Cfg; +mark_dead_bb([L|Ls], Cfg, Liveness, Roots) -> + Block = hipe_rtl_cfg:bb(Cfg, L), + BlockCode = hipe_bb:code(Block), + LiveOut = ordsets:from_list(strip(hipe_rtl_liveness:liveout(Liveness, L))), + NewBlockCode = do_mark_dead_bb(BlockCode, LiveOut, Roots, []), + %% Update the CFG + NewBB = hipe_bb:code_update(Block, NewBlockCode), + NewCFG = hipe_rtl_cfg:bb_add(Cfg, L, NewBB), + mark_dead_bb(Ls, NewCFG, Liveness, Roots). + +do_mark_dead_bb([], _LiveOut, _Roots, NewBlockCode) -> + lists:reverse(NewBlockCode); +do_mark_dead_bb([I|Is], LiveOut ,Roots, NewBlockCode) -> + Uses = ordsets:from_list(strip(hipe_rtl:uses(I))), + %% GC roots that are used in this instruction + RootsUsed = ordsets:intersection(Roots, Uses), + UsedAfter_ = strip(lists:flatten([hipe_rtl:uses(V) || V <- Is])), + UsedAfter = ordsets:from_list(UsedAfter_), + %% GC roots that are live after this instruction + LiveAfter = ordsets:union(LiveOut, UsedAfter), + %% GC roots that their last use is in this instruction + DeadRoots = ordsets:subtract(RootsUsed, LiveAfter), + %% Recreate the RTL variable from the corresponding Index + OldVars = [hipe_rtl:mk_var(V1) || V1 <- DeadRoots], + %% Mark the RTL variable as DEAD (last use) + NewVars = [kill_var(V2) || V2 <- OldVars], + %% Create a list with the substitution of the old vars with the new + %% ones which are marked with the dead keyword + Subtitution = lists:zip(OldVars, NewVars), + NewI = case Subtitution of + [] -> I; + _ -> hipe_rtl:subst_uses_llvm(Subtitution, I) + end, + do_mark_dead_bb(Is, LiveOut, Roots, [NewI|NewBlockCode]). + +%% Update the liveness of a var,in order to mark that this is the last use. +kill_var(Var) -> hipe_rtl:var_liveness_update(Var, dead). + +%% We are only interested for rtl_vars, since only rtl_vars are possible gc +%% roots. +strip(L) -> [Y || {rtl_var, Y, _} <- L]. diff --git a/lib/hipe/llvm/hipe_llvm_main.erl b/lib/hipe/llvm/hipe_llvm_main.erl new file mode 100644 index 0000000000..e911fb89c9 --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm_main.erl @@ -0,0 +1,514 @@ +%% -*- erlang-indent-level: 2 -*- +-module(hipe_llvm_main). + +-export([rtl_to_native/4]). + +-include("../../kernel/src/hipe_ext_format.hrl"). +-include("hipe_llvm_arch.hrl"). +-include("elf_format.hrl"). + +%% @doc Translation of RTL to a loadable object. This function takes the RTL +%% code and calls hipe_rtl_to_llvm:translate/2 to translate the RTL code to +%% LLVM code. After this, LLVM asm is printed to a file and the LLVM tool +%% chain is invoked in order to produce an object file. +rtl_to_native(MFA, RTL, Roots, Options) -> + %% Compile to LLVM and get Instruction List (along with infos) + {LLVMCode, RelocsDict, ConstTab} = + hipe_rtl_to_llvm:translate(RTL, Roots), + %% Fix function name to an acceptable LLVM identifier (needed for closures) + {_Module, Fun, Arity} = hipe_rtl_to_llvm:fix_mfa_name(MFA), + %% Write LLVM Assembly to intermediate file (on disk) + {ok, Dir, ObjectFile} = + compile_with_llvm(Fun, Arity, LLVMCode, Options, false), + %% + %% Extract information from object file + %% + ObjBin = open_object_file(ObjectFile), + %% Read and set the ELF class + elf_format:set_architecture_flag(ObjBin), + %% Get labels info (for switches and jump tables) + Labels = elf_format:get_rodata_relocs(ObjBin), + {Switches, Closures} = get_tables(ObjBin), + %% Associate Labels with Switches and Closures with stack args + {SwitchInfos, ExposedClosures} = + correlate_labels(Switches ++ Closures, Labels), + %% SwitchInfos: [{"table_50", [Labels]}] + %% ExposedClosures: [{"table_closures", [Labels]}] + + %% Labelmap contains the offsets of the labels in the code that are + %% used for switch's jump tables + LabelMap = create_labelmap(MFA, SwitchInfos, RelocsDict), + %% Get relocation info + TextRelocs = elf_format:get_text_relocs(ObjBin), + %% AccRefs contains the offsets of all references to relocatable symbols in + %% the code: + AccRefs = fix_relocations(TextRelocs, RelocsDict, MFA), + %% Get stack descriptors + SDescs = get_sdescs(ObjBin), + %% FixedSDescs are the stack descriptors after correcting calls that have + %% arguments in the stack + FixedSDescs = + fix_stack_descriptors(RelocsDict, AccRefs, SDescs, ExposedClosures), + Refs = AccRefs ++ FixedSDescs, + %% Get binary code from object file + BinCode = elf_format:extract_text(ObjBin), + %% Remove temp files (if needed) + ok = remove_temp_folder(Dir, Options), + %% Return the code together with information that will be used in the + %% hipe_llvm_merge module to produce the final binary that will be loaded + %% by the hipe unified loader. + {MFA, BinCode, byte_size(BinCode), ConstTab, Refs, LabelMap}. + +%%------------------------------------------------------------------------------ +%% LLVM tool chain +%%------------------------------------------------------------------------------ + +%% @doc Compile function FunName/Arity to LLVM. Return Dir (in order to remove +%% it if we do not want to store temporary files) and ObjectFile name that +%% is created by the LLVM tools. +compile_with_llvm(FunName, Arity, LLVMCode, Options, UseBuffer) -> + Filename = atom_to_list(FunName) ++ "_" ++ integer_to_list(Arity), + %% Save temp files in a unique folder + Dir = unique_folder(FunName, Arity, Options), + ok = file:make_dir(Dir), + %% Print LLVM assembly to file + OpenOpts = [append, raw] ++ + case UseBuffer of + %% true -> [delayed_write]; % Use delayed_write! + false -> [] + end, + {ok, File_llvm} = file:open(Dir ++ Filename ++ ".ll", OpenOpts), + hipe_llvm:pp_ins_list(File_llvm, LLVMCode), + %% delayed_write can cause file:close not to do a close, hence the two calls + ok = file:close(File_llvm), + __ = file:close(File_llvm), + %% Invoke LLVM compiler tools to produce an object file + llvm_opt(Dir, Filename, Options), + llvm_llc(Dir, Filename, Options), + compile(Dir, Filename, "gcc"), %%FIXME: use llc -filetype=obj and skip this! + {ok, Dir, Dir ++ Filename ++ ".o"}. + +%% @doc Invoke opt tool to optimize the bitcode (_name.ll -> _name.bc). +llvm_opt(Dir, Filename, Options) -> + Source = Dir ++ Filename ++ ".ll", + Dest = Dir ++ Filename ++ ".bc", + OptLevel = trans_optlev_flag(opt, Options), + OptFlags = [OptLevel, "-mem2reg", "-strip"], + Command = "opt " ++ fix_opts(OptFlags) ++ " " ++ Source ++ " -o " ++ Dest, + %% io:format("OPT: ~s~n", [Command]), + case os:cmd(Command) of + "" -> ok; + Error -> exit({?MODULE, opt, Error}) + end. + +%% @doc Invoke llc tool to compile the bitcode to object file +%% (_name.bc -> _name.o). +llvm_llc(Dir, Filename, Options) -> + Source = Dir ++ Filename ++ ".bc", + OptLevel = trans_optlev_flag(llc, Options), + Align = find_stack_alignment(), + LlcFlags = [OptLevel, "-code-model=medium", "-stack-alignment=" ++ Align + , "-tailcallopt", "-filetype=asm"], %%FIXME + Command = "llc " ++ fix_opts(LlcFlags) ++ " " ++ Source, + %% io:format("LLC: ~s~n", [Command]), + case os:cmd(Command) of + "" -> ok; + Error -> exit({?MODULE, llc, Error}) + end. + +%% @doc Invoke the compiler tool ("gcc", "llvmc", etc.) to generate an object +%% file from native assembly. +compile(Dir, Fun_Name, Compiler) -> + Source = Dir ++ Fun_Name ++ ".s", + Dest = Dir ++ Fun_Name ++ ".o", + Command = Compiler ++ " -c " ++ Source ++ " -o " ++ Dest, + %% io:format("~s: ~s~n", [Compiler, Command]), + case os:cmd(Command) of + "" -> ok; + Error -> exit({?MODULE, cc, Error}) + end. + +find_stack_alignment() -> + case get(hipe_target_arch) of + x86 -> "4"; + amd64 -> "8"; + _ -> exit({?MODULE, find_stack_alignment, "Unimplemented architecture"}) + end. + +%% @doc Join options. +fix_opts(Opts) -> + string:join(Opts, " "). + +%% @doc Translate optimization-level flag (default is "O2"). +trans_optlev_flag(Tool, Options) -> + Flag = case Tool of + opt -> llvm_opt; + llc -> llvm_llc + end, + case proplists:get_value(Flag, Options) of + o0 -> ""; % "-O0" does not exist in opt tool + o1 -> "-O1"; + o2 -> "-O2"; + o3 -> "-O3"; + undefined -> "-O2" + end. + +%%------------------------------------------------------------------------------ +%% Functions to manage Relocations +%%------------------------------------------------------------------------------ + +%% @doc Get switch table and closure table. +get_tables(Elf) -> + %% Search Symbol Table for an entry with name prefixed with "table_": + Triples = elf_format:get_tab_entries(Elf), + Switches = [T || T={"table_" ++ _, _, _} <- Triples], + Closures = [T || T={"table_closures" ++ _, _, _} <- Switches], + {Switches, Closures}. + +%% @doc This function associates symbols who point to some table of labels with +%% the corresponding offsets of the labels in the code. These tables can +%% either be jump tables for switches or a table which contains the labels +%% of blocks that contain closure calls with more than ?NR_ARG_REGS. +correlate_labels([], _L) -> {[], []}; +correlate_labels(Tables, Labels) -> + %% Sort "Tables" based on "ValueOffsets" + OffsetSortedTb = lists:ukeysort(2, Tables), + %% Unzip offset-sorted list of "Switches" + {Names, _Offsets, TablesSizeList} = lists:unzip3(OffsetSortedTb), + %% Associate switch names with labels + L = split_list(Labels, TablesSizeList), + %% Zip back! (to [{SwitchName, Values}]) + NamesValues = lists:zip(Names, L), + case lists:keytake("table_closures", 1, NamesValues) of + false -> %% No closures in the code, no closure table + {NamesValues, []}; + {value, ClosureTableNV, SwitchesNV} -> + {SwitchesNV, ClosureTableNV} + end. + +%% @doc Create a gb_tree which contains information about the labels that used +%% for switch's jump tables. The keys of the gb_tree are of the form +%% {MFA, Label} and the values are the actual Offsets. +create_labelmap(MFA, SwitchInfos, RelocsDict) -> + create_labelmap(MFA, SwitchInfos, RelocsDict, gb_trees:empty()). + +create_labelmap(_, [], _, LabelMap) -> LabelMap; +create_labelmap(MFA, [{Name, Offsets} | Rest], RelocsDict, LabelMap) -> + case dict:fetch(Name, RelocsDict) of + {switch, {_TableType, LabelList, _NrLabels, _SortOrder}, _JTabLab} -> + KVDict = lists:ukeysort(1, lists:zip(LabelList, Offsets)), + NewLabelMap = insert_to_labelmap(KVDict, LabelMap), + create_labelmap(MFA, Rest, RelocsDict, NewLabelMap); + _ -> + exit({?MODULE, create_labelmap, "Not a jump table!"}) + end. + +%% @doc Insert a list of [{Key,Value}] to a LabelMap (gb_tree). +insert_to_labelmap([], LabelMap) -> LabelMap; +insert_to_labelmap([{Key, Value}|Rest], LabelMap) -> + case gb_trees:lookup(Key, LabelMap) of + none -> + insert_to_labelmap(Rest, gb_trees:insert(Key, Value, LabelMap)); + {value, Value} -> %% Exists with the *exact* same Value. + insert_to_labelmap(Rest, LabelMap) + end. + +%% @doc Correlate object file relocation symbols with info from translation to +%% llvm code. +fix_relocations(Relocs, RelocsDict, MFA) -> + fix_relocs(Relocs, RelocsDict, MFA, []). + +fix_relocs([], _, _, RelocAcc) -> RelocAcc; +fix_relocs([{Name, Offset}|Rs], RelocsDict, {ModName,_,_}=MFA, RelocAcc) -> + case dict:fetch(Name, RelocsDict) of + {atom, AtomName} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?LOAD_ATOM, Offset, AtomName}|RelocAcc]); + {constant, Label} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?LOAD_ADDRESS, Offset, {constant, Label}}|RelocAcc]); + {switch, _, JTabLab} -> %% Treat switch exactly as constant + fix_relocs(Rs, RelocsDict, MFA, + [{?LOAD_ADDRESS, Offset, {constant, JTabLab}}|RelocAcc]); + {closure, _}=Closure -> + fix_relocs(Rs, RelocsDict, MFA, + [{?LOAD_ADDRESS, Offset, Closure}|RelocAcc]); + {call, {bif, BifName, _}} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?CALL_LOCAL, Offset, BifName}|RelocAcc]); + %% MFA calls to functions in the same module are of type 3, while all + %% other MFA calls are of type 2. + {call, {ModName,_F,_A}=CallMFA} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?CALL_LOCAL, Offset, CallMFA}|RelocAcc]); + {call, CallMFA} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?CALL_REMOTE, Offset, CallMFA}|RelocAcc]); + Other -> + exit({?MODULE, fix_relocs, + {"Relocation not in relocation dictionary", Other}}) + end. + +%%------------------------------------------------------------------------------ +%% Functions to manage Stack Descriptors +%%------------------------------------------------------------------------------ + +%% @doc This function takes an ELF Object File binary and returns a proper sdesc +%% list for Erlang/OTP System's loader. The return value should be of the +%% form: +%% { +%% 4, Safepoint Address, +%% {ExnLabel OR [], FrameSize, StackArity, {Liveroot stack frame indexes}}, +%% } +get_sdescs(Elf) -> + case elf_format:extract_note(Elf, ?NOTE_ERLGC_NAME) of + <<>> -> % Object file has no ".note.gc" section! + []; + NoteGC_bin -> + %% Get safe point addresses (stored in ".rela.note.gc" section): + RelaNoteGC = elf_format:extract_rela(Elf, ?NOTE(?NOTE_ERLGC_NAME)), + SPCount = length(RelaNoteGC), + T = SPCount * ?SP_ADDR_SIZE, + %% Pattern match fields of ".note.gc": + <<SPCount:(?bits(?SP_COUNT_SIZE))/integer-little, % Sanity check! + SPAddrs:T/binary, % NOTE: In 64bit they are relocs! + StkFrameSize:(?bits(?SP_STKFRAME_SIZE))/integer-little, + StkArity:(?bits(?SP_STKARITY_SIZE))/integer-little, + _LiveRootCount:(?bits(?SP_LIVEROOTCNT_SIZE))/integer-little, % Skip + Roots/binary>> = NoteGC_bin, + LiveRoots = get_liveroots(Roots, []), + %% Extract information about the safe point addresses: + SPOffs = + case elf_format:is64bit() of + true -> %% Find offsets in ".rela.note.gc": + elf_format:get_rela_addends(RelaNoteGC); + false -> %% Find offsets in SPAddrs (in ".note.gc"): + get_spoffs(SPAddrs, []) + end, + %% Extract Exception Handler labels: + ExnHandlers = elf_format:get_exn_handlers(Elf), + %% Combine ExnHandlers and Safe point addresses (return addresses): + ExnAndSPOffs = combine_ras_and_exns(ExnHandlers, SPOffs, []), + create_sdesc_list(ExnAndSPOffs, StkFrameSize, StkArity, LiveRoots, []) + end. + +%% @doc Extracts a bunch of integers (live roots) from a binary. Returns a tuple +%% as need for stack descriptors. +get_liveroots(<<>>, Acc) -> + list_to_tuple(Acc); +get_liveroots(<<Root:?bits(?LR_STKINDEX_SIZE)/integer-little, + MoreRoots/binary>>, Acc) -> + get_liveroots(MoreRoots, [Root | Acc]). + +%% @doc Extracts a bunch of integers (safepoint offsets) from a binary. Returns +%% a tuple as need for stack descriptors. +get_spoffs(<<>>, Acc) -> + lists:reverse(Acc); +get_spoffs(<<SPOff:?bits(?SP_ADDR_SIZE)/integer-little, More/binary>>, Acc) -> + get_spoffs(More, [SPOff | Acc]). + +combine_ras_and_exns(_, [], Acc) -> + lists:reverse(Acc); +combine_ras_and_exns(ExnHandlers, [RA | MoreRAs], Acc) -> + %% FIXME: do something better than O(n^2) by taking advantage of the property + %% ||ExnHandlers|| <= ||RAs|| + Handler = find_exn_handler(RA, ExnHandlers), + combine_ras_and_exns(ExnHandlers, MoreRAs, [{Handler, RA} | Acc]). + +find_exn_handler(_, []) -> + []; +find_exn_handler(RA, [{Start, End, Handler} | MoreExnHandlers]) -> + case (RA >= Start andalso RA =< End) of + true -> + Handler; + false -> + find_exn_handler(RA, MoreExnHandlers) + end. + +create_sdesc_list([], _, _, _, Acc) -> + lists:reverse(Acc); +create_sdesc_list([{ExnLbl, SPOff} | MoreExnAndSPOffs], + StkFrameSize, StkArity, LiveRoots, Acc) -> + Hdlr = case ExnLbl of + 0 -> []; + N -> N + end, + create_sdesc_list(MoreExnAndSPOffs, StkFrameSize, StkArity, LiveRoots, + [{?SDESC, SPOff, {Hdlr, StkFrameSize, StkArity, LiveRoots}} + | Acc]). + +%% @doc This function is responsible for correcting the stack descriptors of +%% the calls that are found in the code and have more than NR_ARG_REGS +%% (thus, some of their arguments are passed to the stack). Because of the +%% Reserved Call Frame feature that the LLVM uses, the stack descriptors +%% are not correct since at the point of call the frame size is reduced +%% proportionally to the number of arguments that are passed on the stack. +%% Also the offsets of the roots need to be re-adjusted. +fix_stack_descriptors(_, _, [], _) -> + []; +fix_stack_descriptors(RelocsDict, Relocs, SDescs, ExposedClosures) -> + %% NamedCalls are MFA and BIF calls that need fix + NamedCalls = calls_with_stack_args(RelocsDict), + NamedCallsOffs = calls_offsets_arity(Relocs, NamedCalls), + ExposedClosures1 = + case dict:is_key("table_closures", RelocsDict) of + true -> %% A Table with closures exists + {table_closures, ArityList} = dict:fetch("table_closures", RelocsDict), + case ExposedClosures of + {_, Offsets} -> + lists:zip(Offsets, ArityList); + _ -> + exit({?MODULE, fix_stack_descriptors, + {"Wrong exposed closures", ExposedClosures}}) + end; + false -> + [] + end, + ClosuresOffs = closures_offsets_arity(ExposedClosures1, SDescs), + fix_sdescs(NamedCallsOffs ++ ClosuresOffs, SDescs). + +%% @doc This function takes as argument the relocation dictionary as produced by +%% the translation of RTL code to LLVM and finds the names of the calls +%% (MFA and BIF calls) that have more than NR_ARG_REGS. +calls_with_stack_args(Dict) -> + calls_with_stack_args(dict:to_list(Dict), []). + +calls_with_stack_args([], Calls) -> Calls; +calls_with_stack_args([ {_Name, {call, {M, F, A}}} | Rest], Calls) + when A > ?NR_ARG_REGS -> + Call = + case M of + bif -> {F,A}; + _ -> {M,F,A} + end, + calls_with_stack_args(Rest, [Call|Calls]); +calls_with_stack_args([_|Rest], Calls) -> + calls_with_stack_args(Rest, Calls). + +%% @doc This function extracts the stack arity and the offset in the code of +%% the named calls (MFAs, BIFs) that have stack arguments. +calls_offsets_arity(AccRefs, CallsWithStackArgs) -> + calls_offsets_arity(AccRefs, CallsWithStackArgs, []). + +calls_offsets_arity([], _, Acc) -> Acc; +calls_offsets_arity([{Type, Offset, Term} | Rest], CallsWithStackArgs, Acc) + when Type =:= ?CALL_REMOTE orelse Type =:= ?CALL_LOCAL -> + case lists:member(Term, CallsWithStackArgs) of + true -> + Arity = + case Term of + {_M, _F, A} -> A; + {_F, A} -> A + end, + calls_offsets_arity(Rest, CallsWithStackArgs, + [{Offset + 4, Arity - ?NR_ARG_REGS} | Acc]); + false -> + calls_offsets_arity(Rest, CallsWithStackArgs, Acc) + end; +calls_offsets_arity([_|Rest], CallsWithStackArgs, Acc) -> + calls_offsets_arity(Rest, CallsWithStackArgs, Acc). + +%% @doc This function extracts the stack arity and the offsets of closures that +%% have stack arity. The Closures argument represents the +%% hipe_bifs:llvm_exposure_closure/0 calls in the code. The actual closure +%% is the next call in the code, so the offset of the next call must be +%% calculated from the stack descriptors. +closures_offsets_arity([], _) -> + []; +closures_offsets_arity(ExposedClosures, SDescs) -> + Offsets = [Offset || {_, Offset, _} <- SDescs], + %% Offsets and closures must be sorted in order for find_offsets/3 to work + SortedOffsets = lists:sort(Offsets), + SortedExposedClosures = lists:keysort(1, ExposedClosures), + find_offsets(SortedExposedClosures, SortedOffsets, []). + +find_offsets([], _, Acc) -> Acc; +find_offsets([{Off,Arity}|Rest], Offsets, Acc) -> + [I | RestOffsets] = lists:dropwhile(fun (Y) -> Y<Off end, Offsets), + find_offsets(Rest, RestOffsets, [{I, Arity}|Acc]). + +%% The functions below correct the arity of calls, that are identified +%% by offset, in the stack descriptors. +fix_sdescs([], SDescs) -> SDescs; +fix_sdescs([{Offset, Arity} | Rest], SDescs) -> + case lists:keyfind(Offset, 2, SDescs) of + false -> + fix_sdescs(Rest, SDescs); + {?SDESC, Offset, SDesc} -> + {ExnHandler, FrameSize, StkArity, Roots} = SDesc, + DecRoot = fun(X) -> X-Arity end, + NewRootsList = lists:map(DecRoot, tuple_to_list(Roots)), + NewSDesc = + case length(NewRootsList) > 0 andalso hd(NewRootsList) >= 0 of + true -> + {?SDESC, Offset, {ExnHandler, FrameSize-Arity, StkArity, + list_to_tuple(NewRootsList)}}; + false -> + {?SDESC, Offset, {ExnHandler, FrameSize, StkArity, Roots}} + end, + RestSDescs = lists:keydelete(Offset, 2, SDescs), + fix_sdescs(Rest, [NewSDesc | RestSDescs]) + end. + + +%%------------------------------------------------------------------------------ +%% Miscellaneous functions +%%------------------------------------------------------------------------------ + +%% @doc A function that opens a file as binary. The function takes as argument +%% the name of the file and returns an Erlang binary. +-spec open_object_file(string()) -> binary(). +open_object_file(ObjFile) -> + case file:read_file(ObjFile) of + {ok, Binary} -> + Binary; + {error, Reason} -> + exit({?MODULE, open_file, Reason}) + end. + +remove_temp_folder(Dir, Options) -> + case proplists:get_bool(llvm_save_temps, Options) of + true -> ok; + false -> spawn(fun () -> "" = os:cmd("rm -rf " ++ Dir) end), ok + end. + +unique_id(FunName, Arity) -> + integer_to_list(erlang:phash2({FunName, Arity, now()})). + +unique_folder(FunName, Arity, Options) -> + DirName = "llvm_" ++ unique_id(FunName, Arity) ++ "/", + Dir = + case proplists:get_bool(llvm_save_temps, Options) of + true -> %% Store folder in current directory + DirName; + false -> %% Temporarily store folder in tempfs (/dev/shm/) + "/dev/shm/" ++ DirName + end, + %% Make sure it does not exist + case dir_exists(Dir) of + true -> %% Dir already exists! Generate again. + unique_folder(FunName, Arity, Options); + false -> + Dir + end. + +%% @doc Function that checks that a given Filename is an existing Directory +%% Name (from http://rosettacode.org/wiki/Ensure_that_a_file_exists#Erlang) +dir_exists(Filename) -> + {Flag, Info} = file:read_file_info(Filename), + (Flag =:= ok) andalso (element(3, Info) =:= directory). + +%% @doc Function that takes as arguments a list of integers and a list with +%% numbers indicating how many items should each tuple have and splits +%% the original list to a list of lists of integers (with the specified +%% number of elements), i.e. [ [...], [...] ]. +-spec split_list([integer()], [integer()]) -> [ [integer()] ]. +split_list(List, ElemsPerTuple) -> + split_list(List, ElemsPerTuple, []). + +-spec split_list([integer()], [integer()], [ [integer()] ]) -> [ [integer()] ]. +split_list([], [], Acc) -> + lists:reverse(Acc); +split_list(List, [NumOfElems | MoreNums], Acc) -> + {L1, L2} = lists:split(NumOfElems, List), + split_list(L2, MoreNums, [ L1 | Acc]). diff --git a/lib/hipe/llvm/hipe_llvm_merge.erl b/lib/hipe/llvm/hipe_llvm_merge.erl new file mode 100644 index 0000000000..3ababfc21a --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm_merge.erl @@ -0,0 +1,114 @@ +%%% -*- erlang-indent-level: 2 -*- +-module(hipe_llvm_merge). + +-export([finalize/3]). + +-include("hipe_llvm_arch.hrl"). +-include("../../kernel/src/hipe_ext_format.hrl"). +-include("../rtl/hipe_literals.hrl"). +-include("../main/hipe.hrl"). + +finalize(CompiledCode, Closures, Exports) -> + CompiledCode1 = [CodePack || {_, CodePack} <- CompiledCode], + Code = [{MFA, [], ConstTab} + || {MFA, _, _ , ConstTab, _, _} <- CompiledCode1], + {ConstAlign, ConstSize, ConstMap, RefsFromConsts} = + hipe_pack_constants:pack_constants(Code, ?ARCH_REGISTERS:alignment()), + %% Compute total code size separately as a sanity check for alignment + CodeSize = compute_code_size(CompiledCode1, 0), + %% io:format("Code Size (pre-computed): ~w~n", [CodeSize]), + {CodeBinary, ExportMap} = merge_mfas(CompiledCode1, 0, <<>>, []), + %% io:format("Code Size (post-computed): ~w~n", [byte_size(CodeBinary)]), + ?VERBOSE_ASSERT(CodeSize =:= byte_size(CodeBinary)), + AccRefs = merge_refs(CompiledCode1, ConstMap, 0, []), + %% Bring CompiledCode to a combine_label_maps-acceptable form. + LabelMap = combine_label_maps(CompiledCode1, 0, gb_trees:empty()), + SC = hipe_pack_constants:slim_constmap(ConstMap), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap, Closures, Exports), + SlimRefs = hipe_pack_constants:slim_refs(AccRefs), + term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, + ConstAlign, ConstSize, + SC, % ConstMap + DataRelocs, % LabelMap + SSE, % ExportMap + CodeSize, CodeBinary, SlimRefs, + 0,[] % ColdCodeSize, SlimColdRefs + ]). + +%% Copied from hipe_x86_assemble.erl +nr_pad_bytes(Address) -> + (4 - (Address rem 4)) rem 4. % XXX: 16 or 32 instead? + +align_entry(Address) -> + Address + nr_pad_bytes(Address). + +compute_code_size([{_MFA, _BinaryCode, CodeSize, _, _, _}|Code], Size) -> + compute_code_size(Code, align_entry(Size+CodeSize)); +compute_code_size([], Size) -> Size. + +combine_label_maps([{MFA, _, CodeSize, _, _, LabelMap}|Code], Address, CLM) -> + NewCLM = merge_label_map(gb_trees:to_list(LabelMap), MFA, Address, CLM), + combine_label_maps(Code, align_entry(Address+CodeSize), NewCLM); +combine_label_maps([], _Address, CLM) -> CLM. + +merge_label_map([{Label,Offset}|Rest], MFA, Address, CLM) -> + NewCLM = gb_trees:insert({MFA,Label}, Address+Offset, CLM), + merge_label_map(Rest, MFA, Address, NewCLM); +merge_label_map([], _MFA, _Address, CLM) -> CLM. + +%% @doc Merge the MFAs' binary code to one continuous binary and compute the +%% size of this binary. At the same time create an exportmap in a form +%% of {Address, M, F, A}. +%% XXX: Is alignment correct/optimal for X86/AMD64? +merge_mfas([{{M,F,A}, CodeBinary, CodeSize, _, _, _}|Code], + Address, AccCode, AccExportMap) -> + ?VERBOSE_ASSERT(CodeSize =:= byte_size(CodeBinary)), + {Address1, Code1} = + case nr_pad_bytes(Address + CodeSize) of + 0 -> %% Retains alignment: + {Address + CodeSize, CodeBinary}; + NrPadBytes -> %% Needs padding! + Padding = list_to_binary(lists:duplicate(NrPadBytes, 0)), + {Address + CodeSize + NrPadBytes, % =:= align_entry(Address+CodeSize) + <<CodeBinary/binary, Padding/binary>>} + end, + ?VERBOSE_ASSERT(Address1 =:= + align_entry(Address + CodeSize)), %XXX: Should address be aligned? + AccCode1 = <<AccCode/binary, Code1/binary>>, + merge_mfas(Code, Address1, AccCode1, [{Address, M, F, A}|AccExportMap]); +merge_mfas([], _Address, AccCode, AccExportMap) -> + {AccCode, AccExportMap}. + +%% @doc Merge the references of relocatable symbols in the binary code. The +%% offsets must be updated because of the merging of the code binaries! +merge_refs([], _ConstMap, _Addr, AccRefs) -> AccRefs; +merge_refs([{MFA, _, CodeSize, _, Refs, _}|Rest], ConstMap, Address, AccRefs) -> + %% Important!: The hipe_pack_constants:pack_constants/2 function assignes + %% unique numbers to constants (ConstNo). This numbers are used from now on, + %% instead of labels that were used before. So, in order to be compatible, we + %% must change all the constant labels in the Refs to the corresponding + %% ConstNo, that can be found in the ConstMap (#pcm_entry{}). + UpdatedRefs = [update_ref(label_to_constno(Ref, MFA, ConstMap), Address) + || Ref <- Refs], + merge_refs(Rest, ConstMap, align_entry(Address+CodeSize), + UpdatedRefs++AccRefs). + +label_to_constno({Type, Offset, {constant, Label}}, MFA, ConstMap) -> + ConstNo = hipe_pack_constants:find_const({MFA, Label}, ConstMap), + {Type, Offset, {constant, ConstNo}}; +label_to_constno(Other, _MFA, _ConstMap) -> + Other. + +%% @doc Update offset to a reference. In case of stack descriptors we must check +%% if there exists an exception handler, because it must also be updated. +update_ref({?SDESC, Offset, SDesc}, CodeAddr) -> + NewRefAddr = Offset+CodeAddr, + case SDesc of + {[], _, _, _} -> % No handler; only update offset + {?SDESC, NewRefAddr, SDesc}; + {ExnHandler, FrameSize, StackArity, Roots} -> % Update exception handler + {?SDESC, NewRefAddr, {ExnHandler+CodeAddr, FrameSize, StackArity, Roots}} + end; +update_ref({Type, Offset, Term}, CodeAddr) -> + {Type, Offset+CodeAddr, Term}. diff --git a/lib/hipe/llvm/hipe_rtl_to_llvm.erl b/lib/hipe/llvm/hipe_rtl_to_llvm.erl new file mode 100644 index 0000000000..ba76e1d815 --- /dev/null +++ b/lib/hipe/llvm/hipe_rtl_to_llvm.erl @@ -0,0 +1,1612 @@ +%% -*- erlang-indent-level: 2 -*- + +-module(hipe_rtl_to_llvm). +-author("Chris Stavrakakis, Yiannis Tsiouris"). + +-export([translate/2]). % the main function of this module +-export([fix_mfa_name/1]). % a help function used in hipe_llvm_main + +-include("../rtl/hipe_rtl.hrl"). +-include("../rtl/hipe_literals.hrl"). +-include("hipe_llvm_arch.hrl"). + +-define(WORD_WIDTH, (?bytes_to_bits(hipe_rtl_arch:word_size()))). +-define(BRANCH_META_TAKEN, "0"). +-define(BRANCH_META_NOT_TAKEN, "1"). + +%%------------------------------------------------------------------------------ +%% @doc Main function for translating an RTL function to LLVM Assembly. Takes as +%% input the RTL code and the variable indexes of possible garbage +%% collection roots and returns the corresponing LLVM, a dictionary with +%% all the relocations in the code and a hipe_consttab() with informaton +%% about data. +%%------------------------------------------------------------------------------ +translate(RTL, Roots) -> + Fun = hipe_rtl:rtl_fun(RTL), + Params = hipe_rtl:rtl_params(RTL), + Data = hipe_rtl:rtl_data(RTL), + Code = hipe_rtl:rtl_code(RTL), + %% Init unique symbol generator and initialize the label counter to the last + %% RTL label. + hipe_gensym:init(llvm), + {_, MaxLabel} = hipe_rtl:rtl_label_range(RTL), + put({llvm,label_count}, MaxLabel + 1), + %% Put first label of RTL code in process dictionary + find_code_entry_label(Code), + %% Initialize relocations symbol dictionary + Relocs = dict:new(), + %% Print RTL to file + %% {ok, File_rtl} = file:open("rtl_" ++integer_to_list(random:uniform(2000)) + %% ++ ".rtl", [write]), + %% hipe_rtl:pp(File_rtl, RTL), + %% file:close(File_rtl), + + %% Pass on RTL code to handle exception handling and identify labels of Fail + %% Blocks + {Code1, FailLabels} = fix_code(Code), + %% Allocate stack slots for each virtual register and declare gc roots + AllocaStackCode = alloca_stack(Code1, Params, Roots), + %% Translate Code + {LLVM_Code1, Relocs1, NewData} = + translate_instr_list(Code1, [], Relocs, Data), + %% Create LLVM code to declare relocation symbols as external symbols along + %% with local variables in order to use them as just any other variable + {FinalRelocs, ExternalDecl, LocalVars} = + handle_relocations(Relocs1, Data, Fun), + %% Pass on LLVM code in order to create Fail blocks and a landingpad + %% instruction to each one + LLVM_Code2 = add_landingpads(LLVM_Code1, FailLabels), + %% Create LLVM Code for the compiled function + LLVM_Code3 = create_function_definition(Fun, Params, LLVM_Code2, + AllocaStackCode ++ LocalVars), + %% Final Code = CompiledFunction + External Declarations + FinalLLVMCode = [LLVM_Code3 | ExternalDecl], + {FinalLLVMCode, FinalRelocs, NewData}. + +find_code_entry_label([]) -> + exit({?MODULE, find_code_entry_label, "Empty code"}); +find_code_entry_label([I|_]) -> + case hipe_rtl:is_label(I) of + true -> + put(first_label, hipe_rtl:label_name(I)); + false -> + exit({?MODULE, find_code_entry_label, "First instruction is not a label"}) + end. + +%% @doc Create a stack slot for each virtual register. The stack slots +%% that correspond to possible garbage collection roots must be +%% marked as such. +alloca_stack(Code, Params, Roots) -> + %% Find all assigned virtual registers + Destinations = collect_destinations(Code), + %% Declare virtual registers, and declare garbage collection roots + do_alloca_stack(Destinations++Params, Params, Roots). + +collect_destinations(Code) -> + lists:usort(lists:flatmap(fun insn_dst/1, Code)). + +do_alloca_stack(Destinations, Params, Roots) -> + do_alloca_stack(Destinations, Params, Roots, []). + +do_alloca_stack([], _, _, Acc) -> + Acc; +do_alloca_stack([D|Ds], Params, Roots, Acc) -> + {Name, _I} = trans_dst(D), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + case hipe_rtl:is_var(D) of + true -> + Num = hipe_rtl:var_index(D), + I1 = hipe_llvm:mk_alloca(Name, WordTy, [], []), + case lists:member(Num, Roots) of + true -> %% Variable is a possible Root + T1 = mk_temp(), + BYTE_TYPE_PP = hipe_llvm:mk_pointer(ByteTyPtr), + I2 = + hipe_llvm:mk_conversion(T1, bitcast, WordTyPtr, Name, BYTE_TYPE_PP), + GcRootArgs = [{BYTE_TYPE_PP, T1}, {ByteTyPtr, "@gc_metadata"}], + I3 = hipe_llvm:mk_call([], false, [], [], hipe_llvm:mk_void(), + "@llvm.gcroot", GcRootArgs, []), + I4 = case lists:member(D, Params) of + false -> + hipe_llvm:mk_store(WordTy, "-5", WordTyPtr, Name, + [], [], false); + true -> [] + end, + do_alloca_stack(Ds, Params, Roots, [I1, I2, I3, I4 | Acc]); + false -> + do_alloca_stack(Ds, Params, Roots, [I1|Acc]) + end; + false -> + case hipe_rtl:is_reg(D) andalso isPrecoloured(D) of + true -> %% Precoloured registers are mapped to "special" stack slots + do_alloca_stack(Ds, Params, Roots, Acc); + false -> + I1 = case hipe_rtl:is_fpreg(D) of + true -> + FloatTy = hipe_llvm:mk_double(), + hipe_llvm:mk_alloca(Name, FloatTy, [], []); + false -> hipe_llvm:mk_alloca(Name, WordTy, [], []) + end, + do_alloca_stack(Ds, Params, Roots, [I1|Acc]) + end + end. + +%%------------------------------------------------------------------------------ +%% @doc Translation of the linearized RTL Code. Each RTL instruction is +%% translated to a list of LLVM Assembly instructions. The relocation +%% dictionary is updated when needed. +%%------------------------------------------------------------------------------ +translate_instr_list([], Acc, Relocs, Data) -> + {lists:reverse(lists:flatten(Acc)), Relocs, Data}; +translate_instr_list([I | Is], Acc, Relocs, Data) -> + {Acc1, NewRelocs, NewData} = translate_instr(I, Relocs, Data), + translate_instr_list(Is, [Acc1 | Acc], NewRelocs, NewData). + +translate_instr(I, Relocs, Data) -> + case I of + #alu{} -> + {I2, Relocs2} = trans_alu(I, Relocs), + {I2, Relocs2, Data}; + #alub{} -> + {I2, Relocs2} = trans_alub(I, Relocs), + {I2, Relocs2, Data}; + #branch{} -> + {I2, Relocs2} = trans_branch(I, Relocs), + {I2, Relocs2, Data}; + #call{} -> + {I2, Relocs2} = + case hipe_rtl:call_fun(I) of + %% In AMD64 this instruction does nothing! + %% TODO: chech use of fwait in other architectures! + fwait -> + {[], Relocs}; + _ -> + trans_call(I, Relocs) + end, + {I2, Relocs2, Data}; + #comment{} -> + {I2, Relocs2} = trans_comment(I, Relocs), + {I2, Relocs2, Data}; + #enter{} -> + {I2, Relocs2} = trans_enter(I, Relocs), + {I2, Relocs2, Data}; + #fconv{} -> + {I2, Relocs2} = trans_fconv(I, Relocs), + {I2, Relocs2, Data}; + #fload{} -> + {I2, Relocs2} = trans_fload(I, Relocs), + {I2, Relocs2, Data}; + #fmove{} -> + {I2, Relocs2} = trans_fmove(I, Relocs), + {I2, Relocs2, Data}; + #fp{} -> + {I2, Relocs2} = trans_fp(I, Relocs), + {I2, Relocs2, Data}; + #fp_unop{} -> + {I2, Relocs2} = trans_fp_unop(I, Relocs), + {I2, Relocs2, Data}; + #fstore{} -> + {I2, Relocs2} = trans_fstore(I, Relocs), + {I2, Relocs2, Data}; + #goto{} -> + {I2, Relocs2} = trans_goto(I, Relocs), + {I2, Relocs2, Data}; + #label{} -> + {I2, Relocs2} = trans_label(I, Relocs), + {I2, Relocs2, Data}; + #load{} -> + {I2, Relocs2} = trans_load(I, Relocs), + {I2, Relocs2, Data}; + #load_address{} -> + {I2, Relocs2} = trans_load_address(I, Relocs), + {I2, Relocs2, Data}; + #load_atom{} -> + {I2, Relocs2} = trans_load_atom(I, Relocs), + {I2, Relocs2, Data}; + #move{} -> + {I2, Relocs2} = trans_move(I, Relocs), + {I2, Relocs2, Data}; + #return{} -> + {I2, Relocs2} = trans_return(I, Relocs), + {I2, Relocs2, Data}; + #store{} -> + {I2, Relocs2} = trans_store(I, Relocs), + {I2, Relocs2, Data}; + #switch{} -> %% Only switch instruction updates Data + {I2, Relocs2, NewData} = trans_switch(I, Relocs, Data), + {I2, Relocs2, NewData}; + Other -> + exit({?MODULE, translate_instr, {"Unknown RTL instruction", Other}}) + end. + +%% +%% alu +%% +trans_alu(I, Relocs) -> + RtlDst = hipe_rtl:alu_dst(I), + TmpDst = mk_temp(), + {Src1, I1} = trans_src(hipe_rtl:alu_src1(I)), + {Src2, I2} = trans_src(hipe_rtl:alu_src2(I)), + Op = trans_op(hipe_rtl:alu_op(I)), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I3 = hipe_llvm:mk_operation(TmpDst, Op, WordTy, Src1, Src2, []), + I4 = store_stack_dst(TmpDst, RtlDst), + {[I4, I3, I2, I1], Relocs}. + +%% +%% alub +%% +trans_alub(I, Relocs) -> + case hipe_rtl:alub_cond(I) of + Op when Op =:= overflow orelse Op =:= not_overflow -> + trans_alub_overflow(I, signed, Relocs); + ltu -> %% ltu means unsigned overflow + trans_alub_overflow(I, unsigned, Relocs); + _ -> + trans_alub_no_overflow(I, Relocs) + end. + +trans_alub_overflow(I, Sign, Relocs) -> + {Src1, I1} = trans_src(hipe_rtl:alub_src1(I)), + {Src2, I2} = trans_src(hipe_rtl:alub_src2(I)), + RtlDst = hipe_rtl:alub_dst(I), + TmpDst = mk_temp(), + Name = trans_alub_op(I, Sign), + NewRelocs = relocs_store(Name, {call, {llvm, Name, 2}}, Relocs), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + ReturnType = hipe_llvm:mk_struct([WordTy, hipe_llvm:mk_int(1)]), + T1 = mk_temp(), + I3 = hipe_llvm:mk_call(T1, false, [], [], ReturnType, "@" ++ Name, + [{WordTy, Src1}, {WordTy, Src2}], []), + %% T1{0}: result of the operation + I4 = hipe_llvm:mk_extractvalue(TmpDst, ReturnType, T1 , "0", []), + I5 = store_stack_dst(TmpDst, RtlDst), + T2 = mk_temp(), + %% T1{1}: Boolean variable indicating overflow + I6 = hipe_llvm:mk_extractvalue(T2, ReturnType, T1, "1", []), + case hipe_rtl:alub_cond(I) of + Op when Op =:= overflow orelse Op =:= ltu -> + True_label = mk_jump_label(hipe_rtl:alub_true_label(I)), + False_label = mk_jump_label(hipe_rtl:alub_false_label(I)), + MetaData = branch_metadata(hipe_rtl:alub_pred(I)); + not_overflow -> + True_label = mk_jump_label(hipe_rtl:alub_false_label(I)), + False_label = mk_jump_label(hipe_rtl:alub_true_label(I)), + MetaData = branch_metadata(1 - hipe_rtl:alub_pred(I)) + end, + I7 = hipe_llvm:mk_br_cond(T2, True_label, False_label, MetaData), + {[I7, I6, I5, I4, I3, I2, I1], NewRelocs}. + +trans_alub_op(I, Sign) -> + Name = + case Sign of + signed -> + case hipe_rtl:alub_op(I) of + add -> "llvm.sadd.with.overflow."; + mul -> "llvm.smul.with.overflow."; + sub -> "llvm.ssub.with.overflow."; + Op -> exit({?MODULE, trans_alub_op, {"Unknown alub operator", Op}}) + end; + unsigned -> + case hipe_rtl:alub_op(I) of + add -> "llvm.uadd.with.overflow."; + mul -> "llvm.umul.with.overflow."; + sub -> "llvm.usub.with.overflow."; + Op -> exit({?MODULE, trans_alub_op, {"Unknown alub operator", Op}}) + end + end, + Type = + case hipe_rtl_arch:word_size() of + 4 -> "i32"; + 8 -> "i64" + %% Other -> exit({?MODULE, trans_alub_op, {"Unknown type", Other}}) + end, + Name ++ Type. + +trans_alub_no_overflow(I, Relocs) -> + %% alu + T = hipe_rtl:mk_alu(hipe_rtl:alub_dst(I), hipe_rtl:alub_src1(I), + hipe_rtl:alub_op(I), hipe_rtl:alub_src2(I)), + %% A trans_alu instruction cannot change relocations + {I1, _} = trans_alu(T, Relocs), + %% icmp + %% Translate destination as src, to match with the semantics of instruction + {Dst, I2} = trans_src(hipe_rtl:alub_dst(I)), + Cond = trans_rel_op(hipe_rtl:alub_cond(I)), + T3 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I5 = hipe_llvm:mk_icmp(T3, Cond, WordTy, Dst, "0"), + %% br + Metadata = branch_metadata(hipe_rtl:alub_pred(I)), + True_label = mk_jump_label(hipe_rtl:alub_true_label(I)), + False_label = mk_jump_label(hipe_rtl:alub_false_label(I)), + I6 = hipe_llvm:mk_br_cond(T3, True_label, False_label, Metadata), + {[I6, I5, I2, I1], Relocs}. + +%% +%% branch +%% +trans_branch(I, Relocs) -> + {Src1, I1} = trans_src(hipe_rtl:branch_src1(I)), + {Src2, I2} = trans_src(hipe_rtl:branch_src2(I)), + Cond = trans_rel_op(hipe_rtl:branch_cond(I)), + %% icmp + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I3 = hipe_llvm:mk_icmp(T1, Cond, WordTy, Src1, Src2), + %% br + True_label = mk_jump_label(hipe_rtl:branch_true_label(I)), + False_label = mk_jump_label(hipe_rtl:branch_false_label(I)), + Metadata = branch_metadata(hipe_rtl:branch_pred(I)), + I4 = hipe_llvm:mk_br_cond(T1, True_label, False_label, Metadata), + {[I4, I3, I2, I1], Relocs}. + +branch_metadata(X) when X =:= 0.5 -> []; +branch_metadata(X) when X > 0.5 -> ?BRANCH_META_TAKEN; +branch_metadata(X) when X < 0.5 -> ?BRANCH_META_NOT_TAKEN. + +%% +%% call +%% +trans_call(I, Relocs) -> + RtlCallArgList= hipe_rtl:call_arglist(I), + RtlCallName = hipe_rtl:call_fun(I), + {I0, Relocs1} = expose_closure(RtlCallName, RtlCallArgList, Relocs), + TmpDst = mk_temp(), + {CallArgs, I1} = trans_call_args(RtlCallArgList), + FixedRegs = fixed_registers(), + {LoadedFixedRegs, I2} = load_fixed_regs(FixedRegs), + FinalArgs = fix_reg_args(LoadedFixedRegs) ++ CallArgs, + {Name, I3, Relocs2} = + trans_call_name(RtlCallName, Relocs1, CallArgs, FinalArgs), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + I4 = + case hipe_rtl:call_fail(I) of + %% Normal Call + [] -> + hipe_llvm:mk_call(T1, false, "cc 11", [], FunRetTy, Name, FinalArgs, + []); + %% Call With Exception + FailLabelNum -> + TrueLabel = "L" ++ integer_to_list(hipe_rtl:call_normal(I)), + FailLabel = "%FL" ++ integer_to_list(FailLabelNum), + II1 = + hipe_llvm:mk_invoke(T1, "cc 11", [], FunRetTy, Name, FinalArgs, [], + "%" ++ TrueLabel, FailLabel), + II2 = hipe_llvm:mk_label(TrueLabel), + [II2, II1] + end, + I5 = store_fixed_regs(FixedRegs, T1), + I6 = + case hipe_rtl:call_dstlist(I) of + [] -> []; %% No return value + [Destination] -> + II3 = + hipe_llvm:mk_extractvalue(TmpDst, FunRetTy, T1, + integer_to_list(?NR_PINNED_REGS), []), + II4 = store_stack_dst(TmpDst, Destination), + [II4, II3] + end, + I7 = + case hipe_rtl:call_continuation(I) of + [] -> []; %% No continuation + CC -> + {II5, _} = trans_goto(hipe_rtl:mk_goto(CC), Relocs2), + II5 + end, + {[I7, I6, I5, I4, I3, I2, I1, I0], Relocs2}. + +%% In case of call to a register (closure call) with more than ?NR_ARG_REGS +%% arguments we must track the offset this call in the code, in order to +%% to correct the stack descriptor. So, we insert a new Label and add this label +%% to the "table_closures" +%% --------------------------------|-------------------------------------------- +%% Old Code | New Code +%% --------------------------------|-------------------------------------------- +%% | br %ClosureLabel +%% call %reg(Args) | ClosureLabel: +%% | call %reg(Args) +expose_closure(CallName, CallArgs, Relocs) -> + CallArgsNr = length(CallArgs), + case hipe_rtl:is_reg(CallName) andalso CallArgsNr > ?NR_ARG_REGS of + true -> + LabelNum = hipe_gensym:new_label(llvm), + ClosureLabel = hipe_llvm:mk_label(mk_label(LabelNum)), + JumpIns = hipe_llvm:mk_br(mk_jump_label(LabelNum)), + Relocs1 = + relocs_store({CallName, LabelNum}, + {closure_label, LabelNum, CallArgsNr - ?NR_ARG_REGS}, + Relocs), + {[ClosureLabel, JumpIns], Relocs1}; + false -> + {[], Relocs} + end. + +trans_call_name(RtlCallName, Relocs, CallArgs, FinalArgs) -> + case RtlCallName of + PrimOp when is_atom(PrimOp) -> + LlvmName = trans_prim_op(PrimOp), + Relocs1 = relocs_store(LlvmName, {call, {bif, PrimOp, length(CallArgs)}}, + Relocs), + {"@" ++ LlvmName, [], Relocs1}; + {M, F, A} when is_atom(M), is_atom(F), is_integer(A) -> + LlvmName = trans_mfa_name({M,F,A}), + Relocs1 = relocs_store(LlvmName, {call, {M,F,A}}, Relocs), + {"@" ++ LlvmName, [], Relocs1}; + Reg -> + case hipe_rtl:is_reg(Reg) of + true -> + %% In case of a closure call, the register holding the address + %% of the closure must be converted to function type in + %% order to make the call + TT1 = mk_temp(), + {RegName, II1} = trans_src(Reg), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + II2 = + hipe_llvm:mk_conversion(TT1, inttoptr, WordTy, RegName, WordTyPtr), + TT2 = mk_temp(), + ArgsTypeList = lists:duplicate(length(FinalArgs), WordTy), + FunRetTy = + hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + FunType = hipe_llvm:mk_fun(FunRetTy, ArgsTypeList), + FunTypeP = hipe_llvm:mk_pointer(FunType), + II3 = hipe_llvm:mk_conversion(TT2, bitcast, WordTyPtr, TT1, FunTypeP), + {TT2, [II3, II2, II1], Relocs}; + false -> + exit({?MODULE, trans_call, {"Unimplemented call to", RtlCallName}}) + end + end. + +%% +trans_call_args(ArgList) -> + {Args, I} = lists:unzip(trans_args(ArgList)), + %% Reverse arguments that are passed to stack to match with the Erlang + %% calling convention. (Propably not needed in prim calls.) + ReversedArgs = + case erlang:length(Args) > ?NR_ARG_REGS of + false -> + Args; + true -> + {ArgsInRegs, ArgsInStack} = lists:split(?NR_ARG_REGS, Args), + ArgsInRegs ++ lists:reverse(ArgsInStack) + end, + %% Reverse I, because some of the arguments may go out of scope and + %% should be killed(store -5). When two or more arguments are they + %% same, then order matters! + {ReversedArgs, lists:reverse(I)}. + +%% +%% trans_comment +%% +trans_comment(I, Relocs) -> + I1 = hipe_llvm:mk_comment(hipe_rtl:comment_text(I)), + {I1, Relocs}. + +%% +%% enter +%% +trans_enter(I, Relocs) -> + {CallArgs, I0} = trans_call_args(hipe_rtl:enter_arglist(I)), + FixedRegs = fixed_registers(), + {LoadedFixedRegs, I1} = load_fixed_regs(FixedRegs), + FinalArgs = fix_reg_args(LoadedFixedRegs) ++ CallArgs, + {Name, I2, NewRelocs} = + trans_call_name(hipe_rtl:enter_fun(I), Relocs, CallArgs, FinalArgs), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + I3 = hipe_llvm:mk_call(T1, true, "cc 11", [], FunRetTy, Name, FinalArgs, []), + I4 = hipe_llvm:mk_ret([{FunRetTy, T1}]), + {[I4, I3, I2, I1, I0], NewRelocs}. + +%% +%% fconv +%% +trans_fconv(I, Relocs) -> + %% XXX: Can a fconv destination be a precoloured reg? + RtlDst = hipe_rtl:fconv_dst(I), + TmpDst = mk_temp(), + {Src, I1} = trans_float_src(hipe_rtl:fconv_src(I)), + FloatTy = hipe_llvm:mk_double(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I2 = hipe_llvm:mk_conversion(TmpDst, sitofp, WordTy, Src, FloatTy), + I3 = store_float_stack(TmpDst, RtlDst), + {[I3, I2, I1], Relocs}. + + +%% TODO: fload, fstore, fmove, and fp are almost the same with load, store, move +%% and alu. Maybe we should join them. + +%% +%% fload +%% +trans_fload(I, Relocs) -> + RtlDst = hipe_rtl:fload_dst(I), + RtlSrc = hipe_rtl:fload_src(I), + _Offset = hipe_rtl:fload_offset(I), + TmpDst = mk_temp(), + {Src, I1} = trans_float_src(RtlSrc), + {Offset, I2} = trans_float_src(_Offset), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FloatTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_double()), + I3 = hipe_llvm:mk_operation(T1, add, WordTy, Src, Offset, []), + T2 = mk_temp(), + I4 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, FloatTyPtr), + I5 = hipe_llvm:mk_load(TmpDst, FloatTyPtr, T2, [], [], false), + I6 = store_float_stack(TmpDst, RtlDst), + {[I6, I5, I4, I3, I2, I1], Relocs}. + +%% +%% fmove +%% +trans_fmove(I, Relocs) -> + RtlDst = hipe_rtl:fmove_dst(I), + RtlSrc = hipe_rtl:fmove_src(I), + {Src, I1} = trans_float_src(RtlSrc), + I2 = store_float_stack(Src, RtlDst), + {[I2, I1], Relocs}. + +%% +%% fp +%% +trans_fp(I, Relocs) -> + %% XXX: Just copied trans_alu...think again.. + RtlDst = hipe_rtl:fp_dst(I), + RtlSrc1 = hipe_rtl:fp_src1(I), + RtlSrc2 = hipe_rtl:fp_src2(I), + %% Destination cannot be a precoloured register + FloatTy = hipe_llvm:mk_double(), + FloatTyPtr = hipe_llvm:mk_pointer(FloatTy), + TmpDst = mk_temp(), + {Src1, I1} = trans_float_src(RtlSrc1), + {Src2, I2} = trans_float_src(RtlSrc2), + Op = trans_fp_op(hipe_rtl:fp_op(I)), + I3 = hipe_llvm:mk_operation(TmpDst, Op, FloatTy, Src1, Src2, []), + I4 = store_float_stack(TmpDst, RtlDst), + %% Synchronization for floating point exceptions + I5 = hipe_llvm:mk_store(FloatTy, TmpDst, FloatTyPtr, "%exception_sync", [], + [], true), + T1 = mk_temp(), + I6 = hipe_llvm:mk_load(T1, FloatTyPtr, "%exception_sync", [], [], true), + {[I6, I5, I4, I3, I2, I1], Relocs}. + +%% +%% fp_unop +%% +trans_fp_unop(I, Relocs) -> + RtlDst = hipe_rtl:fp_unop_dst(I), + RtlSrc = hipe_rtl:fp_unop_src(I), + %% Destination cannot be a precoloured register + TmpDst = mk_temp(), + {Src, I1} = trans_float_src(RtlSrc), + Op = trans_fp_op(hipe_rtl:fp_unop_op(I)), + FloatTy = hipe_llvm:mk_double(), + I2 = hipe_llvm:mk_operation(TmpDst, Op, FloatTy, "0.0", Src, []), + I3 = store_float_stack(TmpDst, RtlDst), + {[I3, I2, I1], Relocs}. +%% TODO: Fix fp_unop in a way like the following. You must change trans_dest, +%% in order to call float_to_list in a case of float constant. Maybe the type +%% check is expensive... +%% Dst = hipe_rtl:fp_unop_dst(I), +%% Src = hipe_rtl:fp_unop_src(I), +%% Op = hipe_rtl:fp_unop_op(I), +%% Zero = hipe_rtl:mk_imm(0.0), +%% I1 = hipe_rtl:mk_fp(Dst, Zero, Op, Src), +%% trans_fp(I, Relocs1). + +%% +%% fstore +%% +trans_fstore(I, Relocs) -> + Base = hipe_rtl:fstore_base(I), + case isPrecoloured(Base) of + true -> + trans_fstore_reg(I, Relocs); + false -> + exit({?MODULE, trans_fstore ,{"Not implemented yet", false}}) + end. + +trans_fstore_reg(I, Relocs) -> + {Base, I0} = trans_reg(hipe_rtl:fstore_base(I), dst), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + FloatTy = hipe_llvm:mk_double(), + FloatTyPtr = hipe_llvm:mk_pointer(FloatTy), + T1 = mk_temp(), + I1 = hipe_llvm:mk_load(T1, WordTyPtr, Base, [], [], false), + {Offset, I2} = trans_src(hipe_rtl:fstore_offset(I)), + T2 = mk_temp(), + I3 = hipe_llvm:mk_operation(T2, add, WordTy, T1, Offset, []), + T3 = mk_temp(), + I4 = hipe_llvm:mk_conversion(T3, inttoptr, WordTy, T2, FloatTyPtr), + {Value, I5} = trans_src(hipe_rtl:fstore_src(I)), + I6 = hipe_llvm:mk_store(FloatTy, Value, FloatTyPtr, T3, [], [], false), + {[I6, I5, I4, I3, I2, I1, I0], Relocs}. + +%% +%% goto +%% +trans_goto(I, Relocs) -> + I1 = hipe_llvm:mk_br(mk_jump_label(hipe_rtl:goto_label(I))), + {I1, Relocs}. + +%% +%% label +%% +trans_label(I, Relocs) -> + Label = mk_label(hipe_rtl:label_name(I)), + I1 = hipe_llvm:mk_label(Label), + {I1, Relocs}. + +%% +%% load +%% +trans_load(I, Relocs) -> + RtlDst = hipe_rtl:load_dst(I), + TmpDst = mk_temp(), + %% XXX: Why translate them independently? ------------------------ + {Src, I1} = trans_src(hipe_rtl:load_src(I)), + {Offset, I2} = trans_src(hipe_rtl:load_offset(I)), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + I3 = hipe_llvm:mk_operation(T1, add, WordTy, Src, Offset, []), + %%---------------------------------------------------------------- + I4 = case hipe_rtl:load_size(I) of + word -> + T2 = mk_temp(), + II1 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, WordTyPtr), + II2 = hipe_llvm:mk_load(TmpDst, WordTyPtr, T2, [], [], false), + [II2, II1]; + Size -> + LoadType = llvm_type_from_size(Size), + LoadTypeP = hipe_llvm:mk_pointer(LoadType), + T2 = mk_temp(), + II1 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, LoadTypeP), + T3 = mk_temp(), + LoadTypePointer = hipe_llvm:mk_pointer(LoadType), + II2 = hipe_llvm:mk_load(T3, LoadTypePointer, T2, [], [], false), + Conversion = + case hipe_rtl:load_sign(I) of + signed -> sext; + unsigned -> zext + end, + II3 = + hipe_llvm:mk_conversion(TmpDst, Conversion, LoadType, T3, WordTy), + [II3, II2, II1] + end, + I5 = store_stack_dst(TmpDst, RtlDst), + {[I5, I4, I3, I2, I1], Relocs}. + +%% +%% load_address +%% +trans_load_address(I, Relocs) -> + RtlDst = hipe_rtl:load_address_dst(I), + RtlAddr = hipe_rtl:load_address_addr(I), + {Addr, NewRelocs} = + case hipe_rtl:load_address_type(I) of + constant -> + {"%DL" ++ integer_to_list(RtlAddr) ++ "_var", Relocs}; + closure -> + {{_, ClosureName, _}, _, _} = RtlAddr, + FixedClosureName = fix_closure_name(ClosureName), + Relocs1 = relocs_store(FixedClosureName, {closure, RtlAddr}, Relocs), + {"%" ++ FixedClosureName ++ "_var", Relocs1}; + type -> + exit({?MODULE, trans_load_address, + {"Type not implemented in load_address", RtlAddr}}) + end, + I1 = store_stack_dst(Addr, RtlDst), + {[I1], NewRelocs}. + +%% +%% load_atom +%% +trans_load_atom(I, Relocs) -> + RtlDst = hipe_rtl:load_atom_dst(I), + RtlAtom = hipe_rtl:load_atom_atom(I), + AtomName = "atom_" ++ make_llvm_id(atom_to_list(RtlAtom)), + AtomVar = "%" ++ AtomName ++ "_var", + NewRelocs = relocs_store(AtomName, {atom, RtlAtom}, Relocs), + I1 = store_stack_dst(AtomVar, RtlDst), + {[I1], NewRelocs}. + +%% +%% move +%% +trans_move(I, Relocs) -> + RtlDst = hipe_rtl:move_dst(I), + RtlSrc = hipe_rtl:move_src(I), + {Src, I1} = trans_src(RtlSrc), + I2 = store_stack_dst(Src, RtlDst), + {[I2, I1], Relocs}. + +%% +%% return +%% +trans_return(I, Relocs) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + {VarRet, I1} = + case hipe_rtl:return_varlist(I) of + [] -> + {[], []}; + [A] -> + {Name, II1} = trans_src(A), + {[{WordTy, Name}], II1} + end, + FixedRegs = fixed_registers(), + {LoadedFixedRegs, I2} = load_fixed_regs(FixedRegs), + FixedRet = [{WordTy, X} || X <- LoadedFixedRegs], + Ret = FixedRet ++ VarRet, + {RetTypes, _RetNames} = lists:unzip(Ret), + Type = hipe_llvm:mk_struct(RetTypes), + {RetStruct, I3} = mk_return_struct(Ret, Type), + I4 = hipe_llvm:mk_ret([{Type, RetStruct}]), + {[I4, I3, I2, I1], Relocs}. + +%% @doc Create a structure to hold the return value and the precoloured +%% registers. +mk_return_struct(RetValues, Type) -> + mk_return_struct(RetValues, Type, [], "undef", 0). + +mk_return_struct([], _, Acc, StructName, _) -> + {StructName, Acc}; +mk_return_struct([{ElemType, ElemName}|Rest], Type, Acc, StructName, Index) -> + T1 = mk_temp(), + I1 = hipe_llvm:mk_insertvalue(T1, Type, StructName, ElemType, ElemName, + integer_to_list(Index), []), + mk_return_struct(Rest, Type, [I1 | Acc], T1, Index+1). + +%% +%% store +%% +trans_store(I, Relocs) -> + {Base, I1} = trans_src(hipe_rtl:store_base(I)), + {Offset, I2} = trans_src(hipe_rtl:store_offset(I)), + {Value, I3} = trans_src(hipe_rtl:store_src(I)), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + I4 = hipe_llvm:mk_operation(T1, add, WordTy, Base, Offset, []), + I5 = + case hipe_rtl:store_size(I) of + word -> + T2 = mk_temp(), + II1 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, WordTyPtr), + II2 = hipe_llvm:mk_store(WordTy, Value, WordTyPtr, T2, [], [], + false), + [II2, II1]; + Size -> + %% XXX: Is always trunc correct ? + LoadType = llvm_type_from_size(Size), + LoadTypePointer = hipe_llvm:mk_pointer(LoadType), + T2 = mk_temp(), + II1 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, LoadTypePointer), + T3 = mk_temp(), + II2 = hipe_llvm:mk_conversion(T3, 'trunc', WordTy, Value, LoadType), + II3 = hipe_llvm:mk_store(LoadType, T3, LoadTypePointer, T2, [], [], false), + [II3, II2, II1] + end, + {[I5, I4, I3, I2, I1], Relocs}. + +%% +%% switch +%% +trans_switch(I, Relocs, Data) -> + RtlSrc = hipe_rtl:switch_src(I), + {Src, I1} = trans_src(RtlSrc), + Labels = hipe_rtl:switch_labels(I), + JumpLabels = [mk_jump_label(L) || L <- Labels], + SortOrder = hipe_rtl:switch_sort_order(I), + NrLabels = length(Labels), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + TableType = hipe_llvm:mk_array(NrLabels, ByteTyPtr), + TableTypeP = hipe_llvm:mk_pointer(TableType), + TypedJumpLabels = [{hipe_llvm:mk_label_type(), X} || X <- JumpLabels], + T1 = mk_temp(), + {Src2, []} = trans_dst(RtlSrc), + TableName = "table_" ++ tl(Src2), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I2 = hipe_llvm:mk_getelementptr(T1, TableTypeP, "@"++TableName, + [{WordTy, "0"}, {WordTy, Src}], false), + T2 = mk_temp(), + BYTE_TYPE_PP = hipe_llvm:mk_pointer(ByteTyPtr), + I3 = hipe_llvm:mk_load(T2, BYTE_TYPE_PP, T1, [], [], false), + I4 = hipe_llvm:mk_indirectbr(ByteTyPtr, T2, TypedJumpLabels), + LMap = [{label, L} || L <- Labels], + %% Update data with the info for the jump table + {NewData, JTabLab} = + case hipe_rtl:switch_sort_order(I) of + [] -> + hipe_consttab:insert_block(Data, word, LMap); + SortOrder -> + hipe_consttab:insert_sorted_block(Data, word, LMap, SortOrder) + end, + Relocs2 = relocs_store(TableName, {switch, {TableType, Labels, NrLabels, + SortOrder}, JTabLab}, Relocs), + {[I4, I3, I2, I1], Relocs2, NewData}. + +%% @doc Pass on RTL code in order to fix invoke and closure calls. +fix_code(Code) -> + fix_calls(Code). + +%% @doc Fix invoke calls and closure calls with more than ?NR_ARG_REGS +%% arguments. +fix_calls(Code) -> + fix_calls(Code, [], []). + +fix_calls([], Acc, FailLabels) -> + {lists:reverse(Acc), FailLabels}; +fix_calls([I | Is], Acc, FailLabels) -> + case hipe_rtl:is_call(I) of + true -> + {NewCall, NewFailLabels} = + case hipe_rtl:call_fail(I) of + [] -> + {I, FailLabels}; + FailLabel -> + fix_invoke_call(I, FailLabel, FailLabels) + end, + fix_calls(Is, [NewCall|Acc], NewFailLabels); + false -> + fix_calls(Is, [I|Acc], FailLabels) + end. + +%% @doc When a call has a fail continuation label it must be extended with a +%% normal continuation label to go with the LLVM's invoke instruction. +%% FailLabels is the list of labels of all fail blocks, which are needed to +%% be declared as landing pads. Furtermore, we must add to fail labels a +%% call to hipe_bifs:llvm_fix_pinned_regs/0 in order to avoid reloading old +%% values of pinned registers. This may happen because the result of an +%% invoke instruction is not available at fail-labels, and, thus, we cannot +%% get the correct values of pinned registers. Finally, the stack needs to +%% be re-adjusted when there are stack arguments. +fix_invoke_call(I, FailLabel, FailLabels) -> + NewLabel = hipe_gensym:new_label(llvm), + NewCall1 = hipe_rtl:call_normal_update(I, NewLabel), + SpAdj = find_sp_adj(hipe_rtl:call_arglist(I)), + case lists:keyfind(FailLabel, 1, FailLabels) of + %% Same fail label with same Stack Pointer adjustment + {FailLabel, NewFailLabel, SpAdj} -> + NewCall2 = hipe_rtl:call_fail_update(NewCall1, NewFailLabel), + {NewCall2, FailLabels}; + %% Same fail label but with different Stack Pointer adjustment + {_, _, _} -> + NewFailLabel = hipe_gensym:new_label(llvm), + NewCall2 = hipe_rtl:call_fail_update(NewCall1, NewFailLabel), + {NewCall2, [{FailLabel, NewFailLabel, SpAdj} | FailLabels]}; + %% New Fail label + false -> + NewFailLabel = hipe_gensym:new_label(llvm), + NewCall2 = hipe_rtl:call_fail_update(NewCall1, NewFailLabel), + {NewCall2, [{FailLabel, NewFailLabel, SpAdj} | FailLabels]} + end. + +find_sp_adj(ArgList) -> + NrArgs = length(ArgList), + case NrArgs > ?NR_ARG_REGS of + true -> + (NrArgs - ?NR_ARG_REGS) * hipe_rtl_arch:word_size(); + false -> + 0 + end. + +%% @doc Add landingpad instruction in Fail Blocks. +add_landingpads(LLVM_Code, FailLabels) -> + FailLabels2 = [convert_label(T) || T <- FailLabels], + add_landingpads(LLVM_Code, FailLabels2, []). + +add_landingpads([], _, Acc) -> + lists:reverse(Acc); +add_landingpads([I | Is], FailLabels, Acc) -> + case hipe_llvm:is_label(I) of + true -> + Label = hipe_llvm:label_label(I), + Ins = create_fail_blocks(Label, FailLabels), + add_landingpads(Is, FailLabels, [I | Ins] ++ Acc); + false -> + add_landingpads(Is, FailLabels, [I | Acc]) + end. + +convert_label({X,Y,Z}) -> + {"L" ++ integer_to_list(X), "FL" ++ integer_to_list(Y), Z}. + +%% @doc Create a fail block wich. +create_fail_blocks(_, []) -> []; +create_fail_blocks(Label, FailLabels) -> + create_fail_blocks(Label, FailLabels, []). + +create_fail_blocks(Label, FailLabels, Acc) -> + case lists:keytake(Label, 1, FailLabels) of + false -> + Acc; + {value, {Label, FailLabel, SpAdj}, RestFailLabels} -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I1 = hipe_llvm:mk_label(FailLabel), + LP = hipe_llvm:mk_landingpad(), + I2 = + case SpAdj > 0 of + true -> + StackPointer = ?ARCH_REGISTERS:reg_name(?ARCH_REGISTERS:sp()), + hipe_llvm:mk_adj_stack(integer_to_list(SpAdj), StackPointer, + WordTy); + false -> [] + end, + T1 = mk_temp(), + FixedRegs = fixed_registers(), + FunRetTy = + hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + I3 = hipe_llvm:mk_call(T1, false, "cc 11", [], FunRetTy, + "@hipe_bifs.llvm_fix_pinned_regs.0", [], []), + I4 = store_fixed_regs(FixedRegs, T1), + I5 = hipe_llvm:mk_br("%" ++ Label), + Ins = lists:flatten([I5, I4, I3, I2, LP,I1]), + create_fail_blocks(Label, RestFailLabels, Ins ++ Acc) + end. + +%%------------------------------------------------------------------------------ +%% Miscellaneous Functions +%%------------------------------------------------------------------------------ + +%% @doc Convert RTL argument list to LLVM argument list. +trans_args(ArgList) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + MakeArg = + fun(A) -> + {Name, I1} = trans_src(A), + {{WordTy, Name}, I1} + end, + [MakeArg(A) || A <- ArgList]. + +%% @doc Convert a list of Precoloured registers to LLVM argument list. +fix_reg_args(ArgList) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + [{WordTy, A} || A <- ArgList]. + +%% @doc Load Precoloured registers. +load_fixed_regs(RegList) -> + Names = [mk_temp_reg(R) || R <- RegList], + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + Fun1 = + fun (X, Y) -> + hipe_llvm:mk_load(X, WordTyPtr, "%" ++ Y ++ "_reg_var", [], [], false) + end, + Ins = lists:zipwith(Fun1, Names, RegList), + {Names, Ins}. + +%% @doc Store Precoloured registers. +store_fixed_regs(RegList, Name) -> + Names = [mk_temp_reg(R) || R <- RegList], + Indexes = lists:seq(0, erlang:length(RegList) - 1), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + Fun1 = + fun(X,Y) -> + hipe_llvm:mk_extractvalue(X, FunRetTy, Name, integer_to_list(Y), []) + end, + I1 = lists:zipwith(Fun1, Names, Indexes), + Fun2 = + fun (X, Y) -> + hipe_llvm:mk_store(WordTy, X, WordTyPtr, "%" ++ Y ++ "_reg_var", [], [], + false) + end, + I2 = lists:zipwith(Fun2, Names, RegList), + [I2, I1]. + +%%------------------------------------------------------------------------------ +%% Translation of Names +%%------------------------------------------------------------------------------ + +%% @doc Fix F in MFA tuple to acceptable LLVM identifier (case of closure). +-spec fix_mfa_name(mfa()) -> mfa(). +fix_mfa_name({Mod_Name, Closure_Name, Arity}) -> + Fun_Name = list_to_atom(fix_closure_name(Closure_Name)), + {Mod_Name, Fun_Name, Arity}. + +%% @doc Make an acceptable LLVM identifier for a closure name. +fix_closure_name(ClosureName) -> + make_llvm_id(atom_to_list(ClosureName)). + +%% @doc Create an acceptable LLVM identifier. +make_llvm_id(Name) -> + case Name of + "" -> "Empty"; + Other -> lists:flatten([llvm_id(C) || C <- Other]) + end. + +llvm_id(C) when C=:=46; C>47 andalso C<58; C>64 andalso C<91; C=:=95; + C>96 andalso C<123 -> + C; +llvm_id(C) -> + io_lib:format("_~2.16.0B_",[C]). + +%% @doc Create an acceptable LLVM identifier for an MFA. +trans_mfa_name({M,F,A}) -> + N = atom_to_list(M) ++ "." ++ atom_to_list(F) ++ "." ++ integer_to_list(A), + make_llvm_id(N). + +%%------------------------------------------------------------------------------ +%% Creation of Labels and Temporaries +%%------------------------------------------------------------------------------ +mk_label(N) -> + "L" ++ integer_to_list(N). + +mk_jump_label(N) -> + "%L" ++ integer_to_list(N). + +mk_temp() -> + "%t" ++ integer_to_list(hipe_gensym:new_var(llvm)). + +mk_temp_reg(Name) -> + "%" ++ Name ++ integer_to_list(hipe_gensym:new_var(llvm)). + +%%---------------------------------------------------------------------------- +%% Translation of Operands +%%---------------------------------------------------------------------------- + +store_stack_dst(TempDst, Dst) -> + {Dst2, II1} = trans_dst(Dst), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + II2 = hipe_llvm:mk_store(WordTy, TempDst, WordTyPtr, Dst2, [], [], false), + [II2, II1]. + +store_float_stack(TempDst, Dst) -> + {Dst2, II1} = trans_dst(Dst), + FloatTy = hipe_llvm:mk_double(), + FloatTyPtr = hipe_llvm:mk_pointer(FloatTy), + II2 = hipe_llvm:mk_store(FloatTy, TempDst, FloatTyPtr, Dst2, [], [], false), + [II2, II1]. + +trans_float_src(Src) -> + case hipe_rtl:is_const_label(Src) of + true -> + Name = "@DL" ++ integer_to_list(hipe_rtl:const_label_label(Src)), + T1 = mk_temp(), + %% XXX: Hardcoded offset + ByteTy = hipe_llvm:mk_int(8), + ByteTyPtr = hipe_llvm:mk_pointer(ByteTy), + I1 = hipe_llvm:mk_getelementptr(T1, ByteTyPtr, Name, + [{ByteTy, integer_to_list(?FLOAT_OFFSET)}], true), + T2 = mk_temp(), + FloatTy = hipe_llvm:mk_double(), + FloatTyPtr = hipe_llvm:mk_pointer(FloatTy), + I2 = hipe_llvm:mk_conversion(T2, bitcast, ByteTyPtr, T1, FloatTyPtr), + T3 = mk_temp(), + I3 = hipe_llvm:mk_load(T3, FloatTyPtr, T2, [], [], false), + {T3, [I3, I2, I1]}; + false -> + trans_src(Src) + end. + +trans_src(A) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + case hipe_rtl:is_imm(A) of + true -> + Value = integer_to_list(hipe_rtl:imm_value(A)), + {Value, []}; + false -> + case hipe_rtl:is_reg(A) of + true -> + case isPrecoloured(A) of + true -> trans_reg(A, src); + false -> + {Name, []} = trans_reg(A, src), + T1 = mk_temp(), + I1 = hipe_llvm:mk_load(T1, WordTyPtr, Name, [], [], false), + {T1, [I1]} + end; + false -> + case hipe_rtl:is_var(A) of + true -> + RootName = "%vr" ++ integer_to_list(hipe_rtl:var_index(A)), + T1 = mk_temp(), + I1 = hipe_llvm:mk_load(T1, WordTyPtr, RootName, [], [], false), + I2 = + case hipe_rtl:var_liveness(A) of + live -> + []; + dead -> + NilValue = hipe_tagscheme:mk_nil(), + hipe_llvm:mk_store(WordTy, integer_to_list(NilValue), WordTyPtr, RootName, + [], [], false) + end, + {T1, [I2, I1]}; + false -> + case hipe_rtl:is_fpreg(A) of + true -> + {Name, []} = trans_dst(A), + T1 = mk_temp(), + FloatTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_double()), + I1 = hipe_llvm:mk_load(T1, FloatTyPtr, Name, [], [], false), + {T1, [I1]}; + false -> trans_dst(A) + end + end + end + end. + +trans_dst(A) -> + case hipe_rtl:is_reg(A) of + true -> + trans_reg(A, dst); + false -> + Name = case hipe_rtl:is_var(A) of + true -> + "%vr" ++ integer_to_list(hipe_rtl:var_index(A)); + false -> + case hipe_rtl:is_fpreg(A) of + true -> "%fr" ++ integer_to_list(hipe_rtl:fpreg_index(A)); + false -> + case hipe_rtl:is_const_label(A) of + true -> + "%DL" ++ integer_to_list(hipe_rtl:const_label_label(A)) ++ "_var"; + false -> + exit({?MODULE, trans_dst, {"Bad RTL argument",A}}) + end + end + end, + {Name, []} + end. + +%% @doc Translate a register. If it is precoloured it must be mapped to the +%% correct stack slot that holds the precoloured register value. +trans_reg(Arg, Position) -> + Index = hipe_rtl:reg_index(Arg), + case isPrecoloured(Arg) of + true -> + Name = map_precoloured_reg(Index), + case Position of + src -> fix_reg_src(Name); + dst -> fix_reg_dst(Name) + end; + false -> + {hipe_rtl_arch:reg_name(Index), []} + end. + +map_precoloured_reg(Index) -> + case hipe_rtl_arch:reg_name(Index) of + "%r15" -> "%hp_reg_var"; + "%rbp" -> "%p_reg_var"; + "%esi" -> "%hp_reg_var"; + "%ebp" -> "%p_reg_var"; + "%fcalls" -> + {"%p_reg_var", ?ARCH_REGISTERS:proc_offset(?ARCH_REGISTERS:fcalls())}; + "%hplim" -> + {"%p_reg_var", ?ARCH_REGISTERS:proc_offset(?ARCH_REGISTERS:heap_limit())}; + _ -> + exit({?MODULE, map_precoloured_reg, {"Register not mapped yet", Index}}) + end. + +%% @doc Load precoloured dst register. +fix_reg_dst(Register) -> + case Register of + {Name, Offset} -> %% Case of %fcalls, %hplim + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + pointer_from_reg(Name, WordTy, Offset); + Name -> %% Case of %p and %hp + {Name, []} + end. + +%% @doc Load precoloured src register. +fix_reg_src(Register) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + case Register of + {Name, Offset} -> %% Case of %fcalls, %hplim + {T1, I1} = pointer_from_reg(Name, WordTy, Offset), + T2 = mk_temp(), + I2 = hipe_llvm:mk_load(T2, WordTyPtr, T1, [], [] , false), + {T2, [I2, I1]}; + Name -> %% Case of %p and %hp + T1 = mk_temp(), + {T1, hipe_llvm:mk_load(T1, WordTyPtr, Name, [], [], false)} + end. + +%% @doc Load %fcalls and %hplim. +pointer_from_reg(RegName, Type, Offset) -> + PointerType = hipe_llvm:mk_pointer(Type), + T1 = mk_temp(), + I1 = hipe_llvm:mk_load(T1, PointerType, RegName, [], [] ,false), + T2 = mk_temp(), + I2 = hipe_llvm:mk_conversion(T2, inttoptr, Type, T1, PointerType), + T3 = mk_temp(), + %% XXX: Offsets should be a power of 2. + I3 = hipe_llvm:mk_getelementptr(T3, PointerType, T2, + [{Type, integer_to_list(Offset div hipe_rtl_arch:word_size())}], true), + {T3, [I3, I2, I1]}. + +isPrecoloured(X) -> + hipe_rtl_arch:is_precoloured(X). + +%%------------------------------------------------------------------------------ +%% Translation of operators +%%------------------------------------------------------------------------------ + +trans_op(Op) -> + case Op of + add -> add; + sub -> sub; + 'or' -> 'or'; + 'and' -> 'and'; + 'xor' -> 'xor'; + sll -> shl; + srl -> lshr; + sra -> ashr; + mul -> mul; + 'fdiv' -> fdiv; + 'sdiv' -> sdiv; + 'srem' -> srem; + Other -> exit({?MODULE, trans_op, {"Unknown RTL operator", Other}}) + end. + +trans_rel_op(Op) -> + case Op of + eq -> eq; + ne -> ne; + gtu -> ugt; + geu -> uge; + ltu -> ult; + leu -> ule; + gt -> sgt; + ge -> sge; + lt -> slt; + le -> sle + end. + +trans_prim_op(Op) -> + case Op of + '+' -> "bif_add"; + '-' -> "bif_sub"; + '*' -> "bif_mul"; + 'div' -> "bif_div"; + '/' -> "bif_div"; + Other -> atom_to_list(Other) + end. + +trans_fp_op(Op) -> + case Op of + fadd -> fadd; + fsub -> fsub; + fdiv -> fdiv; + fmul -> fmul; + fchs -> fsub; + Other -> exit({?MODULE, trans_fp_op, {"Unknown RTL float operator",Other}}) + end. + +%% Misc. +insn_dst(I) -> + case I of + #alu{} -> + [hipe_rtl:alu_dst(I)]; + #alub{} -> + [hipe_rtl:alub_dst(I)]; + #call{} -> + case hipe_rtl:call_dstlist(I) of + [] -> []; + [Dst] -> [Dst] + end; + #load{} -> + [hipe_rtl:load_dst(I)]; + #load_address{} -> + [hipe_rtl:load_address_dst(I)]; + #load_atom{} -> + [hipe_rtl:load_atom_dst(I)]; + #move{} -> + [hipe_rtl:move_dst(I)]; + #phi{} -> + [hipe_rtl:phi_dst(I)]; + #fconv{} -> + [hipe_rtl:fconv_dst(I)]; + #fload{} -> + [hipe_rtl:fload_dst(I)]; + #fmove{} -> + [hipe_rtl:fmove_dst(I)]; + #fp{} -> + [hipe_rtl:fp_dst(I)]; + #fp_unop{} -> + [hipe_rtl:fp_unop_dst(I)]; + _ -> + [] + end. + +llvm_type_from_size(Size) -> + case Size of + byte -> hipe_llvm:mk_int(8); + int16 -> hipe_llvm:mk_int(16); + int32 -> hipe_llvm:mk_int(32); + word -> hipe_llvm:mk_int(64) + end. + +%% @doc Create definition for the compiled function. The parameters that are +%% passed to the stack must be reversed to match with the CC. Also +%% precoloured registers that are passed as arguments must be stored to +%% the corresonding stack slots. +create_function_definition(Fun, Params, Code, LocalVars) -> + FunctionName = trans_mfa_name(Fun), + FixedRegs = fixed_registers(), + %% Reverse parameters to match with the Erlang calling convention + ReversedParams = + case erlang:length(Params) > ?NR_ARG_REGS of + false -> + Params; + true -> + {ParamsInRegs, ParamsInStack} = lists:split(?NR_ARG_REGS, Params), + ParamsInRegs ++ lists:reverse(ParamsInStack) + end, + Args = header_regs(FixedRegs) ++ header_params(ReversedParams), + EntryLabel = hipe_llvm:mk_label("Entry"), + FloatTy = hipe_llvm:mk_double(), + ExceptionSync = hipe_llvm:mk_alloca("%exception_sync", FloatTy, [], []), + I2 = load_regs(FixedRegs), + I3 = hipe_llvm:mk_br(mk_jump_label(get(first_label))), + StoredParams = store_params(Params), + EntryBlock = + lists:flatten([EntryLabel, ExceptionSync, I2, LocalVars, StoredParams, I3]), + Final_Code = EntryBlock ++ Code, + FunctionOptions = [nounwind, noredzone, list_to_atom("gc \"erlang\"")], + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + hipe_llvm:mk_fun_def([], [], "cc 11", [], FunRetTy, FunctionName, Args, + FunctionOptions, [], Final_Code). + +header_params(Params) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + [{WordTy, "%v" ++ integer_to_list(hipe_rtl:var_index(P))} || P <- Params]. + +store_params(Params) -> + Fun1 = + fun(X) -> + Index = hipe_rtl:var_index(X), + {Name, _} = trans_dst(X), + ParamName = "%v" ++ integer_to_list(Index), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + hipe_llvm:mk_store(WordTy, ParamName, WordTyPtr, Name, [], [], false) + end, + lists:map(Fun1, Params). + +fixed_registers() -> + case get(hipe_target_arch) of + x86 -> + ["hp", "p"]; + amd64 -> + ["hp", "p"]; + Other -> + exit({?MODULE, map_registers, {"Unknown architecture", Other}}) + end. + +header_regs(Registers) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + [{WordTy, "%" ++ X ++ "_in"} || X <- Registers]. + +load_regs(Registers) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + Fun1 = + fun(X) -> + I1 = hipe_llvm:mk_alloca("%" ++ X ++ "_reg_var", WordTy, [], []), + I2 = hipe_llvm:mk_store(WordTy, "%" ++ X ++ "_in", WordTyPtr, + "%" ++ X ++ "_reg_var", [], [], false), + [I1, I2] + end, + lists:map(Fun1, Registers). + +%%------------------------------------------------------------------------------ +%% Relocation-specific Stuff +%%------------------------------------------------------------------------------ + +relocs_store(Key, Value, Relocs) -> + dict:store(Key, Value, Relocs). + +relocs_to_list(Relocs) -> + dict:to_list(Relocs). + +%% @doc This function is responsible for the actions needed to handle +%% relocations: +%% 1) Updates relocations with constants and switch jump tables. +%% 2) Creates LLVM code to declare relocations as external +%% functions/constants. +%% 3) Creates LLVM code in order to create local variables for the external +%% constants/labels. +handle_relocations(Relocs, Data, Fun) -> + RelocsList = relocs_to_list(Relocs), + %% Seperate Relocations according to their type + {CallList, AtomList, ClosureList, ClosureLabels, SwitchList} = + seperate_relocs(RelocsList), + %% Create code to declare atoms + AtomDecl = [declare_atom(A) || A <- AtomList], + %% Create code to create local name for atoms + AtomLoad = [load_atom(A) || A <- AtomList], + %% Create code to declare closures + ClosureDecl = [declare_closure(C) || C <- ClosureList], + %% Create code to create local name for closures + ClosureLoad = [load_closure(C) || C <- ClosureList], + %% Find function calls + IsExternalCall = fun (X) -> is_external_call(X, Fun) end, + ExternalCallList = lists:filter(IsExternalCall, CallList), + %% Create code to declare external function + FunDecl = fixed_fun_decl() ++ [call_to_decl(C) || C <- ExternalCallList], + %% Extract constant labels from Constant Map (remove duplicates) + ConstLabels = hipe_consttab:labels(Data), + %% Create code to declare constants + ConstDecl = [declare_constant(C) || C <- ConstLabels], + %% Create code to create local name for constants + ConstLoad = [load_constant(C) || C <- ConstLabels], + %% Create code to create jump tables + SwitchDecl = declare_switches(SwitchList, Fun), + %% Create code to create a table with the labels of all closure calls + {ClosureLabelDecl, Relocs1} = + declare_closure_labels(ClosureLabels, Relocs, Fun), + %% Enter constants to relocations + Relocs2 = lists:foldl(fun const_to_dict/2, Relocs1, ConstLabels), + %% Temporary Store inc_stack and llvm_fix_pinned_regs to Dictionary + %% TODO: Remove this + Relocs3 = dict:store("inc_stack_0", {call, {bif, inc_stack_0, 0}}, Relocs2), + Relocs4 = dict:store("hipe_bifs.llvm_fix_pinned_regs.0", + {call, {hipe_bifs, llvm_fix_pinned_regs, 0}}, Relocs3), + BranchMetaData = [ + hipe_llvm:mk_branch_meta(?BRANCH_META_TAKEN, "99", "1") + , hipe_llvm:mk_branch_meta(?BRANCH_META_NOT_TAKEN, "1", "99") + ], + ExternalDeclarations = AtomDecl ++ ClosureDecl ++ ConstDecl ++ FunDecl ++ + ClosureLabelDecl ++ SwitchDecl ++ BranchMetaData, + LocalVariables = AtomLoad ++ ClosureLoad ++ ConstLoad, + {Relocs4, ExternalDeclarations, LocalVariables}. + +%% @doc Seperate relocations according to their type. +seperate_relocs(Relocs) -> + seperate_relocs(Relocs, [], [], [], [], []). + +seperate_relocs([], CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc) -> + {CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc}; +seperate_relocs([R|Rs], CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc) -> + case R of + {_, {call, _}} -> + seperate_relocs(Rs, [R | CallAcc], AtomAcc, ClosureAcc, LabelAcc, + JmpTableAcc); + {_, {atom, _}} -> + seperate_relocs(Rs, CallAcc, [R | AtomAcc], ClosureAcc, LabelAcc, + JmpTableAcc); + {_, {closure, _}} -> + seperate_relocs(Rs, CallAcc, AtomAcc, [R | ClosureAcc], LabelAcc, + JmpTableAcc); + {_, {switch, _, _}} -> + seperate_relocs(Rs, CallAcc, AtomAcc, ClosureAcc, LabelAcc, + [R | JmpTableAcc]); + {_, {closure_label, _, _}} -> + seperate_relocs(Rs, CallAcc, AtomAcc, ClosureAcc, [R | LabelAcc], + JmpTableAcc) + end. + +%% @doc External declaration of an atom. +declare_atom({AtomName, _}) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + hipe_llvm:mk_const_decl("@" ++ AtomName, "external constant", WordTy, ""). + +%% @doc Creation of local variable for an atom. +load_atom({AtomName, _}) -> + Dst = "%" ++ AtomName ++ "_var", + Name = "@" ++ AtomName, + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + hipe_llvm:mk_conversion(Dst, ptrtoint, WordTyPtr, Name, WordTy). + +%% @doc External declaration of a closure. +declare_closure({ClosureName, _})-> + ByteTy = hipe_llvm:mk_int(8), + hipe_llvm:mk_const_decl("@" ++ ClosureName, "external constant", ByteTy, ""). + +%% @doc Creation of local variable for a closure. +load_closure({ClosureName, _})-> + Dst = "%" ++ ClosureName ++ "_var", + Name = "@" ++ ClosureName, + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + hipe_llvm:mk_conversion(Dst, ptrtoint, ByteTyPtr, Name, WordTy). + +%% @doc Declaration of a local variable for a switch jump table. +declare_switches(JumpTableList, Fun) -> + FunName = trans_mfa_name(Fun), + [declare_switch_table(X, FunName) || X <- JumpTableList]. + +declare_switch_table({Name, {switch, {TableType, Labels, _, _}, _}}, FunName) -> + LabelList = [mk_jump_label(L) || L <- Labels], + Fun1 = fun(X) -> "i8* blockaddress(@" ++ FunName ++ ", " ++ X ++ ")" end, + List2 = lists:map(Fun1, LabelList), + List3 = string:join(List2, ",\n"), + List4 = "[\n" ++ List3 ++ "\n]\n", + hipe_llvm:mk_const_decl("@" ++ Name, "constant", TableType, List4). + +%% @doc Declaration of a variable for a table with the labels of all closure +%% calls in the code. +declare_closure_labels([], Relocs, _Fun) -> + {[], Relocs}; +declare_closure_labels(ClosureLabels, Relocs, Fun) -> + FunName = trans_mfa_name(Fun), + {LabelList, ArityList} = + lists:unzip([{mk_jump_label(Label), A} || + {_, {closure_label, Label, A}} <- ClosureLabels]), + Relocs1 = relocs_store("table_closures", {table_closures, ArityList}, Relocs), + List2 = + ["i8* blockaddress(@" ++ FunName ++ ", " ++ L ++ ")" || L <- LabelList], + List3 = string:join(List2, ",\n"), + List4 = "[\n" ++ List3 ++ "\n]\n", + NrLabels = length(LabelList), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + TableType = hipe_llvm:mk_array(NrLabels, ByteTyPtr), + ConstDecl = + hipe_llvm:mk_const_decl("@table_closures", "constant", TableType, List4), + {[ConstDecl], Relocs1}. + +%% @doc A call is treated as non external only in a case of a recursive +%% function. +is_external_call({_, {call, Fun}}, Fun) -> false; +is_external_call(_, _) -> true. + +%% @doc External declaration of a function. +call_to_decl({Name, {call, MFA}}) -> + {M, _F, A} = MFA, + CConv = "cc 11", + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + {Type, Args} = + case M of + llvm -> + {hipe_llvm:mk_struct([WordTy, hipe_llvm:mk_int(1)]), [1, 2]}; + %% +precoloured regs + _ -> + {FunRetTy, lists:seq(1, A + ?NR_PINNED_REGS)} + end, + ArgsTypes = lists:duplicate(length(Args), WordTy), + hipe_llvm:mk_fun_decl([], [], CConv, [], Type, "@" ++ Name, ArgsTypes, []). + +%% @doc These functions are always declared, even if not used. +fixed_fun_decl() -> + ByteTy = hipe_llvm:mk_int(8), + ByteTyPtr = hipe_llvm:mk_pointer(ByteTy), + LandPad = hipe_llvm:mk_fun_decl([], [], [], [], hipe_llvm:mk_int(32), + "@__gcc_personality_v0", [hipe_llvm:mk_int(32), hipe_llvm:mk_int(64), + ByteTyPtr, ByteTyPtr], []), + GCROOTDecl = hipe_llvm:mk_fun_decl([], [], [], [], hipe_llvm:mk_void(), + "@llvm.gcroot", [hipe_llvm:mk_pointer(ByteTyPtr), ByteTyPtr], []), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + FixPinnedRegs = hipe_llvm:mk_fun_decl([], [], [], [], FunRetTy, + "@hipe_bifs.llvm_fix_pinned_regs.0", [], []), + GcMetadata = hipe_llvm:mk_const_decl("@gc_metadata", "external constant", + ByteTy, ""), + [LandPad, GCROOTDecl, FixPinnedRegs, GcMetadata]. + +%% @doc Declare an External Consant. We declare all constants as i8 in order to +%% be able to calcucate pointers of the form DL+6, with the getelementptr +%% instruction. Otherwise we have to convert constants form pointers to +%% values, add the offset and convert them again to pointers. +declare_constant(Label) -> + Name = "@DL" ++ integer_to_list(Label), + ByteTy = hipe_llvm:mk_int(8), + hipe_llvm:mk_const_decl(Name, "external constant", ByteTy, ""). + +%% @doc Load a constant is achieved by converting a pointer to an integer of +%% the correct width. +load_constant(Label) -> + Dst = "%DL" ++ integer_to_list(Label) ++ "_var", + Name = "@DL" ++ integer_to_list(Label), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + hipe_llvm:mk_conversion(Dst, ptrtoint, ByteTyPtr, Name, WordTy). + +%% @doc Store external constants and calls to dictionary. +const_to_dict(Elem, Dict) -> + Name = "DL" ++ integer_to_list(Elem), + dict:store(Name, {'constant', Elem}, Dict). diff --git a/lib/hipe/main/hipe.app.src b/lib/hipe/main/hipe.app.src index 61d167d01a..e81212d4dc 100644 --- a/lib/hipe/main/hipe.app.src +++ b/lib/hipe/main/hipe.app.src @@ -30,6 +30,7 @@ cerl_prettypr, cerl_to_icode, cerl_typean, + elf_format, erl_bif_types, erl_types, hipe, @@ -108,6 +109,10 @@ hipe_ig, hipe_ig_moves, hipe_jit, + hipe_llvm, + hipe_llvm_liveness, + hipe_llvm_main, + hipe_llvm_merge, hipe_ls_regalloc, hipe_main, hipe_moves, @@ -159,6 +164,7 @@ hipe_rtl_symbolic, hipe_rtl_to_amd64, hipe_rtl_to_arm, + hipe_rtl_to_llvm, hipe_rtl_to_ppc, hipe_rtl_to_sparc, hipe_rtl_to_x86, diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index 434d5c3061..d47eced6d8 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -821,7 +821,9 @@ finalize_fun_sequential({MFA, Icode}, Opts, Servers) -> ?debug_msg("Compiled ~w in ~.2f s\n", [MFA,(T2-T1)/1000])), {MFA, Code}; {rtl, LinearRtl} -> - {MFA, LinearRtl} + {MFA, LinearRtl}; + {llvm_binary, Binary} -> + {MFA, Binary} catch error:Error -> ?when_option(verbose, Opts, ?debug_untagged_msg("\n", [])), @@ -890,21 +892,27 @@ do_load(Mod, Bin, BeamBinOrPath) when is_binary(BeamBinOrPath); end. assemble(CompiledCode, Closures, Exports, Options) -> - case get(hipe_target_arch) of - ultrasparc -> - hipe_sparc_assemble:assemble(CompiledCode, Closures, Exports, Options); - powerpc -> - hipe_ppc_assemble:assemble(CompiledCode, Closures, Exports, Options); - ppc64 -> - hipe_ppc_assemble:assemble(CompiledCode, Closures, Exports, Options); - arm -> - hipe_arm_assemble:assemble(CompiledCode, Closures, Exports, Options); - x86 -> - hipe_x86_assemble:assemble(CompiledCode, Closures, Exports, Options); - amd64 -> - hipe_amd64_assemble:assemble(CompiledCode, Closures, Exports, Options); - Arch -> - ?EXIT({executing_on_an_unsupported_architecture, Arch}) + case proplists:get_bool(to_llvm, Options) of + false -> + case get(hipe_target_arch) of + ultrasparc -> + hipe_sparc_assemble:assemble(CompiledCode, Closures, Exports, Options); + powerpc -> + hipe_ppc_assemble:assemble(CompiledCode, Closures, Exports, Options); + ppc64 -> + hipe_ppc_assemble:assemble(CompiledCode, Closures, Exports, Options); + arm -> + hipe_arm_assemble:assemble(CompiledCode, Closures, Exports, Options); + x86 -> + hipe_x86_assemble:assemble(CompiledCode, Closures, Exports, Options); + amd64 -> + hipe_amd64_assemble:assemble(CompiledCode, Closures, Exports, Options); + Arch -> + ?EXIT({executing_on_an_unsupported_architecture, Arch}) + end; + true -> + %% Merge already compiled code (per MFA) to a single binary. + hipe_llvm_merge:finalize(CompiledCode, Closures, Exports) end. %% -------------------------------------------------------------------- @@ -1330,6 +1338,11 @@ opt_keys() -> timeregalloc, timers, to_rtl, + to_llvm, % Use the LLVM backend for compilation. + llvm_save_temps, % Save the LLVM intermediate files in the current + % directory; useful for debugging. + llvm_llc, % Specify llc optimization-level: o1, o2, o3, undefined. + llvm_opt, % Specify opt optimization-level: o1, o2, o3, undefined. use_indexing, use_inline_atom_search, use_callgraph, @@ -1468,11 +1481,19 @@ opt_expansions() -> [{o1, o1_opts()}, {o2, o2_opts()}, {o3, o3_opts()}, + {to_llvm, llvm_opts(o3)}, + {{to_llvm, o0}, llvm_opts(o0)}, + {{to_llvm, o1}, llvm_opts(o1)}, + {{to_llvm, o2}, llvm_opts(o2)}, + {{to_llvm, o3}, llvm_opts(o3)}, {x87, [x87, inline_fp]}, {inline_fp, case get(hipe_target_arch) of %% XXX: Temporary until x86 x86 -> [x87, inline_fp]; %% has sse2 _ -> [inline_fp] end}]. +llvm_opts(O) -> + [to_llvm, {llvm_opt, O}, {llvm_llc, O}]. + %% This expands "basic" options, which may be tested early and cannot be %% in conflict with options found in the source code. diff --git a/lib/hipe/main/hipe_main.erl b/lib/hipe/main/hipe_main.erl index 99028cc3c1..89b79998be 100644 --- a/lib/hipe/main/hipe_main.erl +++ b/lib/hipe/main/hipe_main.erl @@ -49,7 +49,7 @@ %%===================================================================== -type comp_icode_ret() :: {'native',hipe_architecture(),{'unprofiled',_}} - | {'rtl',tuple()}. + | {'rtl',tuple()} | {'llvm_binary',term()}. %%===================================================================== @@ -115,11 +115,18 @@ compile_icode(MFA, LinearIcode0, Options, Servers, DebugState) -> pp(IcodeCfg7, MFA, icode_liveness, pp_icode_liveness, Options, Servers), FinalIcode = hipe_icode_cfg:cfg_to_linear(IcodeCfg7), ?opt_stop_timer("Icode"), - LinearRTL = ?option_time(icode_to_rtl(MFA,FinalIcode,Options, Servers), - "RTL", Options), + {LinearRTL, Roots} = ?option_time(icode_to_rtl(MFA, FinalIcode, Options, Servers), + "RTL", Options), case proplists:get_bool(to_rtl, Options) of false -> - rtl_to_native(MFA, LinearRTL, Options, DebugState); + case proplists:get_bool(to_llvm, Options) of + false -> + rtl_to_native(MFA, LinearRTL, Options, DebugState); + true -> + %% The LLVM backend returns binary code, unlike the rest of the HiPE + %% backends which return native assembly. + rtl_to_llvm_to_binary(MFA, LinearRTL, Roots, Options, DebugState) + end; true -> put(hipe_debug, DebugState), {rtl, LinearRTL} @@ -385,11 +392,21 @@ icode_to_rtl(MFA, Icode, Options, Servers) -> %% hipe_rtl_cfg:pp(RtlCfg3), pp(RtlCfg3, MFA, rtl_liveness, pp_rtl_liveness, Options, Servers), RtlCfg4 = rtl_lcm(RtlCfg3, Options), - pp(RtlCfg4, MFA, rtl, pp_rtl, Options, Servers), - LinearRTL1 = hipe_rtl_cfg:linearize(RtlCfg4), + %% LLVM: A liveness analysis on RTL must be performed in order to find the GC + %% roots and explicitly mark them (in RTL) when they go out of scope (only + %% when the LLVM backend is used). + {RtlCfg5, Roots} = + case proplists:get_bool(to_llvm, Options) of + false -> + {RtlCfg4, []}; + true -> + hipe_llvm_liveness:analyze(RtlCfg4) + end, + pp(RtlCfg5, MFA, rtl, pp_rtl, Options, Servers), + LinearRTL1 = hipe_rtl_cfg:linearize(RtlCfg5), LinearRTL2 = hipe_rtl_cleanup_const:cleanup(LinearRTL1), %% hipe_rtl:pp(standard_io, LinearRTL2), - LinearRTL2. + {LinearRTL2, Roots}. translate_to_rtl(Icode, Options) -> %% GC tests should have been added in the conversion to Icode. @@ -540,6 +557,17 @@ rtl_to_native(MFA, LinearRTL, Options, DebugState) -> put(hipe_debug, DebugState), LinearNativeCode. +%% Translate Linear RTL to binary code using LLVM. +rtl_to_llvm_to_binary(MFA, LinearRTL, Roots, Options, DebugState) -> + ?opt_start_timer("LLVM native code"), + %% BinaryCode is a tuple, as defined in llvm/hipe_llvm_main module, which + %% contains the binary code together with info needed by the loader, e.g. + %% ConstTab, Refs, LabelMap, etc. + BinaryCode = hipe_llvm_main:rtl_to_native(MFA, LinearRTL, Roots, Options), + ?opt_stop_timer("LLVM native code"), + put(hipe_debug, DebugState), + {llvm_binary, BinaryCode}. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Debugging stuff ... %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/hipe/misc/hipe_gensym.erl b/lib/hipe/misc/hipe_gensym.erl index 84fc8fa7e8..4d2a237188 100644 --- a/lib/hipe/misc/hipe_gensym.erl +++ b/lib/hipe/misc/hipe_gensym.erl @@ -44,7 +44,7 @@ %% Types of allowable entities to set global variables for %%----------------------------------------------------------------------- --type gvarname() :: 'icode' | 'rtl' | 'arm' | 'ppc' | 'sparc' | 'x86'. +-type gvarname() :: 'icode' | 'rtl' | 'arm' | 'ppc' | 'sparc' | 'x86' | 'llvm'. %%----------------------------------------------------------------------- diff --git a/lib/hipe/misc/hipe_pack_constants.erl b/lib/hipe/misc/hipe_pack_constants.erl index ca8a9e6bf7..300f9ae43a 100644 --- a/lib/hipe/misc/hipe_pack_constants.erl +++ b/lib/hipe/misc/hipe_pack_constants.erl @@ -20,30 +20,48 @@ %% -module(hipe_pack_constants). --export([pack_constants/2, slim_refs/1, slim_constmap/1]). +-export([pack_constants/2, slim_refs/1, slim_constmap/1, + find_const/2, mk_data_relocs/2, slim_sorted_exportmap/3]). -include("hipe_consttab.hrl"). -include("../../kernel/src/hipe_ext_format.hrl"). +-include("../main/hipe.hrl"). % Needed for the EXIT macro in find_const/2. %%----------------------------------------------------------------------------- --type raw_data() :: binary() | number() | list() | tuple(). --type tbl_ref() :: {hipe_constlbl(), non_neg_integer()}. +-type const_num() :: non_neg_integer(). +-type raw_data() :: binary() | number() | list() | tuple(). + +-type addr() :: non_neg_integer(). +-type ref_p() :: {DataPos :: hipe_constlbl(), CodeOffset :: addr()}. +-type ref() :: ref_p() | {'sorted', Base :: addr(), [ref_p()]}. + +-type mfa_refs() :: {mfa(), [ref()]}. + +%% XXX: these types may not belong here: FIX! +-type fa() :: {atom(), arity()}. +-type export_map() :: [{addr(), module(), atom(), arity()}]. -record(pcm_entry, {mfa :: mfa(), label :: hipe_constlbl(), - const_num :: non_neg_integer(), - start :: non_neg_integer(), + const_num :: const_num(), + start :: addr(), type :: 0 | 1 | 2, raw_data :: raw_data()}). +-type pcm_entry() :: #pcm_entry{}. + +-type label_map() :: gb_trees:tree({mfa(), hipe_constlbl()}, addr()). + +%% Some of the following types may possibly need to be exported +-type data_relocs() :: [ref()]. +-type packed_const_map() :: [pcm_entry()]. +-type mfa_refs_map() :: [mfa_refs()]. +-type slim_export_map() :: [addr() | module() | atom() | arity() | boolean()]. %%----------------------------------------------------------------------------- -spec pack_constants([{mfa(),[_],hipe_consttab()}], ct_alignment()) -> - {ct_alignment(), - non_neg_integer(), - [#pcm_entry{}], - [{mfa(),[tbl_ref() | {'sorted',non_neg_integer(),[tbl_ref()]}]}]}. + {ct_alignment(), non_neg_integer(), packed_const_map(), mfa_refs_map()}. pack_constants(Data, Align) -> pack_constants(Data, 0, Align, 0, [], []). @@ -194,13 +212,12 @@ compact_dests([], Dest, AccofDest, Acc) -> %% to the slimmed and flattened format ConstMap which is put in object %% files. %% --spec slim_constmap([#pcm_entry{}]) -> [raw_data()]. +-spec slim_constmap(packed_const_map()) -> [raw_data()]. slim_constmap(Map) -> slim_constmap(Map, gb_sets:new(), []). --spec slim_constmap([#pcm_entry{}], gb_sets:set(), [raw_data()]) -> [raw_data()]. -slim_constmap([#pcm_entry{const_num=ConstNo, start=Offset, - type=Type, raw_data=Term}|Rest], Inserted, Acc) -> +slim_constmap([#pcm_entry{const_num = ConstNo, start = Offset, + type = Type, raw_data = Term}|Rest], Inserted, Acc) -> case gb_sets:is_member(ConstNo, Inserted) of true -> slim_constmap(Rest, Inserted, Acc); @@ -209,3 +226,60 @@ slim_constmap([#pcm_entry{const_num=ConstNo, start=Offset, slim_constmap(Rest, NewInserted, [ConstNo, Offset, Type, Term|Acc]) end; slim_constmap([], _Inserted, Acc) -> Acc. + +%% +%% Lookup a constant in a ConstMap. +%% +-spec find_const({mfa(), hipe_constlbl()}, packed_const_map()) -> const_num(). + +find_const({MFA, Label}, [E = #pcm_entry{mfa = MFA, label = Label}|_]) -> + E#pcm_entry.const_num; +find_const(N, [_|R]) -> + find_const(N, R); +find_const(C, []) -> + ?EXIT({constant_not_found, C}). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%% +%% Functions to build and handle Refs, ExportMap and LabelMap. +%% Note: Moved here because they are used by all backends in +%% hipe_{arm,sparc,ppc,x86}_assemble.erl +%% XXX: Is this the right place for them? +%% + +-spec mk_data_relocs(mfa_refs_map(), label_map()) -> data_relocs(). + +mk_data_relocs(RefsFromConsts, LabelMap) -> + lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). + +mk_data_relocs([{MFA, Labels} | Rest], LabelMap, Acc) -> + Map = [case Label of + {L,Pos} -> + Offset = find({MFA,L}, LabelMap), + {Pos,Offset}; + {sorted,Base,OrderedLabels} -> + {sorted, Base, [begin + Offset = find({MFA,L}, LabelMap), + {Order, Offset} + end + || {L,Order} <- OrderedLabels]} + end + || Label <- Labels], + %% msg("Map: ~w Map\n", [Map]), + mk_data_relocs(Rest, LabelMap, [Map,Acc]); +mk_data_relocs([], _, Acc) -> Acc. + +find({MFA,L}, LabelMap) -> + gb_trees:get({MFA,L}, LabelMap). + +-spec slim_sorted_exportmap(export_map(), [mfa()], [fa()]) -> slim_export_map(). + +slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> + IsClosure = lists:member({M,F,A}, Closures), + IsExported = is_exported(F, A, Exports), + [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; +slim_sorted_exportmap([], _, _) -> []. + +is_exported(F, A, Exports) -> + lists:member({F,A}, Exports). diff --git a/lib/hipe/ppc/hipe_ppc_assemble.erl b/lib/hipe/ppc/hipe_ppc_assemble.erl index b2fd50517b..3ad91f4051 100644 --- a/lib/hipe/ppc/hipe_ppc_assemble.erl +++ b/lib/hipe/ppc/hipe_ppc_assemble.erl @@ -46,8 +46,8 @@ assemble(CompiledCode, Closures, Exports, Options) -> print("Total num bytes=~w\n", [CodeSize], Options), %% SC = hipe_pack_constants:slim_constmap(ConstMap), - DataRelocs = mk_data_relocs(RefsFromConsts, LabelMap), - SSE = slim_sorted_exportmap(ExportMap,Closures,Exports), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap,Closures,Exports), SlimRefs = hipe_pack_constants:slim_refs(AccRefs), Bin = term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, ConstAlign, ConstSize, @@ -288,7 +288,7 @@ do_pseudo_li(I, MFA, ConstMap) -> %%% end, %%% {load_address, {Tag,untag_mfa_or_prim(MFAorPrim)}}; {Label,constant} -> - ConstNo = find_const({MFA,Label}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,Label}, ConstMap), {load_address, {constant,ConstNo}}; {Label,closure} -> {load_address, {closure,Label}}; @@ -574,37 +574,6 @@ mk_y(Pred, BD) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -mk_data_relocs(RefsFromConsts, LabelMap) -> - lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). - -mk_data_relocs([{MFA,Labels} | Rest], LabelMap, Acc) -> - Map = [case Label of - {L,Pos} -> - Offset = find({MFA,L}, LabelMap), - {Pos,Offset}; - {sorted,Base,OrderedLabels} -> - {sorted, Base, [begin - Offset = find({MFA,L}, LabelMap), - {Order, Offset} - end - || {L,Order} <- OrderedLabels]} - end - || Label <- Labels], - %% msg("Map: ~w Map\n",[Map]), - mk_data_relocs(Rest, LabelMap, [Map,Acc]); -mk_data_relocs([],_,Acc) -> Acc. - -find({_MFA,_L} = MFAL,LabelMap) -> - gb_trees:get(MFAL, LabelMap). - -slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> - IsClosure = lists:member({M,F,A}, Closures), - IsExported = is_exported(F, A, Exports), - [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; -slim_sorted_exportmap([],_,_) -> []. - -is_exported(F, A, Exports) -> lists:member({F,A}, Exports). - %%% %%% Assembly listing support (pp_asm option). %%% @@ -642,14 +611,3 @@ fill_spaces(N) when N > 0 -> fill_spaces(N-1); fill_spaces(0) -> []. - -%%% -%%% Lookup a constant in a ConstMap. -%%% - -find_const({MFA,Label}, [{pcm_entry,MFA,Label,ConstNo,_,_,_}|_]) -> - ConstNo; -find_const(N, [_|R]) -> - find_const(N, R); -find_const(C, []) -> - ?EXIT({constant_not_found,C}). diff --git a/lib/hipe/rtl/hipe_icode2rtl.erl b/lib/hipe/rtl/hipe_icode2rtl.erl index 034153a3cb..6ab40adcc8 100644 --- a/lib/hipe/rtl/hipe_icode2rtl.erl +++ b/lib/hipe/rtl/hipe_icode2rtl.erl @@ -427,8 +427,6 @@ gen_type_test([X], Type, TrueLbl, FalseLbl, Pred, ConstTab) -> hipe_rtl:mk_branch(X, eq, TmpF, TrueLbl, FalseLbl, Pred)], ConstTab}; cons -> {hipe_tagscheme:test_cons(X, TrueLbl, FalseLbl, Pred), ConstTab}; - constant -> - {hipe_tagscheme:test_constant(X, TrueLbl, FalseLbl, Pred), ConstTab}; fixnum -> {hipe_tagscheme:test_fixnum(X, TrueLbl, FalseLbl, Pred), ConstTab}; float -> diff --git a/lib/hipe/rtl/hipe_rtl.erl b/lib/hipe/rtl/hipe_rtl.erl index 4bf4eb6bd7..bc61bec0bd 100644 --- a/lib/hipe/rtl/hipe_rtl.erl +++ b/lib/hipe/rtl/hipe_rtl.erl @@ -29,7 +29,7 @@ %% <li> {alu, Dst, Src1, Op, Src2} </li> %% <li> {alub, Dst, Src1, Op, Src2, RelOp, TrueLabel, FalseLabel, P} </li> %% <li> {branch, Src1, Src2, RelOp, TrueLabel, FalseLabel, P} </li> -%% <li> {call, DsListt, Fun, ArgList, Type, Continuation, FailContinuation} +%% <li> {call, DsListt, Fun, ArgList, Type, Continuation, FailContinuation, NormalContinuation} %% Type is one of {local, remote, primop, closure} </li> %% <li> {comment, Text} </li> %% <li> {enter, Fun, ArgList, Type} @@ -106,7 +106,7 @@ %% rtl_data_update/2, %% rtl_var_range/1, %% rtl_var_range_update/2, - %% rtl_label_range/1, + rtl_label_range/1, %% rtl_label_range_update/2, rtl_info/1, rtl_info_update/2]). @@ -226,6 +226,7 @@ %% goto_label_update/2, mk_call/6, + mk_call/7, call_fun/1, call_dstlist/1, call_dstlist_update/2, @@ -233,8 +234,10 @@ call_continuation/1, call_fail/1, call_type/1, + call_normal/1, + call_normal_update/2, %% call_continuation_update/2, - %% call_fail_update/2, + call_fail_update/2, is_call/1, mk_enter/3, @@ -290,10 +293,13 @@ %% fconv_src_update/2, %% is_fconv/1, - %% mk_var/1, + mk_var/1, + mk_var/2, mk_new_var/0, is_var/1, var_index/1, + var_liveness/1, + var_liveness_update/2, %% change_vars_to_regs/1, @@ -350,10 +356,15 @@ %% move_dst_update/2, fixnumop_dst_update/2, pp_instr/2, - %% pp_arg/2, + %% Uber hack! + pp_var/2, + pp_reg/2, + pp_arg/2, phi_arglist_update/2, phi_redirect_pred/3]). +-export([subst_uses_llvm/2]). + -export_type([alub_cond/0]). %% @@ -387,7 +398,7 @@ rtl_data(#rtl{data=Data}) -> Data. %% rtl_data_update(Rtl, Data) -> Rtl#rtl{data=Data}. %% rtl_var_range(#rtl{var_range=VarRange}) -> VarRange. %% rtl_var_range_update(Rtl, VarRange) -> Rtl#rtl{var_range=VarRange}. -%% rtl_label_range(#rtl{label_range=LabelRange}) -> LabelRange. +rtl_label_range(#rtl{label_range=LabelRange}) -> LabelRange. %% rtl_label_range_update(Rtl, LabelRange) -> Rtl#rtl{label_range=LabelRange}. rtl_info(#rtl{info=Info}) -> Info. rtl_info_update(Rtl, Info) -> Rtl#rtl{info=Info}. @@ -643,6 +654,17 @@ is_goto(_) -> false. %% call %% +%% LLVM: Call with normal continuation +mk_call(DstList, Fun, ArgList, Continuation, FailContinuation, + NormalContinuation, Type) -> + case Type of + remote -> ok; + not_remote -> ok + end, + #call{dstlist=DstList, 'fun'=Fun, arglist=ArgList, type=Type, + continuation=Continuation, failcontinuation=FailContinuation, + normalcontinuation=NormalContinuation}. + mk_call(DstList, Fun, ArgList, Continuation, FailContinuation, Type) -> case Type of remote -> ok; @@ -651,6 +673,10 @@ mk_call(DstList, Fun, ArgList, Continuation, FailContinuation, Type) -> #call{dstlist=DstList, 'fun'=Fun, arglist=ArgList, type=Type, continuation=Continuation, failcontinuation=FailContinuation}. + +call_normal(#call{normalcontinuation=NormalContinuation}) -> NormalContinuation. +call_normal_update(C, NewNormalContinuation) -> + C#call{normalcontinuation=NewNormalContinuation}. call_dstlist(#call{dstlist=DstList}) -> DstList. call_dstlist_update(C, NewDstList) -> C#call{dstlist=NewDstList}. call_fun(#call{'fun'=Fun}) -> Fun. @@ -853,11 +879,14 @@ reg_is_gcsafe(#rtl_reg{is_gc_safe=IsGcSafe}) -> IsGcSafe. is_reg(#rtl_reg{}) -> true; is_reg(_) -> false. --record(rtl_var, {index :: non_neg_integer()}). +-record(rtl_var, {index :: non_neg_integer(), liveness=live :: dead | live}). mk_var(Num) when is_integer(Num), Num >= 0 -> #rtl_var{index=Num}. +mk_var(Num, Liveness) when is_integer(Num), Num>=0 -> #rtl_var{index=Num, liveness=Liveness}. mk_new_var() -> mk_var(hipe_gensym:get_next_var(rtl)). var_index(#rtl_var{index=Index}) -> Index. +var_liveness(#rtl_var{liveness=Liveness}) -> Liveness. +var_liveness_update(RtlVar, Liveness) -> RtlVar#rtl_var{liveness=Liveness}. is_var(#rtl_var{}) -> true; is_var(_) -> false. @@ -1077,6 +1106,131 @@ subst_uses(Subst, I) -> switch_src_update(I, subst1(Subst, switch_src(I))) end. +subst_uses_llvm(Subst, I) -> + case I of + #alu{} -> + {NewSrc2, Subst1} = subst1_llvm(Subst, alu_src2(I)), + {NewSrc1, _ } = subst1_llvm(Subst1, alu_src1(I)), + I0 = alu_src1_update(I, NewSrc1), + alu_src2_update(I0, NewSrc2); + #alub{} -> + {NewSrc2, Subst1} = subst1_llvm(Subst, alub_src2(I)), + {NewSrc1, _ } = subst1_llvm(Subst1, alub_src1(I)), + I0 = alub_src1_update(I, NewSrc1), + alub_src2_update(I0, NewSrc2); + #branch{} -> + {NewSrc2, Subst1} = subst1_llvm(Subst, branch_src2(I)), + {NewSrc1, _ } = subst1_llvm(Subst1, branch_src1(I)), + I0 = branch_src1_update(I, NewSrc1), + branch_src2_update(I0, NewSrc2); + #call{} -> + case call_is_known(I) of + false -> + {NewFun, Subst1} = subst1_llvm(Subst, call_fun(I)), + {NewArgList, _} = subst_list_llvm(Subst1, call_arglist(I)), + I0 = call_fun_update(I, NewFun), + call_arglist_update(I0, NewArgList); + true -> + {NewArgList, _} = subst_list_llvm(Subst, call_arglist(I)), + call_arglist_update(I, NewArgList) + end; + #comment{} -> + I; + #enter{} -> + case enter_is_known(I) of + false -> + {NewFun, Subst1} = subst1_llvm(Subst, enter_fun(I)), + {NewArgList, _} = subst_list_llvm(Subst1, enter_arglist(I)), + I0 = enter_fun_update(I, NewFun), + enter_arglist_update(I0, NewArgList); + true -> + {NewArgList, _} = subst_list_llvm(Subst, enter_arglist(I)), + enter_arglist_update(I, NewArgList) + end; + #fconv{} -> + {NewSrc, _ } = subst1_llvm(Subst, fconv_src(I)), + fconv_src_update(I, NewSrc); + #fixnumop{} -> + {NewSrc, _ } = subst1_llvm(Subst, fixnumop_src(I)), + fixnumop_src_update(I, NewSrc); + #fload{} -> + {NewSrc, Subst1} = subst1_llvm(Subst, fload_src(I)), + {NewOffset, _ } = subst1_llvm(Subst1, fload_offset(I)), + I0 = fload_src_update(I, NewSrc), + fload_offset_update(I0, NewOffset); + #fmove{} -> + {NewSrc, _ } = subst1_llvm(Subst, fmove_src(I)), + fmove_src_update(I, NewSrc); + #fp{} -> + {NewSrc2, Subst1} = subst1_llvm(Subst, fp_src2(I)), + {NewSrc1, _ } = subst1_llvm(Subst1, fp_src1(I)), + I0 = fp_src1_update(I, NewSrc1), + fp_src2_update(I0, NewSrc2); + #fp_unop{} -> + {NewSrc, _ } = subst1_llvm(Subst, fp_unop_src(I)), + fp_unop_src_update(I, NewSrc); + #fstore{} -> + {NewSrc, Subst1} = subst1_llvm(Subst, fstore_src(I)), + {NewBase, Subst2} = subst1_llvm(Subst1, fstore_base(I)), + {NewOffset, _ } = subst1_llvm(Subst2, fstore_offset(I)), + I0 = fstore_src_update(I, NewSrc), + I1 = fstore_base_update(I0, NewBase), + fstore_offset_update(I1, NewOffset); + #goto{} -> + I; + #goto_index{} -> + I; + #gctest{} -> + {NewWords, _ } = subst1_llvm(Subst, gctest_words(I)), + gctest_words_update(I, NewWords); + #label{} -> + I; + #load{} -> + {NewSrc, Subst1} = subst1_llvm(Subst, load_src(I)), + {NewOffset, _ } = subst1_llvm(Subst1, load_offset(I)), + I0 = load_src_update(I, NewSrc), + load_offset_update(I0, NewOffset); + #load_address{} -> + I; + #load_atom{} -> + I; + #load_word_index{} -> + I; + #move{} -> + {NewSrc, _ } = subst1_llvm(Subst, move_src(I)), + move_src_update(I, NewSrc); + #multimove{} -> + {NewSrcList, _} = subst_list_llvm(Subst, multimove_srclist(I)), + multimove_srclist_update(I, NewSrcList); + #phi{} -> + phi_argvar_subst(I, Subst); + #return{} -> + {NewVarList, _} = subst_list_llvm(Subst, return_varlist(I)), + return_varlist_update(I, NewVarList); + #store{} -> + {NewSrc, Subst1} = subst1_llvm(Subst, store_src(I)), + {NewBase, Subst2} = subst1_llvm(Subst1, store_base(I)), + {NewOffset, _ } = subst1_llvm(Subst2, store_offset(I)), + I0 = store_src_update(I, NewSrc), + I1 = store_base_update(I0, NewBase), + store_offset_update(I1, NewOffset); + #switch{} -> + {NewSrc, _ } = subst1_llvm(Subst, switch_src(I)), + switch_src_update(I, NewSrc) + end. + +subst_list_llvm(S,X) -> subst_list_llvm(S, lists:reverse(X), []). +subst_list_llvm(S, [], Acc) -> {Acc, S}; +subst_list_llvm(S, [X|Xs], Acc) -> + {NewX, RestS} = subst1_llvm(S, X), + subst_list_llvm(RestS, Xs, [NewX|Acc]). + +subst1_llvm(A,B) -> subst1_llvm(A,B,[]). + +subst1_llvm([], X, Acc) -> {X, Acc}; +subst1_llvm([{X,Y}|Rs], X, Acc) -> {Y, Acc++Rs}; +subst1_llvm([R|Xs], X, Acc) -> subst1_llvm(Xs,X,[R|Acc]). + subst_defines(Subst, I)-> case I of #alu{} -> @@ -1614,7 +1768,11 @@ pp_var(Dev, Arg) -> true -> pp_hard_reg(Dev, var_index(Arg)); false -> - io:format(Dev, "v~w", [var_index(Arg)]) + io:format(Dev, "v~w", [var_index(Arg)]), + case var_liveness(Arg) of + dead -> io:format(Dev, "(dead)", []); + _ -> ok + end end. pp_arg(Dev, A) -> diff --git a/lib/hipe/rtl/hipe_rtl.hrl b/lib/hipe/rtl/hipe_rtl.hrl index 974e40f830..fbdf9ac524 100644 --- a/lib/hipe/rtl/hipe_rtl.hrl +++ b/lib/hipe/rtl/hipe_rtl.hrl @@ -28,7 +28,8 @@ -record(alu, {dst, src1, op, src2}). -record(alub, {dst, src1, op, src2, 'cond', true_label, false_label, p}). -record(branch, {src1, src2, 'cond', true_label, false_label, p}). --record(call, {dstlist, 'fun', arglist, type, continuation, failcontinuation}). +-record(call, {dstlist, 'fun', arglist, type, continuation, + failcontinuation, normalcontinuation = []}). -record(comment, {text}). -record(enter, {'fun', arglist, type}). -record(fconv, {dst, src}). diff --git a/lib/hipe/rtl/hipe_rtl_liveness.erl b/lib/hipe/rtl/hipe_rtl_liveness.erl index 3cfada9d6c..0c4b6b2e11 100644 --- a/lib/hipe/rtl/hipe_rtl_liveness.erl +++ b/lib/hipe/rtl/hipe_rtl_liveness.erl @@ -34,7 +34,8 @@ -module(hipe_rtl_liveness). -%% -define(LIVEOUT_NEEDED,true). % needed for liveness.inc below. +%% -define(DEBUG_LIVENESS,true). +-define(LIVEOUT_NEEDED,true). % needed for liveness.inc below. -define(PRETTY_PRINT,false). -include("hipe_rtl.hrl"). diff --git a/lib/hipe/rtl/hipe_tagscheme.erl b/lib/hipe/rtl/hipe_tagscheme.erl index f1e8d1ef41..4725889d8d 100644 --- a/lib/hipe/rtl/hipe_tagscheme.erl +++ b/lib/hipe/rtl/hipe_tagscheme.erl @@ -40,13 +40,12 @@ test_any_pid/4, test_any_port/4, test_ref/4, test_fun/4, test_fun2/5, test_matchstate/4, test_binary/4, test_bitstr/4, test_list/4, - test_integer/4, test_number/4, test_constant/4, test_tuple_N/5]). + test_integer/4, test_number/4, test_tuple_N/5]). -export([realtag_fixnum/2, tag_fixnum/2, realuntag_fixnum/2, untag_fixnum/2]). -export([test_two_fixnums/3, test_fixnums/4, unsafe_fixnum_add/3, unsafe_fixnum_sub/3, fixnum_gt/5, fixnum_lt/5, fixnum_ge/5, fixnum_le/5, fixnum_val/1, - fixnum_mul/4, - fixnum_addsub/5, fixnum_andorxor/4, fixnum_not/2, + fixnum_mul/4, fixnum_addsub/5, fixnum_andorxor/4, fixnum_not/2, fixnum_bsr/3, fixnum_bsl/3]). -export([unsafe_car/2, unsafe_cdr/2, unsafe_constant_element/3, unsafe_update_element/3, element/6]). @@ -405,17 +404,6 @@ test_number(X, TrueLab, FalseLab, Pred) -> hipe_rtl:mk_branch(Tmp, 'eq', hipe_rtl:mk_imm(HeaderFlonum), TrueLab, FalseLab, Pred)]. -%% CONS, NIL, and TUPLE are not constants, everything else is -test_constant(X, TrueLab, FalseLab, Pred) -> - Lab1 = hipe_rtl:mk_new_label(), - Lab2 = hipe_rtl:mk_new_label(), - Pred1 = 1-Pred, - [test_cons(X, FalseLab, hipe_rtl:label_name(Lab1), Pred1), - Lab1, - test_nil(X, FalseLab, hipe_rtl:label_name(Lab2), Pred1), - Lab2, - test_tuple(X, FalseLab, TrueLab, Pred1)]. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% tag_fixnum(DestVar, SrcReg) -> diff --git a/lib/hipe/sparc/hipe_sparc_assemble.erl b/lib/hipe/sparc/hipe_sparc_assemble.erl index b534fe20ec..68a4e1b349 100644 --- a/lib/hipe/sparc/hipe_sparc_assemble.erl +++ b/lib/hipe/sparc/hipe_sparc_assemble.erl @@ -45,8 +45,8 @@ assemble(CompiledCode, Closures, Exports, Options) -> print("Total num bytes=~w\n", [CodeSize], Options), %% SC = hipe_pack_constants:slim_constmap(ConstMap), - DataRelocs = mk_data_relocs(RefsFromConsts, LabelMap), - SSE = slim_sorted_exportmap(ExportMap,Closures,Exports), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap,Closures,Exports), SlimRefs = hipe_pack_constants:slim_refs(AccRefs), Bin = term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, ConstAlign, ConstSize, @@ -222,7 +222,7 @@ do_pseudo_set(I, MFA, ConstMap) -> %%% end, %%% {load_address, {Tag,untag_mfa_or_prim(MFAorPrim)}}; {Label,constant} -> - ConstNo = find_const({MFA,Label}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,Label}, ConstMap), {load_address, {constant,ConstNo}}; {Label,closure} -> {load_address, {closure,Label}}; @@ -507,37 +507,6 @@ px({pred,Pred}) -> % XXX: use pt/pn throughout entire backend %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -mk_data_relocs(RefsFromConsts, LabelMap) -> - lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). - -mk_data_relocs([{MFA,Labels} | Rest], LabelMap, Acc) -> - Map = [case Label of - {L,Pos} -> - Offset = find({MFA,L}, LabelMap), - {Pos,Offset}; - {sorted,Base,OrderedLabels} -> - {sorted, Base, [begin - Offset = find({MFA,L}, LabelMap), - {Order, Offset} - end - || {L,Order} <- OrderedLabels]} - end - || Label <- Labels], - %% msg("Map: ~w Map\n",[Map]), - mk_data_relocs(Rest, LabelMap, [Map,Acc]); -mk_data_relocs([],_,Acc) -> Acc. - -find({_MFA,_L} = MFAL, LabelMap) -> - gb_trees:get(MFAL, LabelMap). - -slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> - IsClosure = lists:member({M,F,A}, Closures), - IsExported = is_exported(F, A, Exports), - [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; -slim_sorted_exportmap([],_,_) -> []. - -is_exported(F, A, Exports) -> lists:member({F,A}, Exports). - %%% %%% Assembly listing support (pp_asm option). %%% @@ -575,14 +544,3 @@ fill_spaces(N) when N > 0 -> fill_spaces(N-1); fill_spaces(0) -> []. - -%%% -%%% Lookup a constant in a ConstMap. -%%% - -find_const({MFA,Label},[{pcm_entry,MFA,Label,ConstNo,_,_,_}|_]) -> - ConstNo; -find_const(N,[_|R]) -> - find_const(N,R); -find_const(C,[]) -> - ?EXIT({constant_not_found,C}). diff --git a/lib/hipe/x86/hipe_x86_assemble.erl b/lib/hipe/x86/hipe_x86_assemble.erl index 7878c7219d..3f756769c4 100644 --- a/lib/hipe/x86/hipe_x86_assemble.erl +++ b/lib/hipe/x86/hipe_x86_assemble.erl @@ -21,7 +21,6 @@ %%% %%% TODO: %%% - Simplify combine_label_maps and mk_data_relocs. -%%% - Move find_const to hipe_pack_constants? -ifdef(HIPE_AMD64). -define(HIPE_X86_ASSEMBLE, hipe_amd64_assemble). @@ -80,8 +79,8 @@ assemble(CompiledCode, Closures, Exports, Options) -> %% ?debug_msg("Constants are ~w bytes\n",[ConstSize])), %% SC = hipe_pack_constants:slim_constmap(ConstMap), - DataRelocs = mk_data_relocs(RefsFromConsts, LabelMap), - SSE = slim_sorted_exportmap(ExportMap,Closures,Exports), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap,Closures,Exports), SlimRefs = hipe_pack_constants:slim_refs(AccRefs), Bin = term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, ConstAlign, ConstSize, @@ -442,7 +441,7 @@ translate_imm(#x86_imm{value=Imm}, Context, MayTrunc8) -> case Imm of {Label,constant} -> {MFA,ConstMap} = Context, - ConstNo = find_const({MFA,Label}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,Label}, ConstMap), {constant,ConstNo}; {Label,closure} -> {closure,Label}; @@ -712,7 +711,7 @@ resolve_jmp_switch_arg(I, _Context) -> {rm64,hipe_amd64_encode:rm_mem(EA)}. -else. resolve_jmp_switch_arg(I, {MFA,ConstMap}) -> - ConstNo = find_const({MFA,hipe_x86:jmp_switch_jtab(I)}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,hipe_x86:jmp_switch_jtab(I)}, ConstMap), Disp32 = {?LOAD_ADDRESS,{constant,ConstNo}}, SINDEX = ?HIPE_X86_ENCODE:sindex(2, hipe_x86:temp_reg(hipe_x86:jmp_switch_temp(I))), EA = ?HIPE_X86_ENCODE:ea_disp32_sindex(Disp32, SINDEX), % this creates a SIB implicitly @@ -932,37 +931,6 @@ resolve_x87_binop_args(Src=#x86_fpreg{}, Dst=#x86_fpreg{})-> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -mk_data_relocs(RefsFromConsts, LabelMap) -> - lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). - -mk_data_relocs([{MFA,Labels} | Rest], LabelMap, Acc) -> - Map = [case Label of - {L,Pos} -> - Offset = find({MFA,L}, LabelMap), - {Pos,Offset}; - {sorted,Base,OrderedLabels} -> - {sorted, Base, [begin - Offset = find({MFA,L}, LabelMap), - {Order, Offset} - end - || {L,Order} <- OrderedLabels]} - end - || Label <- Labels], - %% msg("Map: ~w Map\n",[Map]), - mk_data_relocs(Rest, LabelMap, [Map,Acc]); -mk_data_relocs([],_,Acc) -> Acc. - -find({MFA,L},LabelMap) -> - gb_trees:get({MFA,L}, LabelMap). - -slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> - IsClosure = lists:member({M,F,A}, Closures), - IsExported = is_exported(F, A, Exports), - [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; -slim_sorted_exportmap([],_,_) -> []. - -is_exported(F, A, Exports) -> lists:member({F,A}, Exports). - %%% %%% Assembly listing support (pp_asm option). %%% @@ -1001,14 +969,3 @@ fill_spaces(N) when N > 0 -> fill_spaces(N-1); fill_spaces(0) -> []. - -%%% -%%% Lookup a constant in a ConstMap. -%%% - -find_const({MFA,Label},[{pcm_entry,MFA,Label,ConstNo,_,_,_}|_]) -> - ConstNo; -find_const(N,[_|R]) -> - find_const(N,R); -find_const(C,[]) -> - ?EXIT({constant_not_found,C}). diff --git a/lib/inets/src/http_client/httpc_cookie.erl b/lib/inets/src/http_client/httpc_cookie.erl index 69900bae65..134115bdfa 100644 --- a/lib/inets/src/http_client/httpc_cookie.erl +++ b/lib/inets/src/http_client/httpc_cookie.erl @@ -335,7 +335,8 @@ add_domain(Str, #http_cookie{domain = Domain}) -> Str ++ "; $Domain=" ++ Domain. parse_set_cookies(CookieHeaders, DefaultPathDomain) -> - SetCookieHeaders = [Value || {"set-cookie", Value} <- CookieHeaders], + %% empty Set-Cookie header is invalid according to RFC but some sites violate it + SetCookieHeaders = [Value || {"set-cookie", Value} <- CookieHeaders, Value /= ""], Cookies = [parse_set_cookie(SetCookieHeader, DefaultPathDomain) || SetCookieHeader <- SetCookieHeaders], %% print_cookies("Parsed Cookies", Cookies), diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 5598185ad3..88e08be789 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1119,15 +1119,8 @@ handle_http_body(Body, #state{headers = Headers, handle_response(State#state{headers = NewHeaders, body = NewBody}) end; - Encoding when is_list(Encoding) -> - ?hcrt("handle_http_body - encoding", [{encoding, Encoding}]), - NewState = answer_request(Request, - httpc_response:error(Request, - unknown_encoding), - State), - {stop, normal, NewState}; - _ -> - ?hcrt("handle_http_body - other", []), + Enc when Enc =:= "identity"; Enc =:= undefined -> + ?hcrt("handle_http_body - identity", []), Length = list_to_integer(Headers#http_response_h.'content-length'), case ((Length =< MaxBodySize) orelse (MaxBodySize =:= nolimit)) of @@ -1149,12 +1142,19 @@ handle_http_body(Body, #state{headers = Headers, body_too_big), State), {stop, normal, NewState} - end + end; + Encoding when is_list(Encoding) -> + ?hcrt("handle_http_body - other", [{encoding, Encoding}]), + NewState = answer_request(Request, + httpc_response:error(Request, + unknown_encoding), + State), + {stop, normal, NewState} end. handle_response(#state{status = new} = State) -> ?hcrd("handle response - status = new", []), - handle_response(check_persistent(State)); + handle_response(try_to_enable_pipeline_or_keep_alive(State)); handle_response(#state{request = Request, status = Status, @@ -1429,22 +1429,39 @@ is_keep_alive_enabled_server(_,_) -> is_keep_alive_connection(Headers, #session{client_close = ClientClose}) -> (not ((ClientClose) orelse httpc_response:is_server_closing(Headers))). -check_persistent( - #state{session = #session{type = Type} = Session, +try_to_enable_pipeline_or_keep_alive( + #state{session = Session, + request = #request{method = Method}, status_line = {Version, _, _}, headers = Headers, - profile_name = ProfileName} = State) -> + profile_name = ProfileName} = State) -> + ?hcrd("try to enable pipeline or keep-alive", + [{version, Version}, + {headers, Headers}, + {session, Session}]), case is_keep_alive_enabled_server(Version, Headers) andalso - is_keep_alive_connection(Headers, Session) of + is_keep_alive_connection(Headers, Session) of true -> - mark_persistent(ProfileName, Session), - State#state{status = Type}; + case (is_pipeline_enabled_client(Session) andalso + httpc_request:is_idempotent(Method)) of + true -> + insert_session(Session, ProfileName), + State#state{status = pipeline}; + false -> + insert_session(Session, ProfileName), + %% Make sure type is keep_alive in session + %% as it in this case might be pipeline + NewSession = Session#session{type = keep_alive}, + State#state{status = keep_alive, + session = NewSession} + end; false -> State#state{status = close} end. answer_request(#request{id = RequestId, from = From} = Request, Msg, - #state{timers = Timers, + #state{session = Session, + timers = Timers, profile_name = ProfileName} = State) -> ?hcrt("answer request", [{request, Request}, {msg, Msg}]), httpc_response:send(From, Msg), @@ -1454,14 +1471,19 @@ answer_request(#request{id = RequestId, from = From} = Request, Msg, Timer = {RequestId, TimerRef}, cancel_timer(TimerRef, {timeout, Request#request.id}), httpc_manager:request_done(RequestId, ProfileName), + NewSession = maybe_make_session_available(ProfileName, Session), Timers2 = Timers#timers{request_timers = lists:delete(Timer, RequestTimers)}, State#state{request = Request#request{from = answer_sent}, + session = NewSession, timers = Timers2}. -mark_persistent(ProfileName, Session) -> - update_session(ProfileName, Session, #session.persistent, true), - Session#session{persistent = true}. +maybe_make_session_available(ProfileName, + #session{available = false} = Session) -> + update_session(ProfileName, Session, #session.available, true), + Session#session{available = true}; +maybe_make_session_available(_ProfileName, Session) -> + Session. cancel_timers(#timers{request_timers = ReqTmrs, queue_timer = QTmr}) -> cancel_timer(QTmr, timeout_queue), diff --git a/lib/inets/src/http_client/httpc_internal.hrl b/lib/inets/src/http_client/httpc_internal.hrl index d5b3dd2a2a..add5d11dfa 100644 --- a/lib/inets/src/http_client/httpc_internal.hrl +++ b/lib/inets/src/http_client/httpc_internal.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -143,8 +143,8 @@ %% true | false %% This will be true, when a response has been received for - %% the first request and the server has not closed the connection - persistent = false + %% the first request. See type above. + available = false }). diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index a3ed371e61..48a9c32454 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2013. All Rights Reserved. +%% Copyright Ericsson AB 2002-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -451,7 +451,7 @@ do_init(ProfileName, CookiesDir) -> %%-------------------------------------------------------------------- handle_call({request, Request}, _, State) -> ?hcri("request", [{request, Request}]), - case (catch handle_request(Request, State, false)) of + case (catch handle_request(Request, State)) of {reply, Msg, NewState} -> {reply, Msg, NewState}; Error -> @@ -511,7 +511,7 @@ handle_cast({retry_or_redirect_request, {Time, Request}}, {noreply, State}; handle_cast({retry_or_redirect_request, Request}, State) -> - case (catch handle_request(Request, State, true)) of + case (catch handle_request(Request, State)) of {reply, {ok, _}, NewState} -> {noreply, NewState}; Error -> @@ -724,7 +724,7 @@ get_handler_info(Tab) -> handle_request(#request{settings = #http_options{version = "HTTP/0.9"}} = Request, - State, _) -> + State) -> %% Act as an HTTP/0.9 client that does not know anything %% about persistent connections @@ -737,7 +737,7 @@ handle_request(#request{settings = handle_request(#request{settings = #http_options{version = "HTTP/1.0"}} = Request, - State, _) -> + State) -> %% Act as an HTTP/1.0 client that does not %% use persistent connections @@ -748,13 +748,13 @@ handle_request(#request{settings = start_handler(NewRequest#request{headers = NewHeaders}, State), {reply, {ok, NewRequest#request.id}, State}; -handle_request(Request, State = #state{options = Options}, Retry) -> +handle_request(Request, State = #state{options = Options}) -> NewRequest = handle_cookies(generate_request_id(Request), State), SessionType = session_type(Options), case select_session(Request#request.method, Request#request.address, - Request#request.scheme, SessionType, State, Retry) of + Request#request.scheme, SessionType, State) of {ok, HandlerPid} -> pipeline_or_keep_alive(NewRequest, HandlerPid, State); no_connection -> @@ -778,7 +778,6 @@ start_handler(#request{id = Id, #state{profile_name = ProfileName, handler_db = HandlerDb, options = Options}) -> - ClientClose = httpc_request:is_client_closing(Request#request.headers), {ok, Pid} = case is_inets_manager() of true -> @@ -789,18 +788,13 @@ start_handler(#request{id = Id, end, HandlerInfo = {Id, Pid, From}, ets:insert(HandlerDb, HandlerInfo), - insert_session(#session{id = {Request#request.address, Pid}, - scheme = Request#request.scheme, - client_close = ClientClose, - type = session_type(Options) - }, ProfileName), erlang:monitor(process, Pid). select_session(Method, HostPort, Scheme, SessionType, #state{options = #options{max_pipeline_length = MaxPipe, max_keep_alive_length = MaxKeepAlive}, - session_db = SessionDb}, Retry) -> + session_db = SessionDb}) -> ?hcrd("select session", [{session_type, SessionType}, {max_pipeline_length, MaxPipe}, {max_keep_alive_length, MaxKeepAlive}]), @@ -813,23 +807,13 @@ select_session(Method, HostPort, Scheme, SessionType, %% client_close, scheme and type specified. %% The fields id (part of: HandlerPid) and queue_length %% specified. - Pattern = case (Retry andalso SessionType == pipeline) of - true -> - #session{id = {HostPort, '$1'}, - client_close = false, - scheme = Scheme, - queue_length = '$2', - type = SessionType, - persistent = true, - _ = '_'}; - false -> - #session{id = {HostPort, '$1'}, - client_close = false, - scheme = Scheme, - queue_length = '$2', - type = SessionType, - _ = '_'} - end, + Pattern = #session{id = {HostPort, '$1'}, + client_close = false, + scheme = Scheme, + queue_length = '$2', + type = SessionType, + available = true, + _ = '_'}, %% {'_', {HostPort, '$1'}, false, Scheme, '_', '$2', SessionTyp}, Candidates = ets:match(SessionDb, Pattern), ?hcrd("select session", [{host_port, HostPort}, diff --git a/lib/inets/src/http_server/httpd_example.erl b/lib/inets/src/http_server/httpd_example.erl index 16a080f8e2..6fc07f033c 100644 --- a/lib/inets/src/http_server/httpd_example.erl +++ b/lib/inets/src/http_server/httpd_example.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -66,7 +66,7 @@ get_bin(_Env,_Input) -> <INPUT TYPE=\"text\" NAME=\"input2\"> <INPUT TYPE=\"submit\"><BR> </FORM>" ++ "\n"), - footer()]. + list_to_binary(footer())]. post(_Env,[]) -> [header(), diff --git a/lib/inets/test/Makefile b/lib/inets/test/Makefile index c156b34406..609396273d 100644 --- a/lib/inets/test/Makefile +++ b/lib/inets/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2013. All Rights Reserved. +# Copyright Ericsson AB 1997-2014. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index fe6edd504e..b1b799c953 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2013. All Rights Reserved. +%% Copyright Ericsson AB 2004-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -91,6 +91,7 @@ only_simulated() -> [ cookie, cookie_profile, + empty_set_cookie, trace, stream_once, no_content_204, @@ -104,6 +105,7 @@ only_simulated() -> remote_socket_close, remote_socket_close_async, transfer_encoding, + transfer_encoding_identity, redirect_loop, redirect_moved_permanently, redirect_multiple_choises, @@ -296,6 +298,9 @@ trace(Config) when is_list(Config) -> pipeline(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, {ok, _} = httpc:request(get, Request, [], [], pipeline), + + %% Make sure pipeline session is registerd + test_server:sleep(4000), keep_alive_requests(Request, pipeline). %%-------------------------------------------------------------------- @@ -303,6 +308,9 @@ pipeline(Config) when is_list(Config) -> persistent_connection(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, {ok, _} = httpc:request(get, Request, [], [], persistent), + + %% Make sure pipeline session is registerd + test_server:sleep(4000), keep_alive_requests(Request, persistent). %%------------------------------------------------------------------------- @@ -530,6 +538,19 @@ cookie_profile(Config) when is_list(Config) -> inets:stop(httpc, cookie_test). %%------------------------------------------------------------------------- +empty_set_cookie() -> + [{doc, "Test empty Set-Cookie header."}]. +empty_set_cookie(Config) when is_list(Config) -> + ok = httpc:set_options([{cookies, enabled}]), + + Request0 = {url(group_name(Config), "/empty_set_cookie.html", Config), []}, + + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, Request0, [], []), + + ok = httpc:set_options([{cookies, disabled}]). + +%%------------------------------------------------------------------------- headers_as_is(doc) -> ["Test the option headers_as_is"]; headers_as_is(Config) when is_list(Config) -> @@ -624,6 +645,12 @@ transfer_encoding(Config) when is_list(Config) -> %%------------------------------------------------------------------------- +transfer_encoding_identity(Config) when is_list(Config) -> + URL = url(group_name(Config), "/identity_transfer_encoding.html", Config), + {ok, {{_,200,_}, [_|_], "IDENTITY"}} = httpc:request(URL). + +%%------------------------------------------------------------------------- + empty_response_header() -> [{doc, "Test the case that the HTTP server does not send any headers. Solves OTP-6830"}]. empty_response_header(Config) when is_list(Config) -> @@ -1609,6 +1636,13 @@ handle_uri(_,"/capital_transfer_encoding.html",_,_,Socket,_) -> send(Socket, http_chunk:encode("obar</BODY></HTML>")), http_chunk:encode_last(); +handle_uri(_,"/identity_transfer_encoding.html",_,_,_,_) -> + "HTTP/1.0 200 OK\r\n" + "Transfer-Encoding:identity\r\n" + "Content-Length:8\r\n" + "\r\n" + "IDENTITY"; + handle_uri(_,"/cookie.html",_,_,_,_) -> "HTTP/1.1 200 ok\r\n" ++ "set-cookie:" ++ "test_cookie=true; path=/;" ++ @@ -1616,6 +1650,12 @@ handle_uri(_,"/cookie.html",_,_,_,_) -> "Content-Length:32\r\n\r\n"++ "<HTML><BODY>foobar</BODY></HTML>"; +handle_uri(_,"/empty_set_cookie.html",_,_,_,_) -> + "HTTP/1.1 200 ok\r\n" ++ + "set-cookie: \r\n" ++ + "Content-Length:32\r\n\r\n"++ + "<HTML><BODY>foobar</BODY></HTML>"; + handle_uri(_,"/missing_crlf.html",_,_,_,_) -> "HTTP/1.1 200 ok" ++ "Content-Length:32\r\n" ++ diff --git a/lib/inets/test/httpd_1_0.erl b/lib/inets/test/httpd_1_0.erl index 53f23b12e0..0836c9e881 100644 --- a/lib/inets/test/httpd_1_0.erl +++ b/lib/inets/test/httpd_1_0.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,7 +20,7 @@ -module(httpd_1_0). --export([host/4]). +-export([host/4, trace/4]). %%------------------------------------------------------------------------- %% Test cases @@ -31,3 +31,8 @@ host(Type, Port, Host, Node) -> "GET / HTTP/1.0\r\n\r\n", [{statuscode, 200}, {version, "HTTP/1.0"}]). +trace(Type, Port, Host, Node)-> + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "TRACE / HTTP/1.0\r\n\r\n", + [{statuscode, 501}, + {version, "HTTP/1.0"}]). diff --git a/lib/inets/test/httpd_1_1.erl b/lib/inets/test/httpd_1_1.erl index 4b2a5f619d..6a5fc4a18f 100644 --- a/lib/inets/test/httpd_1_1.erl +++ b/lib/inets/test/httpd_1_1.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,7 +22,7 @@ -include_lib("kernel/include/file.hrl"). --export([host/4, chunked/4, expect/4, range/4, if_test/5, http_trace/4, +-export([host/4, chunked/4, expect/4, range/4, if_test/5, trace/4, head/4, mod_cgi_chunked_encoding_test/5]). %% -define(all_keys_lower_case,true). @@ -152,13 +152,13 @@ if_test(Type, Port, Host, Node, DocRoot)-> calendar:datetime_to_gregorian_seconds(FileInfo#file_info.mtime), Mod = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec-1)), + CreatedSec-1)), %% Test that we get the data when the file is modified ok = httpd_test_lib:verify_request(Type, Host, Port, Node, "GET / HTTP/1.1\r\nHost:" ++ Host ++ - "\r\nIf-Modified-Since:" ++ - Mod ++ "\r\n\r\n", + "\r\nIf-Modified-Since:" ++ + Mod ++ "\r\n\r\n", [{statuscode, 200}]), Mod1 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( CreatedSec+100)), @@ -168,74 +168,69 @@ if_test(Type, Port, Host, Node, DocRoot)-> ++ Mod1 ++"\r\n\r\n", [{statuscode, 304}]), - + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, "GET / HTTP/1.1\r\nHost:" ++ Host ++ - "\r\nIf-Modified-Since:" ++ - "AAA[...]AAAA" ++ "\r\n\r\n", + "\r\nIf-Modified-Since:" ++ + "AAA[...]AAAA" ++ "\r\n\r\n", [{statuscode, 400}]), - - - Mod2 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( + + Mod2 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( CreatedSec+1)), - %% Control that the If-Unmodified-Header lmits the response - ok = httpd_test_lib:verify_request(Type,Host,Port,Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ - "\r\nIf-Unmodified-Since:" ++ Mod2 - ++ "\r\n\r\n", - [{statuscode, 200}]), - Mod3 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( + %% Control that the If-Unmodified-Header lmits the response + ok = httpd_test_lib:verify_request(Type,Host,Port,Node, + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ + "\r\nIf-Unmodified-Since:" ++ Mod2 + ++ "\r\n\r\n", + [{statuscode, 200}]), + Mod3 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( CreatedSec-1)), ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ - "\r\nIf-Unmodified-Since:"++ Mod3 + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ + "\r\nIf-Unmodified-Since:"++ Mod3 ++"\r\n\r\n", - [{statuscode, 412}]), + [{statuscode, 412}]), - %% Control that we get the body when the etag match + %% Control that we get the body when the etag match ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ Host - ++"\r\n"++ - "If-Match:"++ - httpd_util:create_etag(FileInfo)++ - "\r\n\r\n", - [{statuscode, 200}]), - ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ - Host ++ "\r\n"++ - "If-Match:NotEtag\r\n\r\n", - [{statuscode, 412}]), + "GET / HTTP/1.1\r\nHost:" ++ Host + ++"\r\n"++ + "If-Match:"++ + httpd_util:create_etag(FileInfo)++ + "\r\n\r\n", + [{statuscode, 200}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "GET / HTTP/1.1\r\nHost:" ++ + Host ++ "\r\n"++ + "If-Match:NotEtag\r\n\r\n", + [{statuscode, 412}]), - %% Control the response when the if-none-match header is there - ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++"\r\n"++ - "If-None-Match:NoTaag," ++ - httpd_util:create_etag(FileInfo) ++ - "\r\n\r\n", - [{statuscode, 304}]), + %% Control the response when the if-none-match header is there + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "GET / HTTP/1.1\r\nHost:" + ++ Host ++"\r\n"++ + "If-None-Match:NoTaag," ++ + httpd_util:create_etag(FileInfo) ++ + "\r\n\r\n", + [{statuscode, 304}]), ok = httpd_test_lib:verify_request(Type, Host, Port, Node, "GET / HTTP/1.1\r\nHost:" - ++ Host ++ "\r\n"++ - "If-None-Match:NotEtag," - "NeihterEtag\r\n\r\n", + ++ Host ++ "\r\n"++ + "If-None-Match:NotEtag," + "NeihterEtag\r\n\r\n", [{statuscode,200}]), ok. - -http_trace(Type, Port, Host, Node)-> + +trace(Type, Port, Host, Node)-> ok = httpd_test_lib:verify_request(Type, Host, Port, Node, "TRACE / HTTP/1.1\r\n" ++ "Host:" ++ Host ++ "\r\n" ++ "Max-Forwards:2\r\n\r\n", - [{statuscode, 200}]), - ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "TRACE / HTTP/1.0\r\n\r\n", - [{statuscode, 501}, - {version, "HTTP/1.0"}]). + [{statuscode, 200}]). head(Type, Port, Host, Node)-> %% mod_include ok = httpd_test_lib:verify_request(Type, Host, Port, Node, @@ -283,7 +278,7 @@ mod_cgi_chunked_encoding_test(Type, Port, Host, Node, [Request| Rest])-> %%-------------------------------------------------------------------- validateRangeRequest(Socket,Response,ValidBody,C,O,DE)-> receive - {tcp,Socket,Data} -> + {_,Socket,Data} -> case string:str(Data,"\r\n") of 0-> validateRangeRequest(Socket, @@ -312,7 +307,7 @@ validateRangeRequest1(Socket, Response, ValidBody) -> case end_of_header(Response) of false -> receive - {tcp,Socket,Data} -> + {_,Socket,Data} -> validateRangeRequest1(Socket, Response ++ Data, ValidBody); _-> @@ -331,10 +326,10 @@ validateRangeRequest2(Socket, Head, Body, ValidBody, {multiPart,Boundary})-> validateMultiPartRangeRequest(Body, ValidBody, Boundary); false-> receive - {tcp, Socket, Data} -> + {_, Socket, Data} -> validateRangeRequest2(Socket, Head, Body ++ Data, ValidBody, {multiPart, Boundary}); - {tcp_closed, Socket} -> + {_, Socket} -> error; _ -> error @@ -353,7 +348,7 @@ validateRangeRequest2(Socket, Head, Body, ValidBody, BodySize) end; Size when Size < BodySize -> receive - {tcp, Socket, Data} -> + {_, Socket, Data} -> validateRangeRequest2(Socket, Head, Body ++ Data, ValidBody, BodySize); _ -> diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index c0d73663d3..3eb8a0818f 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -26,6 +26,7 @@ -include_lib("kernel/include/file.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("public_key/include/public_key.hrl"). -include("inets_test_lib.hrl"). %% Note: This directive should only be used in test suites. @@ -33,6 +34,11 @@ -record(httpd_user, {user_name, password, user_data}). -record(httpd_group, {group_name, userlist}). +-define(MAX_HEADER_SIZE, 256). +%% Minutes before failed auths timeout. +-define(FAIL_EXPIRE_TIME,1). +%% Seconds before successful auths timeout. +-define(AUTH_TIMEOUT,5). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- @@ -42,22 +48,59 @@ suite() -> all() -> [ - {group, http}, - {group, http_limit} - %%{group, https} + {group, http_basic}, + {group, https_basic}, + {group, http_limit}, + {group, https_limit}, + {group, http_basic_auth}, + {group, https_basic_auth}, + {group, http_auth_api}, + {group, https_auth_api}, + {group, http_auth_api_dets}, + {group, https_auth_api_dets}, + {group, http_auth_api_mnesia}, + {group, https_auth_api_mnesia}, + {group, http_htaccess}, + {group, https_htaccess}, + {group, http_security}, + {group, https_security} ]. groups() -> [ - {http, [], all_groups()}, - %%{https, [], all_groups()}, - {http_limit, [], [max_clients_1_1, max_clients_1_0, max_clients_0_9]}, - {http_1_1, [], [host, chunked, expect, cgi] ++ http_head() ++ http_get()}, - {http_1_0, [], [host, cgi] ++ http_head() ++ http_get()}, - {http_0_9, [], http_head() ++ http_get()} + {http_basic, [], basic_groups()}, + {https_basic, [], basic_groups()}, + {http_limit, [], [{group, limit}]}, + {https_limit, [], [{group, limit}]}, + {http_basic_auth, [], [{group, basic_auth}]}, + {https_basic_auth, [], [{group, basic_auth}]}, + {http_auth_api, [], [{group, auth_api}]}, + {https_auth_api, [], [{group, auth_api}]}, + {http_auth_api_dets, [], [{group, auth_api_dets}]}, + {https_auth_api_dets, [], [{group, auth_api_dets}]}, + {http_auth_api_mnesia, [], [{group, auth_api_mnesia}]}, + {https_auth_api_mnesia, [], [{group, auth_api_mnesia}]}, + {http_htaccess, [], [{group, htaccess}]}, + {https_htaccess, [], [{group, htaccess}]}, + {http_security, [], [{group, security}]}, + {https_security, [], [{group, security}]}, + {limit, [], [max_clients_1_1, max_clients_1_0, max_clients_0_9]}, + {basic_auth, [], [basic_auth_1_1, basic_auth_1_0, basic_auth_0_9]}, + {auth_api, [], [auth_api_1_1, auth_api_1_0, auth_api_0_9 + ]}, + {auth_api_dets, [], [auth_api_1_1, auth_api_1_0, auth_api_0_9 + ]}, + {auth_api_mnesia, [], [auth_api_1_1, auth_api_1_0, auth_api_0_9 + ]}, + {htaccess, [], [htaccess_1_1, htaccess_1_0, htaccess_0_9]}, + {security, [], [security_1_1, security_1_0]}, %% Skip 0.9 as causes timing issus in test code + {http_1_1, [], [host, chunked, expect, cgi, cgi_chunked_encoding_test, + trace, range, if_modified_since] ++ http_head() ++ http_get() ++ load()}, + {http_1_0, [], [host, cgi, trace] ++ http_head() ++ http_get() ++ load()}, + {http_0_9, [], http_head() ++ http_get() ++ load()} ]. -all_groups ()-> +basic_groups ()-> [{group, http_1_1}, {group, http_1_0}, {group, http_0_9} @@ -66,15 +109,27 @@ all_groups ()-> http_head() -> [head]. http_get() -> - [alias, get, - basic_auth, - esi, ssi]. + [alias, + get, + %%actions, Add configuration so that this test mod_action + esi, + ssi, + content_length, + bad_hex, + missing_CR, + max_header, + ipv6 + ]. +load() -> + [light, medium + %%,heavy + ]. + init_per_suite(Config) -> PrivDir = ?config(priv_dir, Config), DataDir = ?config(data_dir, Config), inets_test_lib:stop_apps([inets]), - inets_test_lib:start_apps([inets]), ServerRoot = filename:join(PrivDir, "server_root"), inets_test_lib:del_dirs(ServerRoot), DocRoot = filename:join(ServerRoot, "htdocs"), @@ -82,21 +137,31 @@ init_per_suite(Config) -> [{server_root, ServerRoot}, {doc_root, DocRoot}, {node, node()}, - {host, inets_test_lib:hostname()} | Config]. + {host, inets_test_lib:hostname()}, + {address, getaddr()} | Config]. end_per_suite(_Config) -> ok. %%-------------------------------------------------------------------- -init_per_group(https = Group, Config0) -> - case start_apps(Group) of - ok -> - init_httpd(Group, [{type, ssl} | Config0]); - _ -> - {skip, "Could not start https apps"} - end; - -init_per_group(Group, Config0) when Group == http; Group == http_limit -> +init_per_group(Group, Config0) when Group == https_basic; + Group == https_limit; + Group == https_basic_auth; + Group == https_auth_api; + Group == https_auth_api_dets; + Group == https_auth_api_mnesia; + Group == https_security + -> + init_ssl(Group, Config0); +init_per_group(Group, Config0) when Group == http_basic; + Group == http_limit; + Group == http_basic_auth; + Group == http_auth_api; + Group == http_auth_api_dets; + Group == http_auth_api_mnesia; + Group == http_security + -> + ok = start_apps(Group), init_httpd(Group, [{type, ip_comm} | Config0]); init_per_group(http_1_1, Config) -> [{http_version, "HTTP/1.1"} | Config]; @@ -104,22 +169,57 @@ init_per_group(http_1_0, Config) -> [{http_version, "HTTP/1.0"} | Config]; init_per_group(http_0_9, Config) -> [{http_version, "HTTP/0.9"} | Config]; +init_per_group(http_htaccess = Group, Config) -> + Path = ?config(doc_root, Config), + catch remove_htaccess(Path), + create_htaccess_data(Path, ?config(address, Config)), + ok = start_apps(Group), + init_httpd(Group, [{type, ip_comm} | Config]); +init_per_group(https_htaccess = Group, Config) -> + Path = ?config(doc_root, Config), + catch remove_htaccess(Path), + create_htaccess_data(Path, ?config(address, Config)), + init_ssl(Group, Config); +init_per_group(auth_api, Config) -> + [{auth_prefix, ""} | Config]; +init_per_group(auth_api_dets, Config) -> + [{auth_prefix, "dets_"} | Config]; +init_per_group(auth_api_mnesia, Config) -> + start_mnesia(?config(node, Config)), + [{auth_prefix, "mnesia_"} | Config]; init_per_group(_, Config) -> Config. -end_per_group(http, _Config) -> - ok; -end_per_group(https, _Config) -> - ssl:stop(); + +end_per_group(Group, _Config) when Group == http_basic; + Group == http_limit; + Group == http_basic_auth; + Group == http_auth_api; + Group == http_auth_api_dets; + Group == http_auth_api_mnesia; + Group == http_htaccess; + Group == http_security + -> + inets:stop(); +end_per_group(Group, _Config) when Group == https_basic; + Group == https_limit; + Group == https_basic_auth; + Group == https_auth_api; + Group == http_auth_api_dets; + Group == http_auth_api_mnesia; + Group == https_htaccess; + Group == http_security + -> + ssl:stop(), + inets:stop(); + +end_per_group(auth_api_mnesia, _) -> + cleanup_mnesia(); + end_per_group(_, _) -> ok. -init_httpd(Group, Config0) -> - Config1 = proplists:delete(port, Config0), - Config = proplists:delete(server_pid, Config1), - {Pid, Port} = server_start(Group, server_config(Group, Config)), - [{server_pid, Pid}, {port, Port} | Config]. %%-------------------------------------------------------------------- -init_per_testcase(host, Config) -> +init_per_testcase(Case, Config) when Case == host; Case == trace -> Prop = ?config(tc_group_properties, Config), Name = proplists:get_value(name, Prop), Cb = case Name of @@ -129,15 +229,15 @@ init_per_testcase(host, Config) -> httpd_1_1 end, [{version_cb, Cb} | proplists:delete(version_cb, Config)]; + +init_per_testcase(range, Config) -> + DocRoot = ?config(doc_root, Config), + create_range_data(DocRoot), + Config; + init_per_testcase(_, Config) -> Config. -%% init_per_testcase(basic_auth = Case, Config) -> -%% start_mnesia(?config(node, Config)), -%% common_init_per_test_case(Case, Config); - -%% end_per_testcase(basic_auth, Config) -> -%% cleanup_mnesia(); end_per_testcase(_Case, _Config) -> ok. @@ -163,8 +263,11 @@ get() -> get(Config) when is_list(Config) -> Version = ?config(http_version, Config), Host = ?config(host, Config), + Type = ?config(type, Config), ok = httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), http_request("GET /index.html ", Version, Host), [{statuscode, 200}, {header, "Content-Type", "text/html"}, @@ -172,6 +275,15 @@ get(Config) when is_list(Config) -> {header, "Server"}, {version, Version}]). +basic_auth_1_1(Config) when is_list(Config) -> + basic_auth([{http_version, "HTTP/1.1"} | Config]). + +basic_auth_1_0(Config) when is_list(Config) -> + basic_auth([{http_version, "HTTP/1.0"} | Config]). + +basic_auth_0_9(Config) when is_list(Config) -> + basic_auth([{http_version, "HTTP/0.9"} | Config]). + basic_auth() -> [{doc, "Test Basic authentication with WWW-Authenticate header"}]. @@ -203,13 +315,211 @@ basic_auth(Config) -> Config, [{statuscode, 200}]), %% Authentication still required! basic_auth_requiered(Config). - + +auth_api_1_1(Config) when is_list(Config) -> + auth_api([{http_version, "HTTP/1.1"} | Config]). + +auth_api_1_0(Config) when is_list(Config) -> + auth_api([{http_version, "HTTP/1.0"} | Config]). + +auth_api_0_9(Config) when is_list(Config) -> + auth_api([{http_version, "HTTP/0.9"} | Config]). + +auth_api() -> + [{doc, "Test mod_auth API"}]. + +auth_api(Config) when is_list(Config) -> + Prefix = ?config(auth_prefix, Config), + do_auth_api(Prefix, Config). + +do_auth_api(AuthPrefix, Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + Port = ?config(port, Config), + Node = ?config(node, Config), + ServerRoot = ?config(server_root, Config), + ok = http_status("GET / ", Config, + [{statuscode, 200}]), + ok = auth_status(auth_request("/", "one", "WrongPassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Make sure Authenticate header is received even the second time + %% we try a incorrect password! Otherwise a browser client will hang! + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "dummy", "WrongPassword", Version, Host), Config, + [{statuscode, 401}, + {header, "WWW-Authenticate"}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", "dummy", "WrongPassword", + Version, Host), Config, [{statuscode, 401}, + {header, "WWW-Authenticate"}]), + + %% Change the password to DummyPassword then try to add a user + %% Get an error and set it to NoPassword + ok = update_password(Node, ServerRoot, Host, Port, AuthPrefix, + "open", "NoPassword", "DummyPassword"), + {error,bad_password} = + add_user(Node, ServerRoot, Port, AuthPrefix, "open", "one", + "onePassword", []), + ok = update_password(Node, ServerRoot, Host, Port, AuthPrefix, "open", + "DummyPassword", "NoPassword"), + + %% Test /*open, require user one Aladdin + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "open"), + + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "one", "onePassword", Version, Host), Config, + [{statuscode, 401}]), + + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "two", "twoPassword", Version, Host), Config, + [{statuscode, 401}]), + + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "Aladdin", "onePassword", Version, Host), + Config, [{statuscode, 401}]), + + true = add_user(Node, ServerRoot, Port, AuthPrefix, "open", "one", + "onePassword", []), + true = add_user(Node, ServerRoot, Port, AuthPrefix, "open", "two", + "twoPassword", []), + true = add_user(Node, ServerRoot, Port, AuthPrefix, "open", "Aladdin", + "AladdinPassword", []), + {ok, [_|_]} = list_users(Node, ServerRoot, Host, Port, + AuthPrefix, "open"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "one", "WrongPassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "one", "onePassword", Version, Host), + Config, [{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "two", "twoPassword", Version, Host), + Config,[{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "Aladdin", "WrongPassword", Version, Host), + Config,[{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "Aladdin", "AladdinPassword", Version, Host), + Config, [{statuscode, 200}]), + + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "open"), + {ok, []} = list_users(Node, ServerRoot, Host, Port, + AuthPrefix, "open"), + + %% Phase 2 + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "secret"), + {ok, []} = list_users(Node, ServerRoot, Host, Port, AuthPrefix, + "secret"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "one", "onePassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "two", "twoPassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "three", "threePassword", Version, Host), + Config, [{statuscode, 401}]), + add_user(Node, ServerRoot, Port, AuthPrefix, "secret", "one", + "onePassword", + []), + add_user(Node, ServerRoot, Port, AuthPrefix, "secret", + "two", "twoPassword", []), + add_user(Node, ServerRoot, Port, AuthPrefix, "secret", "Aladdin", + "AladdinPassword",[]), + add_group_member(Node, ServerRoot, Port, AuthPrefix, "secret", + "one", "group1"), + add_group_member(Node, ServerRoot, Port, AuthPrefix, "secret", + "two", "group1"), + add_group_member(Node, ServerRoot, Port, AuthPrefix, + "secret", "Aladdin", "group2"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "one", "onePassword", Version, Host), + Config, [{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "two", "twoPassword", Version, Host), + Config,[{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "Aladdin", "AladdinPassword", Version, Host), + Config, [{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "three", "threePassword", Version, Host), + Config, [{statuscode, 401}]), + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "secret"), + {ok, []} = list_users(Node, ServerRoot, Host, Port, + AuthPrefix, "secret"), + remove_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret"), + + {ok, []} = list_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret"), + + %% Phase 3 + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + remove_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", + "three", "threePassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", "two", "twoPassword", Version, Host), + Config, [{statuscode, 401}]), + add_user(Node, ServerRoot, Port, AuthPrefix, + "secret/top_secret","three", + "threePassword",[]), + add_user(Node, ServerRoot, Port, AuthPrefix, "secret/top_secret", + "two","twoPassword", []), + add_group_member(Node, ServerRoot, Port, AuthPrefix, "secret/top_secret", "three", "group3"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", "three", "threePassword", + Version, Host), + Config, [{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", "two", "twoPassword", Version, Host), + Config, [{statuscode, 401}]), + add_group_member(Node, ServerRoot, Port, AuthPrefix, "secret/top_secret", "two", "group3"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", + "two", "twoPassword", Version, Host), + Config, [{statuscode, 200}]), + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + {ok, []} = list_users(Node, ServerRoot, Host, Port, + AuthPrefix, "secret/top_secret"), + remove_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + {ok, []} = list_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", "two", "twoPassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/","three", "threePassword", Version, Host), + Config, [{statuscde, 401}]). +%%------------------------------------------------------------------------- +ipv6() -> + [{require, ipv6_hosts}, + {doc,"Test ipv6."}]. +ipv6(Config) when is_list(Config) -> + {ok, Hostname0} = inet:gethostname(), + case lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) of + true -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + URI = http_request("GET /", Version, Host), + httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), [inet6], + ?config(code, Config), + URI, + [{statuscode, 200}, {version, Version}]); + false -> + {skip, "Host does not support IPv6"} + end. + +%%------------------------------------------------------------------------- ssi() -> [{doc, "HTTP GET server side include test"}]. ssi(Config) when is_list(Config) -> Version = ?config(http_version, Config), Host = ?config(host, Config), + Type = ?config(type, Config), ok = httpd_test_lib:verify_request(?config(type, Config), Host, ?config(port, Config), + transport_opts(Type, Config), ?config(node, Config), http_request("GET /fsize.shtml ", Version, Host), [{statuscode, 200}, @@ -217,6 +527,131 @@ ssi(Config) when is_list(Config) -> {header, "Date"}, {header, "Server"}, {version, Version}]). +%%------------------------------------------------------------------------- +htaccess_1_1(Config) when is_list(Config) -> + htaccess([{http_version, "HTTP/1.1"} | Config]). + +htaccess_1_0(Config) when is_list(Config) -> + htaccess([{http_version, "HTTP/1.0"} | Config]). + +htaccess_0_9(Config) when is_list(Config) -> + htaccess([{http_version, "HTTP/0.9"} | Config]). + +htaccess() -> + [{doc, "Test mod_auth API"}]. + +htaccess(Config) when is_list(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + Type = ?config(type, Config), + Port = ?config(port, Config), + Node = ?config(node, Config), + %% Control that authentication required! + %% Control that the pages that shall be + %% authenticated really need authenticatin + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/open/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/secret/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/secret/top_secret/ ", + Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + + %% Make sure Authenticate header is received even the second time + %% we try a incorrect password! Otherwise a browser client will hang! + ok = auth_status(auth_request("/ht/open/", + "dummy", "WrongPassword", Version, Host), Config, + [{statuscode, 401}, + {header, "WWW-Authenticate"}]), + ok = auth_status(auth_request("/ht/open/", + "dummy", "WrongPassword", Version, Host), Config, + [{statuscode, 401}, + {header, "WWW-Authenticate"}]), + + %% Control that not just the first user in the list is valid + %% Control the first user + %% Authennticating ["one:OnePassword" user first in user list] + ok = auth_status(auth_request("/ht/open/dummy.html", "one", "OnePassword", + Version, Host), Config, + [{statuscode, 200}]), + + %% Control the second user + %% Authentication OK and a directory listing is supplied! + %% ["Aladdin:open sesame" user second in user list] + ok = auth_status(auth_request("/ht/open/","Aladdin", + "AladdinPassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Contro that bad passwords and userids get a good denial + %% User correct but wrong password! ["one:one" user first in user list] + ok = auth_status(auth_request("/ht/open/", "one", "one", Version, Host), Config, + [{statuscode, 401}]), + %% Neither user or password correct! ["dummy:dummy"] + ok = auth_status(auth_request("/ht/open/", "dummy", "dummy", Version, Host), Config, + [{statuscode, 401}]), + + %% Control that authetication still works, even if its a member in a group + %% Authentication OK! ["two:TwoPassword" user in first group] + ok = auth_status(auth_request("/ht/secret/dummy.html", "two", + "TwoPassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Authentication OK and a directory listing is supplied! + %% ["three:ThreePassword" user in second group] + ok = auth_status(auth_request("/ht/secret/", "three", + "ThreePassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Deny users with bad passwords even if the user is a group member + %% User correct but wrong password! ["two:two" user in first group] + ok = auth_status(auth_request("/ht/secret/", "two", "two", Version, Host), Config, + [{statuscode, 401}]), + %% Neither user or password correct! ["dummy:dummy"] + ok = auth_status(auth_request("/ht/secret/", "dummy", "dummy", Version, Host), Config, + [{statuscode, 401}]), + + %% control that we deny the users that are in subnet above the allowed + ok = auth_status(auth_request("/ht/blocknet/dummy.html", "four", + "FourPassword", Version, Host), Config, + [{statuscode, 403}]), + %% Control that we only applies the rules to the right methods + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("HEAD /ht/blocknet/dummy.html ", Version, Host), + [{statuscode, head_status(Version)}, + {version, Version}]), + + %% Control that the rerquire directive can be overrideen + ok = auth_status(auth_request("/ht/secret/top_secret/ ", "Aladdin", "AladdinPassword", + Version, Host), Config, + [{statuscode, 401}]), + + %% Authentication still required! + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/open/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/secret/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/secret/top_secret/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]). + +%%------------------------------------------------------------------------- host() -> [{doc, "Test host header"}]. @@ -224,21 +659,21 @@ host(Config) when is_list(Config) -> Cb = ?config(version_cb, Config), Cb:host(?config(type, Config), ?config(port, Config), ?config(host, Config), ?config(node, Config)). - +%%------------------------------------------------------------------------- chunked() -> [{doc, "Check that the server accepts chunked requests."}]. chunked(Config) when is_list(Config) -> httpd_1_1:chunked(?config(type, Config), ?config(port, Config), ?config(host, Config), ?config(node, Config)). - +%%------------------------------------------------------------------------- expect() -> ["Check that the server handles request with the expect header " "field appropiate"]. expect(Config) when is_list(Config) -> httpd_1_1:expect(?config(type, Config), ?config(port, Config), ?config(host, Config), ?config(node, Config)). - +%%------------------------------------------------------------------------- max_clients_1_1() -> [{doc, "Test max clients limit"}]. @@ -256,7 +691,7 @@ max_clients_0_9() -> max_clients_0_9(Config) when is_list(Config) -> do_max_clients([{http_version, "HTTP/0.9"} | Config]). - +%%------------------------------------------------------------------------- esi() -> [{doc, "Test mod_esi"}]. @@ -286,7 +721,7 @@ esi(Config) when is_list(Config) -> ok = http_status("GET /cgi-bin/erl/httpd_example:get ", Config, [{statuscode, 200}, {no_header, "cache-control"}]). - +%%------------------------------------------------------------------------- cgi() -> [{doc, "Test mod_cgi"}]. @@ -361,7 +796,27 @@ cgi(Config) when is_list(Config) -> ok = http_status("GET /cgi-bin/" ++ Script ++ " ", Config, [{statuscode, 200}, {no_header, "cache-control"}]). - +%%------------------------------------------------------------------------- +cgi_chunked_encoding_test() -> + [{doc, "Test chunked encoding together with mod_cgi "}]. +cgi_chunked_encoding_test(Config) when is_list(Config) -> + Host = ?config(host, Config), + Script = + case test_server:os_type() of + {win32, _} -> + "/cgi-bin/printenv.bat"; + _ -> + "/cgi-bin/printenv.sh" + end, + Requests = + ["GET " ++ Script ++ " HTTP/1.1\r\nHost:"++ Host ++"\r\n\r\n", + "GET /cgi-bin/erl/httpd_example/newformat HTTP/1.1\r\nHost:" + ++ Host ++"\r\n\r\n"], + httpd_1_1:mod_cgi_chunked_encoding_test(?config(type, Config), ?config(port, Config), + Host, + ?config(node, Config), + Requests). +%%------------------------------------------------------------------------- alias() -> [{doc, "Test mod_alias"}]. @@ -389,160 +844,246 @@ alias(Config) when is_list(Config) -> [{statuscode, 301}, {header, "Location"}, {header, "Content-Type","text/html"}]). +%%------------------------------------------------------------------------- +actions() -> + [{doc, "Test mod_actions"}]. +actions(Config) when is_list(Config) -> + ok = http_status("GET /", Config, [{statuscode, 200}]). -%% auth_api() -> -%% [{doc, "Test mod_auth API"}]. - -%% auth_api(Config) when is_list(Config) -> -%% Version = ?config(http_version, Config), -%% Host = ?config(host, Config), -%% ok = http_status("GET / ", Config, -%% [{statuscode, 200}]), -%% ok = auth_status(auth_request("/", "one", "WrongPassword", Version, Host), Config, -%% [{statuscode, 200}]), - -%% %% Make sure Authenticate header is received even the second time -%% %% we try a incorrect password! Otherwise a browser client will hang! -%% ok = auth_status(auth_request("/" ++ AuthStoreType ++ "open/", -%% "dummy", "WrongPassword", Host), Config, -%% [{statuscode, 401}, -%% {header, "WWW-Authenticate"}]), -%% ok = auth_status(auth_request("/" ++ AuthStoreType ++ "open/", "dummy", "WrongPassword", -%% Host), Config, [{statuscode, 401}, -%% {header, "WWW-Authenticate"}]), - -%% %% Change the password to DummyPassword then try to add a user -%% %% Get an error and set it to NoPassword -%% ok = update_password(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "open", "NoPassword", "DummyPassword"), -%% {error,bad_password} = -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "open", "one", -%% "onePassword", []), -%% ok = update_password(Node, ServerRoot, Host, Port, AuthStoreType ++"open", -%% "DummyPassword", "NoPassword"), - -%% %% Test /*open, require user one Aladdin -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType ++ "open"), +%%------------------------------------------------------------------------- +range() -> + [{doc, "Test Range header"}]. + +range(Config) when is_list(Config) -> + httpd_1_1:range(?config(type, Config), ?config(port, Config), + ?config(host, Config), ?config(node, Config)). + +%%------------------------------------------------------------------------- +if_modified_since() -> + [{doc, "Test If-Modified-Since header"}]. + +if_modified_since(Config) when is_list(Config) -> + httpd_1_1:if_test(?config(type, Config), ?config(port, Config), + ?config(host, Config), ?config(node, Config), + ?config(doc_root, Config)). +%%------------------------------------------------------------------------- +trace() -> + [{doc, "Test TRACE method"}]. + +trace(Config) when is_list(Config) -> + Cb = ?config(version_cb, Config), + Cb:trace(?config(type, Config), ?config(port, Config), + ?config(host, Config), ?config(node, Config)). + +%%------------------------------------------------------------------------- +light() -> + ["Test light load"]. +light(Config) when is_list(Config) -> + httpd_load:load_test(?config(type, Config), ?config(port, Config), ?config(host, Config), + ?config(node, Config), 10). +%%------------------------------------------------------------------------- +medium() -> + ["Test medium load"]. +medium(Config) when is_list(Config) -> + httpd_load:load_test(?config(type, Config), ?config(port, Config), ?config(host, Config), + ?config(node, Config), 100). +%%------------------------------------------------------------------------- +heavy() -> + ["Test heavy load"]. +heavy(Config) when is_list(Config) -> + httpd_load:load_test(?config(type, Config), ?config(port, Config), ?config(host, Config), + ?config(node, Config), + 1000). +%%------------------------------------------------------------------------- +content_length() -> + ["Tests that content-length is correct OTP-5775"]. +content_length(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + ok = httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), ?config(node, Config), + http_request("GET /cgi-bin/erl/httpd_example:get_bin ", + Version, Host), + [{statuscode, 200}, + {content_length, 274}, + {version, Version}]). +%%------------------------------------------------------------------------- +bad_hex() -> + ["Tests that a URI with a bad hexadecimal code is handled OTP-6003"]. +bad_hex(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + ok = httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), ?config(node, Config), + http_request("GET http://www.erlang.org/%skalle ", + Version, Host), + [{statuscode, 400}, + {version, Version}]). +%%------------------------------------------------------------------------- +missing_CR() -> + ["Tests missing CR in delimiter OTP-7304"]. +missing_CR(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + ok = httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), ?config(node, Config), + http_request_missing_CR("GET /index.html ", Version, Host), + [{statuscode, 200}, + {version, Version}]). + +%%------------------------------------------------------------------------- +max_header() -> + ["Denial Of Service (DOS) attack, prevented by max_header"]. +max_header(Config) when is_list(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + case Version of + "HTTP/0.9" -> + {skip, no_implemented}; + _ -> + dos_hostname(?config(type, Config), ?config(port, Config), Host, + ?config(node, Config), Version, ?MAX_HEADER_SIZE) + end. + +%%------------------------------------------------------------------------- +security_1_1(Config) when is_list(Config) -> + security([{http_version, "HTTP/1.1"} | Config]). + +security_1_0(Config) when is_list(Config) -> + security([{http_version, "HTTP/1.0"} | Config]). + +security() -> + ["Test mod_security"]. +security(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + Port = ?config(port, Config), + Node = ?config(node, Config), + ServerRoot = ?config(server_root, Config), + + global:register_name(mod_security_test, self()), % Receive events + + test_server:sleep(5000), + + OpenDir = filename:join([ServerRoot, "htdocs", "open"]), + + %% Test blocking / unblocking of users. -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "one", "onePassword", [{statuscode, 401}]), + %% /open, require user one Aladdin + remove_users(Node, ServerRoot, Host, Port, "", "open"), + + ok = auth_status(auth_request("/open/", + "one", "onePassword", Version, Host), Config, + [{statuscode, 401}]), + + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "one"}, {password, "onePassword"}]}, + Node, Port), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "two", "twoPassword", [{statuscode, 401}]), + ok = auth_status(auth_request("/open/", + "two", "twoPassword", Version, Host), Config, + [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "Aladdin", "onePassword", [{statuscode, 401}]), - -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "open", "one", -%% "onePassword", []), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "open", "two", -%% "twoPassword", []), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "open", "Aladdin", -%% "AladdinPassword", []), + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "two"}, {password, "twoPassword"}]}, + Node, Port), + + ok = auth_status(auth_request("/open/", + "Aladdin", "AladdinPassword", Version, Host), + Config, [{statuscode, 401}]), + + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "Aladdin"}, + {password, "AladdinPassword"}]}, + Node, Port), + + add_user(Node, ServerRoot, Port, "", "open", "one", "onePassword", []), + add_user(Node, ServerRoot, Port, "", "open", "two", "twoPassword", []), + + ok = auth_status(auth_request("/open/", "one", "WrongPassword", Version, Host), Config, + [{statuscode, 401}]), + + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "one"}, {password, "WrongPassword"}]}, + Node, Port), + + ok = auth_status(auth_request("/open/", "one", "WrongPassword", Version, Host), Config, + [{statuscode, 401}]), + + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "one"}, {password, "WrongPassword"}]}, + Node, Port), + receive_security_event({event, user_block, Port, OpenDir, + [{user, "one"}]}, Node, Port), + + global:unregister_name(mod_security_test), % No more events. + + ok = auth_status(auth_request("/open/", "one", "WrongPassword", Version, Host), Config, + [{statuscode, 401}]), + + %% User "one" should be blocked now.. + case list_blocked_users(Node, Port) of + [{"one",_, Port, OpenDir,_}] -> + ok; + Blocked -> + ct:fail({unexpected_blocked, Blocked}) + end, + + [{"one",_, Port, OpenDir,_}] = list_blocked_users(Node, Port, OpenDir), + + true = unblock_user(Node, "one", Port, OpenDir), + %% User "one" should not be blocked any more. + + [] = list_blocked_users(Node, Port), + + ok = auth_status(auth_request("/open/", "one", "onePassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Test list_auth_users & auth_timeout + + ["one"] = list_auth_users(Node, Port), + + ok = auth_status(auth_request("/open/", "two", "onePassword", Version, Host), Config, + [{statuscode, 401}]), + + ["one"] = list_auth_users(Node, Port), + -%% {ok, [_|_]} = list_users(Node, ServerRoot, Host, Port, -%% AuthStoreType++"open"), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ "open/", -%% "one", "WrongPassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ "open/", -%% "one", "onePassword", [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "two", "twoPassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ "open/", -%% "Aladdin", "WrongPassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "Aladdin", "AladdinPassword", [{statuscode, 200}]), + ["one"] = list_auth_users(Node, Port, OpenDir), + -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType++"open"), -%% {ok, []} = list_users(Node, ServerRoot, Host, Port, -%% AuthStoreType++"open"), - -%% %% Phase 2 -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType++"secret"), -%% {ok, []} = list_users(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret"), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "one", "onePassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "two", "twoPassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ "secret/", -%% "three", "threePassword", [{statuscode, 401}]), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "secret", "one", -%% "onePassword", -%% []), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "secret", -%% "two", "twoPassword", []), -%% add_user(Node, ServerRoot, Port, AuthStoreType++"secret", "Aladdin", -%% "AladdinPassword",[]), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ "secret", -%% "one", "group1"), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ "secret", -%% "two", "group1"), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ -%% "secret", "Aladdin", "group2"), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "one", "onePassword", [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "two", "twoPassword", [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "Aladdin", "AladdinPassword", [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "three", "threePassword", [{statuscode, 401}]), -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType ++ "secret"), -%% {ok, []} = list_users(Node, ServerRoot, Host, Port, -%% AuthStoreType ++ "secret"), -%% remove_groups(Node, ServerRoot, Host, Port, AuthStoreType ++ "secret"), -%% Directory = filename:join([ServerRoot, "htdocs", AuthStoreType ++ -%% "secret"]), -%% {ok, []} = list_groups(Node, ServerRoot, Host, Port, Directory), - -%% %% Phase 3 -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret/top_secret"), -%% remove_groups(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret/top_secret"), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", -%% "three", "threePassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", "two", "twoPassword", -%% [{statuscode, 401}]), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ -%% "secret/top_secret","three", -%% "threePassword",[]), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "secret/top_secret", -%% "two","twoPassword", []), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ -%% "secret/top_secret", -%% "three", "group3"), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", "three", "threePassword", -%% [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", "two", "twoPassword", -%% [{statuscode, 401}]), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ -%% "secret/top_secret", -%% "two", "group3"), -%% auth_request(Type,Host,Port,Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", -%% "two", "twoPassword", [{statuscode, 200}]), -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret/top_secret"), -%% {ok, []} = list_users(Node, ServerRoot, Host, Port, -%% AuthStoreType ++ "secret/top_secret"), -%% remove_groups(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret/top_secret"), -%% Directory2 = filename:join([ServerRoot, "htdocs", -%% AuthStoreType ++ "secret/top_secret"]), -%% {ok, []} = list_groups(Node, ServerRoot, Host, Port, Directory2), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ -%% "secret/top_secret/", "two", "twoPassword", -%% [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ -%% "secret/top_secret/","three", "threePassword", -%% [{statuscode, 401}]). + ok = auth_status(auth_request("/open/", "two", "twoPassword", Version, Host), Config, + [{statuscode, 401}]), + + ["one"] = list_auth_users(Node, Port), + + + ["one"] = list_auth_users(Node, Port, OpenDir), + + %% Wait for successful auth to timeout. + test_server:sleep(?AUTH_TIMEOUT*1001), + + [] = list_auth_users(Node, Port), + + [] = list_auth_users(Node, Port, OpenDir), + + %% "two" is blocked. + + true = unblock_user(Node, "two", Port, OpenDir), + + + %% Test explicit blocking. Block user 'two'. + + [] = list_blocked_users(Node,Port,OpenDir), + + true = block_user(Node, "two", Port, OpenDir, 10), + + ok = auth_status(auth_request("/open/", "two", "twoPassword", Version, Host), Config, + [{statuscode, 401}]), + + true = unblock_user(Node, "two", Port, OpenDir). + %%-------------------------------------------------------------------- @@ -550,21 +1091,34 @@ alias(Config) when is_list(Config) -> %%-------------------------------------------------------------------- do_max_clients(Config) -> Version = ?config(http_version, Config), - Host = ?config(host, Config), - start_blocker(Config), - ok = httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), - http_request("GET /index.html ", Version, Host), + Host = ?config(host, Config), + Port = ?config(port, Config), + Type = ?config(type, Config), + + Request = http_request("GET /index.html ", Version, Host), + BlockRequest = http_request("GET /eval?httpd_example:delay(2000) ", Version, Host), + {ok, Socket} = inets_test_lib:connect_bin(Type, Host, Port, transport_opts(Type, Config)), + inets_test_lib:send(Type, Socket, BlockRequest), + ct:sleep(100), + ok = httpd_test_lib:verify_request(Type, Host, + Port, + transport_opts(Type, Config), + ?config(node, Config), + Request, [{statuscode, 503}, {version, Version}]), receive - after 2000 -> - ok = httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), - http_request("GET /index.html ", Version, Host), - [{statuscode, 200}, - {version, Version}]) - end. + {_, Socket, _Msg} -> + ok + end, + inets_test_lib:close(Type, Socket), + ok = httpd_test_lib:verify_request(Type, Host, + Port, + transport_opts(Type, Config), + ?config(node, Config), + Request, + [{statuscode, 200}, + {version, Version}]). setup_server_dirs(ServerRoot, DocRoot, DataDir) -> CgiDir = filename:join(ServerRoot, "cgi-bin"), @@ -604,10 +1158,24 @@ setup_server_dirs(ServerRoot, DocRoot, DataDir) -> ok = file:write_file_info(EnvCGI, FileInfo1#file_info{mode = 8#00755}). -start_apps(https) -> - inets_test_lib:start_apps([crypto, public_key, ssl]); -start_apps(_) -> - ok. +start_apps(Group) when Group == https_basic; + Group == https_limit; + Group == https_basic_auth; + Group == https_auth_api; + Group == https_auth_api_dets; + Group == https_auth_api_mnesia; + Group == http_htaccess; + Group == http_security -> + inets_test_lib:start_apps([inets, asn1, crypto, public_key, ssl]); +start_apps(Group) when Group == http_basic; + Group == http_limit; + Group == http_basic_auth; + Group == http_auth_api; + Group == http_auth_api_dets; + Group == http_auth_api_mnesia; + Group == https_htaccess; + Group == https_security -> + inets_test_lib:start_apps([inets]). server_start(_, HttpdConfig) -> {ok, Pid} = inets:start(httpd, HttpdConfig), @@ -615,6 +1183,80 @@ server_start(_, HttpdConfig) -> {value, {_, _, Info}} = lists:keysearch(Pid, 2, Serv), {Pid, proplists:get_value(port, Info)}. +init_ssl(Group, Config) -> + PrivDir = ?config(priv_dir, Config), + CaKey = {_Trusted,_} = + erl_make_certs:make_cert([{key, dsa}, + {subject, + [{name, "Public Key"}, + {?'id-at-name', + {printableString, "public_key"}}, + {?'id-at-pseudonym', + {printableString, "pubkey"}}, + {city, "Stockholm"}, + {country, "SE"}, + {org, "erlang"}, + {org_unit, "testing dep"} + ]} + ]), + ok = erl_make_certs:write_pem(PrivDir, "public_key_cacert", CaKey), + + CertK1 = {_Cert1, _} = erl_make_certs:make_cert([{issuer, CaKey}]), + CertK2 = {_Cert2,_} = erl_make_certs:make_cert([{issuer, CertK1}, + {digest, md5}, + {extensions, false}]), + ok = erl_make_certs:write_pem(PrivDir, "public_key_cert", CertK2), + + case start_apps(Group) of + ok -> + init_httpd(Group, [{type, ssl} | Config]); + _ -> + {skip, "Could not start https apps"} + end. + +server_config(http_basic, Config) -> + basic_conf() ++ server_config(http, Config); +server_config(https_basic, Config) -> + basic_conf() ++ server_config(https, Config); +server_config(http_limit, Config) -> + [{max_clients, 1}] ++ server_config(http, Config); +server_config(https_limit, Config) -> + [{max_clients, 1}] ++ server_config(https, Config); +server_config(http_basic_auth, Config) -> + ServerRoot = ?config(server_root, Config), + auth_conf(ServerRoot) ++ server_config(http, Config); +server_config(https_basic_auth, Config) -> + ServerRoot = ?config(server_root, Config), + auth_conf(ServerRoot) ++ server_config(https, Config); +server_config(http_auth_api, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, plain) ++ server_config(http, Config); +server_config(https_auth_api, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, plain) ++ server_config(https, Config); +server_config(http_auth_api_dets, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, dets) ++ server_config(http, Config); +server_config(https_auth_api_dets, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, dets) ++ server_config(https, Config); +server_config(http_auth_api_mnesia, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, mnesia) ++ server_config(http, Config); +server_config(https_auth_api_mnesia, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, mnesia) ++ server_config(https, Config); +server_config(http_htaccess, Config) -> + auth_access_conf() ++ server_config(http, Config); +server_config(https_htaccess, Config) -> + auth_access_conf() ++ server_config(https, Config); +server_config(http_security, Config) -> + ServerRoot = ?config(server_root, Config), + tl(auth_conf(ServerRoot)) ++ security_conf(ServerRoot) ++ server_config(http, Config); +server_config(https_security, Config) -> + ServerRoot = ?config(server_root, Config), + tl(auth_conf(ServerRoot)) ++ security_conf(ServerRoot) ++ server_config(https, Config); + server_config(http, Config) -> ServerRoot = ?config(server_root, Config), [{port, 0}, @@ -625,6 +1267,7 @@ server_config(http, Config) -> {ipfamily, inet}, {max_header_size, 256}, {max_header_action, close}, + {directory_index, ["index.html", "welcome.html"]}, {mime_types, [{"html","text/html"},{"htm","text/html"}, {"shtml","text/html"}, {"gif", "image/gif"}]}, {alias, {"/icons/", filename:join(ServerRoot,"icons") ++ "/"}}, @@ -633,13 +1276,24 @@ server_config(http, Config) -> {script_alias, {"/htbin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}}, {erl_script_alias, {"/cgi-bin/erl", [httpd_example, io]}}, {eval_script_alias, {"/eval", [httpd_example, io]}} - ] ++ auth_conf(ServerRoot); + ]; -server_config(http_limit, Config) -> - [{max_clients, 1}] ++ server_config(http, Config); - -server_config(_, _) -> - []. +server_config(https, Config) -> + PrivDir = ?config(priv_dir, Config), + [{socket_type, {essl, + [{cacertfile, + filename:join(PrivDir, "public_key_cacert.pem")}, + {certfile, + filename:join(PrivDir, "public_key_cert.pem")}, + {keyfile, + filename:join(PrivDir, "public_key_cert_key.pem")} + ]}}] ++ server_config(http, Config). + +init_httpd(Group, Config0) -> + Config1 = proplists:delete(port, Config0), + Config = proplists:delete(server_pid, Config1), + {Pid, Port} = server_start(Group, server_config(Group, Config)), + [{server_pid, Pid}, {port, Port} | Config]. http_request(Request, "HTTP/1.1" = Version, Host, {Headers, Body}) -> Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n" ++ Headers ++ "\r\n" ++ Body; @@ -662,19 +1316,33 @@ auth_request(Path, User, Passwd, Version, _Host) -> base64:encode_to_string(User++":"++Passwd) ++ "\r\n\r\n". +http_request_missing_CR(Request, "HTTP/1.1" = Version, Host) -> + Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n\r\n\n"; +http_request_missing_CR(Request, Version, _) -> + Request ++ Version ++ "\r\n\n". + head_status("HTTP/0.9") -> 501; %% Not implemented in HTTP/0.9 head_status(_) -> 200. +basic_conf() -> + [{modules, [mod_alias, mod_range, mod_responsecontrol, + mod_trace, mod_esi, mod_cgi, mod_dir, mod_get, mod_head]}]. + +auth_access_conf() -> + [{modules, [mod_alias, mod_htaccess, mod_dir, mod_get, mod_head]}, + {access_files, [".htaccess"]}]. + auth_conf(Root) -> - [{directory, {filename:join(Root, "htdocs/open"), + [{modules, [mod_alias, mod_auth, mod_dir, mod_get, mod_head]}, + {directory, {filename:join(Root, "htdocs/open"), [{auth_type, plain}, {auth_name, "Open Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, {auth_group_file, filename:join(Root, "auth/group")}, {require_user, ["one", "Aladdin"]}]}}, - {directory, {filename:join(Root, "htdocs/secret"), + {directory, {filename:join(Root, "htdocs/secret"), [{auth_type, plain}, {auth_name, "Secret Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, @@ -685,43 +1353,134 @@ auth_conf(Root) -> {auth_name, "Top Secret Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, {auth_group_file, filename:join(Root, "auth/group")}, - {require_group, ["group3"]}]}}, + {require_group, ["group3"]}]}}]. + +auth_api_conf(Root, plain) -> + [{modules, [mod_alias, mod_auth, mod_dir, mod_get, mod_head]}, {directory, {filename:join(Root, "htdocs/open"), - [{auth_type, mnesia}, + [{auth_type, plain}, {auth_name, "Open Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, {auth_group_file, filename:join(Root, "auth/group")}, {require_user, ["one", "Aladdin"]}]}}, {directory, {filename:join(Root, "htdocs/secret"), - [{auth_type, mnesia}, + [{auth_type, plain}, {auth_name, "Secret Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, {auth_group_file, filename:join(Root, "auth/group")}, - {require_group, ["group1", "group2"]}]}} - ]. + {require_group, ["group1", "group2"]}]}}, + {directory, {filename:join(Root, "htdocs/secret/top_secret"), + [{auth_type, plain}, + {auth_name, "Top Secret Area"}, + {auth_user_file, filename:join(Root, "auth/passwd")}, + {auth_group_file, filename:join(Root, "auth/group")}, + {require_group, ["group3"]}]}}]; +auth_api_conf(Root, dets) -> + [ + {modules, [mod_alias, mod_auth, mod_dir, mod_get, mod_head]}, + {directory, {filename:join(Root, "htdocs/dets_open"), + [{auth_type, dets}, + {auth_name, "Dets Open Area"}, + {auth_user_file, filename:join(Root, "passwd")}, + {auth_group_file, filename:join(Root, "group")}, + {require_user, ["one", "Aladdin"]}]}}, + {directory, {filename:join(Root, "htdocs/dets_secret"), + [{auth_type, dets}, + {auth_name, "Dests Secret Area"}, + {auth_user_file, filename:join(Root, "passwd")}, + {auth_group_file, filename:join(Root, "group")}, + {require_group, ["group1", "group2"]}]}}, + {directory, {filename:join(Root, "htdocs/dets_secret/top_secret"), + [{auth_type, dets}, + {auth_name, "Dets Top Secret Area"}, + {auth_user_file, filename:join(Root, "passwd")}, + {auth_group_file, filename:join(Root, "group")}, + {require_group, ["group3"]}]}} + ]; + +auth_api_conf(Root, mnesia) -> + [{modules, [mod_alias, mod_auth, mod_dir, mod_get, mod_head]}, + {directory, {filename:join(Root, "htdocs/mnesia_open"), + [{auth_type, mnesia}, + {auth_name, "Mnesia Open Area"}, + {require_user, ["one", "Aladdin"]}]}}, + {directory, {filename:join(Root, "htdocs/mnesia_secret"), + [{auth_type, mnesia}, + {auth_name, "Mnesia Secret Area"}, + {require_group, ["group1", "group2"]}]}}, + {directory, {filename:join(Root, "htdocs/mnesia_secret/top_secret"), + [{auth_type, mnesia}, + {auth_name, "Mnesia Top Secret Area"}, + {require_group, ["group3"]}]}}]. + +security_conf(Root) -> + SecFile = filename:join(Root, "security_data"), + Open = filename:join(Root, "htdocs/open"), + Secret = filename:join(Root, "htdocs/secret"), + TopSecret = filename:join(Root, "htdocs/secret/top_secret"), + + [{modules, [mod_alias, mod_auth, mod_security, mod_dir, mod_get, mod_head]}, + {security_directory, {Open, + [{auth_name, "Open Area"}, + {auth_user_file, filename:join(Root, "auth/passwd")}, + {auth_group_file, filename:join(Root, "auth/group")}, + {require_user, ["one", "Aladdin"]} | + mod_security_conf(SecFile, Open)]}}, + {security_directory, {Secret, + [{auth_name, "Secret Area"}, + {auth_user_file, filename:join(Root, "auth/passwd")}, + {auth_group_file, filename:join(Root, "auth/group")}, + {require_group, ["group1", "group2"]} | + mod_security_conf(SecFile, Secret)]}}, + {security_directory, {TopSecret, + [{auth_name, "Top Secret Area"}, + {auth_user_file, filename:join(Root, "auth/passwd")}, + {auth_group_file, filename:join(Root, "auth/group")}, + {require_group, ["group3"]} | + mod_security_conf(SecFile, TopSecret)]}}]. + +mod_security_conf(SecFile, Dir) -> + [{data_file, SecFile}, + {max_retries, 3}, + {fail_expire_time, ?FAIL_EXPIRE_TIME}, + {block_time, 1}, + {auth_timeout, ?AUTH_TIMEOUT}, + {callback_module, ?MODULE}, + {path, Dir} %% This is should not be needed, but is atm, awful design! + ]. + http_status(Request, Config, Expected) -> Version = ?config(http_version, Config), Host = ?config(host, Config), + Type = ?config(type, Config), httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), http_request(Request, Version, Host), Expected ++ [{version, Version}]). http_status(Request, HeadersAndBody, Config, Expected) -> Version = ?config(http_version, Config), - Host = ?config(host, Config), + Host = ?config(host, Config), + Type = ?config(type, Config), httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), http_request(Request, Version, Host, HeadersAndBody), Expected ++ [{version, Version}]). auth_status(AuthRequest, Config, Expected) -> Version = ?config(http_version, Config), Host = ?config(host, Config), + Type = ?config(type, Config), httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), AuthRequest, Expected ++ [{version, Version}]). @@ -772,23 +1531,258 @@ cleanup_mnesia() -> mnesia:delete_schema([node()]), ok. -start_blocker(Config) -> - spawn(httpd_SUITE, init_blocker, [self(), Config]), - receive - blocker_start -> +transport_opts(ssl, Config) -> + PrivDir = ?config(priv_dir, Config), + [{cacertfile, filename:join(PrivDir, "public_key_cacert.pem")}]; +transport_opts(_, _) -> + []. + + +%%% mod_range +create_range_data(Path) -> + PathAndFileName=filename:join([Path,"range.txt"]), + case file:read_file(PathAndFileName) of + {error, enoent} -> + file:write_file(PathAndFileName,list_to_binary(["12345678901234567890", + "12345678901234567890", + "12345678901234567890", + "12345678901234567890", + "12345678901234567890"])); + _ -> ok end. + +%%% mod_htaccess +create_htaccess_data(Path, IpAddress)-> + create_htaccess_dirs(Path), + + create_html_file(filename:join([Path,"ht/open/dummy.html"])), + create_html_file(filename:join([Path,"ht/blocknet/dummy.html"])), + create_html_file(filename:join([Path,"ht/secret/dummy.html"])), + create_html_file(filename:join([Path,"ht/secret/top_secret/dummy.html"])), -init_blocker(From, Config) -> - From ! blocker_start, - block(Config). + create_htaccess_file(filename:join([Path,"ht/open/.htaccess"]), + Path, "user one Aladdin"), + create_htaccess_file(filename:join([Path,"ht/secret/.htaccess"]), + Path, "group group1 group2"), + create_htaccess_file(filename:join([Path, + "ht/secret/top_secret/.htaccess"]), + Path, "user four"), + create_htaccess_file(filename:join([Path,"ht/blocknet/.htaccess"]), + Path, nouser, IpAddress), + + create_user_group_file(filename:join([Path,"ht","users.file"]), + "one:OnePassword\ntwo:TwoPassword\nthree:" + "ThreePassword\nfour:FourPassword\nAladdin:" + "AladdinPassword"), + create_user_group_file(filename:join([Path,"ht","groups.file"]), + "group1: two one\ngroup2: two three"). + +create_html_file(PathAndFileName)-> + file:write_file(PathAndFileName,list_to_binary( + "<html><head><title>test</title></head> + <body>testar</body></html>")). + +create_htaccess_file(PathAndFileName, BaseDir, RequireData)-> + file:write_file(PathAndFileName, + list_to_binary( + "AuthUserFile "++ BaseDir ++ + "/ht/users.file\nAuthGroupFile "++ BaseDir + ++ "/ht/groups.file\nAuthName Test\nAuthType" + " Basic\n<Limit>\nrequire " ++ RequireData ++ + "\n</Limit>")). + +create_htaccess_file(PathAndFileName, BaseDir, nouser, IpAddress)-> + file:write_file(PathAndFileName,list_to_binary( + "AuthUserFile "++ BaseDir ++ + "/ht/users.file\nAuthGroupFile " ++ + BaseDir ++ "/ht/groups.file\nAuthName" + " Test\nAuthType" + " Basic\n<Limit GET>\n\tallow from " ++ + format_ip(IpAddress, + string:rchr(IpAddress,$.)) ++ + "\n</Limit>")). + +create_user_group_file(PathAndFileName, Data)-> + file:write_file(PathAndFileName, list_to_binary(Data)). + +create_htaccess_dirs(Path)-> + ok = file:make_dir(filename:join([Path,"ht"])), + ok = file:make_dir(filename:join([Path,"ht/open"])), + ok = file:make_dir(filename:join([Path,"ht/blocknet"])), + ok = file:make_dir(filename:join([Path,"ht/secret"])), + ok = file:make_dir(filename:join([Path,"ht/secret/top_secret"])). + +remove_htaccess_dirs(Path)-> + file:del_dir(filename:join([Path,"ht/secret/top_secret"])), + file:del_dir(filename:join([Path,"ht/secret"])), + file:del_dir(filename:join([Path,"ht/blocknet"])), + file:del_dir(filename:join([Path,"ht/open"])), + file:del_dir(filename:join([Path,"ht"])). + +format_ip(IpAddress,Pos)when Pos > 0-> + case lists:nth(Pos,IpAddress) of + $.-> + case lists:nth(Pos-2,IpAddress) of + $.-> + format_ip(IpAddress,Pos-3); + _-> + lists:sublist(IpAddress,Pos-2) ++ "." + end; + _ -> + format_ip(IpAddress,Pos-1) + end; + +format_ip(IpAddress, _Pos)-> + "1" ++ IpAddress. + +remove_htaccess(Path)-> + file:delete(filename:join([Path,"ht/open/dummy.html"])), + file:delete(filename:join([Path,"ht/secret/dummy.html"])), + file:delete(filename:join([Path,"ht/secret/top_secret/dummy.html"])), + file:delete(filename:join([Path,"ht/blocknet/dummy.html"])), + file:delete(filename:join([Path,"ht/blocknet/.htaccess"])), + file:delete(filename:join([Path,"ht/open/.htaccess"])), + file:delete(filename:join([Path,"ht/secret/.htaccess"])), + file:delete(filename:join([Path,"ht/secret/top_secret/.htaccess"])), + file:delete(filename:join([Path,"ht","users.file"])), + file:delete(filename:join([Path,"ht","groups.file"])), + remove_htaccess_dirs(Path). + +dos_hostname(Type, Port, Host, Node, Version, Max) -> + TooLongHeader = lists:append(lists:duplicate(Max + 1, "a")), + + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + dos_hostname_request("", Version), + [{statuscode, 200}, + {version, Version}]), + + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + dos_hostname_request("dummy-host.ericsson.se", Version), + [{statuscode, 200}, + {version, Version}]), + + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + dos_hostname_request(TooLongHeader, Version), + [{statuscode, dos_code(Version)}, + {version, Version}]). +dos_hostname_request(Host, Version) -> + dos_http_request("GET / ", Version, Host). + +dos_http_request(Request, "HTTP/1.1" = Version, Host) -> + http_request(Request, Version, Host); +dos_http_request(Request, Version, Host) -> + Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n\r\n". + +dos_code("HTTP/1.0") -> + 403; %% 413 not defined in HTTP/1.0 +dos_code(_) -> + 413. + +update_password(Node, ServerRoot, _Address, Port, AuthPrefix, Dir, Old, New)-> + Directory = filename:join([ServerRoot, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, update_password, + [undefined, Port, Directory, Old, New, New]). + +add_user(Node, Root, Port, AuthPrefix, Dir, User, Password, UserData) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, add_user, + [User, Password, UserData, Addr, Port, Directory]). + + +delete_user(Node, Root, _Host, Port, AuthPrefix, Dir, User) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, delete_user, [User, Addr, Port, Directory]). +remove_users(Node, ServerRoot, Host, Port, AuthPrefix, Dir) -> + %% List users, delete them, and make sure they are gone. + case list_users(Node, ServerRoot, Host, Port, AuthPrefix, Dir) of + {ok, Users} -> + lists:foreach(fun(User) -> + delete_user(Node, ServerRoot, Host, + Port, AuthPrefix, Dir, User) + end, + Users), + {ok, []} = list_users(Node, ServerRoot, Host, Port, AuthPrefix, Dir); + _ -> + ok + end. + +list_users(Node, Root, _Host, Port, AuthPrefix, Dir) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, list_users, [Addr, Port, Directory]). + +remove_groups(Node, ServerRoot, Host, Port, AuthPrefix, Dir) -> + {ok, Groups} = list_groups(Node, ServerRoot, Host, Port, AuthPrefix, Dir), + lists:foreach(fun(Group) -> + delete_group(Node, Group, Port, ServerRoot, AuthPrefix, Dir) + end, + Groups), + {ok, []} = list_groups(Node, ServerRoot, Host, Port, AuthPrefix, Dir). + +delete_group(Node, Group, Port, Root, AuthPrefix, Dir) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, delete_group, [Group, Addr, Port, Directory]). + +list_groups(Node, Root, _, Port, AuthPrefix, Dir) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, list_groups, [Addr, Port, Directory]). + +add_group_member(Node, Root, Port, AuthPrefix, Dir, User, Group) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, add_group_member, [Group, User, Addr, Port, + Directory]). +getaddr() -> + {ok,HostName} = inet:gethostname(), + {ok,{A1,A2,A3,A4}} = inet:getaddr(HostName,inet), + lists:flatten(io_lib:format("~p.~p.~p.~p",[A1,A2,A3,A4])). + +receive_security_event(Event, Node, Port) -> + receive + Event -> + ok; + {'EXIT', _, _} -> + receive_security_event(Event, Node, Port) + after 5000 -> + %% Flush the message queue, to see if we got something... + inets_test_lib:flush() + end. + +list_blocked_users(Node,Port) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, list_blocked_users, [Addr,Port]). + +list_blocked_users(Node,Port,Dir) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, list_blocked_users, [Addr,Port,Dir]). + +block_user(Node,User,Port,Dir,Sec) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, block_user, [User, Addr, Port, Dir, Sec]). + +unblock_user(Node,User,Port,Dir) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, unblock_user, [User, Addr, Port, Dir]). + +list_auth_users(Node,Port) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, list_auth_users, [Addr,Port]). + +list_auth_users(Node,Port,Dir) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, list_auth_users, [Addr,Port,Dir]). + +event(What, Port, Dir, Data) -> + Msg = {event, What, Port, Dir, Data}, + case global:whereis_name(mod_security_test) of + undefined -> + ok; + _Pid -> + global:send(mod_security_test, Msg) + end. -block(Config) -> - Version = ?config(http_version, Config), - Host = ?config(host, Config), - httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), - http_request("GET /eval?httpd_example:delay(1000) ", - Version, Host), - [{statuscode, 200}, - {version, Version}]). diff --git a/lib/inets/test/httpd_basic_SUITE.erl b/lib/inets/test/httpd_basic_SUITE.erl index 2d06f3e70c..fbe65145dc 100644 --- a/lib/inets/test/httpd_basic_SUITE.erl +++ b/lib/inets/test/httpd_basic_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2012. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -65,7 +65,8 @@ end_per_group(_GroupName, Config) -> init_per_suite(Config) -> tsp("init_per_suite -> entry with" "~n Config: ~p", [Config]), - ok = inets:start(), + inets_test_lib:stop_apps([inets]), + inets_test_lib:start_apps([inets]), PrivDir = ?config(priv_dir, Config), DataDir = ?config(data_dir, Config), diff --git a/lib/inets/test/httpd_test_lib.erl b/lib/inets/test/httpd_test_lib.erl index 6406eeae79..ed466fd727 100644 --- a/lib/inets/test/httpd_test_lib.erl +++ b/lib/inets/test/httpd_test_lib.erl @@ -92,16 +92,6 @@ verify_request(SocketType, Host, Port, Node, RequestStr, Options, TimeOut) verify_request(SocketType, Host, Port, [], Node, RequestStr, Options, TimeOut). verify_request(SocketType, Host, Port, TranspOpts0, Node, RequestStr, Options, TimeOut) -> - tsp("verify_request -> entry with" - "~n SocketType: ~p" - "~n Host: ~p" - "~n Port: ~p" - "~n TranspOpts: ~p" - "~n Node: ~p" - "~n Options: ~p" - "~n TimeOut: ~p", - [SocketType, Host, Port, TranspOpts0, Node, Options, TimeOut]), - %% For now, until we modernize the httpd tests TranspOpts = case lists:member(inet6, TranspOpts0) of @@ -113,10 +103,7 @@ verify_request(SocketType, Host, Port, TranspOpts0, Node, RequestStr, Options, T try inets_test_lib:connect_bin(SocketType, Host, Port, TranspOpts) of {ok, Socket} -> - tsp("verify_request -> connected - now send message"), SendRes = inets_test_lib:send(SocketType, Socket, RequestStr), - tsp("verify_request -> send result: " - "~n ~p", [SendRes]), State = case inets_regexp:match(RequestStr, "printenv") of nomatch -> #state{}; @@ -127,37 +114,24 @@ verify_request(SocketType, Host, Port, TranspOpts0, Node, RequestStr, Options, T case request(State#state{request = RequestStr, socket = Socket}, TimeOut) of {error, Reason} -> - tsp("verify_request -> request failed: " - "~n Reason: ~p", [Reason]), {error, Reason}; NewState -> - tsp("verify_request -> validate reply: " - "~n NewState: ~p", [NewState]), ValidateResult = validate(RequestStr, NewState, Options, Node, Port), - tsp("verify_request -> validation result: " - "~n ~p", [ValidateResult]), inets_test_lib:close(SocketType, Socket), ValidateResult end; ConnectError -> - tsp("verify_request -> connect error: " - "~n ~p" - "~n", [ConnectError]), - tsf({connect_error, ConnectError, - [SocketType, Host, Port, TranspOpts]}) + ct:fail({connect_error, ConnectError, + [SocketType, Host, Port, TranspOpts]}) catch T:E -> - tsp("verify_request -> connect failed: " - "~n E: ~p" - "~n T: ~p" - "~n", [E, T]), - tsf({connect_failure, - [{type, T}, - {error, E}, - {stacktrace, erlang:get_stacktrace()}, - {args, [SocketType, Host, Port, TranspOpts]}]}) + ct:fail({connect_failure, + [{type, T}, + {error, E}, + {stacktrace, erlang:get_stacktrace()}, + {args, [SocketType, Host, Port, TranspOpts]}]}) end. request(#state{mfa = {Module, Function, Args}, @@ -166,10 +140,6 @@ request(#state{mfa = {Module, Function, Args}, HeadRequest = lists:sublist(RequestStr, 1, 4), receive {tcp, Socket, Data} -> - io:format("~p ~w[~w]request -> received (tcp) data" - "~n Data: ~p" - "~n", [self(), ?MODULE, ?LINE, Data]), - print(tcp, Data, State), case Module:Function([Data | Args]) of {ok, Parsed} -> handle_http_msg(Parsed, State); @@ -179,22 +149,12 @@ request(#state{mfa = {Module, Function, Args}, request(State#state{mfa = NewMFA}, TimeOut) end; {tcp_closed, Socket} when Function =:= whole_body -> - io:format("~p ~w[~w]request -> " - "received (tcp) closed when whole_body" - "~n", [self(), ?MODULE, ?LINE]), - print(tcp, "closed", State), State#state{body = hd(Args)}; {tcp_closed, Socket} -> - io:format("~p ~w[~w]request -> received (tcp) closed" - "~n", [self(), ?MODULE, ?LINE]), exit({test_failed, connection_closed}); {tcp_error, Socket, Reason} -> - io:format("~p ~w[~w]request -> received (tcp) error" - "~n Reason: ~p" - "~n", [self(), ?MODULE, ?LINE, Reason]), ct:fail({tcp_error, Reason}); {ssl, Socket, Data} -> - print(ssl, Data, State), case Module:Function([Data | Args]) of {ok, Parsed} -> handle_http_msg(Parsed, State); @@ -204,28 +164,19 @@ request(#state{mfa = {Module, Function, Args}, request(State#state{mfa = NewMFA}, TimeOut) end; {ssl_closed, Socket} when Function =:= whole_body -> - print(ssl, "closed", State), State#state{body = hd(Args)}; {ssl_closed, Socket} -> exit({test_failed, connection_closed}); {ssl_error, Socket, Reason} -> ct:fail({ssl_error, Reason}) after TimeOut -> - io:format("~p ~w[~w]request -> timeout" - "~n", [self(), ?MODULE, ?LINE]), + ct:pal("~p ~w[~w]request -> timeout" + "~n", [self(), ?MODULE, ?LINE]), ct:fail(connection_timed_out) end. handle_http_msg({Version, StatusCode, ReasonPharse, Headers, Body}, State = #state{request = RequestStr}) -> - io:format("~p ~w[~w]handle_http_msg -> entry with" - "~n Version: ~p" - "~n StatusCode: ~p" - "~n ReasonPharse: ~p" - "~n Headers: ~p" - "~n Body: ~p" - "~n", [self(), ?MODULE, ?LINE, - Version, StatusCode, ReasonPharse, Headers, Body]), case is_expect(RequestStr) of true -> State#state{status_line = {Version, @@ -285,11 +236,6 @@ validate(RequestStr, #state{status_line = {Version, StatusCode, _}, headers = Headers, body = Body}, Options, N, P) -> - tsp("validate -> entry with" - "~n StatusCode: ~p" - "~n Headers: ~p" - "~n Body: ~p", [StatusCode, Headers, Body]), - check_version(Version, Options), case lists:keysearch(statuscode, 1, Options) of {value, _} -> @@ -311,20 +257,20 @@ check_version(Version, Options) -> {value, {version, Version}} -> ok; {value, {version, Ver}} -> - tsf({wrong_version, [{got, Version}, - {expected, Ver}]}); + ct:fail({wrong_version, [{got, Version}, + {expected, Ver}]}); _ -> - case Version of - "HTTP/1.1" -> - ok; + case Version of + "HTTP/1.1" -> + ok; _ -> - tsf({wrong_version, [{got, Version}, - {expected, "HTTP/1.1"}]}) - end + ct:fail({wrong_version, [{got, Version}, + {expected, "HTTP/1.1"}]}) + end end. check_status_code(StatusCode, [], Options) -> - tsf({wrong_status_code, [{got, StatusCode}, {expected, Options}]}); + ct:fail({wrong_status_code, [{got, StatusCode}, {expected, Options}]}); check_status_code(StatusCode, Current = [_ | Rest], Options) -> case lists:keysearch(statuscode, 1, Current) of {value, {statuscode, StatusCode}} -> @@ -332,7 +278,7 @@ check_status_code(StatusCode, Current = [_ | Rest], Options) -> {value, {statuscode, _OtherStatus}} -> check_status_code(StatusCode, Rest, Options); false -> - tsf({wrong_status_code, [{got, StatusCode}, {expected, Options}]}) + ct:fail({wrong_status_code, [{got, StatusCode}, {expected, Options}]}) end. do_validate(_, [], _, _) -> @@ -345,9 +291,9 @@ do_validate(Header, [{header, HeaderField}|Rest], N, P) -> {value, {LowerHeaderField, _Value}} -> ok; false -> - tsf({missing_header_field, LowerHeaderField, Header}); + ct:fail({missing_header_field, LowerHeaderField, Header}); _ -> - tsf({missing_header_field, LowerHeaderField, Header}) + ct:fail({missing_header_field, LowerHeaderField, Header}) end, do_validate(Header, Rest, N, P); do_validate(Header, [{header, HeaderField, Value}|Rest],N,P) -> @@ -356,15 +302,15 @@ do_validate(Header, [{header, HeaderField, Value}|Rest],N,P) -> {value, {LowerHeaderField, Value}} -> ok; false -> - tsf({wrong_header_field_value, LowerHeaderField, Header}); + ct:fail({wrong_header_field_value, LowerHeaderField, Header}); _ -> - tsf({wrong_header_field_value, LowerHeaderField, Header}) + ct:fail({wrong_header_field_value, LowerHeaderField, Header}) end, do_validate(Header, Rest, N, P); do_validate(Header,[{no_header, HeaderField}|Rest],N,P) -> case lists:keysearch(HeaderField,1,Header) of {value,_} -> - tsf({wrong_header_field_value, HeaderField, Header}); + ct:fail({wrong_header_field_value, HeaderField, Header}); _ -> ok end, @@ -382,14 +328,14 @@ is_expect(RequestStr) -> %% OTP-5775, content-length check_body("GET /cgi-bin/erl/httpd_example:get_bin HTTP/1.0\r\n\r\n", 200, "text/html", Length, _Body) when (Length =/= 274) -> - tsf(content_length_error); + ct:fail(content_length_error); check_body("GET /cgi-bin/cgi_echo HTTP/1.0\r\n\r\n", 200, "text/plain", _, Body) -> case size(Body) of 100 -> ok; _ -> - tsf(content_length_error) + ct:fail(content_length_error) end; check_body(RequestStr, 200, "text/html", _, Body) -> @@ -404,16 +350,3 @@ check_body(RequestStr, 200, "text/html", _, Body) -> check_body(_, _, _, _,_) -> ok. -print(Proto, Data, #state{print = true}) -> - ct:pal("Received ~p: ~p~n", [Proto, Data]); -print(_, _, #state{print = false}) -> - ok. - - -tsp(F) -> - inets_test_lib:tsp(F). -tsp(F, A) -> - inets_test_lib:tsp(F, A). - -tsf(Reason) -> - inets_test_lib:tsf(Reason). diff --git a/lib/inets/test/inets_sup_SUITE.erl b/lib/inets/test/inets_sup_SUITE.erl index 12b85a816f..cf28f5a245 100644 --- a/lib/inets/test/inets_sup_SUITE.erl +++ b/lib/inets/test/inets_sup_SUITE.erl @@ -260,10 +260,10 @@ tftpd_worker(suite) -> []; tftpd_worker(Config) when is_list(Config) -> [] = supervisor:which_children(tftp_sup), - {ok, Pid0} = inets:start(tftpd, [{host, "localhost"}, - {port, inet_port()}]), - {ok, _Pid1} = inets:start(tftpd, [{host, "localhost"}, - {port, inet_port()}], stand_alone), + {ok, Pid0} = inets:start(tftpd, [{host, inets_test_lib:hostname()}, + {port, 0}]), + {ok, _Pid1} = inets:start(tftpd, [{host, inets_test_lib:hostname()}, + {port, 0}], stand_alone), [{_,Pid0, worker, _}] = supervisor:which_children(tftp_sup), inets:stop(tftpd, Pid0), @@ -397,13 +397,6 @@ httpc_subtree(Config) when is_list(Config) -> tsp("httpc_subtree -> done"), ok. -inet_port() -> - {ok, Socket} = gen_tcp:listen(0, [{reuseaddr, true}]), - {ok, Port} = inet:port(Socket), - gen_tcp:close(Socket), - Port. - - tsp(F) -> tsp(F, []). tsp(F, A) -> diff --git a/lib/inets/test/old_httpd_SUITE.erl b/lib/inets/test/old_httpd_SUITE.erl index de9aa4562e..3e1a1a3845 100644 --- a/lib/inets/test/old_httpd_SUITE.erl +++ b/lib/inets/test/old_httpd_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -155,65 +155,103 @@ all() -> [ {group, ip}, {group, ssl}, - {group, http_1_1_ip}, - {group, http_1_0_ip}, - {group, http_0_9_ip}, - {group, ipv6}, + %%{group, http_1_1_ip}, + %%{group, http_1_0_ip}, + %%{group, http_0_9_ip}, + %%{group, ipv6}, {group, tickets} ]. groups() -> [ {ip, [], - [ip_mod_alias, ip_mod_actions, ip_mod_security, - ip_mod_auth, ip_mod_auth_api, ip_mod_auth_mnesia_api, - ip_mod_htaccess, ip_mod_cgi, ip_mod_esi, ip_mod_get, - ip_mod_head, ip_mod_all, ip_load_light, ip_load_medium, - ip_load_heavy, ip_dos_hostname, ip_time_test, - ip_restart_no_block, ip_restart_disturbing_block, - ip_restart_non_disturbing_block, - ip_block_disturbing_idle, ip_block_non_disturbing_idle, - ip_block_503, ip_block_disturbing_active, - ip_block_non_disturbing_active, - ip_block_disturbing_active_timeout_not_released, - ip_block_disturbing_active_timeout_released, - ip_block_non_disturbing_active_timeout_not_released, - ip_block_non_disturbing_active_timeout_released, - ip_block_disturbing_blocker_dies, - ip_block_non_disturbing_blocker_dies]}, + [ + %%ip_mod_alias, + ip_mod_actions, + %%ip_mod_security, + %% ip_mod_auth, + %% ip_mod_auth_api, + ip_mod_auth_mnesia_api, + %%ip_mod_htaccess, + %%ip_mod_cgi, + %%ip_mod_esi, + %%ip_mod_get, + %%ip_mod_head, + %%ip_mod_all, + %% ip_load_light, + %% ip_load_medium, + %% ip_load_heavy, + %%ip_dos_hostname, + ip_time_test + %% Replaced by load_config + %% ip_restart_no_block, + %% ip_restart_disturbing_block, + %% ip_restart_non_disturbing_block, + %% ip_block_disturbing_idle, + %% ip_block_non_disturbing_idle, + %% ip_block_503, + %% ip_block_disturbing_active, + %% ip_block_non_disturbing_active, + %% ip_block_disturbing_active_timeout_not_released, + %% ip_block_disturbing_active_timeout_released, + %% ip_block_non_disturbing_active_timeout_not_released, + %% ip_block_non_disturbing_active_timeout_released, + %% ip_block_disturbing_blocker_dies, + %% ip_block_non_disturbing_blocker_dies + ]}, {ssl, [], [{group, essl}]}, {essl, [], - [essl_mod_alias, essl_mod_actions, essl_mod_security, - essl_mod_auth, essl_mod_auth_api, - essl_mod_auth_mnesia_api, essl_mod_htaccess, - essl_mod_cgi, essl_mod_esi, essl_mod_get, essl_mod_head, - essl_mod_all, essl_load_light, essl_load_medium, - essl_load_heavy, essl_dos_hostname, essl_time_test, - essl_restart_no_block, essl_restart_disturbing_block, - essl_restart_non_disturbing_block, - essl_block_disturbing_idle, - essl_block_non_disturbing_idle, essl_block_503, - essl_block_disturbing_active, - essl_block_non_disturbing_active, - essl_block_disturbing_active_timeout_not_released, - essl_block_disturbing_active_timeout_released, - essl_block_non_disturbing_active_timeout_not_released, - essl_block_non_disturbing_active_timeout_released, - essl_block_disturbing_blocker_dies, - essl_block_non_disturbing_blocker_dies]}, - {http_1_1_ip, [], - [ip_host, ip_chunked, ip_expect, ip_range, ip_if_test, - ip_http_trace, ip_http1_1_head, - ip_mod_cgi_chunked_encoding_test]}, - {http_1_0_ip, [], - [ip_head_1_0, ip_get_1_0, ip_post_1_0]}, - {http_0_9_ip, [], [ip_get_0_9]}, - {ipv6, [], [ipv6_hostname_ipcomm, ipv6_address_ipcomm, - ipv6_hostname_essl, ipv6_address_essl]}, + [ + %%essl_mod_alias, + essl_mod_actions, + %% essl_mod_security, + %% essl_mod_auth, + %% essl_mod_auth_api, + essl_mod_auth_mnesia_api, + %%essl_mod_htaccess, + %%essl_mod_cgi, + %%essl_mod_esi, + %%essl_mod_get, + %%essl_mod_head, + %% essl_mod_all, + %% essl_load_light, + %% essl_load_medium, + %% essl_load_heavy, + %%essl_dos_hostname, + essl_time_test + %% Replaced by load_config + %% essl_restart_no_block, + %% essl_restart_disturbing_block, + %% essl_restart_non_disturbing_block, + %% essl_block_disturbing_idle, + %% essl_block_non_disturbing_idle, essl_block_503, + %% essl_block_disturbing_active, + %% essl_block_non_disturbing_active, + %% essl_block_disturbing_active_timeout_not_released, + %% essl_block_disturbing_active_timeout_released, + %% essl_block_non_disturbing_active_timeout_not_released, + %% essl_block_non_disturbing_active_timeout_released, + %% essl_block_disturbing_blocker_dies, + %% essl_block_non_disturbing_blocker_dies + ]}, + %% {http_1_1_ip, [], + %% [ + %% %%ip_host, ip_chunked, ip_expect, + %% %%ip_range, + %% %%ip_if_test + %% %%ip_http_trace, ip_http1_1_head, + %% %%ip_mod_cgi_chunked_encoding_test + %% ]}, + %%{http_1_0_ip, [], + %%[ip_head_1_0, ip_get_1_0, ip_post_1_0]}, + %%{http_0_9_ip, [], [ip_get_0_9]}, + %% {ipv6, [], [ipv6_hostname_ipcomm, ipv6_address_ipcomm, + %% ipv6_hostname_essl, ipv6_address_essl]}, {tickets, [], - [ticket_5775, ticket_5865, ticket_5913, ticket_6003, - ticket_7304]}]. - + [%%ticket_5775, ticket_5865, + ticket_5913%%, ticket_6003, + %%ticket_7304 + ]}]. init_per_group(ipv6 = _GroupName, Config) -> case inets_test_lib:has_ipv6_support() of diff --git a/lib/kernel/doc/src/app.xml b/lib/kernel/doc/src/app.xml index b24a2bd7bd..8575d94048 100644 --- a/lib/kernel/doc/src/app.xml +++ b/lib/kernel/doc/src/app.xml @@ -213,8 +213,8 @@ RTDeps [ApplicationVersion] [] OTP 17.0. The type of its value might be subject to changes during the OTP 17 release.</p></warning> <warning><p>All runtime dependencies specified in OTP applications - during the OTP 17 release might not be completely correct. This - is actively worked on. Declared runtime dependencies in OTP + during the OTP 17 release may not be completely correct. This + is actively being worked on. Declared runtime dependencies in OTP applications are expected to be correct in OTP 18.</p></warning> </item> </taglist> diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 6d4b1cb2db..8dae34431b 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -642,6 +642,11 @@ <item> <p>Symbolic links are not supported on this platform.</p> </item> + <tag><c>eperm</c></tag> + <item> + <p>User does not have privileges to create symbolic links + (<c>SeCreateSymbolicLinkPrivilege</c> on Windows).</p> + </item> </taglist> </desc> </func> diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index 976d5e35cb..e5928c7b63 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -194,6 +194,13 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> CodeSize, CodeBinary, Refs, 0,[] % ColdSize, CRrefs ] = binary_to_term(Bin), + ?debug_msg("***** ErLLVM *****~nVersion: ~s~nCheckSum: ~w~nConstAlign: ~w~n" ++ + "ConstSize: ~w~nConstMap: ~w~nLabelMap: ~w~nExportMap ~w~nRefs ~w~n", + [Version, CheckSum, ConstAlign, ConstSize, ConstMap, LabelMap, ExportMap, + Refs]), + %% Write HiPE binary code to a file in the current directory in order to + %% debug by disassembling. + %% file:write_file("erl.o", CodeBinary, [binary]), %% Check that we are loading up-to-date code. version_check(Version, Mod), case hipe_bifs:check_crc(CheckSum) of @@ -221,6 +228,7 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> {MFAs,Addresses} = exports(ExportMap, CodeAddress), %% Remove references to old versions of the module. ReferencesToPatch = get_refs_from(MFAs, []), + %% io:format("References to patch: ~w~n", [ReferencesToPatch]), ok = remove_refs_from(MFAs), %% Patch all dynamic references in the code. %% Function calls, Atoms, Constants, System calls @@ -246,8 +254,7 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> AddressesOfClosuresToPatch = calculate_addresses(ClosurePatches, CodeAddress, Addresses), export_funs(Addresses), - export_funs(Mod, BeamBinary, Addresses, AddressesOfClosuresToPatch), - ok + export_funs(Mod, BeamBinary, Addresses, AddressesOfClosuresToPatch) end, %% Redirect references to the old module to the new module's BEAM stub. patch_to_emu_step2(OldReferencesToPatch), diff --git a/lib/kernel/src/os.erl b/lib/kernel/src/os.erl index 9ffa9adeab..3bda391b8e 100644 --- a/lib/kernel/src/os.erl +++ b/lib/kernel/src/os.erl @@ -230,7 +230,9 @@ unix_cmd(Cmd) -> %% and the commands are read from standard input. We set the %% $1 parameter for easy identification of the resident shell. %% --define(SHELL, "/bin/sh -s unix:cmd 2>&1"). +-define(ROOT, "/"). +-define(ROOT_ANDROID, "/system"). +-define(SHELL, "bin/sh -s unix:cmd 2>&1"). -define(PORT_CREATOR_NAME, os_cmd_port_creator). %% @@ -280,7 +282,12 @@ start_port_srv(Request) -> end. start_port_srv_handle({Ref,Client}) -> - Reply = try open_port({spawn, ?SHELL},[stream]) of + Path = case lists:reverse(erlang:system_info(system_architecture)) of + % androideabi + "ibaediordna" ++ _ -> filename:join([?ROOT_ANDROID, ?SHELL]); + _ -> filename:join([?ROOT, ?SHELL]) + end, + Reply = try open_port({spawn, Path},[stream]) of Port when is_port(Port) -> (catch port_connect(Port, Client)), unlink(Port), diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 6b52493f46..f6d6cd94ab 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -428,7 +428,13 @@ make_del_dir(Config) when is_list(Config) -> % because there are processes having that directory as current. ?line ok = ?FILE_MODULE:make_dir(NewDir), ?line {ok,CurrentDir} = file:get_cwd(), - ?line ok = ?FILE_MODULE:set_cwd(NewDir), + case {os:type(), length(NewDir) >= 260 } of + {{win32,_}, true} -> + io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH)\n", []), + io:format("\nNewDir = ~p\n", [NewDir]); + _ -> + ?line ok = ?FILE_MODULE:set_cwd(NewDir) + end, try %% Check that we get an error when trying to create... %% a deep directory @@ -485,32 +491,39 @@ cur_dir_0(Config) when is_list(Config) -> atom_to_list(?MODULE) ++"_curdir"), ?line ok = ?FILE_MODULE:make_dir(NewDir), - ?line io:format("cd to ~s",[NewDir]), - ?line ok = ?FILE_MODULE:set_cwd(NewDir), - - %% Create a file in the new current directory, and check that it - %% really is created there - ?line UncommonName = "uncommon.fil", - ?line {ok,Fd} = ?FILE_MODULE:open(UncommonName,read_write), - ?line ok = ?FILE_MODULE:close(Fd), - ?line {ok,NewDirFiles} = ?FILE_MODULE:list_dir("."), - ?line true = lists:member(UncommonName,NewDirFiles), - - %% Delete the directory and return to the old current directory - %% and check that the created file isn't there (too!) - ?line expect({error, einval}, {error, eacces}, - ?FILE_MODULE:del_dir(NewDir)), - ?line ?FILE_MODULE:delete(UncommonName), - ?line {ok,[]} = ?FILE_MODULE:list_dir("."), - ?line ok = ?FILE_MODULE:set_cwd(Dir1), - ?line io:format("cd back to ~s",[Dir1]), - ?line ok = ?FILE_MODULE:del_dir(NewDir), - ?line {error, enoent} = ?FILE_MODULE:set_cwd(NewDir), - ?line ok = ?FILE_MODULE:set_cwd(Dir1), - ?line io:format("cd back to ~s",[Dir1]), - ?line {ok,OldDirFiles} = ?FILE_MODULE:list_dir("."), - ?line false = lists:member(UncommonName,OldDirFiles), - + case {os:type(), length(NewDir) >= 260} of + {{win32,_}, true} -> + io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH):\n"), + io:format("\nNewDir = ~p\n", [NewDir]); + _ -> + io:format("cd to ~s",[NewDir]), + ok = ?FILE_MODULE:set_cwd(NewDir), + + %% Create a file in the new current directory, and check that it + %% really is created there + UncommonName = "uncommon.fil", + {ok,Fd} = ?FILE_MODULE:open(UncommonName,read_write), + ok = ?FILE_MODULE:close(Fd), + {ok,NewDirFiles} = ?FILE_MODULE:list_dir("."), + true = lists:member(UncommonName,NewDirFiles), + + %% Delete the directory and return to the old current directory + %% and check that the created file isn't there (too!) + expect({error, einval}, {error, eacces}, + ?FILE_MODULE:del_dir(NewDir)), + ?FILE_MODULE:delete(UncommonName), + {ok,[]} = ?FILE_MODULE:list_dir("."), + ok = ?FILE_MODULE:set_cwd(Dir1), + io:format("cd back to ~s",[Dir1]), + + ok = ?FILE_MODULE:del_dir(NewDir), + {error, enoent} = ?FILE_MODULE:set_cwd(NewDir), + ok = ?FILE_MODULE:set_cwd(Dir1), + io:format("cd back to ~s",[Dir1]), + {ok,OldDirFiles} = ?FILE_MODULE:list_dir("."), + false = lists:member(UncommonName,OldDirFiles) + end, + %% Try doing some bad things ?line {error, badarg} = ?FILE_MODULE:set_cwd({foo,bar}), ?line {error, enoent} = ?FILE_MODULE:set_cwd(""), @@ -1982,7 +1995,6 @@ names(Config) when is_list(Config) -> ?line Name1 = filename:join(RootDir, FileName), ?line Name2 = [RootDir,"/","foo1",".","fil"], ?line Name3 = [RootDir,"/",foo,$1,[[[],[],'.']],"f",il], - ?line Name4 = list_to_atom(Name1), ?line {ok,Fd0} = ?FILE_MODULE:open(Name1,write), ?line ok = ?FILE_MODULE:close(Fd0), @@ -1995,23 +2007,33 @@ names(Config) when is_list(Config) -> ?line ok = ?FILE_MODULE:close(Fd2), ?line {ok,Fd3} = ?FILE_MODULE:open(Name3,read), ?line ok = ?FILE_MODULE:close(Fd3), - ?line {ok,Fd4} = ?FILE_MODULE:open(Name4,read), - ?line ok = ?FILE_MODULE:close(Fd4), + case length(Name1) > 255 of + true -> + io:format("Path too long for an atom:\n\n~p\n", [Name1]); + false -> + Name4 = list_to_atom(Name1), + {ok,Fd4} = ?FILE_MODULE:open(Name4,read), + ok = ?FILE_MODULE:close(Fd4) + end, %% Try some path names ?line Path1 = RootDir, ?line Path2 = [RootDir], ?line Path3 = ['',[],[RootDir,[[]]]], - ?line Path4 = list_to_atom(Path1), ?line {ok,Fd11,_} = ?FILE_MODULE:path_open([Path1],FileName,read), ?line ok = ?FILE_MODULE:close(Fd11), ?line {ok,Fd12,_} = ?FILE_MODULE:path_open([Path2],FileName,read), ?line ok = ?FILE_MODULE:close(Fd12), ?line {ok,Fd13,_} = ?FILE_MODULE:path_open([Path3],FileName,read), ?line ok = ?FILE_MODULE:close(Fd13), - ?line {ok,Fd14,_} = ?FILE_MODULE:path_open([Path4],FileName,read), - ?line ok = ?FILE_MODULE:close(Fd14), - + case length(Path1) > 255 of + true-> + io:format("Path too long for an atom:\n\n~p\n", [Path1]); + false -> + Path4 = list_to_atom(Path1), + {ok,Fd14,_} = ?FILE_MODULE:path_open([Path4],FileName,read), + ok = ?FILE_MODULE:close(Fd14) + end, ?line [] = flush(), ?line test_server:timetrap_cancel(Dog), ok. @@ -2673,6 +2695,9 @@ symlinks(Config) when is_list(Config) -> case ?FILE_MODULE:make_symlink(Name, Alias) of {error, enotsup} -> {skipped, "Links not supported on this platform"}; + {error, eperm} -> + {win32,_} = os:type(), + {skipped, "Windows user not privileged to create symlinks"}; ok -> ?line {ok, Info1} = ?FILE_MODULE:read_file_info(Name), ?line {ok, Info1} = ?FILE_MODULE:read_file_info(Alias), @@ -3599,7 +3624,11 @@ otp_10852(Config) when is_list(Config) -> ok = rpc_call(Node, list_dir_all, [B]), ok = rpc_call(Node, read_file, [B]), ok = rpc_call(Node, make_link, [B,B]), - ok = rpc_call(Node, make_symlink, [B,B]), + case rpc_call(Node, make_symlink, [B,B]) of + ok -> ok; + {error, E} when (E =:= enotsup) or (E =:= eperm) -> + {win32,_} = os:type() + end, ok = rpc_call(Node, delete, [B]), ok = rpc_call(Node, make_dir, [B]), ok = rpc_call(Node, del_dir, [B]), diff --git a/lib/kernel/test/prim_file_SUITE.erl b/lib/kernel/test/prim_file_SUITE.erl index 97d66eab81..05bd5b3a3d 100644 --- a/lib/kernel/test/prim_file_SUITE.erl +++ b/lib/kernel/test/prim_file_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2013. All Rights Reserved. +%% Copyright Ericsson AB 2000-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -266,7 +266,13 @@ make_del_dir(Config, Handle, Suffix) -> % because there are processes having that directory as current. ?line ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), ?line {ok, CurrentDir} = ?PRIM_FILE_call(get_cwd, Handle, []), - ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), + case {os:type(), length(NewDir) >= 260 } of + {{win32,_}, true} -> + io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH)\n", []), + io:format("\nNewDir = ~p\n", [NewDir]); + _ -> + ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]) + end, try %% Check that we get an error when trying to create... %% a deep directory @@ -335,31 +341,37 @@ cur_dir_0(Config, Handle) -> ?line RootDir = ?config(priv_dir,Config), ?line NewDir = filename:join(RootDir, DirName), ?line ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), - ?line io:format("cd to ~s",[NewDir]), - ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), - - %% Create a file in the new current directory, and check that it - %% really is created there - ?line UncommonName = "uncommon.fil", - ?line {ok,Fd} = ?PRIM_FILE:open(UncommonName, [read, write]), - ?line ok = ?PRIM_FILE:close(Fd), - ?line {ok,NewDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), - ?line true = lists:member(UncommonName,NewDirFiles), - - %% Delete the directory and return to the old current directory - %% and check that the created file isn't there (too!) - ?line expect({error, einval}, {error, eacces}, {error, eexist}, + case {os:type(), length(NewDir) >= 260} of + {{win32,_}, true} -> + io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH):\n"), + io:format("\nNewDir = ~p\n", [NewDir]); + _ -> + io:format("cd to ~s",[NewDir]), + ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), + + %% Create a file in the new current directory, and check that it + %% really is created there + UncommonName = "uncommon.fil", + {ok,Fd} = ?PRIM_FILE:open(UncommonName, [read, write]), + ok = ?PRIM_FILE:close(Fd), + {ok,NewDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), + true = lists:member(UncommonName,NewDirFiles), + + %% Delete the directory and return to the old current directory + %% and check that the created file isn't there (too!) + expect({error, einval}, {error, eacces}, {error, eexist}, ?PRIM_FILE_call(del_dir, Handle, [NewDir])), - ?line ?PRIM_FILE_call(delete, Handle, [UncommonName]), - ?line {ok,[]} = ?PRIM_FILE_call(list_dir, Handle, ["."]), - ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), - ?line io:format("cd back to ~s",[Dir1]), - ?line ok = ?PRIM_FILE_call(del_dir, Handle, [NewDir]), - ?line {error, enoent} = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), - ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), - ?line io:format("cd back to ~s",[Dir1]), - ?line {ok,OldDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), - ?line false = lists:member(UncommonName,OldDirFiles), + ?PRIM_FILE_call(delete, Handle, [UncommonName]), + {ok,[]} = ?PRIM_FILE_call(list_dir, Handle, ["."]), + ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), + io:format("cd back to ~s",[Dir1]), + ok = ?PRIM_FILE_call(del_dir, Handle, [NewDir]), + {error, enoent} = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), + ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), + io:format("cd back to ~s",[Dir1]), + {ok,OldDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), + false = lists:member(UncommonName,OldDirFiles) + end, %% Try doing some bad things ?line {error, badarg} = @@ -1981,6 +1993,9 @@ symlinks(Config, Handle, Suffix) -> case ?PRIM_FILE_call(make_symlink, Handle, [Name, Alias]) of {error, enotsup} -> {skipped, "Links not supported on this platform"}; + {error, eperm} -> + {win32,_} = os:type(), + {skipped, "Windows user not privileged to create links"}; ok -> ?line {ok, Info1} = ?PRIM_FILE_call(read_file_info, Handle, [Name]), diff --git a/lib/os_mon/c_src/memsup.c b/lib/os_mon/c_src/memsup.c index b5114d10ed..409db84aa7 100644 --- a/lib/os_mon/c_src/memsup.c +++ b/lib/os_mon/c_src/memsup.c @@ -324,7 +324,7 @@ get_mem_procfs(memory_ext *me){ /* arch specific functions */ -#if defined(__linux__) /* ifdef SYSINFO */ +#if defined(__linux__) && !defined(__ANDROID__)/* ifdef SYSINFO */ /* sysinfo does not include cached memory which is a problem. */ static int get_extended_mem_sysinfo(memory_ext *me) { @@ -395,8 +395,12 @@ get_extended_mem_sgi(memory_ext *me) { static void get_extended_mem(memory_ext *me) { +/* android */ +#if defined(__ANDROID__) + if (get_mem_procfs(me)) return; + /* linux */ -#if defined(__linux__) +#elif defined(__linux__) if (get_mem_procfs(me)) return; if (get_extended_mem_sysinfo(me)) return; diff --git a/lib/public_key/src/pubkey_cert.erl b/lib/public_key/src/pubkey_cert.erl index dc8d68c78f..ae517ca642 100644 --- a/lib/public_key/src/pubkey_cert.erl +++ b/lib/public_key/src/pubkey_cert.erl @@ -319,6 +319,8 @@ verify_fun(Otpcert, Result, UserState0, VerifyFun) -> %% %% Description: Extracts a specific extension from a list of extensions. %%-------------------------------------------------------------------- +select_extension(_, asn1_NOVALUE) -> + undefined; select_extension(_, []) -> undefined; select_extension(Id, [#'Extension'{extnID = Id} = Extension | _]) -> @@ -342,8 +344,11 @@ match_name(uniformResourceIdentifier, URI, [PermittedName | Rest]) -> incomplete -> false; {_, _, Host, _, _} -> - match_name(fun is_valid_host_or_domain/2, Host, - PermittedName, Rest) + PN = case split_uri(PermittedName) of + {_, _, PNhost, _, _} -> PNhost; + _X -> PermittedName + end, + match_name(fun is_valid_host_or_domain/2, Host, PN, Rest) end; match_name(emailAddress, Name, [PermittedName | Rest]) -> @@ -511,10 +516,10 @@ is_dir_name2(Value, Value) -> true; is_dir_name2({printableString, Value1}, {printableString, Value2}) -> string:to_lower(strip_spaces(Value1)) =:= string:to_lower(strip_spaces(Value2)); -is_dir_name2({utf8String, Value1}, String) -> %% BUGBUG FIX UTF8 conv - is_dir_name2({printableString, binary_to_list(Value1)}, String); -is_dir_name2(String, {utf8String, Value1}) -> %% BUGBUG FIX UTF8 conv - is_dir_name2(String, {printableString, binary_to_list(Value1)}); +is_dir_name2({utf8String, Value1}, String) -> + is_dir_name2({printableString, unicode:characters_to_list(Value1)}, String); +is_dir_name2(String, {utf8String, Value1}) -> + is_dir_name2(String, {printableString, unicode:characters_to_list(Value1)}); is_dir_name2(_, _) -> false. diff --git a/lib/public_key/src/pubkey_crl.erl b/lib/public_key/src/pubkey_crl.erl index eaba5bfa1b..f0df4bc3f2 100644 --- a/lib/public_key/src/pubkey_crl.erl +++ b/lib/public_key/src/pubkey_crl.erl @@ -39,7 +39,13 @@ validate(OtpCert, OtherDPCRLs, DP, {DerCRL, CRL}, {DerDeltaCRL, DeltaCRL}, CertIssuer = TBSCert#'OTPTBSCertificate'.issuer, TBSCRL = CRL#'CertificateList'.tbsCertList, CRLIssuer = TBSCRL#'TBSCertList'.issuer, - AltNames = subject_alt_names(TBSCert#'OTPTBSCertificate'.extensions), + AltNames = case pubkey_cert:select_extension(?'id-ce-subjectAltName', + TBSCert#'OTPTBSCertificate'.extensions) of + undefined -> + []; + Ext -> + Ext#'Extension'.extnValue + end, revoked_status(DP, IDP, {directoryName, CRLIssuer}, [ {directoryName, CertIssuer} | AltNames], SerialNumber, Revoked, DeltaRevoked, RevokedState1); @@ -397,16 +403,18 @@ verify_dp_name(IDPNames, DPorIssuerNames) -> match_one([], _) -> false; match_one([{Type, Name} | Names], CandidateNames) -> - Candidates = [NameName || {NameType, NameName} <- CandidateNames, NameType == Type], + Candidates = [NameName || {NameType, NameName} <- CandidateNames, + NameType == Type], case Candidates of [] -> false; - [_|_] -> case pubkey_cert:match_name(Type, Name, Candidates) of - true -> - true; - false -> - match_one(Names, CandidateNames) - end + [_|_] -> + case pubkey_cert:match_name(Type, Name, Candidates) of + true -> + true; + false -> + match_one(Names, CandidateNames) + end end. verify_dp_bools(TBSCert, IDP) -> @@ -664,6 +672,8 @@ verify_extensions([#'TBSCertList_revokedCertificates_SEQOF'{crlEntryExtensions = verify_extensions(pubkey_cert:extensions_list(Ext)) and verify_extensions(Rest); verify_extensions([]) -> true; +verify_extensions(asn1_NOVALUE) -> + true; verify_extensions([#'Extension'{critical = true, extnID = Id} | Rest]) -> case lists:member(Id, [?'id-ce-authorityKeyIdentifier', ?'id-ce-issuerAltName', @@ -689,13 +699,3 @@ authority_key_identifier(Extensions) -> Enc = extension_value(?'id-ce-authorityKeyIdentifier', 'AuthorityKeyIdentifier', Extensions), pubkey_cert_records:transform(Enc, decode). - -subject_alt_names(Extensions) -> - Enc = extension_value(?'id-ce-subjectAltName', - 'GeneralNames', Extensions), - case Enc of - undefined -> - []; - _ -> - pubkey_cert_records:transform(Enc, decode) - end. diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 3d370a93a5..e5da797efb 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -2233,7 +2233,7 @@ format_error({undefined_applications,Apps}) -> io_lib:format("Undefined applications: ~p~n",[Apps]); format_error({duplicate_modules,Dups}) -> io_lib:format("Duplicated modules: ~n~ts", - [map(fun({{Mod,_,App1,_,_},{Mod,_,App2,_,_}}) -> + [map(fun({{Mod,App1,_},{Mod,App2,_}}) -> io_lib:format("\t~w specified in ~w and ~w~n", [Mod,App1,App2]) end, Dups)]); diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index e3f6933476..1d3a71e94e 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -47,6 +47,7 @@ abnormal_script/1, src_tests_script/1, crazy_script/1, included_script/1, included_override_script/1, included_fail_script/1, included_bug_script/1, exref_script/1, + duplicate_modules_script/1, otp_3065_circular_dependenies/1, included_and_used_sort_script/1]). -export([tar_options/1, normal_tar/1, no_mod_vsn_tar/1, system_files_tar/1, system_files_tar/2, invalid_system_files_tar/1, @@ -84,6 +85,7 @@ groups() -> src_tests_script, crazy_script, included_script, included_override_script, included_fail_script, included_bug_script, exref_script, + duplicate_modules_script, otp_3065_circular_dependenies, included_and_used_sort_script]}, {tar, [], [tar_options, normal_tar, no_mod_vsn_tar, system_files_tar, @@ -822,6 +824,33 @@ no_hipe({ok, Value}) -> {ok, Value} end. +%% duplicate_modules_script: Check that make_script rejects two +%% applications providing the same module. +duplicate_modules_script(Config) when is_list(Config) -> + {ok, OldDir} = file:get_cwd(), + + {LatestDir, LatestName} = create_script(duplicate_modules,Config), + + DataDir = filename:absname(?copydir), + + ok = file:set_cwd(LatestDir), + LibDir = fname([DataDir, d_duplicate_modules, lib]), + P = [fname([LibDir, 'app1-1.0', ebin]), + fname([LibDir, 'app2-1.0', ebin])], + + %% Check wrong app vsn + error = systools:make_script(LatestName, [{path, P}]), + {error, + systools_make, + {duplicate_modules, [ + {{myapp,app1,_}, {myapp,app2,_}} + ] + } + } = systools:make_script(LatestName, [silent, {path, P}]), + + ok = file:set_cwd(OldDir), + ok. + %% tar_options: Check illegal tar options. tar_options(Config) when is_list(Config) -> {'EXIT',{{badarg,[{path,["Path",12,"Another"]}]}, _}} = @@ -2186,7 +2215,10 @@ create_script(current_all_future_sasl,Config) -> do_create_script(current_all_future_sasl,Config,current,Apps); create_script({unicode,RelVsn},Config) -> Apps = core_apps(current) ++ [{ua,"1.0"}], - do_create_script(unicode,RelVsn,Config,current,Apps). + do_create_script(unicode,RelVsn,Config,current,Apps); +create_script(duplicate_modules,Config) -> + Apps = core_apps(current) ++ [{app1,"1.0"},{app2,"1.0"}], + do_create_script(duplicate_modules,Config,current,Apps). do_create_script(Id,Config,ErtsVsn,AppVsns) -> diff --git a/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app1-1.0/ebin/app1.app b/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app1-1.0/ebin/app1.app new file mode 100644 index 0000000000..dea9257f2f --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app1-1.0/ebin/app1.app @@ -0,0 +1,7 @@ +{application, app1, + [{description, "Application 1"}, + {vsn, "1.0"}, + {modules, [myapp]}, + {registered, []}, + {applications, []}, + {env, []}]}. diff --git a/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app1-1.0/src/myapp.erl b/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app1-1.0/src/myapp.erl new file mode 100644 index 0000000000..bf2ab7c79c --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app1-1.0/src/myapp.erl @@ -0,0 +1,2 @@ +-module(myapp). +-vsn("1.0"). diff --git a/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app2-1.0/ebin/app2.app b/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app2-1.0/ebin/app2.app new file mode 100644 index 0000000000..476750d8b2 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app2-1.0/ebin/app2.app @@ -0,0 +1,7 @@ +{application, app2, + [{description, "Application 2"}, + {vsn, "1.0"}, + {modules, [myapp]}, + {registered, []}, + {applications, []}, + {env, []}]}. diff --git a/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app2-1.0/src/myapp.erl b/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app2-1.0/src/myapp.erl new file mode 100644 index 0000000000..bf2ab7c79c --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_duplicate_modules/lib/app2-1.0/src/myapp.erl @@ -0,0 +1,2 @@ +-module(myapp). +-vsn("1.0"). diff --git a/lib/snmp/test/Makefile b/lib/snmp/test/Makefile index f22b7ea8ee..7bc9dd07d4 100644 --- a/lib/snmp/test/Makefile +++ b/lib/snmp/test/Makefile @@ -93,10 +93,10 @@ ifeq ($(SNMP_DEBUG),e) SNMP_FLAGS += -Dsnmp_error endif ifeq ($(SNMP_DEBUG),l) - SNMP_FLAGS += -Dsnmp_log + SNMP_FLAGS += -Dsnmp_error -Dsnmp_log endif ifeq ($(SNMP_DEBUG),d) - SNMP_FLAGS += -Dsnmp_debug + SNMP_FLAGS += -Dsnmp_error -Dsnmp_log -Dsnmp_debug endif ifeq ($(DONT_USE_TS),true) diff --git a/lib/snmp/test/modules.mk b/lib/snmp/test/modules.mk index 3d658bf8e8..fd8315ec4d 100644 --- a/lib/snmp/test/modules.mk +++ b/lib/snmp/test/modules.mk @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2004-2012. All Rights Reserved. +# Copyright Ericsson AB 2004-2014. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -42,6 +42,7 @@ TEST_UTIL_MODULES = \ snmp_test_manager \ snmp_test_mgr \ snmp_test_mgr_misc \ + snmp_test_mgr_counter_server \ sa \ klas3 \ test1 \ diff --git a/lib/snmp/test/snmp_agent_test.erl b/lib/snmp/test/snmp_agent_test.erl index 89a6ce1253..2a9f2e842d 100644 --- a/lib/snmp/test/snmp_agent_test.erl +++ b/lib/snmp/test/snmp_agent_test.erl @@ -19,10 +19,6 @@ -module(snmp_agent_test). -%% TODO -%% * Test fault-tolerance (kill master etc) -%% - -export([ all/0, groups/0, @@ -41,7 +37,7 @@ v1_processing/1, big/1, big2/1, - loop_mib/1, + loop_mib_1/1, api/1, subagent/1, mnesia/1, @@ -394,8 +390,9 @@ usm_read/0, usm_del_user/0, usm_bad/0, - loop_mib_1/0, - loop_mib_2/0, + loop_mib_1_test/0, + loop_mib_2_test/0, + loop_mib_3_test/0, otp_1129_i/1, otp_1162_test/0, otp_1131_test/0, @@ -546,8 +543,9 @@ groups() -> init_per_suite(Config0) when is_list(Config0) -> - ?DBG("init_per_suite -> entry with" - "~n Config0: ~p", [Config0]), + p("init_per_suite -> entry with" + "~n Config: ~p" + "~n Nodes: ~p", [Config0, erlang:nodes()]), Config1 = snmp_test_lib:init_suite_top_dir(?MODULE, Config0), Config2 = snmp_test_lib:fix_data_dir(Config1), @@ -558,16 +556,32 @@ init_per_suite(Config0) when is_list(Config0) -> Config3 = [{mib_dir, MibDir}, {std_mib_dir, StdMibDir} | Config2], - ?DBG("init_per_suite -> end with" - "~n Config3: ~p", [Config3]), + snmp_test_mgr_counter_server:start(), + + p("init_per_suite -> end when" + "~n Config: ~p" + "~n Nodes: ~p", [Config3, erlang:nodes()]), Config3. end_per_suite(Config) when is_list(Config) -> - ?DBG("end_per_suite -> entry with" - "~n Config: ~p", [Config]), + p("end_per_suite -> entry with" + "~n Config: ~p" + "~n Nodes: ~p", [Config, erlang:nodes()]), + + case snmp_test_mgr_counter_server:stop() of + {ok, _Counters} -> + p("end_per_suite -> sucessfully stopped counter server" + "~n Counters: ~p", [_Counters]); + + {error, Reason} -> + p("end_per_suite -> failed stopping counter server" + "~n Reason: ~p", [Reason]) + end, + p("end_per_suite -> end when" + "~n Nodes: ~p", [erlang:nodes()]), Config. @@ -675,10 +689,16 @@ end_per_group(_GroupName, Config) -> %% ---- Init Per TestCase ---- init_per_testcase(Case, Config) when is_list(Config) -> - ?DBG("init_per_testcase -> entry with" - "~n Config: ~p", [Config]), + p("init_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p", [Config, erlang:nodes()]), - init_per_testcase1(Case, Config). + Result = init_per_testcase1(Case, Config), + + p("init_per_testcase -> done when" + "~n Result: ~p" + "~n Nodes: ~p", [Result, erlang:nodes()]), + Result. init_per_testcase1(otp8395 = Case, Config) when is_list(Config) -> ?DBG("init_per_testcase1 -> entry with" @@ -719,12 +739,18 @@ init_per_testcase1(_Case, Config) when is_list(Config) -> %% ---- End Per TestCase ---- end_per_testcase(Case, Config) when is_list(Config) -> - ?DBG("end_per_testcase -> entry with" - "~n Config: ~p", [Config]), + p("end_per_testcase -> entry with" + "~n Config: ~p" + "~n Nodes: ~p", [Config, erlang:nodes()]), display_log(Config), - end_per_testcase1(Case, Config). + Result = end_per_testcase1(Case, Config), + + p("end_per_testcase -> done with" + "~n Result: ~p" + "~n Nodes: ~p", [Result, erlang:nodes()]), + Result. end_per_testcase1(otp8395, Config) when is_list(Config) -> otp8395({fin, Config}); @@ -1173,7 +1199,7 @@ mse_simple(X) -> ?P(mse_simple), simple(X). mse_v1_processing(X) -> ?P(mse_v1_processing), v1_processing(X). mse_big(X) -> ?P(mse_big), big(X). mse_big2(X) -> ?P(mse_big2), big2(X). -mse_loop_mib(X) -> ?P(mse_loop_mib), loop_mib(X). +mse_loop_mib(X) -> ?P(mse_loop_mib), loop_mib_1(X). mse_api(X) -> ?P(mse_api), api(X). mse_sa_register(X) -> ?P(mse_sa_register), sa_register(X). mse_v1_trap(X) -> ?P(mse_v1_trap), v1_trap(X). @@ -1194,7 +1220,7 @@ msd_simple(X) -> ?P(msd_simple), simple(X). msd_v1_processing(X) -> ?P(msd_v1_processing), v1_processing(X). msd_big(X) -> ?P(msd_big), big(X). msd_big2(X) -> ?P(msd_big2), big2(X). -msd_loop_mib(X) -> ?P(msd_loop_mib), loop_mib(X). +msd_loop_mib(X) -> ?P(msd_loop_mib), loop_mib_1(X). msd_api(X) -> ?P(msd_api), api(X). msd_sa_register(X) -> ?P(msd_sa_register), sa_register(X). msd_v1_trap(X) -> ?P(msd_v1_trap), v1_trap(X). @@ -1215,7 +1241,7 @@ msm_simple(X) -> ?P(msm_simple), simple(X). msm_v1_processing(X) -> ?P(msm_v1_processing), v1_processing(X). msm_big(X) -> ?P(msm_big2), big(X). msm_big2(X) -> ?P(msm_loop_mib), big2(X). -msm_loop_mib(X) -> ?P(msm_loop_mib), loop_mib(X). +msm_loop_mib(X) -> ?P(msm_loop_mib), loop_mib_1(X). msm_api(X) -> ?P(msm_api), api(X). msm_sa_register(X) -> ?P(msm_sa_register), sa_register(X). msm_v1_trap(X) -> ?P(msm_v1_trap), v1_trap(X). @@ -1618,7 +1644,7 @@ v1_cases() -> v1_processing, big, big2, - loop_mib, + loop_mib_1, api, subagent, mnesia, @@ -2095,9 +2121,9 @@ await_dummy_manager_started(Pid) -> {ok,Pid,Port}; {'EXIT', Pid, Reason} -> {error, Pid, Reason}; - O -> + _O -> ?LOG("dummy_manager_start -> received unknown message:" - "~n ~p",[O]), + "~n ~p",[_O]), await_dummy_manager_started(Pid) end. @@ -2120,16 +2146,16 @@ dummy_manager_send_trap2(Pid) -> dummy_manager_await_trap2_ack() -> ?DBG("dummy_manager_await_trap2 -> entry",[]), receive - {received_trap,Trap} -> - ?LOG("dummy_manager_await_trap2 -> received trap: ~p",[Trap]), + {received_trap, _Trap} -> + ?LOG("dummy_manager_await_trap2 -> received trap: ~p", [_Trap]), %% Note: %% Without this sleep the v2_inform_i testcase failes! There %% is no relation between these two test cases as far as I %% able to figure out... ?SLEEP(60000), ok; - O -> - ?ERR("dummy_manager_await_trap2 -> unexpected message: ~p",[O]), + _O -> + ?ERR("dummy_manager_await_trap2 -> unexpected message: ~p",[_O]), ok after 10000 -> ?ERR("dummy_manager_await_trap2 -> timeout",[]), @@ -2155,32 +2181,34 @@ dummy_manager_loop(P,S,MA) -> "~n Trap: ~p",[Trap]), snmpa:send_trap(MA, Trap, "standard trap"), dummy_manager_loop(P,S,MA); - {udp, _UdpId, Ip, UdpPort, Bytes} -> + {udp, _UdpId, _Ip, _UdpPort, Bytes} -> ?LOG("dummy_manager_loop -> received upd message" "~n from: ~p:~p" "~n size: ~p", - [Ip, UdpPort, dummy_manager_message_sz(Bytes)]), + [_Ip, _UdpPort, dummy_manager_message_sz(Bytes)]), R = dummy_manager_handle_message(Bytes), - ?DBG("dummy_manager_loop -> R: ~p",[R]), + ?DBG("dummy_manager_loop -> R: ~p", [R]), P ! R, - dummy_manager_loop(P,S,MA); + dummy_manager_loop(P, S, MA); stop -> ?DBG("dummy_manager_loop -> received stop request",[]), P ! {dummy_manager_stopping, self()}, gen_udp:close(S), exit(normal); - O -> + _O -> ?LOG("dummy_manager_loop -> received unknown message:" - "~n ~p",[O]), - dummy_manager_loop(P,S,MA) + "~n ~p", [_O]), + dummy_manager_loop(P, S, MA) end. +-ifdef(snmp_log). dummy_manager_message_sz(B) when is_binary(B) -> size(B); dummy_manager_message_sz(L) when is_list(L) -> length(L); dummy_manager_message_sz(_) -> undefined. +-endif. dummy_manager_handle_message(Bytes) -> case (catch snmp_pdus:dec_message(Bytes)) of @@ -3398,11 +3426,11 @@ simple_standard_test() -> db_notify_client(suite) -> []; db_notify_client(Config) when is_list(Config) -> ?P(db_notify_client), - {SaNode, MgrNode, MibDir} = init_case(Config), + {_SaNode, _MgrNode, _MibDir} = init_case(Config), ?DBG("db_notify_client -> case initiated: " "~n SaNode: ~p" "~n MgrNode: ~p" - "~n MibDir: ~p", [SaNode, MgrNode, MibDir]), + "~n MibDir: ~p", [_SaNode, _MgrNode, _MibDir]), ?DBG("db_notify_client -> maximize verbosity", []), snmpa_local_db:verbosity(trace), Self = self(), @@ -4153,8 +4181,8 @@ ma_v2_inform1(MA) -> CmdExp = fun(ok) -> ok; - ({ok, Val}) -> - ?DBG("ma_v2_inform -> [cmd2] Val: ~p", [Val]), + ({ok, _Val}) -> + ?DBG("ma_v2_inform -> [cmd2] Val: ~p", [_Val]), ok; ({error, Id, Extra}) -> {error, {unexpected, Id, Extra}}; @@ -4189,10 +4217,10 @@ ma_v2_inform1(MA) -> CmdSnmpTargets = fun(T) -> receive - {snmp_targets, T, [Addr]} -> + {snmp_targets, T, [_Addr]} -> ?DBG("ma_v2_inform1 -> " "received expected snmp_targets " - "~n with receiver: ~p",[Addr]), + "~n with receiver: ~p", [_Addr]), ok; {snmp_targets, T, Addrs} -> ?ERR("ma_v2_inform1 -> " @@ -4210,16 +4238,16 @@ ma_v2_inform1(MA) -> Cmd06 = fun() -> receive - {snmp_notification, Tag03, {got_response, Addr}} -> + {snmp_notification, Tag03, {got_response, _Addr}} -> ?DBG("ma_v2_inform1 -> " "received expected snmp_notification " - "[with manager response] from: ~n ~p",[Addr]), + "[with manager response] from: ~n ~p", [_Addr]), ok; - {snmp_notification, Tag03, {no_response, Addr}} -> + {snmp_notification, Tag03, {no_response, _Addr}} -> ?ERR("ma_v2_inform1 -> " "received unexpected snmp_notification " "[without manager response] from: ~n ~p", - [Addr]), + [_Addr]), {error, no_response} after 20000 -> @@ -4249,16 +4277,16 @@ ma_v2_inform1(MA) -> Cmd10 = fun() -> receive - {snmp_notification, Tag07, {got_response, Addr}} -> + {snmp_notification, Tag07, {got_response, _Addr}} -> ?ERR("ma_v2_inform1 -> " "received unexpected snmp_notification " - "[with manager response] from: ~n ~p", [Addr]), + "[with manager response] from: ~n ~p", [_Addr]), {error, got_response}; - {snmp_notification, Tag07, {no_response, Addr}} -> + {snmp_notification, Tag07, {no_response, _Addr}} -> ?DBG("ma_v2_inform1 -> " "received expected snmp_notification " "[without manager response] from: ~n ~p", - [Addr]), + [_Addr]), ok after 240000 -> @@ -4302,8 +4330,8 @@ ma_v2_inform2(MA) -> CmdExp = fun(ok) -> ok; - ({ok, Val}) -> - ?DBG("ma_v2_inform -> [cmd2] Val: ~p", [Val]), + ({ok, _Val}) -> + ?DBG("ma_v2_inform -> [cmd2] Val: ~p", [_Val]), ok; ({error, Id, Extra}) -> {error, {unexpected, Id, Extra}}; @@ -4383,8 +4411,8 @@ ma_v2_inform3(MA) -> "~n send notification: testTrapv22", [MA]), CmdExpectInform = - fun(No, Response) -> - ?DBG("CmdExpectInform -> ~p: ~n~p", [No, Response]), + fun(_No, Response) -> + ?DBG("CmdExpectInform -> ~p: ~n~p", [_No, Response]), ?expect2({inform, Response}, [{[sysUpTime, 0], any}, {[snmpTrapOID, 0], ?system ++ [0,1]}]) @@ -4393,8 +4421,8 @@ ma_v2_inform3(MA) -> CmdExp = fun(ok) -> ok; - ({ok, Val}) -> - ?DBG("CmdExp -> Val: ~p", [Val]), + ({ok, _Val}) -> + ?DBG("CmdExp -> Val: ~p", [_Val]), ok; ({error, Id, Extra}) -> {error, {unexpected, Id, Extra}}; @@ -4505,17 +4533,17 @@ delivery_info(Tag, Address, DeliveryResult, Extra) -> command_handler([]) -> ok; -command_handler([{No, Desc, Cmd}|Rest]) -> - ?LOG("command_handler -> command ~w: ~n ~s", [No, Desc]), +command_handler([{_No, _Desc, Cmd}|Rest]) -> + ?LOG("command_handler -> command ~w: ~n ~s", [_No, _Desc]), case (catch Cmd()) of ok -> - ?LOG("command_handler -> ~w: ok",[No]), + ?LOG("command_handler -> ~w: ok", [_No]), command_handler(Rest); {error, Reason} -> - ?ERR("command_handler -> ~w error: ~n~p",[No, Reason]), + ?ERR("command_handler -> ~w error: ~n~p", [_No, Reason]), ?line ?FAIL(Reason); Error -> - ?ERR("command_handler -> ~w unexpected: ~n~p",[No, Error]), + ?ERR("command_handler -> ~w unexpected: ~n~p", [_No, Error]), ?line ?FAIL({unexpected_command_result, Error}) end. @@ -5516,57 +5544,59 @@ usm_bad() -> %% works. %% Load all std mibs that are not loaded by default. %%----------------------------------------------------------------- -loop_mib(suite) -> []; -loop_mib(Config) when is_list(Config) -> - ?P(loop_mib), - ?LOG("loop_mib -> initiate case",[]), +loop_mib_1(suite) -> []; +loop_mib_1(Config) when is_list(Config) -> + ?P(loop_mib_1), + ?LOG("loop_mib_1 -> initiate case",[]), %% snmpa:verbosity(master_agent,debug), %% snmpa:verbosity(mib_server,info), - {SaNode, MgrNode, MibDir} = init_case(Config), - ?DBG("loop_mib -> ~n" + {_SaNode, _MgrNode, _MibDir} = init_case(Config), + ?DBG("loop_mib_1 -> ~n" "\tSaNode: ~p~n" "\tMgrNode: ~p~n" - "\tMibDir: ~p",[SaNode, MgrNode, MibDir]), - ?DBG("loop_mib -> load mib SNMP-COMMUNITY-MIB",[]), + "\tMibDir: ~p", [_SaNode, _MgrNode, _MibDir]), + ?DBG("loop_mib_1 -> load mib SNMP-COMMUNITY-MIB",[]), ?line load_master_std("SNMP-COMMUNITY-MIB"), - ?DBG("loop_mib -> load mib SNMP-MPD-MIB",[]), + ?DBG("loop_mib_1 -> load mib SNMP-MPD-MIB",[]), ?line load_master_std("SNMP-MPD-MIB"), - ?DBG("loop_mib -> load mib SNMP-TARGET-MIB",[]), + ?DBG("loop_mib_1 -> load mib SNMP-TARGET-MIB",[]), ?line load_master_std("SNMP-TARGET-MIB"), - ?DBG("loop_mib -> load mib SNMP-NOTIFICATION-MIB",[]), + ?DBG("loop_mib_1 -> load mib SNMP-NOTIFICATION-MIB",[]), ?line load_master_std("SNMP-NOTIFICATION-MIB"), - ?DBG("loop_mib -> load mib SNMP-FRAMEWORK-MIB",[]), + ?DBG("loop_mib_1 -> load mib SNMP-FRAMEWORK-MIB",[]), ?line load_master_std("SNMP-FRAMEWORK-MIB"), - ?DBG("loop_mib -> load mib SNMP-VIEW-BASED-ACM-MIB",[]), + ?DBG("loop_mib_1 -> load mib SNMP-VIEW-BASED-ACM-MIB",[]), ?line load_master_std("SNMP-VIEW-BASED-ACM-MIB"), - ?DBG("loop_mib -> try",[]), - try_test(loop_mib_1), - ?DBG("loop_mib -> unload mib SNMP-COMMUNITY-MIB",[]), + ?DBG("loop_mib_1 -> try",[]), + + try_test(loop_mib_1_test), + + ?DBG("loop_mib_1 -> unload mib SNMP-COMMUNITY-MIB",[]), ?line unload_master("SNMP-COMMUNITY-MIB"), - ?DBG("loop_mib -> unload mib SNMP-MPD-MIB",[]), + ?DBG("loop_mib_1 -> unload mib SNMP-MPD-MIB",[]), ?line unload_master("SNMP-MPD-MIB"), - ?DBG("loop_mib -> unload mib SNMP-TARGET-MIB",[]), + ?DBG("loop_mib_1 -> unload mib SNMP-TARGET-MIB",[]), ?line unload_master("SNMP-TARGET-MIB"), - ?DBG("loop_mib -> unload mib SNMP-NOTIFICATION-MIB",[]), + ?DBG("loop_mib_1 -> unload mib SNMP-NOTIFICATION-MIB",[]), ?line unload_master("SNMP-NOTIFICATION-MIB"), - ?DBG("loop_mib -> unload mib SNMP-FRAMEWORK-MIB",[]), + ?DBG("loop_mib_1 -> unload mib SNMP-FRAMEWORK-MIB",[]), ?line unload_master("SNMP-FRAMEWORK-MIB"), - ?DBG("loop_mib -> unload mib SNMP-VIEW-BASED-ACM-MIB",[]), + ?DBG("loop_mib_1 -> unload mib SNMP-VIEW-BASED-ACM-MIB",[]), ?line unload_master("SNMP-VIEW-BASED-ACM-MIB"), %% snmpa:verbosity(master_agent,log), %% snmpa:verbosity(mib_server,silence), - ?LOG("loop_mib -> done",[]). + ?LOG("loop_mib_1 -> done",[]). loop_mib_2(suite) -> []; loop_mib_2(Config) when is_list(Config) -> ?P(loop_mib_2), ?LOG("loop_mib_2 -> initiate case",[]), - {SaNode, MgrNode, MibDir} = init_case(Config), - ?DBG("loop_mib_2 -> ~n" + {_SaNode, _MgrNode, _MibDir} = init_case(Config), + ?DBG("do_loop_mib_2 -> ~n" "\tSaNode: ~p~n" "\tMgrNode: ~p~n" - "\tMibDir: ~p",[SaNode, MgrNode, MibDir]), + "\tMibDir: ~p", [_SaNode, _MgrNode, _MibDir]), ?DBG("loop_mib_2 -> load mibs",[]), ?line load_master_std("SNMP-COMMUNITY-MIB"), ?line load_master_std("SNMP-MPD-MIB"), @@ -5574,7 +5604,9 @@ loop_mib_2(Config) when is_list(Config) -> ?line load_master_std("SNMP-NOTIFICATION-MIB"), ?line load_master_std("SNMP-FRAMEWORK-MIB"), ?line load_master_std("SNMP-VIEW-BASED-ACM-MIB"), - try_test(loop_mib_2), + + try_test(loop_mib_2_test), + ?DBG("loop_mib_2 -> unload mibs",[]), ?line unload_master("SNMP-COMMUNITY-MIB"), ?line unload_master("SNMP-MPD-MIB"), @@ -5589,18 +5621,18 @@ loop_mib_3(suite) -> []; loop_mib_3(Config) when is_list(Config) -> ?P(loop_mib_3), ?LOG("loop_mib_3 -> initiate case",[]), - {SaNode, MgrNode, MibDir} = init_case(Config), + {_SaNode, _MgrNode, _MibDir} = init_case(Config), ?DBG("loop_mib_3 -> ~n" "\tSaNode: ~p~n" "\tMgrNode: ~p~n" - "\tMibDir: ~p",[SaNode, MgrNode, MibDir]), + "\tMibDir: ~p", [_SaNode, _MgrNode, _MibDir]), ?DBG("loop_mib_3 -> load mibs",[]), ?line load_master_std("SNMP-TARGET-MIB"), ?line load_master_std("SNMP-NOTIFICATION-MIB"), ?line load_master_std("SNMP-VIEW-BASED-ACM-MIB"), ?line load_master_std("SNMP-USER-BASED-SM-MIB"), - try_test(loop_mib_2), + try_test(loop_mib_3_test), ?DBG("loop_mib_3 -> unload mibs",[]), ?line unload_master("SNMP-TARGET-MIB"), @@ -5611,17 +5643,16 @@ loop_mib_3(Config) when is_list(Config) -> %% Req. As many mibs all possible -loop_mib_1() -> - ?DBG("loop_mib_1 -> entry",[]), +loop_mib_1_test() -> + ?DBG("loop_mib_1_test -> entry",[]), N = loop_it_1([1,1], 0), io:format(user, "found ~w varibles\n", [N]), ?line N = if N < 100 -> 100; true -> N end. - loop_it_1(Oid, N) -> - ?DBG("loop_it_1 -> entry with~n" + ?DBG("loop_it_1_test -> entry with~n" "\tOid: ~p~n" "\tN: ~p",[Oid,N]), case get_next_req([Oid]) of @@ -5629,13 +5660,13 @@ loop_it_1(Oid, N) -> error_status = noError, error_index = 0, varbinds = [#varbind{oid = NOid, - value = Value}]} when NOid > Oid -> - ?DBG("loop_it_1 -> " + value = _Value}]} when NOid > Oid -> + ?DBG("loop_it_1_test -> " "~n NOid: ~p" - "~n Value: ~p",[NOid, Value]), - ?line [Value2] = get_req(1, [NOid]), % must not be same - ?DBG("loop_it_1 -> " - "~n Value2: ~p",[Value2]), + "~n Value: ~p", [NOid, _Value]), + ?line [_Value2] = get_req(1, [NOid]), % must not be same + ?DBG("loop_it_1_test -> " + "~n Value2: ~p", [_Value2]), loop_it_1(NOid, N+1); #pdu{type = 'get-response', @@ -5648,7 +5679,7 @@ loop_it_1(Oid, N) -> error_status = noSuchName, error_index = 1, varbinds = [_]} -> - ?DBG("loop_it_1 -> done: ~p",[N]), + ?DBG("loop_it_1_test -> done: ~p",[N]), N; #pdu{type = 'get-response', @@ -5669,14 +5700,13 @@ loop_it_1(Oid, N) -> %% Req. As many mibs all possible -loop_mib_2() -> - ?DBG("loop_mib_1 -> entry",[]), +loop_mib_2_test() -> + ?DBG("loop_mib_2_test -> entry",[]), N = loop_it_2([1,1], 0), io:format(user, "found ~w varibles\n", [N]), ?line N = if N < 100 -> 100; true -> N end. - loop_it_2(Oid, N) -> ?DBG("loop_it_2 -> entry with" @@ -5686,22 +5716,22 @@ loop_it_2(Oid, N) -> #pdu{type = 'get-response', error_status = noError, error_index = 0, - varbinds = [#varbind{oid = NOid, value = endOfMibView}]} -> + varbinds = [#varbind{oid = _NOid, value = endOfMibView}]} -> ?DBG("loop_it_2 -> " - "~n NOid: ~p",[NOid]), + "~n NOid: ~p", [_NOid]), N; #pdu{type = 'get-response', error_status = noError, error_index = 0, varbinds = [#varbind{oid = NOid, - value = Value}]} when NOid > Oid -> + value = _Value}]} when NOid > Oid -> ?DBG("loop_it_2 -> " "~n NOid: ~p" - "~n Value: ~p",[NOid, Value]), - ?line [Value2] = get_req(1, [NOid]), % must not be same + "~n Value: ~p", [NOid, _Value]), + ?line [_Value2] = get_req(1, [NOid]), % must not be same ?DBG("loop_it_2 -> " - "~n Value2: ~p",[Value2]), + "~n Value2: ~p", [_Value2]), loop_it_2(NOid, N+1); #pdu{type = 'get-response', @@ -5744,6 +5774,10 @@ loop_it_2(Oid, N) -> end. +loop_mib_3_test() -> + ?DBG("loop_mib_3_test -> entry",[]), + loop_mib_2_test(). + %%%----------------------------------------------------------------- %%% Testing of reported bugs and other tickets. @@ -6611,16 +6645,16 @@ otp8395(Config) when is_list(Config) -> AgentNode = ?config(agent_node, Config), AgentLogDir = ?config(agent_log_dir, Config), OutFile = join([AgentLogDir, "otp8395.txt"]), - {ok, LogInfo} = rpc:call(AgentNode, snmpa, log_info, []), - ?DBG("otp8395 -> LogInfo: ~p", [LogInfo]), + {ok, _LogInfo} = rpc:call(AgentNode, snmpa, log_info, []), + ?DBG("otp8395 -> LogInfo: ~p", [_LogInfo]), %% SyncRes = rpc:call(AgentNode, snmp, log_sync, [?audit_trail_log_name]), %% ?DBG("otp8395 -> SyncRes: ~p", [SyncRes]), ok = agent_log_validation(AgentNode), - LTTRes = + _LTTRes = rpc:call(AgentNode, snmpa, log_to_txt, [AgentLogDir, [], OutFile]), - ?DBG("otp8395 -> LTTRes: ~p", [LTTRes]), + ?DBG("otp8395 -> LTTRes: ~p", [_LTTRes]), ?SLEEP(1000), ?DBG("otp8395 -> done", []), @@ -6941,10 +6975,10 @@ stop_stdalone_agent(Pid) when (node(Pid) =/= node()) -> MRef = erlang:monitor(process, Pid), rpc:call(node(Pid), ?MODULE, stop_stdalone_agent, [Pid]), receive - {'DOWN', MRef, process, Pid, Info} -> + {'DOWN', MRef, process, Pid, _Info} -> ?DBG("received expected DOWN message " "regarding snmp agent supervisor: " - "~n Info: ~p", [Info]), + "~n Info: ~p", [_Info]), ok after 5000 -> ?DBG("no DOWN message " @@ -7003,9 +7037,9 @@ do_info(MaNode) -> tree_size_bytes, db_memory]}], verify_info(Info, Keys), - OldInfo = snmpa:old_info_format(Info), - ?DBG("info_test1 -> OldInfo: ~n~p", [OldInfo]), - verify_old_info(OldInfo), + %% OldInfo = snmpa:old_info_format(Info), + %% ?DBG("info_test1 -> OldInfo: ~n~p", [OldInfo]), + %% verify_old_info(OldInfo), ok. verify_info([], []) -> @@ -7048,20 +7082,20 @@ verify_subinfo(Info0, [Key|Keys]) -> verify_subinfo(Info, Keys) end. -verify_old_info(Info) -> - Keys = [vsns, subagents, loaded_mibs, - tree_size_bytes, process_memory, db_memory], - verify_old_info(Keys, Info). - -verify_old_info([], _) -> - ok; -verify_old_info([Key|Keys], Info) -> - case lists:keymember(Key, 1, Info) of - true -> - verify_old_info(Keys, Info); - false -> - ?FAIL({missing_old_info, Key}) - end. +%% verify_old_info(Info) -> +%% Keys = [vsns, subagents, loaded_mibs, +%% tree_size_bytes, process_memory, db_memory], +%% verify_old_info(Keys, Info). + +%% verify_old_info([], _) -> +%% ok; +%% verify_old_info([Key|Keys], Info) -> +%% case lists:keymember(Key, 1, Info) of +%% true -> +%% verify_old_info(Keys, Info); +%% false -> +%% ?FAIL({missing_old_info, Key}) +%% end. %% Index String - string used in index is(S) -> [length(S) | S]. diff --git a/lib/snmp/test/snmp_agent_test_lib.erl b/lib/snmp/test/snmp_agent_test_lib.erl index 122289c28e..d7109253f7 100644 --- a/lib/snmp/test/snmp_agent_test_lib.erl +++ b/lib/snmp/test/snmp_agent_test_lib.erl @@ -299,10 +299,10 @@ call(N,M,F,A) -> "~n Loc: ~p", [Rn, Loc]), put(test_server_loc, Loc), exit(Rn); - {done, Ret, Zed} -> + {done, Ret, _Zed} -> ?DBG("call -> done:" "~n Ret: ~p" - "~n Zed: ~p", [Ret, Zed]), + "~n Zed: ~p", [Ret, _Zed]), case Ret of {error, Reason} -> exit(Reason); @@ -338,8 +338,8 @@ run(Mod, Func, Args, Opts) -> CtxEngineID = snmp_misc:get_option(context_engine_id, Opts, EngineID), Community = snmp_misc:get_option(community, Opts, "all-rights"), ?DBG("run -> start crypto app",[]), - Crypto = ?CRYPTO_START(), - ?DBG("run -> Crypto: ~p", [Crypto]), + _CryptoRes = ?CRYPTO_START(), + ?DBG("run -> Crypto: ~p", [_CryptoRes]), catch snmp_test_mgr:stop(), % If we had a running mgr from a failed case StdM = join(code:priv_dir(snmp), "mibs") ++ "/", Vsn = get(vsn), @@ -676,9 +676,9 @@ stop_agent(Config) when is_list(Config) -> (catch process_info(Sup)), (catch process_info(Par))]), - Info = agent_info(Sup), + _Info = agent_info(Sup), ?DBG("stop_agent -> Agent info: " - "~n ~p", [Info]), + "~n ~p", [_Info]), stop_sup(Sup, Par), @@ -1303,10 +1303,10 @@ get_req(Id, Vars) -> {ok, Val} -> ?DBG("get_req -> response: ~p",[Val]), Val; - {error, _, {ExpFmt, ExpArg}, {ActFmt, ActArg}} -> + {error, _, {_ExpFmt, ExpArg}, {_ActFmt, ActArg}} -> ?DBG("get_req -> error for ~p: " - "~n " ++ ExpFmt ++ - "~n " ++ ActFmt, + "~n " ++ _ExpFmt ++ + "~n " ++ _ActFmt, [Id] ++ ExpArg ++ ActArg), exit({unexpected_response, ExpArg, ActArg}); Error -> @@ -1527,9 +1527,9 @@ rewrite_target_addr_conf(Dir, NewPort) -> case file:read_file_info(TAFile) of {ok, _} -> ok; - {error, R} -> + {error, _R} -> ?ERR("failure reading file info of " - "target address config file: ~p",[R]), + "target address config file: ~p", [_R]), ok end, diff --git a/lib/snmp/test/snmp_manager_test.erl b/lib/snmp/test/snmp_manager_test.erl index 5fe18980bc..3a654a2805 100644 --- a/lib/snmp/test/snmp_manager_test.erl +++ b/lib/snmp/test/snmp_manager_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2013. All Rights Reserved. +%% Copyright Ericsson AB 2003-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -139,6 +139,8 @@ -define(NS_TIMEOUT, 10000). +-define(DEFAULT_MNESIA_DEBUG, none). + %%---------------------------------------------------------------------- %% Records @@ -173,7 +175,9 @@ end_per_suite(Config) when is_list(Config) -> init_per_testcase(Case, Config) when is_list(Config) -> - io:format(user, "~n~n*** INIT ~w:~w ***~n~n", [?MODULE,Case]), + io:format(user, "~n~n*** INIT ~w:~w ***~n~n", [?MODULE, Case]), + p(Case, "init_per_testcase begin when" + "~n Nodes: ~p~n~n", [erlang:nodes()]), %% This version of the API, based on Addr and Port, has been deprecated DeprecatedApiCases = [ @@ -187,16 +191,25 @@ init_per_testcase(Case, Config) when is_list(Config) -> simple_async_get_bulk1, misc_async1 ], - case lists:member(Case, DeprecatedApiCases) of - true -> - %% ?SKIP(api_no_longer_supported); - {skip, api_no_longer_supported}; - false -> - init_per_testcase2(Case, Config) - end. + Result = + case lists:member(Case, DeprecatedApiCases) of + true -> + %% ?SKIP(api_no_longer_supported); + {skip, api_no_longer_supported}; + false -> + init_per_testcase2(Case, Config) + end, + p(Case, "init_per_testcase end when" + "~n Nodes: ~p" + "~n Result: ~p" + "~n~n", [Result, erlang:nodes()]), + Result. init_per_testcase2(Case, Config) -> - ?DBG("init_per_testcase2 -> ~p", [erlang:nodes()]), + ?DBG("init_per_testcase2 -> " + "~n Case: ~p" + "~n Config: ~p" + "~n Nodes: ~p", [Case, Config, erlang:nodes()]), CaseTopDir = snmp_test_lib:init_testcase_top_dir(Case, Config), @@ -314,6 +327,8 @@ init_per_testcase3(Case, Config) -> end. end_per_testcase(Case, Config) when is_list(Config) -> + p(Case, "end_per_testcase begin when" + "~n Nodes: ~p~n~n", [erlang:nodes()]), ?DBG("fin [~w] Nodes [1]: ~p", [Case, erlang:nodes()]), Dog = ?config(watchdog, Config), ?WD_STOP(Dog), @@ -322,6 +337,8 @@ end_per_testcase(Case, Config) when is_list(Config) -> ?DBG("fin [~w] Nodes [2]: ~p", [Case, erlang:nodes()]), %% TopDir = ?config(top_dir, Conf2), %% ?DEL_DIR(TopDir), + p(Case, "end_per_testcase end when" + "~n Nodes: ~p~n~n", [erlang:nodes()]), Conf2. end_per_testcase2(Case, Config) -> @@ -428,10 +445,10 @@ groups() -> {request_tests, [], [ {group, get_tests}, - {group, get_next_tests}, + {group, get_next_tests}, {group, set_tests}, - {group, bulk_tests}, - {group, misc_request_tests} + {group, bulk_tests}, + {group, misc_request_tests} ] }, {request_tests_mt, [], @@ -5303,34 +5320,59 @@ init_manager(AutoInform, Config) -> ?line Node = start_manager_node(), + %% The point with this (try catch block) is to be + %% able to do some cleanup in case we fail to + %% start some of the apps. That is, if we fail to + %% start the apps (mnesia, crypto and snmp agent) + %% we stop the (agent) node! - %% -- - %% Start and initiate crypto on manager node - %% - - ?line ok = init_crypto(Node), + try + begin - %% - %% Write manager config - %% + %% -- + %% Start and initiate crypto on manager node + %% + + ?line ok = init_crypto(Node), + + %% + %% Write manager config + %% + + ?line ok = write_manager_config(Config), + + IRB = case AutoInform of + true -> + auto; + _ -> + user + end, + Conf = [{manager_node, Node}, {irb, IRB} | Config], + Vsns = [v1,v2,v3], + start_manager(Node, Vsns, Conf) + end + catch + T:E -> + StackTrace = ?STACK(), + p("Failure during manager start: " + "~n Error Type: ~p" + "~n Error: ~p" + "~n StackTrace: ~p", [T, E, StackTrace]), + %% And now, *try* to cleanup + (catch stop_node(Node)), + ?FAIL({failed_starting_manager, T, E, StackTrace}) + end. - ?line ok = write_manager_config(Config), - - IRB = case AutoInform of - true -> - auto; - _ -> - user - end, - Conf = [{manager_node, Node}, {irb, IRB} | Config], - Vsns = [v1,v2,v3], - start_manager(Node, Vsns, Conf). - fin_manager(Config) -> Node = ?config(manager_node, Config), - stop_manager(Node, Config), - fin_crypto(Node), - stop_node(Node), + StopMgrRes = stop_manager(Node), + StopCryptoRes = fin_crypto(Node), + StopNode = stop_node(Node), + p("fin_agent -> stop apps and (mgr node ~p) node results: " + "~n SNMP Mgr: ~p" + "~n Crypto: ~p" + "~n Node: ~p", + [Node, StopMgrRes, StopCryptoRes, StopNode]), Config. @@ -5352,52 +5394,93 @@ init_agent(Config) -> ?line Node = start_agent_node(), + %% The point with this (try catch block) is to be + %% able to do some cleanup in case we fail to + %% start some of the apps. That is, if we fail to + %% start the apps (mnesia, crypto and snmp agent) + %% we stop the (agent) node! - %% -- - %% Start and initiate mnesia on agent node - %% - - ?line ok = init_mnesia(Node, Dir), - - - %% -- - %% Start and initiate crypto on agent node - %% - - ?line ok = init_crypto(Node), - - - %% - %% Write agent config - %% - - Vsns = [v1,v2], - ?line ok = write_agent_config(Vsns, Config), - - Conf = [{agent_node, Node}, - {mib_dir, MibDir} | Config], + try + begin + + %% -- + %% Start and initiate mnesia on agent node + %% + + ?line ok = init_mnesia(Node, Dir, ?config(mnesia_debug, Config)), + + + %% -- + %% Start and initiate crypto on agent node + %% + + ?line ok = init_crypto(Node), + + + %% + %% Write agent config + %% + + Vsns = [v1,v2], + ?line ok = write_agent_config(Vsns, Config), + + Conf = [{agent_node, Node}, + {mib_dir, MibDir} | Config], - %% - %% Start the agent - %% - - start_agent(Node, Vsns, Conf). + %% + %% Start the agent + %% + + start_agent(Node, Vsns, Conf) + end + catch + T:E -> + StackTrace = ?STACK(), + p("Failure during agent start: " + "~n Error Type: ~p" + "~n Error: ~p" + "~n StackTrace: ~p", [T, E, StackTrace]), + %% And now, *try* to cleanup + (catch stop_node(Node)), + ?FAIL({failed_starting_agent, T, E, StackTrace}) + end. + fin_agent(Config) -> Node = ?config(agent_node, Config), - stop_agent(Node, Config), - fin_crypto(Node), - fin_mnesia(Node), - stop_node(Node), + StopAgentRes = stop_agent(Node), + StopCryptoRes = fin_crypto(Node), + StopMnesiaRes = fin_mnesia(Node), + StopNode = stop_node(Node), + p("fin_agent -> stop apps and (agent node ~p) node results: " + "~n SNMP Agent: ~p" + "~n Crypto: ~p" + "~n Mnesia: ~p" + "~n Node: ~p", + [Node, StopAgentRes, StopCryptoRes, StopMnesiaRes, StopNode]), Config. -init_mnesia(Node, Dir) -> +init_mnesia(Node, Dir, MnesiaDebug) + when ((MnesiaDebug =/= none) andalso + (MnesiaDebug =/= debug) andalso (MnesiaDebug =/= trace)) -> + init_mnesia(Node, Dir, ?DEFAULT_MNESIA_DEBUG); +init_mnesia(Node, Dir, MnesiaDebug) -> ?DBG("init_mnesia -> load application mnesia", []), ?line ok = load_mnesia(Node), ?DBG("init_mnesia -> application mnesia: set_env dir: ~n~p",[Dir]), ?line ok = set_mnesia_env(Node, dir, filename:join(Dir, "mnesia")), + %% Just in case, only set (known to be) valid values for debug + if + ((MnesiaDebug =:= debug) orelse (MnesiaDebug =:= trace)) -> + ?DBG("init_mnesia -> application mnesia: set_env debug: ~w", + [MnesiaDebug]), + ?line ok = set_mnesia_env(Node, debug, MnesiaDebug); + true -> + ok + end, + ?DBG("init_mnesia -> create mnesia schema",[]), ?line case create_schema(Node) of ok -> @@ -5434,25 +5517,89 @@ fin_crypto(Node) -> %% -- Misc application wrapper functions -- -load_app(Node, App) when (Node =:= node()) andalso is_atom(App) -> - application:load(App); -load_app(Node, App) when is_atom(App) -> - rcall(Node, application, load, [App]). - -start_app(Node, App) when (Node =:= node()) andalso is_atom(App) -> - application:start(App); +load_app(Node, App) -> + VerifySuccess = fun(ok) -> + ok; + ({error, {already_loaded, LoadedApp}}) when (LoadedApp =:= App) -> + ok; + ({error, Reason}) -> + p("failed loading app ~w on ~p: " + "~n ~p", [App, Node, Reason]), + ?FAIL({failed_load, Node, App, Reason}) + end, + do_load_app(Node, App, VerifySuccess). + +do_load_app(Node, App, VerifySuccess) + when (Node =:= node()) andalso is_atom(App) -> + %% Local app + exec(fun() -> application:load(App) end, VerifySuccess); +do_load_app(Node, App, VerifySuccess) -> + %% Remote app + exec(fun() -> rcall(Node, application, load, [App]) end, VerifySuccess). + + start_app(Node, App) -> - rcall(Node, application, start, [App]). + VerifySuccess = fun(ok) -> + ok; + ({error, {already_started, LoadedApp}}) when (LoadedApp =:= App) -> + ok; + ({error, Reason}) -> + p("failed starting app ~w on ~p: " + "~n ~p", [App, Node, Reason]), + ?FAIL({failed_start, Node, App, Reason}) + end, + start_app(Node, App, VerifySuccess). + +start_app(Node, App, VerifySuccess) + when (Node =:= node()) andalso is_atom(App) -> + exec(fun() -> application:start(App) end, VerifySuccess); +start_app(Node, App, VerifySuccess) -> + exec(fun() -> rcall(Node, application, start, [App]) end, VerifySuccess). + + +stop_app(Node, App) -> + VerifySuccess = fun(ok) -> + ok; + ({error, {not_started, LoadedApp}}) when (LoadedApp =:= App) -> + ok; + ({error, Reason}) -> + p("failed stopping app ~w on ~p: " + "~n ~p", [App, Node, Reason]), + ?FAIL({failed_stop, Node, App, Reason}) + end, + stop_app(Node, App, VerifySuccess). + +stop_app(Node, App, VerifySuccess) + when (Node =:= node()) andalso is_atom(App) -> + exec(fun() -> application:stop(App) end, VerifySuccess); +stop_app(Node, App, VerifySuccess) when is_atom(App) -> + exec(fun() -> rcall(Node, application, stop, [App]) end, VerifySuccess). + + +set_app_env(Node, App, Key, Val) -> + VerifySuccess = fun(ok) -> + ok; + ({error, Reason}) -> + p("failed setting app ~w env on ~p" + "~n Key: ~p" + "~n Val: ~p" + "~n Reason: ~p" + "~n ~p", [App, Node, Key, Val, Reason]), + ?FAIL({failed_set_app_env, + Node, App, Key, Val, Reason}) + end, + set_app_env(Node, App, Key, Val, VerifySuccess). -stop_app(Node, App) when (Node =:= node()) andalso is_atom(App) -> - application:stop(App); -stop_app(Node, App) when is_atom(App) -> - rcall(Node, application, stop, [App]). +set_app_env(Node, App, Key, Val, VerifySuccess) + when (Node =:= node()) andalso is_atom(App) -> + exec(fun() -> application:set_env(App, Key, Val) end, VerifySuccess); +set_app_env(Node, App, Key, Val, VerifySuccess) when is_atom(App) -> + exec(fun() -> rcall(Node, application, set_env, [App, Key, Val]) end, + VerifySuccess). -set_app_env(Node, App, Key, Val) when (Node =:= node()) andalso is_atom(App) -> - application:set_env(App, Key, Val); -set_app_env(Node, App, Key, Val) when is_atom(App) -> - rcall(Node, application, set_env, [App, Key, Val]). + +exec(Cmd, VerifySuccess) -> + VerifySuccess(Cmd()). %% -- Misc snmp wrapper functions -- @@ -5900,9 +6047,9 @@ start_manager(Node, Vsns, Conf0, _Opts) -> Conf0. -stop_manager(Node, Conf) -> - stop_snmp(Node), - Conf. +stop_manager(Node) -> + stop_snmp(Node). + %% -- Misc agent wrapper functions -- @@ -5951,9 +6098,8 @@ start_agent(Node, Vsns, Conf0, _Opts) -> ?line ok = start_snmp(Node), Conf0. -stop_agent(Node, Conf) -> - stop_snmp(Node), - Conf. +stop_agent(Node) -> + stop_snmp(Node). agent_load_mib(Node, Mib) -> rcall(Node, snmpa, load_mibs, [[Mib]]). @@ -6015,17 +6161,18 @@ stop_node(Node) -> rpc:cast(Node, erlang, halt, []), await_stopped(Node, 5). -await_stopped(_, 0) -> +await_stopped(Node, 0) -> + p("await_stopped -> ~p still exist: giving up", [Node]), ok; await_stopped(Node, N) -> Nodes = erlang:nodes(), case lists:member(Node, Nodes) of true -> - ?DBG("[~w] ~p still exist", [N, Node]), + p("await_stopped -> ~p still exist: ~w", [Node, N]), ?SLEEP(1000), await_stopped(Node, N-1); false -> - ?DBG("[~w] ~p gone", [N, Node]), + p("await_stopped -> ~p gone: ~w", [Node, N]), ok end. @@ -6271,7 +6418,7 @@ p(F, A) -> p(TName, F, A) -> io:format("*** [~w][~s] ***" - "~n" ++ F ++ "~n", [TName, formated_timestamp()|A]). + "~n " ++ F ++ "~n", [TName, formated_timestamp()|A]). formated_timestamp() -> snmp_test_lib:formated_timestamp(). diff --git a/lib/snmp/test/snmp_test_mgr.erl b/lib/snmp/test/snmp_test_mgr.erl index 40fcbce8f1..d4eb00ff91 100644 --- a/lib/snmp/test/snmp_test_mgr.erl +++ b/lib/snmp/test/snmp_test_mgr.erl @@ -657,7 +657,8 @@ make_vb(Oid) -> #varbind{oid = Oid, variabletype = 'NULL', value = 'NULL'}. make_request_id() -> - random:uniform(16#FFFFFFF-1). + %% random:uniform(16#FFFFFFF-1). + snmp_test_mgr_counter_server:increment(mgr_request_id, 1, 1, 2147483647). echo_pdu(PDU, MiniMIB) -> io:format("~s", [snmp_misc:format_pdu(PDU, MiniMIB)]). diff --git a/lib/snmp/test/snmp_test_mgr_counter_server.erl b/lib/snmp/test/snmp_test_mgr_counter_server.erl new file mode 100644 index 0000000000..db31e0380b --- /dev/null +++ b/lib/snmp/test/snmp_test_mgr_counter_server.erl @@ -0,0 +1,152 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014-2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% The reason for this (test) counter server is that the +%% agent test suite is implemented in such a way that the +%% agent is started once and then used for several test cases. +%% Each request is given a request id which *was* generated using +%% random! It is therefor possible, although unlikely, that a +%% request may get a request id that has recently been used, +%% which will cause the agent to silently reject the request. +%% For this reason, we start this server at the start of the +%% agent suite and stop it at the end and all request ids are +%% generated by this server. +%% + +-module(snmp_test_mgr_counter_server). + +-export([start/0, stop/0, increment/4]). + +-define(SERVER, ?MODULE). +-define(TAB, snmp_test_mgr_counter_tab). + + +%%%------------------------------------------------------------------- +%%% API +%%%------------------------------------------------------------------- + +-spec start() -> ok. + +start() -> + Parent = self(), + ReqIdServer = spawn(fun() -> init(Parent) end), + receive + {ReqIdServer, ok} -> + ok; + {ReqIdServer, {error, Reason}} -> + exit({failed_starting_counter_server, Reason}) + after 5000 -> + exit(ReqIdServer, kill), % Cleanup, just in case + exit({failed_starting_counter_server, timeout}) + end. + +-spec stop() -> {ok, Counters :: list()} | {error, Reason :: term()}. + +stop() -> + request(stop). + + +-spec increment(Counter :: atom(), + Initial :: non_neg_integer(), + Increment :: pos_integer(), + Max :: pos_integer()) -> + Next :: pos_integer(). + +increment(Counter, Initial, Increment, Max) -> + Request = {increment, Counter, Initial, Increment, Max}, + case request(Request) of + {ok, ReqId} -> + ReqId; + {error, Reason} -> + exit(Reason) + end. + + +request(Request) -> + Id = make_ref(), + Msg = {self(), Id, Request}, + try + begin + global:send(?SERVER, Msg), + receive + {reply, Id, Reply} -> + {ok, Reply} + end + end + catch + T:E -> + {error, {T, E}} + end. + + +%%%------------------------------------------------------------------- +%%% Internal functions +%%%------------------------------------------------------------------- + +init(Parent) -> + p("starting"), + case global:register_name(?SERVER, self()) of + yes -> + p("name registration ok"), + Parent ! {self(), ok}; + no -> + p("name registration failed"), + Parent ! {self(), registration_failed}, + exit(registration_failed) + end, + ets:new(?TAB, [set, named_table, {keypos, 1}]), + loop(). + +loop() -> + receive + {From, Id, {increment, Counter, Initial, Increment, Max}} -> + Position = 2, + Threshold = Max, + SetValue = Initial, + UpdateOp = {Position, Increment, Threshold, SetValue}, + NextVal = + try ets:update_counter(?TAB, Counter, UpdateOp) of + Next when is_integer(Next) -> + p("increment ~w: (next) ~w", [Counter, Next]), + Next + catch + error:badarg -> + %% Oups, first time + p("increment ~w: (initial) ~w", [Counter, Initial]), + ets:insert(?TAB, {Counter, Initial}), + Initial + end, + From ! {reply, Id, NextVal}, + loop(); + + {From, Id, stop} -> + p("stop"), + Counters = ets:tab2list(?TAB), + From ! {reply, Id, Counters}, + exit(normal) + end. + + +p(F) -> + p(F, []). + +p(F, A) -> + io:format("*** [~s] COUNTER-SERVER [~w] " ++ F ++ "~n", + [snmp_test_lib:formated_timestamp(), self() | A]). diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 070a2db5a8..d7fff14f92 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -206,7 +206,7 @@ global_request(ConnectionHandler, Type, false = Reply, Data) -> send_all_state_event(ConnectionHandler, {global_request, self(), Type, Reply, Data}). %%-------------------------------------------------------------------- --spec send(pid(), channel_id(), integer(), iolist(), timeout()) -> +-spec send(pid(), channel_id(), integer(), iodata(), timeout()) -> ok | {error, timeout} | {error, closed}. %%-------------------------------------------------------------------- send(ConnectionHandler, ChannelId, Type, Data, Timeout) -> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index da2e076856..57f8dd86d3 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,222 +20,514 @@ %% Internal application API -%%==================================================================== +-behaviour(gen_fsm). + +-include("dtls_connection.hrl"). +-include("dtls_handshake.hrl"). +-include("ssl_alert.hrl"). +-include("dtls_record.hrl"). +-include("ssl_cipher.hrl"). +-include("ssl_api.hrl"). +-include("ssl_internal.hrl"). +-include("ssl_srp.hrl"). +-include_lib("public_key/include/public_key.hrl"). + %% Internal application API -%%==================================================================== +%% Setup +-export([start_fsm/8]). + +%% State transition handling +-export([next_record/1, next_state/4%, + %%next_state_connection/2 + ]). + +%% Handshake handling +-export([%%renegotiate/1, + send_handshake/2, send_change_cipher/2]). + +%% Alert and close handling +-export([send_alert/2, handle_own_alert/4, %%handle_close_alert/3, + handle_normal_shutdown/3 + %%handle_unexpected_message/3, + %%alert_user/5, alert_user/8 + ]). + +%% Data handling +-export([%%write_application_data/3, + read_application_data/2%%, +%% passive_receive/2, next_record_if_active/1 + ]). + +%% Called by tls_connection_sup +-export([start_link/7]). +%% gen_fsm callbacks +-export([init/1, hello/2, certify/2, cipher/2, + abbreviated/2, connection/2, handle_event/3, + handle_sync_event/4, handle_info/3, terminate/3, code_change/4]). -%% %%==================================================================== -%% %% State functions -%% %%==================================================================== - -%% -spec hello(start | #hello_request{} | #client_hello{} | #server_hello{} | term(), -%% #state{}) -> gen_fsm_state_return(). -%% %%-------------------------------------------------------------------- -%% hello(start, #state{host = Host, port = Port, role = client, -%% ssl_options = SslOpts, -%% session = #session{own_certificate = Cert} = Session0, -%% session_cache = Cache, session_cache_cb = CacheCb, -%% connection_states = ConnectionStates0, -%% renegotiation = {Renegotiation, _}, -%% client_cookie = Cookie} = State0) -> -%% Hello = dtls_handshake:client_hello(Host, Port, Cookie, ConnectionStates0, SslOpts, -%% Cache, CacheCb, Renegotiation, Cert), - -%% Version = Hello#client_hello.client_version, -%% State1 = State0#state{negotiated_version = Version, %% Requested version -%% session = -%% Session0#session{session_id = Hello#client_hello.session_id}, -%% dtls_handshake_history = ssl_handshake:init_handshake_history()}, - -%% State2 = send_flight(Hello, waiting, State1), - -%% {Record, State} = next_record(State2), -%% next_state(hello, hello, Record, State); - -%% hello(start, #state{role = server} = State0) -> -%% {Record, State} = next_record(State0), -%% next_state(hello, hello, Record, State); - -%% hello(#hello_request{}, #state{role = client} = State0) -> -%% {Record, State} = next_record(State0), -%% next_state(hello, hello, Record, State); - -%% hello(#server_hello{cipher_suite = CipherSuite, -%% compression_method = Compression} = Hello, -%% #state{session = #session{session_id = OldId}, -%% connection_states = ConnectionStates0, -%% role = client, -%% negotiated_version = ReqVersion, -%% renegotiation = {Renegotiation, _}, -%% ssl_options = SslOptions} = State1) -> -%% State0 = flight_done(State1), -%% case ssl_handshake:hello(Hello, SslOptions, ConnectionStates0, Renegotiation) of -%% #alert{} = Alert -> -%% handle_own_alert(Alert, ReqVersion, hello, State0); -%% {Version, NewId, ConnectionStates, NextProtocol} -> -%% {KeyAlgorithm, _, _, _} = -%% ssl_cipher:suite_definition(CipherSuite), - -%% PremasterSecret = make_premaster_secret(ReqVersion, KeyAlgorithm), - -%% NewNextProtocol = case NextProtocol of -%% undefined -> -%% State0#state.next_protocol; -%% _ -> -%% NextProtocol -%% end, - -%% State = State0#state{key_algorithm = KeyAlgorithm, -%% hashsign_algorithm = default_hashsign(Version, KeyAlgorithm), -%% negotiated_version = Version, -%% connection_states = ConnectionStates, -%% premaster_secret = PremasterSecret, -%% expecting_next_protocol_negotiation = NextProtocol =/= undefined, -%% next_protocol = NewNextProtocol}, - -%% case ssl_session:is_new(OldId, NewId) of -%% true -> -%% handle_new_session(NewId, CipherSuite, Compression, -%% State#state{connection_states = ConnectionStates}); -%% false -> -%% handle_resumed_session(NewId, State#state{connection_states = ConnectionStates}) -%% end -%% end; - -%% hello(#hello_verify_request{cookie = Cookie}, -%% #state{host = Host, port = Port, -%% session = #session{own_certificate = Cert}, -%% session_cache = Cache, session_cache_cb = CacheCb, -%% ssl_options = SslOpts, -%% connection_states = ConnectionStates0, -%% renegotiation = {Renegotiation, _}} = State0) -> -%% Hello = ssl_handshake:client_hello(Host, Port, Cookie, ConnectionStates0, SslOpts, -%% Cache, CacheCb, Renegotiation, Cert), -%% State1 = State0#state{ -%% tls_handshake_history = ssl_handshake:init_handshake_history(), -%% client_cookie = Cookie}, -%% State2 = send_flight(Hello, waiting, State1), - -%% {Record, State} = next_record(State2), -%% next_state(hello, hello, Record, State); - - -%% %%-------------------------------------------------------------------- -%% -spec abbreviated(#hello_request{} | #finished{} | term(), -%% #state{}) -> gen_fsm_state_return(). -%% %%-------------------------------------------------------------------- - -%% abbreviated(timeout, State) -> -%% { next_state, abbreviated, State, hibernate }; - -%% abbreviated(Msg, State) -> -%% handle_unexpected_message(Msg, abbreviated, State). - -%% %%-------------------------------------------------------------------- -%% -spec certify(#hello_request{} | #certificate{} | #server_key_exchange{} | -%% #certificate_request{} | #server_hello_done{} | #client_key_exchange{} | term(), -%% #state{}) -> gen_fsm_state_return(). -%% %%-------------------------------------------------------------------- - - -%% certify(timeout, State) -> -%% { next_state, certify, State, hibernate }; - -%% certify(Msg, State) -> -%% handle_unexpected_message(Msg, certify, State). - - -%% %%-------------------------------------------------------------------- -%% -spec cipher(#hello_request{} | #certificate_verify{} | #finished{} | term(), -%% #state{}) -> gen_fsm_state_return(). -%% %%-------------------------------------------------------------------- - -%% cipher(timeout, State) -> -%% { next_state, cipher, State, hibernate }; - -%% cipher(Msg, State) -> -%% handle_unexpected_message(Msg, cipher, State). - -%% %%-------------------------------------------------------------------- -%% -spec connection(#hello_request{} | #client_hello{} | term(), -%% #state{}) -> gen_fsm_state_return(). -%% %%-------------------------------------------------------------------- - -%% connection(timeout, State) -> -%% {next_state, connection, State, hibernate}; - -%% connection(Msg, State) -> -%% handle_unexpected_message(Msg, connection, State). - -%% %%-------------------------------------------------------------------- -%% %%% Internal functions -%% %%-------------------------------------------------------------------- -%% handle_unexpected_message(Msg, Info, #state{negotiated_version = Version} = State) -> -%% Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE), -%% handle_own_alert(Alert, Version, {Info, Msg}, State). - -%% send_flight(HandshakeRec, FlightState, State) -> -%% send_flight(FlightState, buffer_flight(HandshakeRec, State)). - -%% send_flight(FlightState, State = #state{negotiated_version = Version, -%% flight_buffer = Buffer}) -> - -%% State1 = do_send_flight(queue:to_list(Buffer), [], State), -%% finish_send_flight(Version, FlightState, State1). - -%% resend_flight(State = #state{negotiated_version = Version, -%% flight_state = FlightState, -%% flight_buffer = Buffer}) -%% when FlightState == finished; FlightState == waiting -> -%% State1 = do_send_flight(queue:to_list(Buffer), [], State), -%% finish_send_flight(Version, FlightState, State1); - -%% resend_flight(State) -> -%% State. - -%% flight_done(State) -> -%% cancel_dtls_retransmit_timer(State#state{flight_state = done, -%% flight_buffer = undefined}). - -%% do_send_flight([], BinMsgs, State = #state{transport_cb = Transport, socket = Socket}) -> -%% Transport:send(Socket, lists:reverse(BinMsgs)), -%% State; -%% do_send_flight([{Epoch, MsgSeq, HandshakeRec}|T], BinMsgs0, -%% State = #state{negotiated_version = Version, -%% connection_states = ConnectionStates0}) -> -%% CS0 = ssl_record:connection_state_by_epoch(ConnectionStates0, Epoch, write), -%% {BinMsgs, CS1} = encode_handshake_rec(HandshakeRec, Version, MsgSeq, BinMsgs0, CS0), -%% ConnectionStates1 = ssl_record:set_connection_state_by_epoch(ConnectionStates0, CS1, write), -%% do_send_flight(T, BinMsgs, State#state{connection_states = ConnectionStates1}). - -%% cancel_dtls_retransmit_timer(State = #state{dtls_retransmit_timer = TimerRef}) -> -%% cancel_timer(TimerRef), -%% State#state{dtls_retransmit_timer = undefined}. - -%% rearm_dtls_retransmit_timer(State = #state{dtls_retransmit_timer = undefined}) -> -%% TimerRef = erlang:start_timer(1000, self(), dtls_retransmit), -%% State#state{dtls_retransmit_timer = TimerRef}; -%% rearm_dtls_retransmit_timer(State) -> -%% State. - -%% finish_send_flight({254, _}, waiting, State) -> -%% TimerRef = erlang:start_timer(1000, self(), dtls_retransmit), -%% State#state{ -%% dtls_retransmit_timer = TimerRef, -%% last_retransmit = timestamp(), -%% flight_state = waiting}; - -%% finish_send_flight(_, FlightState, State) -> -%% State#state{flight_state = FlightState}. - -%% timestamp() -> -%% {Mega, Sec, Micro} = erlang:now(), -%% Mega * 1000000 * 1000 + Sec * 1000 + (Micro div 1000). - -%% encode_handshake_rec(HandshakeRec, Version, MsgSeq, BinMsgs0, CS0) -> -%% {_, Fragments} = ssl_handshake:encode_handshake(HandshakeRec, Version, MsgSeq, 1400), -%% lists:foldl(fun(F, {Bin, C0}) -> -%% {B, C1} = ssl_record:encode_handshake(F, Version, C0), -%% {[B|Bin], C1} end, {BinMsgs0, CS0}, Fragments). +%%==================================================================== +%% Internal application API +%%==================================================================== +start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_} = Opts, + User, {CbModule, _,_, _} = CbInfo, + Timeout) -> + try + {ok, Pid} = dtls_connection_sup:start_child([Role, Host, Port, Socket, + Opts, User, CbInfo]), + {ok, SslSocket} = ssl_connection:socket_control(?MODULE, Socket, Pid, CbModule), + ok = ssl_connection:handshake(SslSocket, Timeout), + {ok, SslSocket} + catch + error:{badmatch, {error, _} = Error} -> + Error + end; + +start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = true},_} = Opts, + User, {CbModule, _,_, _} = CbInfo, + Timeout) -> + try + {ok, Pid} = dtls_connection_sup:start_child_dist([Role, Host, Port, Socket, + Opts, User, CbInfo]), + {ok, SslSocket} = ssl_connection:socket_control(?MODULE, Socket, Pid, CbModule), + ok = ssl_connection:handshake(SslSocket, Timeout), + {ok, SslSocket} + catch + error:{badmatch, {error, _} = Error} -> + Error + end. + +send_handshake(Handshake, #state{negotiated_version = Version, + tls_handshake_history = Hist0, + connection_states = ConnectionStates0} = State0) -> + {BinHandshake, ConnectionStates, Hist} = + encode_handshake(Handshake, Version, ConnectionStates0, Hist0), + send_flight(BinHandshake, State0#state{connection_states = ConnectionStates, + tls_handshake_history = Hist + }). + +send_alert(Alert, #state{negotiated_version = Version, + socket = Socket, + transport_cb = Transport, + connection_states = ConnectionStates0} = State0) -> + {BinMsg, ConnectionStates} = + ssl_alert:encode(Alert, Version, ConnectionStates0), + Transport:send(Socket, BinMsg), + State0#state{connection_states = ConnectionStates}. + +send_change_cipher(Msg, #state{connection_states = ConnectionStates0, + socket = Socket, + negotiated_version = Version, + transport_cb = Transport} = State0) -> + {BinChangeCipher, ConnectionStates} = + encode_change_cipher(Msg, Version, ConnectionStates0), + Transport:send(Socket, BinChangeCipher), + State0#state{connection_states = ConnectionStates}. + +%%==================================================================== +%% tls_connection_sup API +%%==================================================================== + +%%-------------------------------------------------------------------- +-spec start_link(atom(), host(), inet:port_number(), port(), list(), pid(), tuple()) -> + {ok, pid()} | ignore | {error, reason()}. +%% +%% Description: Creates a gen_fsm process which calls Module:init/1 to +%% initialize. To ensure a synchronized start-up procedure, this function +%% does not return until Module:init/1 has returned. +%%-------------------------------------------------------------------- +start_link(Role, Host, Port, Socket, Options, User, CbInfo) -> + {ok, proc_lib:spawn_link(?MODULE, init, [[Role, Host, Port, Socket, Options, User, CbInfo]])}. + +init([Role, Host, Port, Socket, {SSLOpts0, _} = Options, User, CbInfo]) -> + process_flag(trap_exit, true), + State0 = initial_state(Role, Host, Port, Socket, Options, User, CbInfo), + Handshake = ssl_handshake:init_handshake_history(), + TimeStamp = calendar:datetime_to_gregorian_seconds({date(), time()}), + try ssl_config:init(SSLOpts0, Role) of + {ok, Ref, CertDbHandle, FileRefHandle, CacheHandle, OwnCert, Key, DHParams} -> + Session = State0#state.session, + State = State0#state{ + tls_handshake_history = Handshake, + session = Session#session{own_certificate = OwnCert, + time_stamp = TimeStamp}, + file_ref_db = FileRefHandle, + cert_db_ref = Ref, + cert_db = CertDbHandle, + session_cache = CacheHandle, + private_key = Key, + diffie_hellman_params = DHParams}, + gen_fsm:enter_loop(?MODULE, [], hello, State, get_timeout(State)) + catch + throw:Error -> + gen_fsm:enter_loop(?MODULE, [], error, {Error,State0}, get_timeout(State0)) + end. + +%%-------------------------------------------------------------------- +%% Description:There should be one instance of this function for each +%% possible state name. Whenever a gen_fsm receives an event sent +%% using gen_fsm:send_event/2, the instance of this function with the +%% same name as the current state name StateName is called to handle +%% the event. It is also called if a timeout occurs. +%% +hello(start, #state{host = Host, port = Port, role = client, + ssl_options = SslOpts, + session = #session{own_certificate = Cert} = Session0, + session_cache = Cache, session_cache_cb = CacheCb, + transport_cb = Transport, socket = Socket, + connection_states = ConnectionStates0, + renegotiation = {Renegotiation, _}} = State0) -> + Hello = dtls_handshake:client_hello(Host, Port, ConnectionStates0, SslOpts, + Cache, CacheCb, Renegotiation, Cert), + + Version = Hello#client_hello.client_version, + Handshake0 = ssl_handshake:init_handshake_history(), + {BinMsg, ConnectionStates, Handshake} = + encode_handshake(Hello, Version, ConnectionStates0, Handshake0), + Transport:send(Socket, BinMsg), + State1 = State0#state{connection_states = ConnectionStates, + negotiated_version = Version, %% Requested version + session = + Session0#session{session_id = Hello#client_hello.session_id}, + tls_handshake_history = Handshake}, + {Record, State} = next_record(State1), + next_state(hello, hello, Record, State); + +hello(Hello = #client_hello{client_version = ClientVersion, + extensions = #hello_extensions{hash_signs = HashSigns}}, + State = #state{connection_states = ConnectionStates0, + port = Port, session = #session{own_certificate = Cert} = Session0, + renegotiation = {Renegotiation, _}, + session_cache = Cache, + session_cache_cb = CacheCb, + ssl_options = SslOpts}) -> + HashSign = ssl_handshake:select_hashsign(HashSigns, Cert), + case dtls_handshake:hello(Hello, SslOpts, {Port, Session0, Cache, CacheCb, + ConnectionStates0, Cert}, Renegotiation) of + {Version, {Type, Session}, + ConnectionStates, + #hello_extensions{ec_point_formats = EcPointFormats, + elliptic_curves = EllipticCurves} = ServerHelloExt} -> + ssl_connection:hello({common_client_hello, Type, ServerHelloExt, HashSign}, + State#state{connection_states = ConnectionStates, + negotiated_version = Version, + session = Session, + client_ecc = {EllipticCurves, EcPointFormats}}, ?MODULE); + #alert{} = Alert -> + handle_own_alert(Alert, ClientVersion, hello, State) + end; +hello(Hello, + #state{connection_states = ConnectionStates0, + negotiated_version = ReqVersion, + role = client, + renegotiation = {Renegotiation, _}, + ssl_options = SslOptions} = State) -> + case dtls_handshake:hello(Hello, SslOptions, ConnectionStates0, Renegotiation) of + #alert{} = Alert -> + handle_own_alert(Alert, ReqVersion, hello, State); + {Version, NewId, ConnectionStates, NextProtocol} -> + ssl_connection:handle_session(Hello, + Version, NewId, ConnectionStates, NextProtocol, State) + end; + +hello(Msg, State) -> + ssl_connection:hello(Msg, State, ?MODULE). + +abbreviated(Msg, State) -> + ssl_connection:abbreviated(Msg, State, ?MODULE). + +certify(Msg, State) -> + ssl_connection:certify(Msg, State, ?MODULE). + +cipher(Msg, State) -> + ssl_connection:cipher(Msg, State, ?MODULE). + +connection(#hello_request{}, #state{host = Host, port = Port, + session = #session{own_certificate = Cert} = Session0, + session_cache = Cache, session_cache_cb = CacheCb, + ssl_options = SslOpts, + connection_states = ConnectionStates0, + renegotiation = {Renegotiation, _}} = State0) -> + Hello = dtls_handshake:client_hello(Host, Port, ConnectionStates0, SslOpts, + Cache, CacheCb, Renegotiation, Cert), + %% TODO DTLS version State1 = send_handshake(Hello, State0), + State1 = State0, + {Record, State} = + next_record( + State1#state{session = Session0#session{session_id + = Hello#client_hello.session_id}}), + next_state(connection, hello, Record, State); + +connection(#client_hello{} = Hello, #state{role = server, allow_renegotiate = true} = State) -> + %% Mitigate Computational DoS attack + %% http://www.educatedguesswork.org/2011/10/ssltls_and_computational_dos.html + %% http://www.thc.org/thc-ssl-dos/ Rather than disabling client + %% initiated renegotiation we will disallow many client initiated + %% renegotiations immediately after each other. + erlang:send_after(?WAIT_TO_ALLOW_RENEGOTIATION, self(), allow_renegotiate), + hello(Hello, State#state{allow_renegotiate = false}); + +connection(#client_hello{}, #state{role = server, allow_renegotiate = false} = State0) -> + Alert = ?ALERT_REC(?WARNING, ?NO_RENEGOTIATION), + State = send_alert(Alert, State0), + next_state_connection(connection, State); + +connection(Msg, State) -> + ssl_connection:connection(Msg, State, tls_connection). + +%%-------------------------------------------------------------------- +%% Description: Whenever a gen_fsm receives an event sent using +%% gen_fsm:send_all_state_event/2, this function is called to handle +%% the event. Not currently used! +%%-------------------------------------------------------------------- +handle_event(_Event, StateName, State) -> + {next_state, StateName, State, get_timeout(State)}. + +%%-------------------------------------------------------------------- +%% Description: Whenever a gen_fsm receives an event sent using +%% gen_fsm:sync_send_all_state_event/2,3, this function is called to handle +%% the event. +%%-------------------------------------------------------------------- +handle_sync_event(Event, From, StateName, State) -> + ssl_connection:handle_sync_event(Event, From, StateName, State). + +%%-------------------------------------------------------------------- +%% Description: This function is called by a gen_fsm when it receives any +%% other message than a synchronous or asynchronous event +%% (or a system message). +%%-------------------------------------------------------------------- + +%% raw data from socket, unpack records +handle_info({Protocol, _, Data}, StateName, + #state{data_tag = Protocol} = State0) -> + %% Simplify for now to avoid dialzer warnings before implementation is compleate + %% case next_tls_record(Data, State0) of + %% {Record, State} -> + %% next_state(StateName, StateName, Record, State); + %% #alert{} = Alert -> + %% handle_normal_shutdown(Alert, StateName, State0), + %% {stop, {shutdown, own_alert}, State0} + %% end; + {Record, State} = next_tls_record(Data, State0), + next_state(StateName, StateName, Record, State); + +handle_info({CloseTag, Socket}, StateName, + #state{socket = Socket, close_tag = CloseTag, + negotiated_version = _Version} = State) -> + handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), + {stop, {shutdown, transport_closed}, State}; + +handle_info(Msg, StateName, State) -> + ssl_connection:handle_info(Msg, StateName, State). + +%%-------------------------------------------------------------------- +%% Description:This function is called by a gen_fsm when it is about +%% to terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_fsm terminates with +%% Reason. The return value is ignored. +%%-------------------------------------------------------------------- +terminate(Reason, StateName, State) -> + ssl_connection:terminate(Reason, StateName, State). + +%%-------------------------------------------------------------------- +%% code_change(OldVsn, StateName, State, Extra) -> {ok, StateName, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +encode_handshake(Handshake, Version, ConnectionStates0, Hist0) -> + Seq = sequence(ConnectionStates0), + {EncHandshake, FragmentedHandshake} = dtls_handshake:encode_handshake(Handshake, Version, + Seq), + Hist = ssl_handshake:update_handshake_history(Hist0, EncHandshake), + {Encoded, ConnectionStates} = + dtls_record:encode_handshake(FragmentedHandshake, + Version, ConnectionStates0), + {Encoded, ConnectionStates, Hist}. + +next_record(#state{%%flight = #flight{state = finished}, + protocol_buffers = + #protocol_buffers{dtls_packets = [], dtls_cipher_texts = [CT | Rest]} + = Buffers, + connection_states = ConnStates0} = State) -> + case dtls_record:decode_cipher_text(CT, ConnStates0) of + {Plain, ConnStates} -> + {Plain, State#state{protocol_buffers = + Buffers#protocol_buffers{dtls_cipher_texts = Rest}, + connection_states = ConnStates}}; + #alert{} = Alert -> + {Alert, State} + end; +next_record(#state{socket = Socket, + transport_cb = Transport} = State) -> %% when FlightState =/= finished + ssl_socket:setopts(Transport, Socket, [{active,once}]), + {no_record, State}; + + +next_record(State) -> + {no_record, State}. + +next_state(Current,_, #alert{} = Alert, #state{negotiated_version = Version} = State) -> + handle_own_alert(Alert, Version, Current, State); + +next_state(_,Next, no_record, State) -> + {next_state, Next, State, get_timeout(State)}; + +%% next_state(_,Next, #ssl_tls{type = ?ALERT, fragment = EncAlerts}, State) -> +%% Alerts = decode_alerts(EncAlerts), +%% handle_alerts(Alerts, {next_state, Next, State, get_timeout(State)}); + +next_state(Current, Next, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, + State0 = #state{protocol_buffers = + #protocol_buffers{dtls_handshake_buffer = Buf0} = Buffers, + negotiated_version = Version}) -> + Handle = + fun({#hello_request{} = Packet, _}, {next_state, connection = SName, State}) -> + %% This message should not be included in handshake + %% message hashes. Starts new handshake (renegotiation) + Hs0 = ssl_handshake:init_handshake_history(), + ?MODULE:SName(Packet, State#state{tls_handshake_history=Hs0, + renegotiation = {true, peer}}); + ({#hello_request{} = Packet, _}, {next_state, SName, State}) -> + %% This message should not be included in handshake + %% message hashes. Already in negotiation so it will be ignored! + ?MODULE:SName(Packet, State); + ({#client_hello{} = Packet, Raw}, {next_state, connection = SName, State}) -> + Version = Packet#client_hello.client_version, + Hs0 = ssl_handshake:init_handshake_history(), + Hs1 = ssl_handshake:update_handshake_history(Hs0, Raw), + ?MODULE:SName(Packet, State#state{tls_handshake_history=Hs1, + renegotiation = {true, peer}}); + ({Packet, Raw}, {next_state, SName, State = #state{tls_handshake_history=Hs0}}) -> + Hs1 = ssl_handshake:update_handshake_history(Hs0, Raw), + ?MODULE:SName(Packet, State#state{tls_handshake_history=Hs1}); + (_, StopState) -> StopState + end, + try + {Packets, Buf} = tls_handshake:get_tls_handshake(Version,Data,Buf0), + State = State0#state{protocol_buffers = + Buffers#protocol_buffers{dtls_packets = Packets, + dtls_handshake_buffer = Buf}}, + handle_dtls_handshake(Handle, Next, State) + catch throw:#alert{} = Alert -> + handle_own_alert(Alert, Version, Current, State0) + end; + +next_state(_, StateName, #ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, State0) -> + %% Simplify for now to avoid dialzer warnings before implementation is compleate + %% case read_application_data(Data, State0) of + %% Stop = {stop,_,_} -> + %% Stop; + %% {Record, State} -> + %% next_state(StateName, StateName, Record, State) + %% end; + {Record, State} = read_application_data(Data, State0), + next_state(StateName, StateName, Record, State); + +next_state(Current, Next, #ssl_tls{type = ?CHANGE_CIPHER_SPEC, fragment = <<1>>} = + _ChangeCipher, + #state{connection_states = ConnectionStates0} = State0) -> + ConnectionStates1 = + ssl_record:activate_pending_connection_state(ConnectionStates0, read), + {Record, State} = next_record(State0#state{connection_states = ConnectionStates1}), + next_state(Current, Next, Record, State); +next_state(Current, Next, #ssl_tls{type = _Unknown}, State0) -> + %% Ignore unknown type + {Record, State} = next_record(State0), + next_state(Current, Next, Record, State). + +handle_dtls_handshake(Handle, StateName, + #state{protocol_buffers = + #protocol_buffers{dtls_packets = [Packet]} = Buffers} = State) -> + FsmReturn = {next_state, StateName, State#state{protocol_buffers = + Buffers#protocol_buffers{dtls_packets = []}}}, + Handle(Packet, FsmReturn); + +handle_dtls_handshake(Handle, StateName, + #state{protocol_buffers = + #protocol_buffers{dtls_packets = [Packet | Packets]} = Buffers} = + State0) -> + FsmReturn = {next_state, StateName, State0#state{protocol_buffers = + Buffers#protocol_buffers{dtls_packets = + Packets}}}, + case Handle(Packet, FsmReturn) of + {next_state, NextStateName, State, _Timeout} -> + handle_dtls_handshake(Handle, NextStateName, State); + {stop, _,_} = Stop -> + Stop + end. + + +send_flight(Fragments, #state{transport_cb = Transport, socket = Socket, + protocol_buffers = _PBuffers} = State) -> + Transport:send(Socket, Fragments), + %% Start retransmission + %% State#state{protocol_buffers = + %% (PBuffers#protocol_buffers){ #flight{state = waiting}}}}. + State. + +handle_own_alert(_,_,_, State) -> %% Place holder + {stop, {shutdown, own_alert}, State}. + +handle_normal_shutdown(_, _, _State) -> %% Place holder + ok. + +encode_change_cipher(#change_cipher_spec{}, Version, ConnectionStates) -> + dtls_record:encode_change_cipher_spec(Version, ConnectionStates). + +initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions}, User, + {CbModule, DataTag, CloseTag, ErrorTag}) -> + ConnectionStates = ssl_record:init_connection_states(Role), + + SessionCacheCb = case application:get_env(ssl, session_cb) of + {ok, Cb} when is_atom(Cb) -> + Cb; + _ -> + ssl_session_cache + end, + + Monitor = erlang:monitor(process, User), + + #state{socket_options = SocketOptions, + %% We do not want to save the password in the state so that + %% could be written in the clear into error logs. + ssl_options = SSLOptions#ssl_options{password = undefined}, + session = #session{is_resumable = new}, + transport_cb = CbModule, + data_tag = DataTag, + close_tag = CloseTag, + error_tag = ErrorTag, + role = Role, + host = Host, + port = Port, + socket = Socket, + connection_states = ConnectionStates, + protocol_buffers = #protocol_buffers{}, + user_application = {Monitor, User}, + user_data_buffer = <<>>, + session_cache_cb = SessionCacheCb, + renegotiation = {false, first}, + start_or_recv_from = undefined, + send_queue = queue:new(), + protocol_cb = ?MODULE + }. +read_application_data(_,State) -> + {#ssl_tls{fragment = <<"place holder">>}, State}. + +next_tls_record(_, State) -> + {#ssl_tls{fragment = <<"place holder">>}, State}. + +get_timeout(_) -> %% Place holder + infinity. + +next_state_connection(_, State) -> %% Place holder + {next_state, connection, State, get_timeout(State)}. + +sequence(_) -> + %%TODO real imp + 1. diff --git a/lib/ssl/src/dtls_connection.hrl b/lib/ssl/src/dtls_connection.hrl index b8dff479d5..08707dc8de 100644 --- a/lib/ssl/src/dtls_connection.hrl +++ b/lib/ssl/src/dtls_connection.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,24 +28,19 @@ -include("ssl_connection.hrl"). -record(protocol_buffers, { - dtls_packets = [] ::[binary()], % Not yet handled decode ssl/tls packets. - dtls_record_buffer :: binary(), % Buffer of incomplete records - dtls_handshake_buffer :: binary(), % Buffer of incomplete handshakes - dtls_cipher_texts :: [binary()], - dtls_cipher_texts_next :: [binary()] % Received for Epoch not yet active + dtls_packets = [], %%::[binary()], % Not yet handled decode ssl/tls packets. + dtls_record_buffer = <<>>, %%:: binary(), % Buffer of incomplete records + dtls_handshake_buffer = <<>>, %%:: binary(), % Buffer of incomplete handshakes + dtls_cipher_texts = [], %%:: [binary()], + dtls_cipher_texts_next %%:: [binary()] % Received for Epoch not yet active }). -record(flight, { last_retransmit, last_read_seq, msl_timer, - flight_state, - flight_buffer, % buffer of not yet ACKed TLS records - }). - --record(message_sequences, { - read = 0, - write = 0 + state, + buffer % buffer of not yet ACKed TLS records }). -endif. % -ifdef(dtls_connection). diff --git a/lib/ssl/src/dtls_connection_sup.erl b/lib/ssl/src/dtls_connection_sup.erl index 9fe545be18..0b4711cfb4 100644 --- a/lib/ssl/src/dtls_connection_sup.erl +++ b/lib/ssl/src/dtls_connection_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -26,8 +26,8 @@ -behaviour(supervisor). %% API --export([start_link/0]). --export([start_child/1]). +-export([start_link/0, start_link_dist/0]). +-export([start_child/1, start_child_dist/1]). %% Supervisor callback -export([init/1]). @@ -38,8 +38,14 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). +start_link_dist() -> + supervisor:start_link({local, dtls_connection_sup_dist}, ?MODULE, []). + start_child(Args) -> supervisor:start_child(?MODULE, Args). + +start_child_dist(Args) -> + supervisor:start_child(dtls_connection_sup_dist, Args). %%%========================================================================= %%% Supervisor callback diff --git a/lib/ssl/src/dtls_handshake.erl b/lib/ssl/src/dtls_handshake.erl index 5db2434753..31d525b295 100644 --- a/lib/ssl/src/dtls_handshake.erl +++ b/lib/ssl/src/dtls_handshake.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,11 +20,15 @@ -include("dtls_handshake.hrl"). -include("dtls_record.hrl"). -include("ssl_internal.hrl"). +-include("ssl_alert.hrl"). --export([client_hello/8, client_hello/9, hello/3, +-export([client_hello/8, client_hello/9, hello/4, get_dtls_handshake/2, - dtls_handshake_new_flight/1, dtls_handshake_new_epoch/1, - encode_handshake/4]). + %%dtls_handshake_new_flight/1, dtls_handshake_new_epoch/1, + encode_handshake/3]). + +-type dtls_handshake() :: #client_hello{} | #hello_verify_request{} | + ssl_handshake:ssl_handshake(). %%==================================================================== %% Internal application API @@ -54,12 +58,12 @@ client_hello(Host, Port, Cookie, ConnectionStates, ciphers = UserSuites } = SslOpts, Cache, CacheCb, Renegotiation, OwnCert) -> - Version = dtls_record:highest_protocol_version(Versions), + Version = dtls_record:highest_protocol_version(Versions), Pending = ssl_record:pending_connection_state(ConnectionStates, read), SecParams = Pending#connection_state.security_parameters, CipherSuites = ssl_handshake:available_suites(UserSuites, Version), - Extensions = ssl_handshake:client_hello_extensions(Host, Version, CipherSuites, + Extensions = ssl_handshake:client_hello_extensions(Host, dtls_v1:corresponding_tls_version(Version), CipherSuites, SslOpts, ConnectionStates, Renegotiation), Id = ssl_session:client_id({Host, Port, SslOpts}, Cache, CacheCb, OwnCert), @@ -73,163 +77,197 @@ client_hello(Host, Port, Cookie, ConnectionStates, extensions = Extensions }. -hello(Address, Port, - #ssl_tls{epoch = _Epoch, record_seq = _Seq, - version = Version} = Record) -> - {[{Hello, _}], _, _} = - get_dtls_handshake(Record, - dtls_handshake_new_flight(undefined)), - #client_hello{client_version = {Major, Minor}, - random = Random, - session_id = SessionId, - cipher_suites = CipherSuites, - compression_methods = CompressionMethods} = Hello, - CookieData = [address_to_bin(Address, Port), - <<?BYTE(Major), ?BYTE(Minor)>>, - Random, SessionId, CipherSuites, CompressionMethods], - Cookie = crypto:hmac(sha, <<"secret">>, CookieData), - - case Hello of - #client_hello{cookie = Cookie} -> - accept; - _ -> - %% generate HelloVerifyRequest - HelloVerifyRequest = encode_handshake(#hello_verify_request{protocol_version = Version, - cookie = Cookie}, - Version, 0, 1400), - {reply, HelloVerifyRequest} - end. +hello(#server_hello{server_version = Version, random = Random, + cipher_suite = CipherSuite, + compression_method = Compression, + session_id = SessionId, extensions = HelloExt}, + #ssl_options{versions = SupportedVersions} = SslOpt, + ConnectionStates0, Renegotiation) -> + case dtls_record:is_acceptable_version(Version, SupportedVersions) of + true -> + handle_server_hello_extensions(Version, SessionId, Random, CipherSuite, + Compression, HelloExt, SslOpt, ConnectionStates0, Renegotiation); + false -> + ?ALERT_REC(?FATAL, ?PROTOCOL_VERSION) + end; -%%-------------------------------------------------------------------- -encode_handshake(Package, Version, MsgSeq, Mss) -> - {MsgType, Bin} = enc_hs(Package, Version), +hello(#client_hello{client_version = ClientVersion}, _Options, {_,_,_,_,ConnectionStates,_}, _Renegotiation) -> + %% Return correct typ to make dialyzer happy until we have time to make the real imp. + {ClientVersion, {new, #session{}}, ConnectionStates, #hello_extensions{}}. + +%% hello(Address, Port, +%% #ssl_tls{epoch = _Epoch, sequence_number = _Seq, +%% version = Version} = Record) -> +%% case get_dtls_handshake(Record, +%% dtls_handshake_new_flight(undefined)) of +%% {[Hello | _], _} -> +%% hello(Address, Port, Version, Hello); +%% {retransmit, HandshakeState} -> +%% {retransmit, HandshakeState} +%% end. + +%% hello(Address, Port, Version, Hello) -> +%% #client_hello{client_version = {Major, Minor}, +%% random = Random, +%% session_id = SessionId, +%% cipher_suites = CipherSuites, +%% compression_methods = CompressionMethods} = Hello, +%% CookieData = [address_to_bin(Address, Port), +%% <<?BYTE(Major), ?BYTE(Minor)>>, +%% Random, SessionId, CipherSuites, CompressionMethods], +%% Cookie = crypto:hmac(sha, <<"secret">>, CookieData), + +%% case Hello of +%% #client_hello{cookie = Cookie} -> +%% accept; +%% _ -> +%% %% generate HelloVerifyRequest +%% HelloVerifyRequest = enc_hs(#hello_verify_request{protocol_version = Version, +%% cookie = Cookie}, +%% Version, 0, 1400), +%% {reply, HelloVerifyRequest} +%% end. + +%% %%-------------------------------------------------------------------- +encode_handshake(Handshake, Version, MsgSeq) -> + {MsgType, Bin} = enc_handshake(Handshake, Version), Len = byte_size(Bin), - HsHistory = [MsgType, ?uint24(Len), ?uint16(MsgSeq), ?uint24(0), ?uint24(Len), Bin], - BinMsg = dtls_split_handshake(Mss, MsgType, Len, MsgSeq, Bin, 0, []), - {HsHistory, BinMsg}. + EncHandshake = [MsgType, ?uint24(Len), ?uint16(MsgSeq), ?uint24(0), ?uint24(Len), Bin], + FragmentedHandshake = dtls_fragment(erlang:iolist_size(EncHandshake), MsgType, Len, MsgSeq, Bin, 0, []), + {EncHandshake, FragmentedHandshake}. -%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- -spec get_dtls_handshake(#ssl_tls{}, #dtls_hs_state{} | binary()) -> - {[dtls_handshake()], #dtls_hs_state{}} | {retransmit, #dtls_hs_state{}}. -% -% Description: Given a DTLS state and new data from ssl_record, collects -% and returns it as a list of handshake messages, also returns a new -% DTLS state -%-------------------------------------------------------------------- -% get_dtls_handshake(Record, <<>>) -> -% get_dtls_handshake_aux(Record, dtls_hs_state_init()); + {[dtls_handshake()], #dtls_hs_state{}} | {retransmit, #dtls_hs_state{}}. +%% +%% Description: Given a DTLS state and new data from ssl_record, collects +%% and returns it as a list of handshake messages, also returns a new +%% DTLS state +%%-------------------------------------------------------------------- +get_dtls_handshake(Record, <<>>) -> + get_dtls_handshake_aux(Record, #dtls_hs_state{}); %% Init handshake state!? get_dtls_handshake(Record, HsState) -> get_dtls_handshake_aux(Record, HsState). -%-------------------------------------------------------------------- --spec dtls_handshake_new_epoch(#dtls_hs_state{}) -> #dtls_hs_state{}. -% -% Description: Reset the DTLS decoder state for a new Epoch -%-------------------------------------------------------------------- -% dtls_handshake_new_epoch(<<>>) -> -% dtls_hs_state_init(); -dtls_handshake_new_epoch(HsState) -> - HsState#dtls_hs_state{highest_record_seq = 0, - starting_read_seq = HsState#dtls_hs_state.current_read_seq, - fragments = gb_trees:empty(), completed = []}. - -%-------------------------------------------------------------------- --spec dtls_handshake_new_flight(integer() | undefined) -> #dtls_hs_state{}. -% -% Description: Init the DTLS decoder state for a new Flight -dtls_handshake_new_flight(ExpectedReadReq) -> - #dtls_hs_state{current_read_seq = ExpectedReadReq, - highest_record_seq = 0, - starting_read_seq = 0, - fragments = gb_trees:empty(), completed = []}. +%% %%-------------------------------------------------------------------- +%% -spec dtls_handshake_new_epoch(#dtls_hs_state{}) -> #dtls_hs_state{}. +%% %% +%% %% Description: Reset the DTLS decoder state for a new Epoch +%% %%-------------------------------------------------------------------- +%% dtls_handshake_new_epoch(<<>>) -> +%% dtls_hs_state_init(); +%% dtls_handshake_new_epoch(HsState) -> +%% HsState#dtls_hs_state{highest_record_seq = 0, +%% starting_read_seq = HsState#dtls_hs_state.current_read_seq, +%% fragments = gb_trees:empty(), completed = []}. + +%% %-------------------------------------------------------------------- +%% -spec dtls_handshake_new_flight(integer() | undefined) -> #dtls_hs_state{}. +%% % +%% % Description: Init the DTLS decoder state for a new Flight +%% dtls_handshake_new_flight(ExpectedReadReq) -> +%% #dtls_hs_state{current_read_seq = ExpectedReadReq, +%% highest_record_seq = 0, +%% starting_read_seq = 0, +%% fragments = gb_trees:empty(), completed = []}. %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- +handle_server_hello_extensions(Version, SessionId, Random, CipherSuite, + Compression, HelloExt, SslOpt, ConnectionStates0, Renegotiation) -> + case ssl_handshake:handle_server_hello_extensions(dtls_record, Random, CipherSuite, + Compression, HelloExt, Version, + SslOpt, ConnectionStates0, Renegotiation) of + #alert{} = Alert -> + Alert; + {ConnectionStates, Protocol} -> + {Version, SessionId, ConnectionStates, Protocol} + end. -dtls_split_handshake(Mss, MsgType, Len, MsgSeq, Bin, Offset, Acc) +dtls_fragment(Mss, MsgType, Len, MsgSeq, Bin, Offset, Acc) when byte_size(Bin) + 12 < Mss -> FragmentLen = byte_size(Bin), BinMsg = [MsgType, ?uint24(Len), ?uint16(MsgSeq), ?uint24(Offset), ?uint24(FragmentLen), Bin], lists:reverse([BinMsg|Acc]); -dtls_split_handshake(Mss, MsgType, Len, MsgSeq, Bin, Offset, Acc) -> +dtls_fragment(Mss, MsgType, Len, MsgSeq, Bin, Offset, Acc) -> FragmentLen = Mss - 12, <<Fragment:FragmentLen/bytes, Rest/binary>> = Bin, BinMsg = [MsgType, ?uint24(Len), ?uint16(MsgSeq), ?uint24(Offset), ?uint24(FragmentLen), Fragment], - dtls_split_handshake(Mss, MsgType, Len, MsgSeq, Rest, Offset + FragmentLen, [BinMsg|Acc]). + dtls_fragment(Mss, MsgType, Len, MsgSeq, Rest, Offset + FragmentLen, [BinMsg|Acc]). get_dtls_handshake_aux(#ssl_tls{version = Version, - record_seq = SeqNo, - fragment = Data}, HsState) -> + sequence_number = SeqNo, + fragment = Data}, HsState) -> get_dtls_handshake_aux(Version, SeqNo, Data, HsState). get_dtls_handshake_aux(Version, SeqNo, - <<?BYTE(Type), ?UINT24(Length), - ?UINT16(MessageSeq), - ?UINT24(FragmentOffset), ?UINT24(FragmentLength), - Body:FragmentLength/binary, Rest/binary>>, - HsState0) -> + <<?BYTE(Type), ?UINT24(Length), + ?UINT16(MessageSeq), + ?UINT24(FragmentOffset), ?UINT24(FragmentLength), + Body:FragmentLength/binary, Rest/binary>>, + HsState0) -> case reassemble_dtls_fragment(SeqNo, Type, Length, MessageSeq, - FragmentOffset, FragmentLength, - Body, HsState0) of - {retransmit, HsState1} -> - case Rest of - <<>> -> - {retransmit, HsState1}; - _ -> - get_dtls_handshake_aux(Version, SeqNo, Rest, HsState1) - end; - {HsState1, HighestSeqNo, MsgBody} -> - HsState2 = dec_dtls_fragment(Version, HighestSeqNo, Type, Length, MessageSeq, MsgBody, HsState1), - HsState3 = process_dtls_fragments(Version, HsState2), - get_dtls_handshake_aux(Version, SeqNo, Rest, HsState3); - HsState2 -> - HsState3 = process_dtls_fragments(Version, HsState2), - get_dtls_handshake_aux(Version, SeqNo, Rest, HsState3) - end; + FragmentOffset, FragmentLength, + Body, HsState0) of + {retransmit, HsState1} -> + case Rest of + <<>> -> + {retransmit, HsState1}; + _ -> + get_dtls_handshake_aux(Version, SeqNo, Rest, HsState1) + end; + {HsState1, HighestSeqNo, MsgBody} -> + HsState2 = dec_dtls_fragment(Version, HighestSeqNo, Type, Length, MessageSeq, MsgBody, HsState1), + HsState3 = process_dtls_fragments(Version, HsState2), + get_dtls_handshake_aux(Version, SeqNo, Rest, HsState3); + HsState2 -> + HsState3 = process_dtls_fragments(Version, HsState2), + get_dtls_handshake_aux(Version, SeqNo, Rest, HsState3) + end; get_dtls_handshake_aux(_Version, _SeqNo, <<>>, HsState) -> - {lists:reverse(HsState#dtls_hs_state.completed), - HsState#dtls_hs_state{completed = []}}. + {lists:reverse(HsState#dtls_hs_state.completed), + HsState#dtls_hs_state{completed = []}}. dec_dtls_fragment(Version, SeqNo, Type, Length, MessageSeq, MsgBody, - HsState = #dtls_hs_state{highest_record_seq = HighestSeqNo, completed = Acc}) -> + HsState = #dtls_hs_state{highest_record_seq = HighestSeqNo, completed = Acc}) -> Raw = <<?BYTE(Type), ?UINT24(Length), ?UINT16(MessageSeq), ?UINT24(0), ?UINT24(Length), MsgBody/binary>>, H = decode_handshake(Version, Type, MsgBody), HsState#dtls_hs_state{completed = [{H,Raw}|Acc], highest_record_seq = erlang:max(HighestSeqNo, SeqNo)}. process_dtls_fragments(Version, - HsState0 = #dtls_hs_state{current_read_seq = CurrentReadSeq, - fragments = Fragments0}) -> + HsState0 = #dtls_hs_state{current_read_seq = CurrentReadSeq, + fragments = Fragments0}) -> case gb_trees:is_empty(Fragments0) of - true -> - HsState0; - _ -> - case gb_trees:smallest(Fragments0) of - {CurrentReadSeq, {SeqNo, Type, Length, CurrentReadSeq, {Length, [{0, Length}], MsgBody}}} -> - HsState1 = dtls_hs_state_process_seq(HsState0), - HsState2 = dec_dtls_fragment(Version, SeqNo, Type, Length, CurrentReadSeq, MsgBody, HsState1), - process_dtls_fragments(Version, HsState2); - _ -> - HsState0 - end - end. + true -> + HsState0; + _ -> + case gb_trees:smallest(Fragments0) of + {CurrentReadSeq, {SeqNo, Type, Length, CurrentReadSeq, {Length, [{0, Length}], MsgBody}}} -> + HsState1 = dtls_hs_state_process_seq(HsState0), + HsState2 = dec_dtls_fragment(Version, SeqNo, Type, Length, CurrentReadSeq, MsgBody, HsState1), + process_dtls_fragments(Version, HsState2); + _ -> + HsState0 + end + end. dtls_hs_state_process_seq(HsState0 = #dtls_hs_state{current_read_seq = CurrentReadSeq, - fragments = Fragments0}) -> + fragments = Fragments0}) -> Fragments1 = gb_trees:delete_any(CurrentReadSeq, Fragments0), HsState0#dtls_hs_state{current_read_seq = CurrentReadSeq + 1, - fragments = Fragments1}. + fragments = Fragments1}. dtls_hs_state_add_fragment(MessageSeq, Fragment, HsState0 = #dtls_hs_state{fragments = Fragments0}) -> Fragments1 = gb_trees:enter(MessageSeq, Fragment, Fragments0), HsState0#dtls_hs_state{fragments = Fragments1}. reassemble_dtls_fragment(SeqNo, Type, Length, MessageSeq, 0, Length, - Body, HsState0 = #dtls_hs_state{current_read_seq = undefined}) + Body, HsState0 = #dtls_hs_state{current_read_seq = undefined}) when Type == ?CLIENT_HELLO; Type == ?SERVER_HELLO; - Type == ?HELLO_VERIFY_REQUEST -> + Type == ?HELLO_VERIFY_REQUEST -> %% First message, should be client hello %% return the current message and set the next expected Sequence %% @@ -245,8 +283,8 @@ reassemble_dtls_fragment(_SeqNo, _Type, Length, _MessageSeq, _, Length, HsState; reassemble_dtls_fragment(SeqNo, _Type, Length, MessageSeq, 0, Length, - Body, HsState0 = - #dtls_hs_state{starting_read_seq = StartingReadSeq}) + Body, HsState0 = + #dtls_hs_state{starting_read_seq = StartingReadSeq}) when MessageSeq < StartingReadSeq -> %% this has to be the start of a new flight, let it through %% @@ -257,69 +295,69 @@ reassemble_dtls_fragment(SeqNo, _Type, Length, MessageSeq, 0, Length, {HsState, SeqNo, Body}; reassemble_dtls_fragment(_SeqNo, _Type, Length, MessageSeq, 0, Length, - _Body, HsState = - #dtls_hs_state{current_read_seq = CurrentReadSeq}) + _Body, HsState = + #dtls_hs_state{current_read_seq = CurrentReadSeq}) when MessageSeq < CurrentReadSeq -> {retransmit, HsState}; reassemble_dtls_fragment(_SeqNo, _Type, Length, MessageSeq, 0, Length, - _Body, HsState = #dtls_hs_state{current_read_seq = CurrentReadSeq}) + _Body, HsState = #dtls_hs_state{current_read_seq = CurrentReadSeq}) when MessageSeq < CurrentReadSeq -> HsState; reassemble_dtls_fragment(SeqNo, _Type, Length, MessageSeq, 0, Length, - Body, HsState0 = #dtls_hs_state{current_read_seq = MessageSeq}) -> + Body, HsState0 = #dtls_hs_state{current_read_seq = MessageSeq}) -> %% Message fully contained and it's the current seq HsState1 = dtls_hs_state_process_seq(HsState0), {HsState1, SeqNo, Body}; reassemble_dtls_fragment(SeqNo, Type, Length, MessageSeq, 0, Length, - Body, HsState) -> + Body, HsState) -> %% Message fully contained and it's the NOT the current seq -> buffer Fragment = {SeqNo, Type, Length, MessageSeq, - dtls_fragment_init(Length, 0, Length, Body)}, + dtls_fragment_init(Length, 0, Length, Body)}, dtls_hs_state_add_fragment(MessageSeq, Fragment, HsState); reassemble_dtls_fragment(_SeqNo, _Type, Length, MessageSeq, FragmentOffset, FragmentLength, - _Body, - HsState = #dtls_hs_state{current_read_seq = CurrentReadSeq}) + _Body, + HsState = #dtls_hs_state{current_read_seq = CurrentReadSeq}) when FragmentOffset + FragmentLength == Length andalso MessageSeq == (CurrentReadSeq - 1) -> {retransmit, HsState}; reassemble_dtls_fragment(_SeqNo, _Type, _Length, MessageSeq, _FragmentOffset, _FragmentLength, - _Body, - HsState = #dtls_hs_state{current_read_seq = CurrentReadSeq}) + _Body, + HsState = #dtls_hs_state{current_read_seq = CurrentReadSeq}) when MessageSeq < CurrentReadSeq -> HsState; reassemble_dtls_fragment(SeqNo, Type, Length, MessageSeq, - FragmentOffset, FragmentLength, - Body, - HsState = #dtls_hs_state{fragments = Fragments0}) -> + FragmentOffset, FragmentLength, + Body, + HsState = #dtls_hs_state{fragments = Fragments0}) -> case gb_trees:lookup(MessageSeq, Fragments0) of - {value, Fragment} -> - dtls_fragment_reassemble(SeqNo, Type, Length, MessageSeq, - FragmentOffset, FragmentLength, - Body, Fragment, HsState); - none -> - dtls_fragment_start(SeqNo, Type, Length, MessageSeq, - FragmentOffset, FragmentLength, - Body, HsState) + {value, Fragment} -> + dtls_fragment_reassemble(SeqNo, Type, Length, MessageSeq, + FragmentOffset, FragmentLength, + Body, Fragment, HsState); + none -> + dtls_fragment_start(SeqNo, Type, Length, MessageSeq, + FragmentOffset, FragmentLength, + Body, HsState) end. dtls_fragment_start(SeqNo, Type, Length, MessageSeq, - FragmentOffset, FragmentLength, - Body, HsState = #dtls_hs_state{fragments = Fragments0}) -> + FragmentOffset, FragmentLength, + Body, HsState = #dtls_hs_state{fragments = Fragments0}) -> Fragment = {SeqNo, Type, Length, MessageSeq, - dtls_fragment_init(Length, FragmentOffset, FragmentLength, Body)}, - Fragments1 = gb_trees:insert(MessageSeq, Fragment, Fragments0), + dtls_fragment_init(Length, FragmentOffset, FragmentLength, Body)}, + Fragments1 = gb_trees:insert(MessageSeq, Fragment, Fragments0), HsState#dtls_hs_state{fragments = Fragments1}. dtls_fragment_reassemble(SeqNo, Type, Length, MessageSeq, FragmentOffset, FragmentLength, - Body, - {LastSeqNo, Type, Length, MessageSeq, FragBuffer0}, - HsState = #dtls_hs_state{fragments = Fragments0}) -> + Body, + {LastSeqNo, Type, Length, MessageSeq, FragBuffer0}, + HsState = #dtls_hs_state{fragments = Fragments0}) -> FragBuffer1 = dtls_fragment_add(FragBuffer0, FragmentOffset, FragmentLength, Body), Fragment = {erlang:max(SeqNo, LastSeqNo), Type, Length, MessageSeq, FragBuffer1}, Fragments1 = gb_trees:enter(MessageSeq, Fragment, Fragments0), @@ -328,8 +366,8 @@ dtls_fragment_reassemble(SeqNo, Type, Length, MessageSeq, %% Type, Length or Seq mismatch, drop everything... %% Note: the RFC is not clear on how to handle this... dtls_fragment_reassemble(_SeqNo, _Type, _Length, MessageSeq, - _FragmentOffset, _FragmentLength, _Body, _Fragment, - HsState = #dtls_hs_state{fragments = Fragments0}) -> + _FragmentOffset, _FragmentLength, _Body, _Fragment, + HsState = #dtls_hs_state{fragments = Fragments0}) -> Fragments1 = gb_trees:delete_any(MessageSeq, Fragments0), HsState#dtls_hs_state{fragments = Fragments1}. @@ -360,7 +398,7 @@ merge_fragment_list(Rest = [{HStart, _HEnd}|_], Frag = {_FStart, FEnd}, Acc) lists:reverse(Acc) ++ [Frag|Rest]; merge_fragment_list([{HStart, HEnd}|Rest], _Frag = {FStart, FEnd}, Acc) - when + when FStart =< HEnd orelse FEnd >= HStart -> Start = erlang:min(HStart, FStart), End = erlang:max(HEnd, FEnd), @@ -370,20 +408,20 @@ merge_fragment_list([{HStart, HEnd}|Rest], _Frag = {FStart, FEnd}, Acc) add_fragment(List, {FragmentOffset, FragmentLength}) -> merge_fragment_list(List, {FragmentOffset, FragmentOffset + FragmentLength}, []). -enc_hs(#hello_verify_request{protocol_version = {Major, Minor}, - cookie = Cookie}, _Version) -> - CookieLength = byte_size(Cookie), +enc_handshake(#hello_verify_request{protocol_version = {Major, Minor}, + cookie = Cookie}, _Version) -> + CookieLength = byte_size(Cookie), {?HELLO_VERIFY_REQUEST, <<?BYTE(Major), ?BYTE(Minor), - ?BYTE(CookieLength), - Cookie/binary>>}; - -enc_hs(#client_hello{client_version = {Major, Minor}, - random = Random, - session_id = SessionID, - cookie = Cookie, - cipher_suites = CipherSuites, - compression_methods = CompMethods, - extensions = HelloExtensions}, Version) -> + ?BYTE(CookieLength), + Cookie/binary>>}; + +enc_handshake(#client_hello{client_version = {Major, Minor}, + random = Random, + session_id = SessionID, + cookie = Cookie, + cipher_suites = CipherSuites, + compression_methods = CompMethods, + extensions = HelloExtensions}, Version) -> SIDLength = byte_size(SessionID), BinCookie = enc_client_hello_cookie(Version, Cookie), BinCompMethods = list_to_binary(CompMethods), @@ -391,13 +429,13 @@ enc_hs(#client_hello{client_version = {Major, Minor}, BinCipherSuites = list_to_binary(CipherSuites), CsLength = byte_size(BinCipherSuites), ExtensionsBin = ssl_handshake:encode_hello_extensions(HelloExtensions), - + {?CLIENT_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, - ?BYTE(SIDLength), SessionID/binary, - BinCookie/binary, - ?UINT16(CsLength), BinCipherSuites/binary, - ?BYTE(CmLength), BinCompMethods/binary, ExtensionsBin/binary>>}; -enc_hs(HandshakeMsg, Version) -> + ?BYTE(SIDLength), SessionID/binary, + BinCookie/binary, + ?UINT16(CsLength), BinCipherSuites/binary, + ?BYTE(CmLength), BinCompMethods/binary, ExtensionsBin/binary>>}; +enc_handshake(HandshakeMsg, Version) -> ssl_handshake:encode_handshake(HandshakeMsg, Version). enc_client_hello_cookie(_, <<>>) -> @@ -407,26 +445,26 @@ enc_client_hello_cookie(_, Cookie) -> <<?BYTE(CookieLength), Cookie/binary>>. decode_handshake(_Version, ?CLIENT_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, - ?BYTE(SID_length), Session_ID:SID_length/binary, - ?BYTE(Cookie_length), Cookie:Cookie_length/binary, - ?UINT16(Cs_length), CipherSuites:Cs_length/binary, - ?BYTE(Cm_length), Comp_methods:Cm_length/binary, - Extensions/binary>>) -> - + ?BYTE(SID_length), Session_ID:SID_length/binary, + ?BYTE(Cookie_length), Cookie:Cookie_length/binary, + ?UINT16(Cs_length), CipherSuites:Cs_length/binary, + ?BYTE(Cm_length), Comp_methods:Cm_length/binary, + Extensions/binary>>) -> + DecodedExtensions = ssl_handshake:decode_hello_extensions(Extensions), - + #client_hello{ client_version = {Major,Minor}, random = Random, - session_id = Session_ID, + session_id = Session_ID, cookie = Cookie, cipher_suites = ssl_handshake:decode_suites('2_bytes', CipherSuites), compression_methods = Comp_methods, extensions = DecodedExtensions - }; + }; decode_handshake(_Version, ?HELLO_VERIFY_REQUEST, <<?BYTE(Major), ?BYTE(Minor), - ?BYTE(CookieLength), Cookie:CookieLength/binary>>) -> + ?BYTE(CookieLength), Cookie:CookieLength/binary>>) -> #hello_verify_request{ protocol_version = {Major,Minor}, @@ -434,7 +472,7 @@ decode_handshake(_Version, ?HELLO_VERIFY_REQUEST, <<?BYTE(Major), ?BYTE(Minor), decode_handshake(Version, Tag, Msg) -> ssl_handshake:decode_handshake(Version, Tag, Msg). -address_to_bin({A,B,C,D}, Port) -> - <<0:80,16#ffff:16,A,B,C,D,Port:16>>; -address_to_bin({A,B,C,D,E,F,G,H}, Port) -> - <<A:16,B:16,C:16,D:16,E:16,F:16,G:16,H:16,Port:16>>. +%% address_to_bin({A,B,C,D}, Port) -> +%% <<0:80,16#ffff:16,A,B,C,D,Port:16>>; +%% address_to_bin({A,B,C,D,E,F,G,H}, Port) -> +%% <<A:16,B:16,C:16,D:16,E:16,F:16,G:16,H:16,Port:16>>. diff --git a/lib/ssl/src/dtls_handshake.hrl b/lib/ssl/src/dtls_handshake.hrl index 5bdf45f627..3b57575b6d 100644 --- a/lib/ssl/src/dtls_handshake.hrl +++ b/lib/ssl/src/dtls_handshake.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -53,6 +53,4 @@ completed }). --type dtls_handshake() :: #client_hello{} | #hello_verify_request{} | ssl_handshake(). - -endif. % -ifdef(dtls_handshake). diff --git a/lib/ssl/src/dtls_record.erl b/lib/ssl/src/dtls_record.erl index b0a7976864..a7bbb6bc40 100644 --- a/lib/ssl/src/dtls_record.erl +++ b/lib/ssl/src/dtls_record.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -35,7 +35,7 @@ -export([decode_cipher_text/2]). %% Encoding --export([encode_plain_text/4]). +-export([encode_plain_text/4, encode_handshake/3, encode_change_cipher_spec/2]). %% Protocol version handling -export([protocol_version/1, lowest_protocol_version/2, @@ -46,6 +46,11 @@ -export([init_connection_state_seq/2, current_connection_state_epoch/2, set_connection_state_by_epoch/3, connection_state_by_epoch/3]). +-export_type([dtls_version/0, dtls_atom_version/0]). + +-type dtls_version() :: ssl_record:ssl_version(). +-type dtls_atom_version() :: dtlsv1 | 'dtlsv1.2'. + -compile(inline). %%==================================================================== @@ -70,7 +75,7 @@ get_dtls_records_aux(<<?BYTE(?APPLICATION_DATA),?BYTE(MajVer),?BYTE(MinVer), Acc) -> get_dtls_records_aux(Rest, [#ssl_tls{type = ?APPLICATION_DATA, version = {MajVer, MinVer}, - epoch = Epoch, record_seq = SequenceNumber, + epoch = Epoch, sequence_number = SequenceNumber, fragment = Data} | Acc]); get_dtls_records_aux(<<?BYTE(?HANDSHAKE),?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Epoch), ?UINT48(SequenceNumber), @@ -78,7 +83,7 @@ get_dtls_records_aux(<<?BYTE(?HANDSHAKE),?BYTE(MajVer),?BYTE(MinVer), Data:Length/binary, Rest/binary>>, Acc) when MajVer >= 128 -> get_dtls_records_aux(Rest, [#ssl_tls{type = ?HANDSHAKE, version = {MajVer, MinVer}, - epoch = Epoch, record_seq = SequenceNumber, + epoch = Epoch, sequence_number = SequenceNumber, fragment = Data} | Acc]); get_dtls_records_aux(<<?BYTE(?ALERT),?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Epoch), ?UINT48(SequenceNumber), @@ -86,7 +91,7 @@ get_dtls_records_aux(<<?BYTE(?ALERT),?BYTE(MajVer),?BYTE(MinVer), Rest/binary>>, Acc) -> get_dtls_records_aux(Rest, [#ssl_tls{type = ?ALERT, version = {MajVer, MinVer}, - epoch = Epoch, record_seq = SequenceNumber, + epoch = Epoch, sequence_number = SequenceNumber, fragment = Data} | Acc]); get_dtls_records_aux(<<?BYTE(?CHANGE_CIPHER_SPEC),?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Epoch), ?UINT48(SequenceNumber), @@ -94,7 +99,7 @@ get_dtls_records_aux(<<?BYTE(?CHANGE_CIPHER_SPEC),?BYTE(MajVer),?BYTE(MinVer), Acc) -> get_dtls_records_aux(Rest, [#ssl_tls{type = ?CHANGE_CIPHER_SPEC, version = {MajVer, MinVer}, - epoch = Epoch, record_seq = SequenceNumber, + epoch = Epoch, sequence_number = SequenceNumber, fragment = Data} | Acc]); get_dtls_records_aux(<<0:1, _CT:7, ?BYTE(_MajVer), ?BYTE(_MinVer), @@ -125,14 +130,15 @@ encode_plain_text(Type, Version, Data, {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), WriteState1 = WriteState0#connection_state{compression_state = CompS1}, MacHash = calc_mac_hash(WriteState1, Type, Version, Epoch, Seq, Comp), - {CipherFragment, WriteState} = ssl_record:cipher(Version, Comp, WriteState1, MacHash), + {CipherFragment, WriteState} = ssl_record:cipher(dtls_v1:corresponding_tls_version(Version), + Comp, WriteState1, MacHash), CipherText = encode_tls_cipher_text(Type, Version, Epoch, Seq, CipherFragment), {CipherText, ConnectionStates#connection_states{current_write = WriteState#connection_state{sequence_number = Seq +1}}}. decode_cipher_text(#ssl_tls{type = Type, version = Version, epoch = Epoch, - record_seq = Seq, + sequence_number = Seq, fragment = CipherFragment} = CipherText, #connection_states{current_read = #connection_state{compression_state = CompressionS0, @@ -141,7 +147,7 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, CompressAlg = SecParams#security_parameters.compression_algorithm, {PlainFragment, Mac, ReadState1} = ssl_record:decipher(dtls_v1:corresponding_tls_version(Version), CipherFragment, ReadState0), - MacHash = calc_mac_hash(Type, Version, Epoch, Seq, PlainFragment, ReadState1), + MacHash = calc_mac_hash(ReadState1, Type, Version, Epoch, Seq, PlainFragment), case ssl_record:is_correct_mac(Mac, MacHash) of true -> {Plain, CompressionS1} = ssl_record:uncompress(CompressAlg, @@ -153,10 +159,27 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, false -> ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC) end. +%%-------------------------------------------------------------------- +-spec encode_handshake(iolist(), dtls_version(), #connection_states{}) -> + {iolist(), #connection_states{}}. +%% +%% Description: Encodes a handshake message to send on the ssl-socket. +%%-------------------------------------------------------------------- +encode_handshake(Frag, Version, ConnectionStates) -> + encode_plain_text(?HANDSHAKE, Version, Frag, ConnectionStates). + +%%-------------------------------------------------------------------- +-spec encode_change_cipher_spec(dtls_version(), #connection_states{}) -> + {iolist(), #connection_states{}}. +%% +%% Description: Encodes a change_cipher_spec-message to send on the ssl socket. +%%-------------------------------------------------------------------- +encode_change_cipher_spec(Version, ConnectionStates) -> + encode_plain_text(?CHANGE_CIPHER_SPEC, Version, <<1:8>>, ConnectionStates). %%-------------------------------------------------------------------- --spec protocol_version(tls_atom_version() | tls_version()) -> - tls_version() | tls_atom_version(). +-spec protocol_version(dtls_atom_version() | dtls_version()) -> + dtls_version() | dtls_atom_version(). %% %% Description: Creates a protocol version record from a version atom %% or vice versa. @@ -170,7 +193,7 @@ protocol_version({254, 253}) -> protocol_version({254, 255}) -> dtlsv1. %%-------------------------------------------------------------------- --spec lowest_protocol_version(tls_version(), tls_version()) -> tls_version(). +-spec lowest_protocol_version(dtls_version(), dtls_version()) -> dtls_version(). %% %% Description: Lowes protocol version of two given versions %%-------------------------------------------------------------------- @@ -183,7 +206,7 @@ lowest_protocol_version(Version = {M,_}, {N, _}) when M > N -> lowest_protocol_version(_,Version) -> Version. %%-------------------------------------------------------------------- --spec highest_protocol_version([tls_version()]) -> tls_version(). +-spec highest_protocol_version([dtls_version()]) -> dtls_version(). %% %% Description: Highest protocol version present in a list %%-------------------------------------------------------------------- @@ -203,7 +226,7 @@ highest_protocol_version(_, [Version | Rest]) -> %%-------------------------------------------------------------------- --spec supported_protocol_versions() -> [tls_version()]. +-spec supported_protocol_versions() -> [dtls_version()]. %% %% Description: Protocol versions supported %%-------------------------------------------------------------------- @@ -234,7 +257,7 @@ supported_connection_protocol_versions([]) -> ?ALL_DATAGRAM_SUPPORTED_VERSIONS. %%-------------------------------------------------------------------- --spec is_acceptable_version(tls_version(), Supported :: [tls_version()]) -> boolean(). +-spec is_acceptable_version(dtls_version(), Supported :: [dtls_version()]) -> boolean(). %% %% Description: ssl version 2 is not acceptable security risks are too big. %% @@ -244,7 +267,7 @@ is_acceptable_version(Version, Versions) -> %%-------------------------------------------------------------------- --spec init_connection_state_seq(tls_version(), #connection_states{}) -> +-spec init_connection_state_seq(dtls_version(), #connection_states{}) -> #connection_state{}. %% %% Description: Copy the read sequence number to the write sequence number @@ -343,5 +366,5 @@ calc_mac_hash(#connection_state{mac_secret = MacSecret, Length, Fragment). mac_hash(Version, MacAlg, MacSecret, SeqNo, Type, Length, Fragment) -> - dtls_v1:mac_hash(MacAlg, MacSecret, SeqNo, Type, Version, + dtls_v1:mac_hash(Version, MacAlg, MacSecret, SeqNo, Type, Length, Fragment). diff --git a/lib/ssl/src/dtls_record.hrl b/lib/ssl/src/dtls_record.hrl index e935d84bdf..edb77fb2b1 100644 --- a/lib/ssl/src/dtls_record.hrl +++ b/lib/ssl/src/dtls_record.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,16 +28,15 @@ -include("ssl_record.hrl"). %% Common TLS and DTLS records and Constantes -%% Used to handle tls_plain_text, tls_compressed and tls_cipher_text +%% Used to handle dtls_plain_text, dtls_compressed and dtls_cipher_text -record(ssl_tls, { type, version, - record_seq, % used in plain_text - epoch, % used in plain_text - message_seq, - fragment_offset, - fragment_length, + epoch, + sequence_number, + offset, + length, fragment }). diff --git a/lib/ssl/src/dtls_v1.erl b/lib/ssl/src/dtls_v1.erl index 6e41641483..5a7ab32887 100644 --- a/lib/ssl/src/dtls_v1.erl +++ b/lib/ssl/src/dtls_v1.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,7 +22,7 @@ -export([suites/1, mac_hash/7, ecc_curves/1, corresponding_tls_version/1]). --spec suites(Minor:: 253|255) -> [cipher_suite()]. +-spec suites(Minor:: 253|255) -> [ssl_cipher:cipher_suite()]. suites(Minor) -> tls_v1:suites(corresponding_minor_tls_version(Minor)). diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 9e098e12c4..a88bf45293 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -276,7 +276,7 @@ controlling_process(#sslsocket{pid = {Listen, Transport:controlling_process(Listen, NewOwner). %%-------------------------------------------------------------------- --spec connection_info(#sslsocket{}) -> {ok, {tls_atom_version(), erl_cipher_suite()}} | +-spec connection_info(#sslsocket{}) -> {ok, {tls_record:tls_atom_version(), ssl_cipher:erl_cipher_suite()}} | {error, reason()}. %% %% Description: Returns ssl protocol and cipher used for the connection @@ -312,7 +312,7 @@ peercert(#sslsocket{pid = {Listen, _}}) when is_port(Listen) -> {error, enotconn}. %%-------------------------------------------------------------------- --spec suite_definition(cipher_suite()) -> erl_cipher_suite(). +-spec suite_definition(ssl_cipher:cipher_suite()) -> ssl_cipher:erl_cipher_suite(). %% %% Description: Return erlang cipher suite definition. %%-------------------------------------------------------------------- @@ -330,8 +330,8 @@ negotiated_next_protocol(#sslsocket{pid = Pid}) -> ssl_connection:negotiated_next_protocol(Pid). %%-------------------------------------------------------------------- --spec cipher_suites() -> [erl_cipher_suite()]. --spec cipher_suites(erlang | openssl | all) -> [erl_cipher_suite()] | [string()]. +-spec cipher_suites() -> [ssl_cipher:erl_cipher_suite()]. +-spec cipher_suites(erlang | openssl | all) -> [ssl_cipher:erl_cipher_suite()] | [string()]. %% Description: Returns all supported cipher suites. %%-------------------------------------------------------------------- @@ -437,8 +437,8 @@ session_info(#sslsocket{pid = {Listen,_}}) when is_port(Listen) -> {error, enotconn}. %%--------------------------------------------------------------- --spec versions() -> [{ssl_app, string()} | {supported, [tls_atom_version()]} | - {available, [tls_atom_version()]}]. +-spec versions() -> [{ssl_app, string()} | {supported, [tls_record:tls_atom_version()]} | + {available, [tls_record:tls_atom_version()]}]. %% %% Description: Returns a list of relevant versions. %%-------------------------------------------------------------------- @@ -558,6 +558,8 @@ handle_options(Opts0, _Role) -> Opts = proplists:expand([{binary, [{mode, binary}]}, {list, [{mode, list}]}], Opts0), assert_proplist(Opts), + RecordCb = record_cb(Opts), + ReuseSessionFun = fun(_, _, _, _) -> true end, DefaultVerifyNoneFun = @@ -600,12 +602,14 @@ handle_options(Opts0, _Role) -> end, CertFile = handle_option(certfile, Opts, <<>>), - + + RecordCb = record_cb(Opts), + Versions = case handle_option(versions, Opts, []) of [] -> - tls_record:supported_protocol_versions(); + RecordCb:supported_protocol_versions(); Vsns -> - [tls_record:protocol_version(Vsn) || Vsn <- Vsns] + [RecordCb:protocol_version(Vsn) || Vsn <- Vsns] end, SSLOptions = #ssl_options{ @@ -1035,6 +1039,13 @@ connection_cb(dtls) -> connection_cb(Opts) -> connection_cb(proplists:get_value(protocol, Opts, tls)). +record_cb(tls) -> + tls_record; +record_cb(dtls) -> + dtls_record; +record_cb(Opts) -> + record_cb(proplists:get_value(protocol, Opts, tls)). + connection_sup(tls_connection) -> tls_connection_sup; connection_sup(dtls_connection) -> diff --git a/lib/ssl/src/ssl_alert.erl b/lib/ssl/src/ssl_alert.erl index 5c842b4d19..db1535b5ec 100644 --- a/lib/ssl/src/ssl_alert.erl +++ b/lib/ssl/src/ssl_alert.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -38,7 +38,7 @@ %%==================================================================== %%-------------------------------------------------------------------- --spec encode(#alert{}, tls_version(), #connection_states{}) -> +-spec encode(#alert{}, ssl_record:ssl_version(), #connection_states{}) -> {iolist(), #connection_states{}}. %% %% Description: @@ -124,5 +124,17 @@ description_txt(?USER_CANCELED) -> "user canceled"; description_txt(?NO_RENEGOTIATION) -> "no renegotiation"; +description_txt(?UNSUPPORTED_EXTENSION) -> + "unsupported extension"; +description_txt(?CERTIFICATE_UNOBTAINABLE) -> + "certificate unobtainable"; +description_txt(?UNRECOGNISED_NAME) -> + "unrecognised name"; +description_txt(?BAD_CERTIFICATE_STATUS_RESPONSE) -> + "bad certificate status response"; +description_txt(?BAD_CERTIFICATE_HASH_VALUE) -> + "bad certificate hash value"; description_txt(?UNKNOWN_PSK_IDENTITY) -> - "unknown psk identity". + "unknown psk identity"; +description_txt(Enum) -> + lists:flatten(io_lib:format("unsupported/unknown alert: ~p", [Enum])). diff --git a/lib/ssl/src/ssl_alert.hrl b/lib/ssl/src/ssl_alert.hrl index 2a8a91aefa..2d1f323085 100644 --- a/lib/ssl/src/ssl_alert.hrl +++ b/lib/ssl/src/ssl_alert.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2012. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -48,7 +48,7 @@ %% unsupported_certificate(43), %% certificate_revoked(44), %% certificate_expired(45), - %% certificate_unknown(46), +%% certificate_unknown(46), %% illegal_parameter(47), %% unknown_ca(48), %% access_denied(49), @@ -60,6 +60,13 @@ %% internal_error(80), %% user_canceled(90), %% no_renegotiation(100), +%% RFC 4366 +%% unsupported_extension(110), +%% certificate_unobtainable(111), +%% unrecognized_name(112), +%% bad_certificate_status_response(113), +%% bad_certificate_hash_value(114), +%% RFC 4366 %% unknown_psk_identity(115), %% (255) %% } AlertDescription; @@ -88,6 +95,11 @@ -define(INTERNAL_ERROR, 80). -define(USER_CANCELED, 90). -define(NO_RENEGOTIATION, 100). +-define(UNSUPPORTED_EXTENSION, 110). +-define(CERTIFICATE_UNOBTAINABLE, 111). +-define(UNRECOGNISED_NAME, 112). +-define(BAD_CERTIFICATE_STATUS_RESPONSE, 113). +-define(BAD_CERTIFICATE_HASH_VALUE, 114). -define(UNKNOWN_PSK_IDENTITY, 115). -define(ALERT_REC(Level,Desc), #alert{level=Level,description=Desc,where={?FILE, ?LINE}}). diff --git a/lib/ssl/src/ssl_api.hrl b/lib/ssl/src/ssl_api.hrl index 607991750f..22185ff60a 100644 --- a/lib/ssl/src/ssl_api.hrl +++ b/lib/ssl/src/ssl_api.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -24,8 +24,6 @@ %% Visible in API -export_type([connect_option/0, listen_option/0, ssl_option/0, transport_option/0, - erl_cipher_suite/0, %% From ssl_cipher.hrl - tls_atom_version/0, %% From ssl_internal.hrl prf_random/0, sslsocket/0]). @@ -39,23 +37,24 @@ -type listen_option() :: socket_listen_option() | ssl_option() | transport_option(). -type socket_listen_option() :: gen_tcp:listen_option(). --type ssl_option() :: {verify, verify_type()} | - {verify_fun, {fun(), InitialUserState::term()}} | - {fail_if_no_peer_cert, boolean()} | {depth, integer()} | - {cert, Der::binary()} | {certfile, path()} | {key, Der::binary()} | - {keyfile, path()} | {password, string()} | {cacerts, [Der::binary()]} | - {cacertfile, path()} | {dh, Der::binary()} | {dhfile, path()} | - {user_lookup_fun, {fun(), InitialUserState::term()}} | - {psk_identity, string()} | - {srp_identity, {string(), string()}} | - {ciphers, ciphers()} | {ssl_imp, ssl_imp()} | {reuse_sessions, boolean()} | - {reuse_session, fun()} | {hibernate_after, integer()|undefined} | - {next_protocols_advertised, list(binary())} | - {client_preferred_next_protocols, binary(), client | server, list(binary())}. +-type ssl_option() :: {versions, ssl_record:ssl_atom_version()} | + {verify, verify_type()} | + {verify_fun, {fun(), InitialUserState::term()}} | + {fail_if_no_peer_cert, boolean()} | {depth, integer()} | + {cert, Der::binary()} | {certfile, path()} | {key, Der::binary()} | + {keyfile, path()} | {password, string()} | {cacerts, [Der::binary()]} | + {cacertfile, path()} | {dh, Der::binary()} | {dhfile, path()} | + {user_lookup_fun, {fun(), InitialUserState::term()}} | + {psk_identity, string()} | + {srp_identity, {string(), string()}} | + {ciphers, ciphers()} | {ssl_imp, ssl_imp()} | {reuse_sessions, boolean()} | + {reuse_session, fun()} | {hibernate_after, integer()|undefined} | + {next_protocols_advertised, list(binary())} | + {client_preferred_next_protocols, binary(), client | server, list(binary())}. -type verify_type() :: verify_none | verify_peer. -type path() :: string(). --type ciphers() :: [erl_cipher_suite()] | +-type ciphers() :: [ssl_cipher:erl_cipher_suite()] | string(). % (according to old API) -type ssl_imp() :: new | old. diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index b2077c662a..78a328ace8 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -38,6 +38,21 @@ openssl_suite/1, openssl_suite_name/1, filter/2, filter_suites/1, hash_algorithm/1, sign_algorithm/1, is_acceptable_hash/2]). +-export_type([cipher_suite/0, + erl_cipher_suite/0, openssl_cipher_suite/0, + key_algo/0]). + +-type cipher() :: null |rc4_128 | idea_cbc | des40_cbc | des_cbc | '3des_ede_cbc' + | aes_128_cbc | aes_256_cbc. +-type hash() :: null | sha | md5 | sha224 | sha256 | sha384 | sha512. +-type key_algo() :: null | rsa | dhe_rsa | dhe_dss | ecdhe_ecdsa| ecdh_ecdsa | ecdh_rsa| srp_rsa| srp_dss | psk | dhe_psk | rsa_psk | dh_anon | ecdh_anon | srp_anon. +-type erl_cipher_suite() :: {key_algo(), cipher(), hash()}. +-type int_cipher_suite() :: {key_algo(), cipher(), hash(), hash() | default_prf}. +-type cipher_suite() :: binary(). +-type cipher_enum() :: integer(). +-type openssl_cipher_suite() :: string(). + + -compile(inline). %%-------------------------------------------------------------------- @@ -51,7 +66,7 @@ security_parameters(?TLS_NULL_WITH_NULL_NULL = CipherSuite, SecParams) -> security_parameters(undefined, CipherSuite, SecParams). %%-------------------------------------------------------------------- --spec security_parameters(tls_version() | undefined, cipher_suite(), #security_parameters{}) -> +-spec security_parameters(ssl_record:ssl_version() | undefined, cipher_suite(), #security_parameters{}) -> #security_parameters{}. %% %% Description: Returns a security parameters record where the @@ -72,7 +87,7 @@ security_parameters(Version, CipherSuite, SecParams) -> hash_size = hash_size(Hash)}. %%-------------------------------------------------------------------- --spec cipher(cipher_enum(), #cipher_state{}, binary(), iolist(), tls_version()) -> +-spec cipher(cipher_enum(), #cipher_state{}, binary(), iodata(), ssl_record:ssl_version()) -> {binary(), #cipher_state{}}. %% %% Description: Encrypts the data and the MAC using chipher described @@ -127,7 +142,7 @@ block_cipher(Fun, BlockSz, #cipher_state{key=Key, iv=IV} = CS0, {T, CS0#cipher_state{iv=NextIV}}. %%-------------------------------------------------------------------- --spec decipher(cipher_enum(), integer(), #cipher_state{}, binary(), tls_version()) -> +-spec decipher(cipher_enum(), integer(), #cipher_state{}, binary(), ssl_record:ssl_version()) -> {binary(), binary(), #cipher_state{}} | #alert{}. %% %% Description: Decrypts the data and the MAC using cipher described @@ -200,7 +215,7 @@ block_decipher(Fun, #cipher_state{key=Key, iv=IV} = CipherState0, ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC) end. %%-------------------------------------------------------------------- --spec suites(tls_version()) -> [cipher_suite()]. +-spec suites(ssl_record:ssl_version()) -> [cipher_suite()]. %% %% Description: Returns a list of supported cipher suites. %%-------------------------------------------------------------------- @@ -229,7 +244,7 @@ anonymous_suites() -> ?TLS_ECDH_anon_WITH_AES_256_CBC_SHA]. %%-------------------------------------------------------------------- --spec psk_suites(tls_version() | integer()) -> [cipher_suite()]. +-spec psk_suites(ssl_record:ssl_version() | integer()) -> [cipher_suite()]. %% %% Description: Returns a list of the PSK cipher suites, only supported %% if explicitly set by user. diff --git a/lib/ssl/src/ssl_cipher.hrl b/lib/ssl/src/ssl_cipher.hrl index 62a5269def..3ce9c19aa9 100644 --- a/lib/ssl/src/ssl_cipher.hrl +++ b/lib/ssl/src/ssl_cipher.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -26,16 +26,6 @@ -ifndef(ssl_cipher). -define(ssl_cipher, true). --type cipher() :: null |rc4_128 | idea_cbc | des40_cbc | des_cbc | '3des_ede_cbc' - | aes_128_cbc | aes_256_cbc. --type hash() :: null | sha | md5 | sha224 | sha256 | sha384 | sha512. --type key_algo() :: null | rsa | dhe_rsa | dhe_dss | ecdhe_ecdsa| ecdh_ecdsa | ecdh_rsa| srp_rsa| srp_dss | psk | dhe_psk | rsa_psk | dh_anon | ecdh_anon | srp_anon. --type erl_cipher_suite() :: {key_algo(), cipher(), hash()}. --type int_cipher_suite() :: {key_algo(), cipher(), hash(), hash() | default_prf}. --type cipher_suite() :: binary(). --type cipher_enum() :: integer(). --type openssl_cipher_suite() :: string(). - %%% SSL cipher protocol %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -define(CHANGE_CIPHER_SPEC_PROTO, 1). % _PROTO to not clash with % SSL record protocol diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index e283e6079e..ed9e4d344f 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -52,6 +52,7 @@ %% SSL all state functions -export([handle_sync_event/4, handle_info/3, terminate/3]). + %%==================================================================== %% Internal application API %%==================================================================== diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index 341a4217e4..b01c6cb1b3 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -48,17 +48,17 @@ socket_options :: #socket_options{}, connection_states :: #connection_states{}, protocol_buffers :: term(), %% #protocol_buffers{} from tls_record.hrl or dtls_recor.hrl - tls_handshake_history ::tls_handshake_history(), + tls_handshake_history :: ssl_handshake:ssl_handshake_history(), cert_db :: reference(), session :: #session{}, session_cache :: db_handle(), session_cache_cb :: atom(), - negotiated_version :: tls_version(), + negotiated_version :: ssl_record:ssl_version(), client_certificate_requested = false :: boolean(), - key_algorithm :: key_algo(), + key_algorithm :: ssl_cipher:key_algo(), hashsign_algorithm = {undefined, undefined}, cert_hashsign_algorithm, - public_key_info ::public_key_info(), + public_key_info ::ssl_handshake:public_key_info(), private_key ::public_key:private_key(), diffie_hellman_params, % PKIX: #'DHParameter'{} relevant for server side diffie_hellman_keys, % {PublicKey, PrivateKey} diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 74ca7ca699..1108edcf48 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -31,6 +31,18 @@ -include("ssl_srp.hrl"). -include_lib("public_key/include/public_key.hrl"). +-export_type([ssl_handshake/0, ssl_handshake_history/0, + public_key_info/0, oid/0]). + +-type oid() :: tuple(). +-type public_key_params() :: #'Dss-Parms'{} | {namedCurve, oid()} | #'ECParameters'{} | term(). +-type public_key_info() :: {oid(), #'RSAPublicKey'{} | integer() | #'ECPoint'{}, public_key_params()}. +-type ssl_handshake_history() :: {[binary()], [binary()]}. + +-type ssl_handshake() :: #server_hello{} | #server_hello_done{} | #certificate{} | #certificate_request{} | + #client_key_exchange{} | #finished{} | #certificate_verify{} | + #hello_request{} | #next_protocol{}. + %% Handshake messages -export([hello_request/0, server_hello/4, server_hello_done/0, certificate/4, certificate_request/4, key_exchange/3, @@ -80,7 +92,7 @@ hello_request() -> #hello_request{}. %%-------------------------------------------------------------------- --spec server_hello(#session{}, tls_version(), #connection_states{}, +-spec server_hello(#session{}, ssl_record:ssl_version(), #connection_states{}, #hello_extensions{}) -> #server_hello{}. %% %% Description: Creates a server hello message. @@ -164,8 +176,8 @@ next_protocol(SelectedProtocol) -> %%-------------------------------------------------------------------- -spec client_certificate_verify(undefined | der_cert(), binary(), - tls_version(), term(), public_key:private_key(), - tls_handshake_history()) -> + ssl_record:ssl_version(), term(), public_key:private_key(), + ssl_handshake_history()) -> #certificate_verify{} | ignore | #alert{}. %% %% Description: Creates a certificate_verify message, called by the client. @@ -188,7 +200,7 @@ client_certificate_verify(OwnCert, MasterSecret, Version, end. %%-------------------------------------------------------------------- --spec certificate_request(erl_cipher_suite(), db_handle(), certdb_ref(), tls_version()) -> +-spec certificate_request(ssl_cipher:erl_cipher_suite(), db_handle(), certdb_ref(), ssl_record:ssl_version()) -> #certificate_request{}. %% %% Description: Creates a certificate_request message, called by the server. @@ -203,7 +215,7 @@ certificate_request(CipherSuite, CertDbHandle, CertDbRef, Version) -> certificate_authorities = Authorities }. %%-------------------------------------------------------------------- --spec key_exchange(client | server, tls_version(), +-spec key_exchange(client | server, ssl_record:ssl_version(), {premaster_secret, binary(), public_key_info()} | {dh, binary()} | {dh, {binary(), binary()}, #'DHParameter'{}, {HashAlgo::atom(), SignAlgo::atom()}, @@ -304,7 +316,7 @@ key_exchange(server, Version, {srp, {PublicKey, _}, ClientRandom, ServerRandom, PrivateKey). %%-------------------------------------------------------------------- --spec finished(tls_version(), client | server, integer(), binary(), tls_handshake_history()) -> +-spec finished(ssl_record:ssl_version(), client | server, integer(), binary(), ssl_handshake_history()) -> #finished{}. %% %% Description: Creates a handshake finished message @@ -331,8 +343,8 @@ verify_server_key(#server_key_params{params_bin = EncParams, verify_signature(Version, Hash, HashSign, Signature, PubKeyInfo). %%-------------------------------------------------------------------- --spec certificate_verify(binary(), public_key_info(), tls_version(), term(), - binary(), tls_handshake_history()) -> valid | #alert{}. +-spec certificate_verify(binary(), public_key_info(), ssl_record:ssl_version(), term(), + binary(), ssl_handshake_history()) -> valid | #alert{}. %% %% Description: Checks that the certificate_verify message is valid. %%-------------------------------------------------------------------- @@ -346,7 +358,7 @@ certificate_verify(Signature, PublicKeyInfo, Version, ?ALERT_REC(?FATAL, ?BAD_CERTIFICATE) end. %%-------------------------------------------------------------------- --spec verify_signature(tls_version(), binary(), {term(), term()}, binary(), +-spec verify_signature(ssl_record:ssl_version(), binary(), {term(), term()}, binary(), public_key_info()) -> true | false. %% %% Description: Checks that a public_key signature is valid. @@ -426,8 +438,8 @@ certify(#certificate{asn1_certificates = ASN1Certs}, CertDbHandle, CertDbRef, end. %%-------------------------------------------------------------------- --spec verify_connection(tls_version(), #finished{}, client | server, integer(), binary(), - tls_handshake_history()) -> verified | #alert{}. +-spec verify_connection(ssl_record:ssl_version(), #finished{}, client | server, integer(), binary(), + ssl_handshake_history()) -> verified | #alert{}. %% %% Description: Checks the ssl handshake finished message to verify %% the connection. @@ -443,7 +455,7 @@ verify_connection(Version, #finished{verify_data = Data}, end. %%-------------------------------------------------------------------- --spec init_handshake_history() -> tls_handshake_history(). +-spec init_handshake_history() -> ssl_handshake_history(). %% %% Description: Initialize the empty handshake history buffer. @@ -452,8 +464,8 @@ init_handshake_history() -> {[], []}. %%-------------------------------------------------------------------- --spec update_handshake_history(tls_handshake_history(), Data ::term()) -> - tls_handshake_history(). +-spec update_handshake_history(ssl_handshake:ssl_handshake_history(), Data ::term()) -> + ssl_handshake:ssl_handshake_history(). %% %% Description: Update the handshake history buffer with Data. %%-------------------------------------------------------------------- @@ -567,7 +579,7 @@ server_key_exchange_hash(md5sha, Value) -> server_key_exchange_hash(Hash, Value) -> crypto:hash(Hash, Value). %%-------------------------------------------------------------------- --spec prf(tls_version(), binary(), binary(), [binary()], non_neg_integer()) -> +-spec prf(ssl_record:ssl_version(), binary(), binary(), [binary()], non_neg_integer()) -> {ok, binary()} | {error, undefined}. %% %% Description: use the TLS PRF to generate key material @@ -611,7 +623,7 @@ select_hashsign(#hash_sign_algos{hash_sign_algos = HashSigns}, Cert) -> HashSign end. %%-------------------------------------------------------------------- --spec select_cert_hashsign(#hash_sign_algos{}| undefined, oid(), tls_version() | {undefined, undefined}) -> +-spec select_cert_hashsign(#hash_sign_algos{}| undefined, oid(), ssl_record:ssl_version() | {undefined, undefined}) -> {atom(), atom()}. %% @@ -631,7 +643,7 @@ select_cert_hashsign(undefined, ?'id-dsa', _) -> {sha, dsa}. %%-------------------------------------------------------------------- --spec master_secret(atom(), tls_version(), #session{} | binary(), #connection_states{}, +-spec master_secret(atom(), ssl_record:ssl_version(), #session{} | binary(), #connection_states{}, client | server) -> {binary(), #connection_states{}} | #alert{}. %% %% Description: Sets or calculates the master secret and calculate keys, @@ -816,7 +828,7 @@ enc_server_key_exchange(Version, Params, {HashAlgo, SignAlgo}, end. %%-------------------------------------------------------------------- --spec decode_client_key(binary(), key_algo(), tls_version()) -> +-spec decode_client_key(binary(), ssl_cipher:key_algo(), ssl_record:ssl_version()) -> #encrypted_premaster_secret{} | #client_diffie_hellman_public{} | #client_ec_diffie_hellman_public{} @@ -831,7 +843,7 @@ decode_client_key(ClientKey, Type, Version) -> dec_client_key(ClientKey, key_exchange_alg(Type), Version). %%-------------------------------------------------------------------- --spec decode_server_key(binary(), key_algo(), tls_version()) -> +-spec decode_server_key(binary(), ssl_cipher:key_algo(), ssl_record:ssl_version()) -> #server_key_params{}. %% %% Description: Decode server_key data and return appropriate type diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index 75160526b9..80284faef0 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -352,18 +352,4 @@ hostname = undefined }). -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Dialyzer types -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - --type oid() :: tuple(). --type public_key_params() :: #'Dss-Parms'{} | {namedCurve, oid()} | #'ECParameters'{} | term(). --type public_key_info() :: {oid(), #'RSAPublicKey'{} | integer() | #'ECPoint'{}, public_key_params()}. --type tls_handshake_history() :: {[binary()], [binary()]}. - --type ssl_handshake() :: #server_hello{} | #server_hello_done{} | #certificate{} | #certificate_request{} | - #client_key_exchange{} | #finished{} | #certificate_verify{} | - #hello_request{} | #next_protocol{}. - - -endif. % -ifdef(ssl_handshake). diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 64b89e9f95..cec5d8fbb1 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -30,8 +30,6 @@ -type from() :: term(). -type host() :: inet:ip_address() | inet:hostname(). -type session_id() :: 0 | binary(). --type tls_version() :: {integer(), integer()}. --type tls_atom_version() :: sslv3 | tlsv1 | 'tlsv1.1' | 'tlsv1.2'. -type certdb_ref() :: reference(). -type db_handle() :: term(). -type der_cert() :: binary(). @@ -73,7 +71,7 @@ -record(ssl_options, { protocol :: tls | dtls, - versions :: ['tlsv1.2' | 'tlsv1.1' | tlsv1 | sslv3] | ['dtlsv1.2' | dtlsv1], + versions :: [ssl_record:ssl_version()], %% ssl_record:atom_version() in API verify :: verify_none | verify_peer, verify_fun, %%:: fun(CertVerifyErrors::term()) -> boolean(), fail_if_no_peer_cert :: boolean(), diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 018c8befe0..b0e9943e6d 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -50,6 +50,11 @@ %% Payload encryption/decryption -export([cipher/4, decipher/3, is_correct_mac/2]). +-export_type([ssl_version/0, ssl_atom_version/0]). + +-type ssl_version() :: {integer(), integer()}. +-type ssl_atom_version() :: tls_record:tls_atom_version(). + %%==================================================================== %% Internal application API %%==================================================================== @@ -299,7 +304,7 @@ set_pending_cipher_state(#connection_states{pending_read = Read, %%-------------------------------------------------------------------- --spec encode_handshake(iolist(), tls_version(), #connection_states{}) -> +-spec encode_handshake(iolist(), ssl_version(), #connection_states{}) -> {iolist(), #connection_states{}}. %% %% Description: Encodes a handshake message to send on the ssl-socket. @@ -308,7 +313,7 @@ encode_handshake(Frag, Version, ConnectionStates) -> encode_plain_text(?HANDSHAKE, Version, Frag, ConnectionStates). %%-------------------------------------------------------------------- --spec encode_alert_record(#alert{}, tls_version(), #connection_states{}) -> +-spec encode_alert_record(#alert{}, ssl_version(), #connection_states{}) -> {iolist(), #connection_states{}}. %% %% Description: Encodes an alert message to send on the ssl-socket. @@ -319,7 +324,7 @@ encode_alert_record(#alert{level = Level, description = Description}, ConnectionStates). %%-------------------------------------------------------------------- --spec encode_change_cipher_spec(tls_version(), #connection_states{}) -> +-spec encode_change_cipher_spec(ssl_version(), #connection_states{}) -> {iolist(), #connection_states{}}. %% %% Description: Encodes a change_cipher_spec-message to send on the ssl socket. @@ -328,7 +333,7 @@ encode_change_cipher_spec(Version, ConnectionStates) -> encode_plain_text(?CHANGE_CIPHER_SPEC, Version, <<1:8>>, ConnectionStates). %%-------------------------------------------------------------------- --spec encode_data(binary(), tls_version(), #connection_states{}) -> +-spec encode_data(binary(), ssl_version(), #connection_states{}) -> {iolist(), #connection_states{}}. %% %% Description: Encodes data to send on the ssl-socket. @@ -356,7 +361,7 @@ compressions() -> [?byte(?NULL)]. %%-------------------------------------------------------------------- --spec cipher(tls_version(), iolist(), #connection_state{}, MacHash::binary()) -> +-spec cipher(ssl_version(), iodata(), #connection_state{}, MacHash::binary()) -> {CipherFragment::binary(), #connection_state{}}. %% %% Description: Payload encryption @@ -372,7 +377,7 @@ cipher(Version, Fragment, ssl_cipher:cipher(BulkCipherAlgo, CipherS0, MacHash, Fragment, Version), {CipherFragment, WriteState0#connection_state{cipher_state = CipherS1}}. %%-------------------------------------------------------------------- --spec decipher(tls_version(), binary(), #connection_state{}) -> {binary(), binary(), #connection_state{}}. +-spec decipher(ssl_version(), binary(), #connection_state{}) -> {binary(), binary(), #connection_state{}}. %% %% Description: Payload decryption %%-------------------------------------------------------------------- diff --git a/lib/ssl/src/ssl_record.hrl b/lib/ssl/src/ssl_record.hrl index c17fa53a62..87ed233c0a 100644 --- a/lib/ssl/src/ssl_record.hrl +++ b/lib/ssl/src/ssl_record.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,7 +20,7 @@ %% %%---------------------------------------------------------------------- %% Purpose: Record and constant defenitions for the SSL-record protocol -%% see RFC 2246 +% see RFC 2246 %%---------------------------------------------------------------------- -ifndef(ssl_record). diff --git a/lib/ssl/src/ssl_v3.erl b/lib/ssl/src/ssl_v3.erl index d477b3df81..68f7f5dee2 100644 --- a/lib/ssl/src/ssl_v3.erl +++ b/lib/ssl/src/ssl_v3.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -130,7 +130,7 @@ setup_keys(MasterSecret, ServerRandom, ClientRandom, HS, KML, _EKML, IVS) -> {ClientWriteMacSecret, ServerWriteMacSecret, ClientWriteKey, ServerWriteKey, ClientIV, ServerIV}. --spec suites() -> [cipher_suite()]. +-spec suites() -> [ssl_cipher:cipher_suite()]. suites() -> [ diff --git a/lib/ssl/src/tls_handshake.erl b/lib/ssl/src/tls_handshake.erl index 01abefca46..183cabcfcd 100644 --- a/lib/ssl/src/tls_handshake.erl +++ b/lib/ssl/src/tls_handshake.erl @@ -33,6 +33,8 @@ -export([client_hello/8, hello/4, get_tls_handshake/3, encode_handshake/2, decode_handshake/3]). +-type tls_handshake() :: #client_hello{} | ssl_handshake:ssl_handshake(). + %%==================================================================== %% Internal application API %%==================================================================== @@ -71,11 +73,11 @@ client_hello(Host, Port, ConnectionStates, #connection_states{} | {inet:port_number(), #session{}, db_handle(), atom(), #connection_states{}, binary() | undefined}, boolean()) -> - {tls_version(), session_id(), #connection_states{}, binary() | undefined}| - {tls_version(), {resumed | new, #session{}}, #connection_states{}, - [binary()] | undefined, - [oid()] | undefined, [oid()] | undefined} | - #alert{}. + {tls_record:tls_version(), session_id(), #connection_states{}, binary() | undefined}| + {tls_record:tls_version(), {resumed | new, #session{}}, #connection_states{}, + [binary()] | undefined, + [ssl_handshake:oid()] | undefined, [ssl_handshake:oid()] | undefined} | + #alert{}. %% %% Description: Handles a recieved hello message %%-------------------------------------------------------------------- @@ -122,7 +124,7 @@ hello(#client_hello{client_version = ClientVersion, end. %%-------------------------------------------------------------------- --spec encode_handshake(tls_handshake(), tls_version()) -> iolist(). +-spec encode_handshake(tls_handshake(), tls_record:tls_version()) -> iolist(). %% %% Description: Encode a handshake packet %%--------------------------------------------------------------------x @@ -132,7 +134,7 @@ encode_handshake(Package, Version) -> [MsgType, ?uint24(Len), Bin]. %%-------------------------------------------------------------------- --spec get_tls_handshake(tls_version(), binary(), binary() | iolist()) -> +-spec get_tls_handshake(tls_record:tls_version(), binary(), binary() | iolist()) -> {[tls_handshake()], binary()}. %% %% Description: Given buffered and new data from ssl_record, collects diff --git a/lib/ssl/src/tls_handshake.hrl b/lib/ssl/src/tls_handshake.hrl index dbe930cb90..1646e5b6f2 100644 --- a/lib/ssl/src/tls_handshake.hrl +++ b/lib/ssl/src/tls_handshake.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -37,6 +37,4 @@ extensions }). --type tls_handshake() :: #client_hello{} | ssl_handshake(). - -endif. % -ifdef(tls_handshake). diff --git a/lib/ssl/src/tls_record.erl b/lib/ssl/src/tls_record.erl index 8c0c4f3c91..4da08e9c51 100644 --- a/lib/ssl/src/tls_record.erl +++ b/lib/ssl/src/tls_record.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -44,6 +44,11 @@ highest_protocol_version/1, supported_protocol_versions/0, is_acceptable_version/1, is_acceptable_version/2]). +-export_type([tls_version/0, tls_atom_version/0]). + +-type tls_version() :: ssl_record:ssl_version(). +-type tls_atom_version() :: sslv3 | tlsv1 | 'tlsv1.1' | 'tlsv1.2'. + -compile(inline). %%==================================================================== diff --git a/lib/ssl/src/tls_v1.erl b/lib/ssl/src/tls_v1.erl index 7b1f53b969..067417d163 100644 --- a/lib/ssl/src/tls_v1.erl +++ b/lib/ssl/src/tls_v1.erl @@ -166,7 +166,7 @@ setup_keys(Version, PrfAlgo, MasterSecret, ServerRandom, ClientRandom, HashSize, {ClientWriteMacSecret, ServerWriteMacSecret, ClientWriteKey, ServerWriteKey, ClientIV, ServerIV}. --spec mac_hash(integer(), binary(), integer(), integer(), tls_version(), +-spec mac_hash(integer(), binary(), integer(), integer(), tls_record:tls_version(), integer(), binary()) -> binary(). mac_hash(Method, Mac_write_secret, Seq_num, Type, {Major, Minor}, @@ -181,7 +181,7 @@ mac_hash(Method, Mac_write_secret, Seq_num, Type, {Major, Minor}, Fragment]), Mac. --spec suites(1|2|3) -> [cipher_suite()]. +-spec suites(1|2|3) -> [ssl_cipher:cipher_suite()]. suites(Minor) when Minor == 1; Minor == 2-> case sufficent_ec_support() of diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index 244eb5ce0a..2f8ff6f04e 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -39,6 +39,7 @@ MODULES = \ ssl_basic_SUITE \ ssl_cipher_SUITE \ ssl_certificate_verify_SUITE\ + ssl_crl_SUITE\ ssl_dist_SUITE \ ssl_handshake_SUITE \ ssl_npn_hello_SUITE \ diff --git a/lib/ssl/test/make_certs.erl b/lib/ssl/test/make_certs.erl index 4603a9f846..c438ae2b87 100644 --- a/lib/ssl/test/make_certs.erl +++ b/lib/ssl/test/make_certs.erl @@ -18,23 +18,68 @@ %% -module(make_certs). +-compile([export_all]). --export([all/2]). +%-export([all/1, all/2, rootCA/2, intermediateCA/3, endusers/3, enduser/3, revoke/3, gencrl/2, verify/3]). --record(dn, {commonName, +-record(config, {commonName, organizationalUnitName = "Erlang OTP", organizationName = "Ericsson AB", localityName = "Stockholm", countryName = "SE", - emailAddress = "[email protected]"}). + emailAddress = "[email protected]", + default_bits = 2048, + v2_crls = true, + ecc_certs = false, + issuing_distribution_point = false, + openssl_cmd = "openssl"}). + + +default_config() -> + #config{}. + +make_config(Args) -> + make_config(Args, #config{}). + +make_config([], C) -> + C; +make_config([{organizationalUnitName, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{organizationalUnitName = Name}); +make_config([{organizationName, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{organizationName = Name}); +make_config([{localityName, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{localityName = Name}); +make_config([{countryName, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{countryName = Name}); +make_config([{emailAddress, Name}|T], C) when is_list(Name) -> + make_config(T, C#config{emailAddress = Name}); +make_config([{default_bits, Bits}|T], C) when is_integer(Bits) -> + make_config(T, C#config{default_bits = Bits}); +make_config([{v2_crls, Bool}|T], C) when is_boolean(Bool) -> + make_config(T, C#config{v2_crls = Bool}); +make_config([{ecc_certs, Bool}|T], C) when is_boolean(Bool) -> + make_config(T, C#config{ecc_certs = Bool}); +make_config([{issuing_distribution_point, Bool}|T], C) when is_boolean(Bool) -> + make_config(T, C#config{issuing_distribution_point = Bool}); +make_config([{openssl_cmd, Cmd}|T], C) when is_list(Cmd) -> + make_config(T, C#config{openssl_cmd = Cmd}). + + +all([DataDir, PrivDir]) -> + all(DataDir, PrivDir). all(DataDir, PrivDir) -> - OpenSSLCmd = "openssl", + all(DataDir, PrivDir, #config{}). + +all(DataDir, PrivDir, C) when is_list(C) -> + all(DataDir, PrivDir, make_config(C)); +all(DataDir, PrivDir, C = #config{}) -> + ok = filelib:ensure_dir(filename:join(PrivDir, "erlangCA")), create_rnd(DataDir, PrivDir), % For all requests - rootCA(PrivDir, OpenSSLCmd, "erlangCA"), - intermediateCA(PrivDir, OpenSSLCmd, "otpCA", "erlangCA"), - endusers(PrivDir, OpenSSLCmd, "otpCA", ["client", "server"]), - collect_certs(PrivDir, ["erlangCA", "otpCA"], ["client", "server"]), + rootCA(PrivDir, "erlangCA", C), + intermediateCA(PrivDir, "otpCA", "erlangCA", C), + endusers(PrivDir, "otpCA", ["client", "server", "revoked"], C), + endusers(PrivDir, "erlangCA", ["localhost"], C), %% Create keycert files SDir = filename:join([PrivDir, "server"]), SC = filename:join([SDir, "cert.pem"]), @@ -46,7 +91,14 @@ all(DataDir, PrivDir) -> CK = filename:join([CDir, "key.pem"]), CKC = filename:join([CDir, "keycert.pem"]), append_files([CK, CC], CKC), - remove_rnd(PrivDir). + RDir = filename:join([PrivDir, "revoked"]), + RC = filename:join([RDir, "cert.pem"]), + RK = filename:join([RDir, "key.pem"]), + RKC = filename:join([RDir, "keycert.pem"]), + revoke(PrivDir, "otpCA", "revoked", C), + append_files([RK, RC], RKC), + remove_rnd(PrivDir), + {ok, C}. append_files(FileNames, ResultFileName) -> {ok, ResultFile} = file:open(ResultFileName, [write]), @@ -59,111 +111,176 @@ do_append_files([F|Fs], RF) -> ok = file:write(RF, Data), do_append_files(Fs, RF). -rootCA(Root, OpenSSLCmd, Name) -> - create_ca_dir(Root, Name, ca_cnf(Name)), - DN = #dn{commonName = Name}, - create_self_signed_cert(Root, OpenSSLCmd, Name, req_cnf(DN)), - ok. +rootCA(Root, Name, C) -> + create_ca_dir(Root, Name, ca_cnf(C#config{commonName = Name})), + create_self_signed_cert(Root, Name, req_cnf(C#config{commonName = Name}), C), + file:copy(filename:join([Root, Name, "cert.pem"]), filename:join([Root, Name, "cacerts.pem"])), + gencrl(Root, Name, C). -intermediateCA(Root, OpenSSLCmd, CA, ParentCA) -> - CA = "otpCA", - create_ca_dir(Root, CA, ca_cnf(CA)), +intermediateCA(Root, CA, ParentCA, C) -> + create_ca_dir(Root, CA, ca_cnf(C#config{commonName = CA})), CARoot = filename:join([Root, CA]), - DN = #dn{commonName = CA}, CnfFile = filename:join([CARoot, "req.cnf"]), - file:write_file(CnfFile, req_cnf(DN)), + file:write_file(CnfFile, req_cnf(C#config{commonName = CA})), KeyFile = filename:join([CARoot, "private", "key.pem"]), ReqFile = filename:join([CARoot, "req.pem"]), - create_req(Root, OpenSSLCmd, CnfFile, KeyFile, ReqFile), + create_req(Root, CnfFile, KeyFile, ReqFile, C), CertFile = filename:join([CARoot, "cert.pem"]), - sign_req(Root, OpenSSLCmd, ParentCA, "ca_cert", ReqFile, CertFile). - -endusers(Root, OpenSSLCmd, CA, Users) -> - lists:foreach(fun(User) -> enduser(Root, OpenSSLCmd, CA, User) end, Users). - -enduser(Root, OpenSSLCmd, CA, User) -> + sign_req(Root, ParentCA, "ca_cert", ReqFile, CertFile, C), + CACertsFile = filename:join(CARoot, "cacerts.pem"), + file:copy(filename:join([Root, ParentCA, "cacerts.pem"]), CACertsFile), + %% append this CA's cert to the cacerts file + {ok, Bin} = file:read_file(CertFile), + {ok, FD} = file:open(CACertsFile, [append]), + file:write(FD, ["\n", Bin]), + file:close(FD), + gencrl(Root, CA, C). + +endusers(Root, CA, Users, C) -> + [enduser(Root, CA, User, C) || User <- Users]. + +enduser(Root, CA, User, C) -> UsrRoot = filename:join([Root, User]), file:make_dir(UsrRoot), CnfFile = filename:join([UsrRoot, "req.cnf"]), - DN = #dn{commonName = User}, - file:write_file(CnfFile, req_cnf(DN)), + file:write_file(CnfFile, req_cnf(C#config{commonName = User})), KeyFile = filename:join([UsrRoot, "key.pem"]), ReqFile = filename:join([UsrRoot, "req.pem"]), - create_req(Root, OpenSSLCmd, CnfFile, KeyFile, ReqFile), + create_req(Root, CnfFile, KeyFile, ReqFile, C), + %create_req(Root, CnfFile, KeyFile, ReqFile), CertFileAllUsage = filename:join([UsrRoot, "cert.pem"]), - sign_req(Root, OpenSSLCmd, CA, "user_cert", ReqFile, CertFileAllUsage), + sign_req(Root, CA, "user_cert", ReqFile, CertFileAllUsage, C), CertFileDigitalSigOnly = filename:join([UsrRoot, "digital_signature_only_cert.pem"]), - sign_req(Root, OpenSSLCmd, CA, "user_cert_digital_signature_only", ReqFile, CertFileDigitalSigOnly). - -collect_certs(Root, CAs, Users) -> - Bins = lists:foldr( - fun(CA, Acc) -> - File = filename:join([Root, CA, "cert.pem"]), - {ok, Bin} = file:read_file(File), - [Bin, "\n" | Acc] - end, [], CAs), - lists:foreach( - fun(User) -> - File = filename:join([Root, User, "cacerts.pem"]), - file:write_file(File, Bins) - end, Users). + sign_req(Root, CA, "user_cert_digital_signature_only", ReqFile, CertFileDigitalSigOnly, C), + CACertsFile = filename:join(UsrRoot, "cacerts.pem"), + file:copy(filename:join([Root, CA, "cacerts.pem"]), CACertsFile), + ok. + +revoke(Root, CA, User, C) -> + UsrCert = filename:join([Root, User, "cert.pem"]), + CACnfFile = filename:join([Root, CA, "ca.cnf"]), + Cmd = [C#config.openssl_cmd, " ca" + " -revoke ", UsrCert, + [" -crl_reason keyCompromise" || C#config.v2_crls ], + " -config ", CACnfFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env), + gencrl(Root, CA, C). + +gencrl(Root, CA, C) -> + CACnfFile = filename:join([Root, CA, "ca.cnf"]), + CACRLFile = filename:join([Root, CA, "crl.pem"]), + Cmd = [C#config.openssl_cmd, " ca" + " -gencrl ", + " -crlhours 24", + " -out ", CACRLFile, + " -config ", CACnfFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env). -create_self_signed_cert(Root, OpenSSLCmd, CAName, Cnf) -> +verify(Root, CA, User, C) -> + CAFile = filename:join([Root, User, "cacerts.pem"]), + CACRLFile = filename:join([Root, CA, "crl.pem"]), + CertFile = filename:join([Root, User, "cert.pem"]), + Cmd = [C#config.openssl_cmd, " verify" + " -CAfile ", CAFile, + " -CRLfile ", CACRLFile, %% this is undocumented, but seems to work + " -crl_check ", + CertFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + try cmd(Cmd, Env) catch + exit:{eval_cmd, _, _} -> + invalid + end. + +create_self_signed_cert(Root, CAName, Cnf, C = #config{ecc_certs = true}) -> CARoot = filename:join([Root, CAName]), CnfFile = filename:join([CARoot, "req.cnf"]), file:write_file(CnfFile, Cnf), KeyFile = filename:join([CARoot, "private", "key.pem"]), CertFile = filename:join([CARoot, "cert.pem"]), - Cmd = [OpenSSLCmd, " req" + Cmd = [C#config.openssl_cmd, " ecparam" + " -out ", KeyFile, + " -name secp521r1 ", + %" -name sect283k1 ", + " -genkey "], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env), + + Cmd2 = [C#config.openssl_cmd, " req" " -new" " -x509" " -config ", CnfFile, - " -keyout ", KeyFile, + " -key ", KeyFile, + " -outform PEM ", " -out ", CertFile], - Env = [{"ROOTDIR", Root}], - cmd(Cmd, Env), - fix_key_file(OpenSSLCmd, KeyFile). - -% openssl 1.0 generates key files in pkcs8 format by default and we don't handle this format -fix_key_file(OpenSSLCmd, KeyFile) -> - KeyFileTmp = KeyFile ++ ".tmp", - Cmd = [OpenSSLCmd, " rsa", - " -in ", - KeyFile, - " -out ", - KeyFileTmp], - cmd(Cmd, []), - ok = file:rename(KeyFileTmp, KeyFile). + cmd(Cmd2, Env); +create_self_signed_cert(Root, CAName, Cnf, C) -> + CARoot = filename:join([Root, CAName]), + CnfFile = filename:join([CARoot, "req.cnf"]), + file:write_file(CnfFile, Cnf), + KeyFile = filename:join([CARoot, "private", "key.pem"]), + CertFile = filename:join([CARoot, "cert.pem"]), + Cmd = [C#config.openssl_cmd, " req" + " -new" + " -x509" + " -config ", CnfFile, + " -keyout ", KeyFile, + " -outform PEM", + " -out ", CertFile], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env). + create_ca_dir(Root, CAName, Cnf) -> CARoot = filename:join([Root, CAName]), + ok = filelib:ensure_dir(CARoot), file:make_dir(CARoot), create_dirs(CARoot, ["certs", "crl", "newcerts", "private"]), create_rnd(Root, filename:join([CAName, "private"])), create_files(CARoot, [{"serial", "01\n"}, + {"crlnumber", "01"}, {"index.txt", ""}, {"ca.cnf", Cnf}]). -create_req(Root, OpenSSLCmd, CnfFile, KeyFile, ReqFile) -> - Cmd = [OpenSSLCmd, " req" +create_req(Root, CnfFile, KeyFile, ReqFile, C = #config{ecc_certs = true}) -> + Cmd = [C#config.openssl_cmd, " ecparam" + " -out ", KeyFile, + " -name secp521r1 ", + %" -name sect283k1 ", + " -genkey "], + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env), + Cmd2 = [C#config.openssl_cmd, " req" + " -new ", + " -key ", KeyFile, + " -outform PEM ", + " -out ", ReqFile, + " -config ", CnfFile], + cmd(Cmd2, Env); + %fix_key_file(KeyFile). +create_req(Root, CnfFile, KeyFile, ReqFile, C) -> + Cmd = [C#config.openssl_cmd, " req" " -new" " -config ", CnfFile, + " -outform PEM ", " -keyout ", KeyFile, " -out ", ReqFile], - Env = [{"ROOTDIR", Root}], - cmd(Cmd, Env), - fix_key_file(OpenSSLCmd, KeyFile). + Env = [{"ROOTDIR", filename:absname(Root)}], + cmd(Cmd, Env). + %fix_key_file(KeyFile). + -sign_req(Root, OpenSSLCmd, CA, CertType, ReqFile, CertFile) -> +sign_req(Root, CA, CertType, ReqFile, CertFile, C) -> CACnfFile = filename:join([Root, CA, "ca.cnf"]), - Cmd = [OpenSSLCmd, " ca" + Cmd = [C#config.openssl_cmd, " ca" " -batch" " -notext" " -config ", CACnfFile, " -extensions ", CertType, " -in ", ReqFile, " -out ", CertFile], - Env = [{"ROOTDIR", Root}], + Env = [{"ROOTDIR", filename:absname(Root)}], cmd(Cmd, Env). %% @@ -194,19 +311,19 @@ cmd(Cmd, Env) -> FCmd = lists:flatten(Cmd), Port = open_port({spawn, FCmd}, [stream, eof, exit_status, stderr_to_stdout, {env, Env}]), - eval_cmd(Port). + eval_cmd(Port, FCmd). -eval_cmd(Port) -> +eval_cmd(Port, Cmd) -> receive {Port, {data, _}} -> - eval_cmd(Port); + eval_cmd(Port, Cmd); {Port, eof} -> ok end, receive {Port, {exit_status, Status}} when Status /= 0 -> %% io:fwrite("exit status: ~w~n", [Status]), - exit({eval_cmd, Status}) + exit({eval_cmd, Cmd, Status}) after 0 -> ok end. @@ -215,7 +332,7 @@ eval_cmd(Port) -> %% Contents of configuration files %% -req_cnf(DN) -> +req_cnf(C) -> ["# Purpose: Configuration for requests (end users and CAs)." "\n" "ROOTDIR = $ENV::ROOTDIR\n" @@ -224,7 +341,7 @@ req_cnf(DN) -> "[req]\n" "input_password = secret\n" "output_password = secret\n" - "default_bits = 1024\n" + "default_bits = ", integer_to_list(C#config.default_bits), "\n" "RANDFILE = $ROOTDIR/RAND\n" "encrypt_key = no\n" "default_md = sha1\n" @@ -235,12 +352,12 @@ req_cnf(DN) -> "\n" "[name]\n" - "commonName = ", DN#dn.commonName, "\n" - "organizationalUnitName = ", DN#dn.organizationalUnitName, "\n" - "organizationName = ", DN#dn.organizationName, "\n" - "localityName = ", DN#dn.localityName, "\n" - "countryName = ", DN#dn.countryName, "\n" - "emailAddress = ", DN#dn.emailAddress, "\n" + "commonName = ", C#config.commonName, "\n" + "organizationalUnitName = ", C#config.organizationalUnitName, "\n" + "organizationName = ", C#config.organizationName, "\n" + "localityName = ", C#config.localityName, "\n" + "countryName = ", C#config.countryName, "\n" + "emailAddress = ", C#config.emailAddress, "\n" "\n" "[ca_ext]\n" @@ -249,8 +366,7 @@ req_cnf(DN) -> "subjectKeyIdentifier = hash\n" "subjectAltName = email:copy\n"]. - -ca_cnf(CA) -> +ca_cnf(C) -> ["# Purpose: Configuration for CAs.\n" "\n" "ROOTDIR = $ENV::ROOTDIR\n" @@ -258,21 +374,23 @@ ca_cnf(CA) -> "\n" "[ca]\n" - "dir = $ROOTDIR/", CA, "\n" + "dir = $ROOTDIR/", C#config.commonName, "\n" "certs = $dir/certs\n" "crl_dir = $dir/crl\n" "database = $dir/index.txt\n" "new_certs_dir = $dir/newcerts\n" "certificate = $dir/cert.pem\n" "serial = $dir/serial\n" - "crl = $dir/crl.pem\n" + "crl = $dir/crl.pem\n", + ["crlnumber = $dir/crlnumber\n" || C#config.v2_crls], "private_key = $dir/private/key.pem\n" "RANDFILE = $dir/private/RAND\n" "\n" - "x509_extensions = user_cert\n" + "x509_extensions = user_cert\n", + ["crl_extensions = crl_ext\n" || C#config.v2_crls], "unique_subject = no\n" "default_days = 3600\n" - "default_md = sha1\n" + "default_md = sha256\n" "preserve = no\n" "policy = policy_match\n" "\n" @@ -286,6 +404,13 @@ ca_cnf(CA) -> "emailAddress = supplied\n" "\n" + "[crl_ext]\n" + "authorityKeyIdentifier=keyid:always,issuer:always\n", + ["issuingDistributionPoint=critical, @idpsec\n" || C#config.issuing_distribution_point], + + "[idpsec]\n" + "fullname=URI:http://localhost:8000/",C#config.commonName,"/crl.pem\n" + "[user_cert]\n" "basicConstraints = CA:false\n" "keyUsage = nonRepudiation, digitalSignature, keyEncipherment\n" @@ -293,6 +418,12 @@ ca_cnf(CA) -> "authorityKeyIdentifier = keyid,issuer:always\n" "subjectAltName = email:copy\n" "issuerAltName = issuer:copy\n" + "crlDistributionPoints=@crl_section\n" + + "[crl_section]\n" + %% intentionally invalid + "URI.1=http://localhost/",C#config.commonName,"/crl.pem\n" + "URI.2=http://localhost:8000/",C#config.commonName,"/crl.pem\n" "\n" "[user_cert_digital_signature_only]\n" @@ -310,4 +441,7 @@ ca_cnf(CA) -> "subjectKeyIdentifier = hash\n" "authorityKeyIdentifier = keyid:always,issuer:always\n" "subjectAltName = email:copy\n" - "issuerAltName = issuer:copy\n"]. + "issuerAltName = issuer:copy\n" + "crlDistributionPoints=@crl_section\n" + ]. + diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 64a93440c7..0148e1f5bc 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -309,7 +309,11 @@ alerts(Config) when is_list(Config) -> ?ILLEGAL_PARAMETER, ?UNKNOWN_CA, ?ACCESS_DENIED, ?DECODE_ERROR, ?DECRYPT_ERROR, ?EXPORT_RESTRICTION, ?PROTOCOL_VERSION, ?INSUFFICIENT_SECURITY, ?INTERNAL_ERROR, ?USER_CANCELED, - ?NO_RENEGOTIATION], + ?NO_RENEGOTIATION, ?UNSUPPORTED_EXTENSION, ?CERTIFICATE_UNOBTAINABLE, + ?UNRECOGNISED_NAME, ?BAD_CERTIFICATE_STATUS_RESPONSE, + ?BAD_CERTIFICATE_HASH_VALUE, ?UNKNOWN_PSK_IDENTITY, + 255 %% Unsupported/unknow alert will result in a description too + ], Alerts = [?ALERT_REC(?WARNING, ?CLOSE_NOTIFY) | [?ALERT_REC(?FATAL, Desc) || Desc <- Descriptions]], lists:foreach(fun(Alert) -> diff --git a/lib/ssl/test/ssl_crl_SUITE.erl b/lib/ssl/test/ssl_crl_SUITE.erl new file mode 100644 index 0000000000..da0349904c --- /dev/null +++ b/lib/ssl/test/ssl_crl_SUITE.erl @@ -0,0 +1,530 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(ssl_crl_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +-define(TIMEOUT, 120000). +-define(LONG_TIMEOUT, 600000). +-define(SLEEP, 1000). +-define(OPENSSL_RENEGOTIATE, "R\n"). +-define(OPENSSL_QUIT, "Q\n"). +-define(OPENSSL_GARBAGE, "P\n"). +-define(EXPIRE, 10). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + {group, basic}, + {group, v1_crl}, + {group, idp_crl} + ]. + +groups() -> + [{basic, [], basic_tests()}, + {v1_crl, [], v1_crl_tests()}, + {idp_crl, [], idp_crl_tests()}]. + +basic_tests() -> + [crl_verify_valid, crl_verify_revoked]. + +v1_crl_tests() -> + [crl_verify_valid, crl_verify_revoked]. + +idp_crl_tests() -> + [crl_verify_valid, crl_verify_revoked]. + +%%%================================================================ +%%% Suite init/end + +init_per_suite(Config0) -> + Dog = ct:timetrap(?LONG_TIMEOUT *2), + case os:find_executable("openssl") of + false -> + {skip, "Openssl not found"}; + _ -> + TLSVersion = ?config(tls_version, Config0), + OpenSSL_version = (catch os:cmd("openssl version")), + ct:log("TLS version: ~p~nOpenSSL version: ~p~n~n~p:module_info(): ~p~n~nssh:module_info(): ~p~n", + [TLSVersion, OpenSSL_version, ?MODULE, ?MODULE:module_info(), ssh:module_info()]), + case ssl_test_lib:enough_openssl_crl_support(OpenSSL_version) of + false -> + {skip, io_lib:format("Bad openssl version: ~p",[OpenSSL_version])}; + _ -> + catch crypto:stop(), + try crypto:start() of + ok -> + ssl:start(), + [{watchdog, Dog}, {openssl_version,OpenSSL_version} | Config0] + catch _C:_E -> + ct:log("crypto:start() caught ~p:~p",[_C,_E]), + {skip, "Crypto did not start"} + end + end + end. + +end_per_suite(_Config) -> + ssl:stop(), + application:stop(crypto). + +%%%================================================================ +%%% Group init/end + +init_per_group(Group, Config) -> + ct:log("~p:~p~nlisteners to port 8000:~n~p~n)",[?MODULE,?LINE,os:cmd("netstat -tln|grep ':8000'")]), + ssl:start(), + inets:start(), + CertDir = filename:join(?config(priv_dir, Config), Group), + DataDir = ?config(data_dir, Config), + ServerRoot = make_dir_path([?config(priv_dir,Config), Group, tmp]), + Result = make_certs:all(DataDir, CertDir, cert_opts(Group)), + ct:log("~p:~p~nmake_certs:all(~n DataDir=~p,~n CertDir=~p,~n ServerRoot=~p~n Opts=~p~n) returned ~p~n", [?MODULE,?LINE,DataDir, CertDir, ServerRoot, cert_opts(Group), Result]), + %% start a HTTP server to serve the CRLs + {ok, Httpd} = inets:start(httpd, [{server_name, "localhost"}, {port, 8000}, + {server_root, ServerRoot}, + {document_root, CertDir}, + {modules, [mod_get]} + ]), + ct:log("~p:~p~nlisteners to port 8000:~n~p~n)",[?MODULE,?LINE,os:cmd("netstat -tln|grep ':8000'")]), + [{make_cert_result, Result}, {cert_dir, CertDir}, {httpd, Httpd} | Config]. + +cert_opts(v1_crl) -> [{v2_crls, false}]; +cert_opts(idp_crl) -> [{issuing_distribution_point, true}]; +cert_opts(_) -> []. + +make_dir_path(PathComponents) -> + lists:foldl(fun(F,P0) -> file:make_dir(P=filename:join(P0,F)), P end, + "", + PathComponents). + + +end_per_group(_GroupName, Config) -> + case ?config(httpd, Config) of + undefined -> ok; + Pid -> + ct:log("Stop httpd ~p",[Pid]), + ok = inets:stop(httpd, Pid) + ,ct:log("Stopped",[]) + end, + inets:stop(), + ct:log("~p:~p~nlisteners to port 8000:~n~p~n)",[?MODULE,?LINE,os:cmd("netstat -tln|grep ':8000'")]), + Config. + +%%%================================================================ +%%% Test cases + +crl_verify_valid() -> + [{doc,"Verify a simple valid CRL chain"}]. +crl_verify_valid(Config) when is_list(Config) -> + process_flag(trap_exit, true), + PrivDir = ?config(cert_dir, Config), + ServerOpts = [{keyfile, filename:join([PrivDir, "server", "key.pem"])}, + {certfile, filename:join([PrivDir, "server", "cert.pem"])}, + {cacertfile, filename:join([PrivDir, "server", "cacerts.pem"])}], + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Data = "From openssl to erlang", + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, erlang_ssl_receive, [Data]}}, + %{mfa, {ssl_test_lib, no_result, []}}, + {options, ServerOpts}]), + ct:log("~p:~p~nreturn from ssl_test_lib:start_server:~n~p",[?MODULE,?LINE,Server]), + Port = ssl_test_lib:inet_port(Server), + + CACerts = load_cert(filename:join([PrivDir, "erlangCA", "cacerts.pem"])), + + ClientOpts = [{cacerts, CACerts}, + {verify, verify_peer}, + {verify_fun, {fun validate_function/3, {CACerts, []}}}], + + + ct:log("~p:~p~ncalling ssl_test_lib:start_client",[?MODULE,?LINE]), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + erlang_ssl_send, [Data]}}, + %{mfa, {ssl_test_lib, no_result, []}}, + {options, ClientOpts}]), + ct:log("~p:~p~nreturn from ssl_test_lib:start_client:~n~p",[?MODULE,?LINE,Client]), + + ssl_test_lib:check_result(Client, ok, Server, ok), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close(Server), + ssl_test_lib:close(Client), + process_flag(trap_exit, false). + +crl_verify_revoked() -> + [{doc,"Verify a simple valid CRL chain"}]. +crl_verify_revoked(Config) when is_list(Config) -> + process_flag(trap_exit, true), + PrivDir = ?config(cert_dir, Config), + ServerOpts = [{keyfile, filename:join([PrivDir, "revoked", "key.pem"])}, + {certfile, filename:join([PrivDir, "revoked", "cert.pem"])}, + {cacertfile, filename:join([PrivDir, "revoked", "cacerts.pem"])}], + ct:log("~p:~p~nserver opts ~p~n", [?MODULE,?LINE, ServerOpts]), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + %{mfa, {?MODULE, erlang_ssl_receive, [Data]}}, + {mfa, {ssl_test_lib, no_result, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + CACerts = load_cert(filename:join([PrivDir, "erlangCA", "cacerts.pem"])), + ClientOpts = [{cacerts, CACerts}, + {verify, verify_peer}, + {verify_fun, {fun validate_function/3, {CACerts, []}}}], + + {connect_failed, _} = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + %{mfa, {?MODULE, + %erlang_ssl_receive, [Data]}}, + {mfa, {ssl_test_lib, no_result, []}}, + {options, ClientOpts}]), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close(Server), + process_flag(trap_exit, false). + +%%%================================================================ +%%% Lib + +erlang_ssl_receive(Socket, Data) -> + ct:log("~p:~p~nConnection info: ~p~n", + [?MODULE,?LINE, ssl:connection_info(Socket)]), + receive + {ssl, Socket, Data} -> + ct:log("~p:~p~nReceived ~p~n",[?MODULE,?LINE, Data]), + %% open_ssl server sometimes hangs waiting in blocking read + ssl:send(Socket, "Got it"), + ok; + {ssl, Socket, Byte} when length(Byte) == 1 -> + erlang_ssl_receive(Socket, tl(Data)); + {Port, {data,Debug}} when is_port(Port) -> + ct:log("~p:~p~nopenssl ~s~n",[?MODULE,?LINE, Debug]), + erlang_ssl_receive(Socket,Data); + Other -> + ct:fail({unexpected_message, Other}) + after 4000 -> + ct:fail({did_not_get, Data}) + end. + + +erlang_ssl_send(Socket, Data) -> + ct:log("~p:~p~nConnection info: ~p~n", + [?MODULE,?LINE, ssl:connection_info(Socket)]), + ssl:send(Socket, Data), + ok. + +load_certs(undefined) -> + undefined; +load_certs(CertDir) -> + case file:list_dir(CertDir) of + {ok, Certs} -> + load_certs(lists:map(fun(Cert) -> filename:join(CertDir, Cert) + end, Certs), []); + {error, _} -> + undefined + end. + +load_certs([], Acc) -> + ct:log("~p:~p~nSuccessfully loaded ~p CA certificates~n", [?MODULE,?LINE, length(Acc)]), + Acc; +load_certs([Cert|Certs], Acc) -> + case filelib:is_dir(Cert) of + true -> + load_certs(Certs, Acc); + _ -> + %ct:log("~p:~p~nLoading certificate ~p~n", [?MODULE,?LINE, Cert]), + load_certs(Certs, load_cert(Cert) ++ Acc) + end. + +load_cert(Cert) -> + {ok, Bin} = file:read_file(Cert), + case filename:extension(Cert) of + ".der" -> + %% no decoding necessary + [Bin]; + _ -> + %% assume PEM otherwise + Contents = public_key:pem_decode(Bin), + [DER || {Type, DER, Cipher} <- Contents, Type == 'Certificate', Cipher == 'not_encrypted'] + end. + +%% @doc Validator function for SSL negotiation. +%% +validate_function(Cert, valid_peer, State) -> + ct:log("~p:~p~nvaliding peer ~p with ~p intermediate certs~n", + [?MODULE,?LINE, get_common_name(Cert), + length(element(2, State))]), + %% peer certificate validated, now check the CRL + Res = (catch check_crl(Cert, State)), + ct:log("~p:~p~nCRL validate result for ~p: ~p~n", + [?MODULE,?LINE, get_common_name(Cert), Res]), + {Res, State}; +validate_function(Cert, valid, {TrustedCAs, IntermediateCerts}=State) -> + case public_key:pkix_is_self_signed(Cert) of + true -> + ct:log("~p:~p~nroot certificate~n",[?MODULE,?LINE]), + %% this is a root cert, no CRL + {valid, {TrustedCAs, [Cert|IntermediateCerts]}}; + false -> + %% check is valid CA certificate, add to the list of + %% intermediates + Res = (catch check_crl(Cert, State)), + ct:log("~p:~p~nCRL intermediate CA validate result for ~p: ~p~n", + [?MODULE,?LINE, get_common_name(Cert), Res]), + {Res, {TrustedCAs, [Cert|IntermediateCerts]}} + end; +validate_function(_Cert, _Event, State) -> + %ct:log("~p:~p~nignoring event ~p~n", [?MODULE,?LINE, _Event]), + {valid, State}. + +%% @doc Given a certificate, find CRL distribution points for the given +%% certificate, fetch, and attempt to validate each CRL through +%% issuer_function/4. +%% +check_crl(Cert, State) -> + %% pull the CRL distribution point(s) out of the certificate, if any + ct:log("~p:~p~ncheck_crl(~n Cert=~p,~nState=~p~n)",[?MODULE,?LINE,Cert,State]), + case pubkey_cert:select_extension( + ?'id-ce-cRLDistributionPoints', + pubkey_cert:extensions_list(Cert#'OTPCertificate'.tbsCertificate#'OTPTBSCertificate'.extensions)) of + undefined -> + ct:log("~p:~p~nno CRL distribution points for ~p~n", + [?MODULE,?LINE, get_common_name(Cert)]), + %% fail; we can't validate if there's no CRL + no_crl; + CRLExtension -> + ct:log("~p:~p~nCRLExtension=~p)",[?MODULE,?LINE,CRLExtension]), + CRLDistPoints = CRLExtension#'Extension'.extnValue, + DPointsAndCRLs = lists:foldl(fun(Point, Acc) -> + %% try to read the CRL over http or from a + %% local file + case fetch_point(Point) of + not_available -> + ct:log("~p:~p~nfetch_point returned~n~p~n)",[?MODULE,?LINE,not_available]), + Acc; + Res -> + ct:log("~p:~p~nfetch_point returned~n~p~n)",[?MODULE,?LINE,Res]), + [{Point, Res} | Acc] + end + end, [], CRLDistPoints), + public_key:pkix_crls_validate(Cert, + DPointsAndCRLs, + [{issuer_fun, + {fun issuer_function/4, State}}]) + end. + +%% @doc Given a list of distribution points for CRLs, certificates and +%% both trusted and intermediary certificates, attempt to build and +%% authority chain back via build_chain to verify that it is valid. +%% +issuer_function(_DP, CRL, _Issuer, {TrustedCAs, IntermediateCerts}) -> + %% XXX the 'Issuer' we get passed here is the AuthorityKeyIdentifier, + %% which we are not currently smart enough to understand + %% Read the CA certs out of the file + ct:log("~p:~p~nissuer_function(~nCRL=~p,~nLast param=~p)",[?MODULE,?LINE,CRL, {TrustedCAs, IntermediateCerts}]), + Certs = [public_key:pkix_decode_cert(DER, otp) || DER <- TrustedCAs], + %% get the real issuer out of the CRL + Issuer = public_key:pkix_normalize_name( + pubkey_cert_records:transform( + CRL#'CertificateList'.tbsCertList#'TBSCertList'.issuer, decode)), + %% assume certificates are ordered from root to tip + case find_issuer(Issuer, IntermediateCerts ++ Certs) of + undefined -> + ct:log("~p:~p~nunable to find certificate matching CRL issuer ~p~n", + [?MODULE,?LINE, Issuer]), + error; + IssuerCert -> + ct:log("~p:~p~nIssuerCert=~p~n)",[?MODULE,?LINE,IssuerCert]), + case build_chain({public_key:pkix_encode('OTPCertificate', + IssuerCert, + otp), + IssuerCert}, IntermediateCerts, Certs, []) of + undefined -> + error; + {OTPCert, Path} -> + {ok, OTPCert, Path} + end + end. + +%% @doc Attempt to build authority chain back using intermediary +%% certificates, falling back on trusted certificates if the +%% intermediary chain of certificates does not fully extend to the +%% root. +%% +%% Returns: {RootCA :: #OTPCertificate{}, Chain :: [der_encoded()]} +%% +build_chain({DER, Cert}, IntCerts, TrustedCerts, Acc) -> + %% check if this cert is self-signed, if it is, we've reached the + %% root of the chain + Issuer = public_key:pkix_normalize_name( + Cert#'OTPCertificate'.tbsCertificate#'OTPTBSCertificate'.issuer), + Subject = public_key:pkix_normalize_name( + Cert#'OTPCertificate'.tbsCertificate#'OTPTBSCertificate'.subject), + case Issuer == Subject of + true -> + case find_issuer(Issuer, TrustedCerts) of + undefined -> + ct:log("~p:~p~nself-signed certificate is NOT trusted~n",[?MODULE,?LINE]), + undefined; + TrustedCert -> + %% return the cert from the trusted list, to prevent + %% issuer spoofing + {TrustedCert, + [public_key:pkix_encode( + 'OTPCertificate', TrustedCert, otp)|Acc]} + end; + false -> + Match = lists:foldl( + fun(C, undefined) -> + S = public_key:pkix_normalize_name(C#'OTPCertificate'.tbsCertificate#'OTPTBSCertificate'.subject), + %% compare the subject to the current issuer + case Issuer == S of + true -> + %% we've found our man + {public_key:pkix_encode('OTPCertificate', C, otp), C}; + false -> + undefined + end; + (_E, A) -> + %% already matched + A + end, undefined, IntCerts), + case Match of + undefined when IntCerts /= TrustedCerts -> + %% continue the chain by using the trusted CAs + ct:log("~p:~p~nRan out of intermediate certs, switching to trusted certs~n",[?MODULE,?LINE]), + build_chain({DER, Cert}, TrustedCerts, TrustedCerts, Acc); + undefined -> + ct:log("Can't construct chain of trust beyond ~p~n", + [?MODULE,?LINE, get_common_name(Cert)]), + %% can't find the current cert's issuer + undefined; + Match -> + build_chain(Match, IntCerts, TrustedCerts, [DER|Acc]) + end + end. + +%% @doc Given a certificate and a list of trusted or intermediary +%% certificates, attempt to find a match in the list or bail with +%% undefined. +find_issuer(Issuer, Certs) -> + lists:foldl( + fun(OTPCert, undefined) -> + %% check if this certificate matches the issuer + Normal = public_key:pkix_normalize_name( + OTPCert#'OTPCertificate'.tbsCertificate#'OTPTBSCertificate'.subject), + case Normal == Issuer of + true -> + OTPCert; + false -> + undefined + end; + (_E, Acc) -> + %% already found a match + Acc + end, undefined, Certs). + +%% @doc Find distribution points for a given CRL and then attempt to +%% fetch the CRL from the first available. +fetch_point(#'DistributionPoint'{distributionPoint={fullName, Names}}) -> + Decoded = [{NameType, + pubkey_cert_records:transform(Name, decode)} + || {NameType, Name} <- Names], + ct:log("~p:~p~ncall fetch(~nDecoded=~p~n)",[?MODULE,?LINE,Decoded]), + fetch(Decoded). + +%% @doc Given a list of locations to retrieve a CRL from, attempt to +%% retrieve either from a file or http resource and bail as soon as +%% it can be found. +%% +%% Currently, only hand a armored PEM or DER encoded file, with +%% defaulting to DER. +%% +fetch([]) -> + not_available; +fetch([{uniformResourceIdentifier, "http"++_=URL}|Rest]) -> + ct:log("~p:~p~ngetting CRL from ~p~n", [?MODULE,?LINE, URL]), + ct:log("~p:~p~nlisteners to port 8000:~n~p~n)",[?MODULE,?LINE,os:cmd("netstat -tln|grep ':8000'")]), + case httpc:request(get, {URL, []}, [], [{body_format, binary}]) of + {ok, {_Status, _Headers, Body}} -> + case Body of + <<"-----BEGIN", _/binary>> -> + ct:log("~p:~p~npublic_key:pem_decode,~nBody=~p~n)",[?MODULE,?LINE,Body]), + [{'CertificateList', + DER, _}=CertList] = public_key:pem_decode(Body), + ct:log("~p:~p~npublic_key:pem_entry_decode,~nCertList=~p~n)",[?MODULE,?LINE,CertList]), + {DER, public_key:pem_entry_decode(CertList)}; + _ -> + ct:log("~p:~p~npublic_key:pem_entry_decode,~nBody=~p~n)",[?MODULE,?LINE,{'CertificateList', Body, not_encrypted}]), + %% assume DER encoded + CertList = public_key:pem_entry_decode( + {'CertificateList', Body, not_encrypted}), + {Body, CertList} + end; + {error, _Reason} -> + ct:log("~p:~p~nfailed to get CRL ~p~n", [?MODULE,?LINE, _Reason]), + fetch(Rest); + Other -> + ct:log("~p:~p~nreally failed to get CRL ~p~n", [?MODULE,?LINE, Other]), + fetch(Rest) + end; +fetch([Loc|Rest]) -> + %% unsupported CRL location + ct:log("~p:~p~nunable to fetch CRL from unsupported location ~p~n", + [?MODULE,?LINE, Loc]), + fetch(Rest). + +%% get the common name attribute out of an OTPCertificate record +get_common_name(OTPCert) -> + %% You'd think there'd be an easier way than this giant mess, but I + %% couldn't find one. + {rdnSequence, Subject} = OTPCert#'OTPCertificate'.tbsCertificate#'OTPTBSCertificate'.subject, + case [Attribute#'AttributeTypeAndValue'.value || [Attribute] <- Subject, + Attribute#'AttributeTypeAndValue'.type == ?'id-at-commonName'] of + [Att] -> + case Att of + {teletexString, Str} -> Str; + {printableString, Str} -> Str; + {utf8String, Bin} -> binary_to_list(Bin) + end; + _ -> + unknown + end. + diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 7ed9adfcd9..7d8ece8d19 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -60,7 +60,7 @@ run_server(Opts) -> Options = proplists:get_value(options, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), - ct:log("ssl:listen(~p, ~p)~n", [Port, Options]), + ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, Options]), {ok, ListenSocket} = rpc:call(Node, Transport, listen, [Port, Options]), Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), @@ -78,13 +78,13 @@ do_run_server(ListenSocket, AcceptSocket, Opts) -> Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), {Module, Function, Args} = proplists:get_value(mfa, Opts), - ct:log("Server: apply(~p,~p,~p)~n", - [Module, Function, [AcceptSocket | Args]]), + ct:log("~p:~p~nServer: apply(~p,~p,~p)~n", + [?MODULE,?LINE, Module, Function, [AcceptSocket | Args]]), case rpc:call(Node, Module, Function, [AcceptSocket | Args]) of no_result_msg -> ok; Msg -> - ct:log("Server Msg: ~p ~n", [Msg]), + ct:log("~p:~p~nServer Msg: ~p ~n", [?MODULE,?LINE, Msg]), Pid ! {self(), Msg} end, receive @@ -93,10 +93,10 @@ do_run_server(ListenSocket, AcceptSocket, Opts) -> {listen, MFA} -> run_server(ListenSocket, [MFA | proplists:delete(mfa, Opts)]); close -> - ct:log("Server closing ~p ~n", [self()]), + ct:log("~p:~p~nServer closing ~p ~n", [?MODULE,?LINE, self()]), Result = rpc:call(Node, Transport, close, [AcceptSocket], 500), Result1 = rpc:call(Node, Transport, close, [ListenSocket], 500), - ct:log("Result ~p : ~p ~n", [Result, Result1]); + ct:log("~p:~p~nResult ~p : ~p ~n", [?MODULE,?LINE, Result, Result1]); {ssl_closed, _} -> ok end. @@ -116,7 +116,7 @@ connect(#sslsocket{} = ListenSocket, Opts) -> end; connect(ListenSocket, Opts) -> Node = proplists:get_value(node, Opts), - ct:log("gen_tcp:accept(~p)~n", [ListenSocket]), + ct:log("~p:~p~ngen_tcp:accept(~p)~n", [?MODULE,?LINE, ListenSocket]), {ok, AcceptSocket} = rpc:call(Node, gen_tcp, accept, [ListenSocket]), AcceptSocket. @@ -124,15 +124,17 @@ connect(ListenSocket, Opts) -> connect(_, _, 0, AcceptSocket, _) -> AcceptSocket; connect(ListenSocket, Node, N, _, Timeout) -> - ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), + ct:log("~p:~p~nssl:transport_accept(~p)~n", [?MODULE,?LINE, ListenSocket]), {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, [ListenSocket]), - ct:log("ssl:ssl_accept(~p, ~p)~n", [AcceptSocket, Timeout]), + ct:log("~p:~p~nssl:ssl_accept(~p, ~p)~n", [?MODULE,?LINE, AcceptSocket, Timeout]), case rpc:call(Node, ssl, ssl_accept, [AcceptSocket, Timeout]) of ok -> +ct:log("~p:~p~nok from ssl:ssl_accept@~p",[?MODULE,?LINE, Node]), connect(ListenSocket, Node, N-1, AcceptSocket, Timeout); Result -> +ct:log("~p:~p~nssl:ssl_accept@~p ret ~p",[?MODULE,?LINE, Node,Result]), Result end. @@ -148,11 +150,13 @@ remove_close_msg(ReconnectTimes) -> start_client(Args) -> Result = spawn_link(?MODULE, run_client_init, [lists:delete(return_socket, Args)]), receive - { connected, Socket } -> - case lists:member(return_socket, Args) of - true -> { Result, Socket }; - false -> Result - end + {connected, Socket} -> + case lists:member(return_socket, Args) of + true -> {Result, Socket}; + false -> Result + end; + {connect_failed, Reason} -> + {connect_failed, Reason} end. run_client_init(Opts) -> @@ -166,27 +170,30 @@ run_client(Opts) -> Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), Options = proplists:get_value(options, Opts), - ct:log("ssl:connect(~p, ~p, ~p)~n", [Host, Port, Options]), + ct:log("~p:~p~nssl:connect(~p, ~p, ~p)~n", [?MODULE,?LINE, Host, Port, Options]), +ct:log("~p:~p~nnet_adm:ping(~p)=~p",[?MODULE,?LINE, Node,net_adm:ping(Node)]), +%%ct:log("~p:~p~n~p:connect(~p, ~p, ~p)@~p~n", [?MODULE,?LINE, Transport, Host, Port, Options, Node]), +ct:log("~p:~p~n~p:connect(~p, ~p, ...)@~p~n", [?MODULE,?LINE, Transport, Host, Port, Node]), case rpc:call(Node, Transport, connect, [Host, Port, Options]) of {ok, Socket} -> - Pid ! { connected, Socket }, - ct:log("Client: connected~n", []), + Pid ! {connected, Socket}, + ct:log("~p:~p~nClient: connected~n", [?MODULE,?LINE]), %% In special cases we want to know the client port, it will %% be indicated by sending {port, 0} in options list! send_selected_port(Pid, proplists:get_value(port, Options), Socket), {Module, Function, Args} = proplists:get_value(mfa, Opts), - ct:log("Client: apply(~p,~p,~p)~n", - [Module, Function, [Socket | Args]]), + ct:log("~p:~p~nClient: apply(~p,~p,~p)~n", + [?MODULE,?LINE, Module, Function, [Socket | Args]]), case rpc:call(Node, Module, Function, [Socket | Args]) of no_result_msg -> ok; Msg -> - ct:log("Client Msg: ~p ~n", [Msg]), + ct:log("~p:~p~nClient Msg: ~p ~n", [?MODULE,?LINE, Msg]), Pid ! {self(), Msg} end, receive close -> - ct:log("Client closing~n", []), + ct:log("~p:~p~nClient closing~n", [?MODULE,?LINE]), rpc:call(Node, Transport, close, [Socket]); {ssl_closed, Socket} -> ok; @@ -196,50 +203,42 @@ run_client(Opts) -> {error, econnrefused = Reason} -> case get(retries) of N when N < 5 -> + ct:log("~p:~p~neconnrefused retries=~p sleep ~p",[?MODULE,?LINE, N,?SLEEP]), put(retries, N+1), ct:sleep(?SLEEP), run_client(Opts); _ -> - ct:log("Client faild several times: connection failed: ~p ~n", [Reason]), + ct:log("~p:~p~nClient faild several times: connection failed: ~p ~n", [?MODULE,?LINE, Reason]), Pid ! {self(), {error, Reason}} end; {error, Reason} -> - ct:log("Client: connection failed: ~p ~n", [Reason]), - Pid ! {self(), {error, Reason}} + ct:log("~p:~p~nClient: connection failed: ~p ~n", [?MODULE,?LINE, Reason]), + Pid ! {connect_failed, Reason}; + {badrpc,BadRPC} -> + ct:log("~p:~p~nBad rpc: ~p",[?MODULE,?LINE, BadRPC]), + Pid ! {connect_failed, {badrpc,BadRPC}} end. close(Pid) -> - ct:log("Close ~p ~n", [Pid]), + ct:log("~p:~p~nClose ~p ~n", [?MODULE,?LINE, Pid]), Monitor = erlang:monitor(process, Pid), Pid ! close, receive {'DOWN', Monitor, process, Pid, Reason} -> erlang:demonitor(Monitor), - ct:log("Pid: ~p down due to:~p ~n", [Pid, Reason]) + ct:log("~p:~p~nPid: ~p down due to:~p ~n", [?MODULE,?LINE, Pid, Reason]) end. check_result(Server, ServerMsg, Client, ClientMsg) -> receive - {Server, ServerMsg} -> - receive - {Client, ClientMsg} -> - ok; - Unexpected -> - Reason = {{expected, {Client, ClientMsg}}, - {got, Unexpected}}, - ct:fail(Reason) - end; - {Client, ClientMsg} -> - receive - {Server, ServerMsg} -> - ok; - Unexpected -> - Reason = {{expected, {Server, ClientMsg}}, - {got, Unexpected}}, - ct:fail(Reason) - end; + {Server, ServerMsg} -> + check_result(Client, ClientMsg); + + {Client, ClientMsg} -> + check_result(Server, ServerMsg); + {Port, {data,Debug}} when is_port(Port) -> - io:format("openssl ~s~n",[Debug]), + ct:log("~p:~p~nopenssl ~s~n",[?MODULE,?LINE, Debug]), check_result(Server, ServerMsg, Client, ClientMsg); Unexpected -> @@ -253,7 +252,7 @@ check_result(Pid, Msg) -> {Pid, Msg} -> ok; {Port, {data,Debug}} when is_port(Port) -> - io:format("openssl ~s~n",[Debug]), + ct:log("~p:~p~nopenssl ~s~n",[?MODULE,?LINE, Debug]), check_result(Pid,Msg); Unexpected -> Reason = {{expected, {Pid, Msg}}, @@ -278,7 +277,7 @@ wait_for_result(Server, ServerMsg, Client, ClientMsg) -> %% Unexpected end; {Port, {data,Debug}} when is_port(Port) -> - io:format("openssl ~s~n",[Debug]), + ct:log("~p:~p~nopenssl ~s~n",[?MODULE,?LINE, Debug]), wait_for_result(Server, ServerMsg, Client, ClientMsg) %% Unexpected -> %% Unexpected @@ -290,7 +289,7 @@ wait_for_result(Pid, Msg) -> {Pid, Msg} -> ok; {Port, {data,Debug}} when is_port(Port) -> - io:format("openssl ~s~n",[Debug]), + ct:log("~p:~p~nopenssl ~s~n",[?MODULE,?LINE, Debug]), wait_for_result(Pid,Msg) %% Unexpected -> %% Unexpected @@ -515,33 +514,33 @@ run_upgrade_server(Opts) -> SslOptions = proplists:get_value(ssl_options, Opts), Pid = proplists:get_value(from, Opts), - ct:log("gen_tcp:listen(~p, ~p)~n", [Port, TcpOptions]), + ct:log("~p:~p~ngen_tcp:listen(~p, ~p)~n", [?MODULE,?LINE, Port, TcpOptions]), {ok, ListenSocket} = rpc:call(Node, gen_tcp, listen, [Port, TcpOptions]), Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), - ct:log("gen_tcp:accept(~p)~n", [ListenSocket]), + ct:log("~p:~p~ngen_tcp:accept(~p)~n", [?MODULE,?LINE, ListenSocket]), {ok, AcceptSocket} = rpc:call(Node, gen_tcp, accept, [ListenSocket]), try {ok, SslAcceptSocket} = case TimeOut of infinity -> - ct:log("ssl:ssl_accept(~p, ~p)~n", - [AcceptSocket, SslOptions]), + ct:log("~p:~p~nssl:ssl_accept(~p, ~p)~n", + [?MODULE,?LINE, AcceptSocket, SslOptions]), rpc:call(Node, ssl, ssl_accept, [AcceptSocket, SslOptions]); _ -> - ct:log("ssl:ssl_accept(~p, ~p, ~p)~n", - [AcceptSocket, SslOptions, TimeOut]), + ct:log("~p:~p~nssl:ssl_accept(~p, ~p, ~p)~n", + [?MODULE,?LINE, AcceptSocket, SslOptions, TimeOut]), rpc:call(Node, ssl, ssl_accept, [AcceptSocket, SslOptions, TimeOut]) end, {Module, Function, Args} = proplists:get_value(mfa, Opts), Msg = rpc:call(Node, Module, Function, [SslAcceptSocket | Args]), - ct:log("Upgrade Server Msg: ~p ~n", [Msg]), + ct:log("~p:~p~nUpgrade Server Msg: ~p ~n", [?MODULE,?LINE, Msg]), Pid ! {self(), Msg}, receive close -> - ct:log("Upgrade Server closing~n", []), + ct:log("~p:~p~nUpgrade Server closing~n", [?MODULE,?LINE]), rpc:call(Node, ssl, close, [SslAcceptSocket]) end catch error:{badmatch, Error} -> @@ -559,24 +558,24 @@ run_upgrade_client(Opts) -> TcpOptions = proplists:get_value(tcp_options, Opts), SslOptions = proplists:get_value(ssl_options, Opts), - ct:log("gen_tcp:connect(~p, ~p, ~p)~n", - [Host, Port, TcpOptions]), + ct:log("~p:~p~ngen_tcp:connect(~p, ~p, ~p)~n", + [?MODULE,?LINE, Host, Port, TcpOptions]), {ok, Socket} = rpc:call(Node, gen_tcp, connect, [Host, Port, TcpOptions]), send_selected_port(Pid, Port, Socket), - ct:log("ssl:connect(~p, ~p)~n", [Socket, SslOptions]), + ct:log("~p:~p~nssl:connect(~p, ~p)~n", [?MODULE,?LINE, Socket, SslOptions]), {ok, SslSocket} = rpc:call(Node, ssl, connect, [Socket, SslOptions]), {Module, Function, Args} = proplists:get_value(mfa, Opts), - ct:log("apply(~p, ~p, ~p)~n", - [Module, Function, [SslSocket | Args]]), + ct:log("~p:~p~napply(~p, ~p, ~p)~n", + [?MODULE,?LINE, Module, Function, [SslSocket | Args]]), Msg = rpc:call(Node, Module, Function, [SslSocket | Args]), - ct:log("Upgrade Client Msg: ~p ~n", [Msg]), + ct:log("~p:~p~nUpgrade Client Msg: ~p ~n", [?MODULE,?LINE, Msg]), Pid ! {self(), Msg}, receive close -> - ct:log("Upgrade Client closing~n", []), + ct:log("~p:~p~nUpgrade Client closing~n", [?MODULE,?LINE]), rpc:call(Node, ssl, close, [SslSocket]) end. @@ -595,21 +594,21 @@ run_upgrade_server_error(Opts) -> SslOptions = proplists:get_value(ssl_options, Opts), Pid = proplists:get_value(from, Opts), - ct:log("gen_tcp:listen(~p, ~p)~n", [Port, TcpOptions]), + ct:log("~p:~p~ngen_tcp:listen(~p, ~p)~n", [?MODULE,?LINE, Port, TcpOptions]), {ok, ListenSocket} = rpc:call(Node, gen_tcp, listen, [Port, TcpOptions]), Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), - ct:log("gen_tcp:accept(~p)~n", [ListenSocket]), + ct:log("~p:~p~ngen_tcp:accept(~p)~n", [?MODULE,?LINE, ListenSocket]), {ok, AcceptSocket} = rpc:call(Node, gen_tcp, accept, [ListenSocket]), Error = case TimeOut of infinity -> - ct:log("ssl:ssl_accept(~p, ~p)~n", - [AcceptSocket, SslOptions]), + ct:log("~p:~p~nssl:ssl_accept(~p, ~p)~n", + [?MODULE,?LINE, AcceptSocket, SslOptions]), rpc:call(Node, ssl, ssl_accept, [AcceptSocket, SslOptions]); _ -> - ct:log("ssl:ssl_accept(~p, ~p, ~p)~n", - [AcceptSocket, SslOptions, TimeOut]), + ct:log("~p:~p~nssl:ssl_accept(~p, ~p, ~p)~n", + [?MODULE,?LINE, AcceptSocket, SslOptions, TimeOut]), rpc:call(Node, ssl, ssl_accept, [AcceptSocket, SslOptions, TimeOut]) end, @@ -628,26 +627,26 @@ run_server_error(Opts) -> Options = proplists:get_value(options, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), - ct:log("ssl:listen(~p, ~p)~n", [Port, Options]), + ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, Options]), case rpc:call(Node, Transport, listen, [Port, Options]) of {ok, #sslsocket{} = ListenSocket} -> %% To make sure error_client will %% get {error, closed} and not {error, connection_refused} Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), - ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), + ct:log("~p:~p~nssl:transport_accept(~p)~n", [?MODULE,?LINE, ListenSocket]), case rpc:call(Node, Transport, transport_accept, [ListenSocket]) of {error, _} = Error -> Pid ! {self(), Error}; {ok, AcceptSocket} -> - ct:log("ssl:ssl_accept(~p)~n", [AcceptSocket]), + ct:log("~p:~p~nssl:ssl_accept(~p)~n", [?MODULE,?LINE, AcceptSocket]), Error = rpc:call(Node, ssl, ssl_accept, [AcceptSocket]), Pid ! {self(), Error} end; {ok, ListenSocket} -> Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), - ct:log("~p:accept(~p)~n", [Transport, ListenSocket]), + ct:log("~p:~p~n~p:accept(~p)~n", [?MODULE,?LINE, Transport, ListenSocket]), case rpc:call(Node, Transport, accept, [ListenSocket]) of {error, _} = Error -> Pid ! {self(), Error} @@ -669,7 +668,7 @@ run_client_error(Opts) -> Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), Options = proplists:get_value(options, Opts), - ct:log("ssl:connect(~p, ~p, ~p)~n", [Host, Port, Options]), + ct:log("~p:~p~nssl:connect(~p, ~p, ~p)~n", [?MODULE,?LINE, Host, Port, Options]), Error = rpc:call(Node, Transport, connect, [Host, Port, Options]), Pid ! {self(), Error}. @@ -892,7 +891,7 @@ der_to_pem(File, Entries) -> cipher_result(Socket, Result) -> Result = ssl:connection_info(Socket), - ct:log("Successfull connect: ~p~n", [Result]), + ct:log("~p:~p~nSuccessfull connect: ~p~n", [?MODULE,?LINE, Result]), %% Importante to send two packets here %% to properly test "cipher state" handling ssl:send(Socket, "Hello\n"), @@ -1061,10 +1060,13 @@ check_sane_openssl_version(Version) -> true end. +enough_openssl_crl_support("OpenSSL 0." ++ _) -> false; +enough_openssl_crl_support(_) -> true. + wait_for_openssl_server() -> receive {Port, {data, Debug}} when is_port(Port) -> - ct:log("openssl ~s~n",[Debug]), + ct:log("~p:~p~nopenssl ~s~n",[?MODULE,?LINE, Debug]), %% openssl has started make sure %% it will be in accept. Parsing %% output is too error prone. (Even diff --git a/lib/stdlib/doc/src/erl_parse.xml b/lib/stdlib/doc/src/erl_parse.xml index 2d5aff3c6c..cf0bff48cd 100644 --- a/lib/stdlib/doc/src/erl_parse.xml +++ b/lib/stdlib/doc/src/erl_parse.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2013</year> + <year>1996</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -173,6 +173,7 @@ </func> <func> <name name="abstract" arity="2"/> + <type name="encoding_func"/> <fsummary>Convert an Erlang term into an abstract form</fsummary> <desc> <p>Converts the Erlang data structure <c><anno>Data</anno></c> into an @@ -183,7 +184,12 @@ selecting which integer lists will be considered as strings. The default is to use the encoding returned by <seealso marker="epp#default_encoding/0"> - <c>epp:default_encoding/0</c></seealso></p> + <c>epp:default_encoding/0</c></seealso>. + The value <c>none</c> means that no integer lists will be + considered as strings. The <c>encoding_func()</c> will be + called with one integer of a list at a time, and if it + returns <c>true</c> for every integer the list will be + considered a string.</p> </desc> </func> </funcs> diff --git a/lib/stdlib/doc/src/sys.xml b/lib/stdlib/doc/src/sys.xml index ab8b380f49..a46fa1289f 100644 --- a/lib/stdlib/doc/src/sys.xml +++ b/lib/stdlib/doc/src/sys.xml @@ -246,6 +246,22 @@ <c>{Module, Id, HandlerState}</c>, where <c>Module</c> is the event handler's module name, <c>Id</c> is the handler's ID (which is the value <c>false</c> if it was registered without an ID), and <c>HandlerState</c> is the handler's state.</p> + <p>If the callback module exports a <c>system_get_state/1</c> function, it will be called in the + target process to get its state. Its argument is the same as the <c>Misc</c> value returned by + <seealso marker="#get_status-1">get_status/1,2</seealso>, and the <c>system_get_state/1</c> + function is expected to extract the callback module's state from it. The <c>system_get_state/1</c> + function must return <c>{ok, State}</c> where <c>State</c> is the callback module's state.</p> + <p>If the callback module does not export a <c>system_get_state/1</c> function, <c>get_state/1,2</c> + assumes the <c>Misc</c> value is the callback module's state and returns it directly instead.</p> + <p>If the callback module's <c>system_get_state/1</c> function crashes or throws an exception, the + caller exits with error <c>{callback_failed, {Module, system_get_state}, {Class, Reason}}</c> where + <c>Module</c> is the name of the callback module and <c>Class</c> and <c>Reason</c> indicate + details of the exception.</p> + <p>The <c>system_get_state/1</c> function is primarily useful for user-defined + behaviours and modules that implement OTP <seealso marker="#special_process">special + processes</seealso>. The <c>gen_server</c>, <c>gen_fsm</c>, and <c>gen_event</c> OTP + behaviour modules export this function, and so callback modules for those behaviours + need not supply their own.</p> <p>To obtain more information about a process, including its state, see <seealso marker="#get_status-1">get_status/1</seealso> and <seealso marker="#get_status-2">get_status/2</seealso>.</p> @@ -289,6 +305,28 @@ function means that only the state of the particular event handler it was working on when it failed or crashed is unchanged; it can still succeed in changing the states of other event handlers registered in the same <c>gen_event</c> process.</p> + <p>If the callback module exports a <c>system_replace_state/2</c> function, it will be called in the + target process to replace its state using <c>StateFun</c>. Its two arguments are <c>StateFun</c> + and <c>Misc</c>, where <c>Misc</c> is the same as the <c>Misc</c> value returned by + <seealso marker="#get_status-1">get_status/1,2</seealso>. A <c>system_replace_state/2</c> function + is expected to return <c>{ok, NewState, NewMisc}</c> where <c>NewState</c> is the callback module's + new state obtained by calling <c>StateFun</c>, and <c>NewMisc</c> is a possibly new value used to + replace the original <c>Misc</c> (required since <c>Misc</c> often contains the callback + module's state within it).</p> + <p>If the callback module does not export a <c>system_replace_state/2</c> function, + <c>replace_state/2,3</c> assumes the <c>Misc</c> value is the callback module's state, passes it + to <c>StateFun</c> and uses the return value as both the new state and as the new value of + <c>Misc</c>.</p> + <p>If the callback module's <c>system_replace_state/2</c> function crashes or throws an exception, + the caller exits with error <c>{callback_failed, {Module, system_replace_state}, {Class, Reason}}</c> + where <c>Module</c> is the name of the callback module and <c>Class</c> and <c>Reason</c> indicate details + of the exception. If the callback module does not provide a <c>system_replace_state/2</c> function and + <c>StateFun</c> crashes or throws an exception, the caller exits with error + <c>{callback_failed, StateFun, {Class, Reason}}</c>.</p> + <p>The <c>system_replace_state/2</c> function is primarily useful for user-defined behaviours and + modules that implement OTP <seealso marker="#special_process">special processes</seealso>. The + <c>gen_server</c>, <c>gen_fsm</c>, and <c>gen_event</c> OTP behaviour modules export this function, + and so callback modules for those behaviours need not supply their own.</p> </desc> </func> <func> @@ -322,7 +360,7 @@ <section> <title>Process Implementation Functions</title> - <p>The following functions are used when implementing a + <p><marker id="special_process"/>The following functions are used when implementing a special process. This is an ordinary process which does not use a standard behaviour, but a process which understands the standard system messages.</p> </section> @@ -375,8 +413,9 @@ process continues the execution, or <c><anno>Module</anno>:system_terminate(Reason, <anno>Parent</anno>, <anno>Debug</anno>, <anno>Misc</anno>)</c> if the process should terminate. The <c><anno>Module</anno></c> must export - <c>system_continue/3</c>, <c>system_terminate/4</c>, and - <c>system_code_change/4</c> (see below). + <c>system_continue/3</c>, <c>system_terminate/4</c>, + <c>system_code_change/4</c>, <c>system_get_state/1</c> and + <c>system_replace_state/2</c> (see below). </p> <p>The <c><anno>Misc</anno></c> argument can be used to save internal data in a process, for example its state. It is sent to @@ -444,6 +483,34 @@ defined, the atom <c>undefined</c> is sent.</p> </desc> </func> + <func> + <name>Mod:system_get_state(Misc) -> {ok, State}</name> + <fsummary>Called when the process should return its current state</fsummary> + <type> + <v>Misc = term()</v> + <v>State = term()</v> + </type> + <desc> + <p>This function is called from <c>sys:handle_system_msg/6</c> when the process + should return a term that reflects its current state. <c>State</c> is the + value returned by <c>sys:get_state/2</c>.</p> + </desc> + </func> + <func> + <name>Mod:system_replace_state(StateFun, Misc) -> {ok, NState, NMisc}</name> + <fsummary>Called when the process should replace its current state</fsummary> + <type> + <v>StateFun = fun((State :: term()) -> NState)</v> + <v>Misc = term()</v> + <v>NState = term()</v> + <v>NMisc = term()</v> + </type> + <desc> + <p>This function is called from <c>sys:handle_system_msg/6</c> when the process + should replace its current state. <c>NState</c> is the value returned by + <c>sys:replace_state/3</c>.</p> + </desc> + </func> </funcs> </erlref> diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index c843ef7736..bebfbd4514 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -848,6 +848,7 @@ Eshell V5.10.1 (abort with ^G) </section> <section> <title>Unicode in Environment and Parameters</title> + <marker id="unicode_in_environment_and_parameters"/> <p>Environment variables and their interpretation is handled much in the same way as file names. If Unicode file names are enabled, environment variables as well as parameters to the Erlang VM are diff --git a/lib/stdlib/src/dict.erl b/lib/stdlib/src/dict.erl index 6088e1a2dd..cf8fb3114a 100644 --- a/lib/stdlib/src/dict.erl +++ b/lib/stdlib/src/dict.erl @@ -55,8 +55,7 @@ -define(exp_size, (?seg_size * ?expand_load)). -define(con_size, (?seg_size * ?contract_load)). --type segs(K, V) :: tuple() - | {K, V}. % dummy +-type segs(_Key, _Value) :: tuple(). %% Define a hashtable. The default values are the standard ones. -record(dict, diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index d212a55b47..9b506b0a44 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -33,23 +33,33 @@ -export_type([source_encoding/0]). --type macros() :: [{atom(), term()}]. +-type macros() :: [atom() | {atom(), term()}]. -type epp_handle() :: pid(). -type source_encoding() :: latin1 | utf8. +-type ifdef() :: 'ifdef' | 'ifndef' | 'else'. + +-type name() :: {'atom', atom()}. +-type argspec() :: 'none' %No arguments + | non_neg_integer(). %Number of arguments +-type tokens() :: [erl_scan:token()]. +-type used() :: {name(), argspec()}. + -define(DEFAULT_ENCODING, utf8). %% Epp state record. --record(epp, {file, %Current file +-record(epp, {file :: file:io_device(), %Current file location=1, %Current location - delta, %Offset from Location (-file) - name="", %Current file name - name2="", %-"-, modified by -file - istk=[], %Ifdef stack - sstk=[], %State stack - path=[], %Include-path - macs = dict:new() :: dict:dict(),%Macros (don't care locations) - uses = dict:new() :: dict:dict(),%Macro use structure + delta=0 :: non_neg_integer(), %Offset from Location (-file) + name="" :: file:name(), %Current file name + name2="" :: file:name(), %-"-, modified by -file + istk=[] :: [ifdef()], %Ifdef stack + sstk=[] :: [#epp{}], %State stack + path=[] :: [file:name()], %Include-path + macs = dict:new() %Macros (don't care locations) + :: dict:dict(name(), {argspec(), tokens()}), + uses = dict:new() %Macro use structure + :: dict:dict(name(), [{argspec(), [used()]}]), default_encoding = ?DEFAULT_ENCODING :: source_encoding(), pre_opened = false :: boolean() }). diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 4c0261a1ad..c4c94fbee4 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -80,13 +80,17 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> -type fa() :: {atom(), arity()}. % function+arity -type ta() :: {atom(), arity()}. % type+arity +-record(typeinfo, {attr, line}). + %% Usage of records, functions, and imports. The variable table, which %% is passed on as an argument, holds the usage of variables. -record(usage, { calls = dict:new(), %Who calls who imported = [], %Actually imported functions - used_records=sets:new() :: sets:set(),%Used record definitions - used_types = dict:new() :: dict:dict()%Used type definitions + used_records = sets:new() %Used record definitions + :: sets:set(atom()), + used_types = dict:new() %Used type definitions + :: dict:dict(ta(), line()) }). %% Define the lint state record. @@ -95,13 +99,17 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> -record(lint, {state=start :: 'start' | 'attribute' | 'function', module=[], %Module behaviour=[], %Behaviour - exports=gb_sets:empty() :: gb_sets:set(),%Exports - imports=[], %Imports + exports=gb_sets:empty() :: gb_sets:set(fa()),%Exports + imports=[] :: [fa()], %Imports, an orddict() compile=[], %Compile flags - records=dict:new() :: dict:dict(), %Record definitions - locals=gb_sets:empty() :: gb_sets:set(),%All defined functions (prescanned) - no_auto=gb_sets:empty() :: gb_sets:set() | 'all',%Functions explicitly not autoimported - defined=gb_sets:empty() :: gb_sets:set(),%Defined fuctions + records=dict:new() %Record definitions + :: dict:dict(atom(), {line(),Fields :: term()}), + locals=gb_sets:empty() %All defined functions (prescanned) + :: gb_sets:set(fa()), + no_auto=gb_sets:empty() %Functions explicitly not autoimported + :: gb_sets:set(fa()) | 'all', + defined=gb_sets:empty() %Defined fuctions + :: gb_sets:set(fa()), on_load=[] :: [fa()], %On-load function on_load_line=0 :: line(), %Line for on_load clashes=[], %Exported functions named as BIFs @@ -116,12 +124,16 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> %outside any fun or lc xqlc= false :: boolean(), %true if qlc.hrl included new = false :: boolean(), %Has user-defined 'new/N' - called= [] :: [{fa(),line()}], %Called functions + called= [] :: [{fa(),line()}], %Called functions usage = #usage{} :: #usage{}, - specs = dict:new() :: dict:dict(), %Type specifications - callbacks = dict:new() :: dict:dict(), %Callback types - types = dict:new() :: dict:dict(), %Type definitions - exp_types=gb_sets:empty():: gb_sets:set()%Exported types + specs = dict:new() %Type specifications + :: dict:dict(mfa(), line()), + callbacks = dict:new() %Callback types + :: dict:dict(mfa(), line()), + types = dict:new() %Type definitions + :: dict:dict(ta(), #typeinfo{}), + exp_types=gb_sets:empty() %Exported types + :: gb_sets:set(ta()) }). -type lint_state() :: #lint{}. @@ -319,10 +331,14 @@ format_error({undefined_type, {TypeName, Arity}}) -> io_lib:format("type ~w~s undefined", [TypeName, gen_type_paren(Arity)]); format_error({unused_type, {TypeName, Arity}}) -> io_lib:format("type ~w~s is unused", [TypeName, gen_type_paren(Arity)]); -format_error({new_builtin_type, {TypeName, Arity}}) -> - io_lib:format("type ~w~s is a new builtin type; " +%% format_error({new_builtin_type, {TypeName, Arity}}) -> +%% io_lib:format("type ~w~s is a new builtin type; " +%% "its (re)definition is allowed only until the next release", +%% [TypeName, gen_type_paren(Arity)]); +format_error({new_var_arity_type, TypeName}) -> + io_lib:format("type ~w is a new builtin type; " "its (re)definition is allowed only until the next release", - [TypeName, gen_type_paren(Arity)]); + [TypeName]); format_error({builtin_type, {TypeName, Arity}}) -> io_lib:format("type ~w~s is a builtin type; it cannot be redefined", [TypeName, gen_type_paren(Arity)]); @@ -1170,7 +1186,7 @@ export_type(Line, ETs, #lint{usage = Usage, exp_types = ETs0} = St0) -> add_error(Line, {bad_export_type, ETs}, St0) end. --spec exports(lint_state()) -> gb_sets:set(). +-spec exports(lint_state()) -> gb_sets:set(fa()). exports(#lint{compile = Opts, defined = Defs, exports = Es}) -> case lists:member(export_all, Opts) of @@ -2574,8 +2590,6 @@ find_field(_F, []) -> error. %% Attr :: 'type' | 'opaque' %% Checks that a type definition is valid. --record(typeinfo, {attr, line}). - type_def(_Attr, _Line, {record, _RecName}, Fields, [], St0) -> %% The record field names and such are checked in the record format. %% We only need to check the types. @@ -2596,23 +2610,30 @@ type_def(Attr, Line, TypeName, ProtoType, Args, St0) -> true -> case is_obsolete_builtin_type(TypePair) of true -> StoreType(St0); - false -> - case is_newly_introduced_builtin_type(TypePair) of - %% allow some types just for bootstrapping - true -> - Warn = {new_builtin_type, TypePair}, - St1 = add_warning(Line, Warn, St0), - StoreType(St1); - false -> - add_error(Line, {builtin_type, TypePair}, St0) - end + false -> add_error(Line, {builtin_type, TypePair}, St0) +%% case is_newly_introduced_builtin_type(TypePair) of +%% %% allow some types just for bootstrapping +%% true -> +%% Warn = {new_builtin_type, TypePair}, +%% St1 = add_warning(Line, Warn, St0), +%% StoreType(St1); +%% false -> +%% add_error(Line, {builtin_type, TypePair}, St0) +%% end end; false -> case - dict:is_key(TypePair, TypeDefs) - orelse is_var_arity_type(TypeName) + dict:is_key(TypePair, TypeDefs) orelse + is_var_arity_type(TypeName) of - true -> add_error(Line, {redefine_type, TypePair}, St0); + true -> + case is_newly_introduced_var_arity_type(TypeName) of + true -> + Warn = {new_var_arity_type, TypeName}, + add_warning(Line, Warn, St0); + false -> + add_error(Line, {redefine_type, TypePair}, St0) + end; false -> St1 = case Attr =:= opaque andalso @@ -2847,8 +2868,10 @@ is_default_type({timeout, 0}) -> true; is_default_type({var, 1}) -> true; is_default_type(_) -> false. -%% OTP 17.0 -is_newly_introduced_builtin_type({Name, _}) when is_atom(Name) -> false. +is_newly_introduced_var_arity_type(map) -> true; +is_newly_introduced_var_arity_type(_) -> false. + +%% is_newly_introduced_builtin_type({Name, _}) when is_atom(Name) -> false. is_obsolete_builtin_type(TypePair) -> obsolete_builtin_type(TypePair) =/= no. diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 6316db7054..1dc5fc52a7 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -919,59 +919,63 @@ normalise_list([]) -> Data :: term(), AbsTerm :: abstract_expr(). abstract(T) -> - abstract(T, 0, epp:default_encoding()). + abstract(T, 0, enc_func(epp:default_encoding())). + +-type encoding_func() :: fun((non_neg_integer()) -> boolean()). %%% abstract/2 takes line and encoding options -spec abstract(Data, Options) -> AbsTerm when Data :: term(), Options :: Line | [Option], Option :: {line, Line} | {encoding, Encoding}, - Encoding :: latin1 | unicode | utf8, + Encoding :: 'latin1' | 'unicode' | 'utf8' | 'none' | encoding_func(), Line :: erl_scan:line(), AbsTerm :: abstract_expr(). abstract(T, Line) when is_integer(Line) -> - abstract(T, Line, epp:default_encoding()); + abstract(T, Line, enc_func(epp:default_encoding())); abstract(T, Options) when is_list(Options) -> Line = proplists:get_value(line, Options, 0), Encoding = proplists:get_value(encoding, Options,epp:default_encoding()), - abstract(T, Line, Encoding). + EncFunc = enc_func(Encoding), + abstract(T, Line, EncFunc). -define(UNICODE(C), - is_integer(C) andalso - (C >= 0 andalso C < 16#D800 orelse + (C < 16#D800 orelse C > 16#DFFF andalso C < 16#FFFE orelse C > 16#FFFF andalso C =< 16#10FFFF)). +enc_func(latin1) -> fun(C) -> C < 256 end; +enc_func(unicode) -> fun(C) -> ?UNICODE(C) end; +enc_func(utf8) -> fun(C) -> ?UNICODE(C) end; +enc_func(none) -> none; +enc_func(Fun) when is_function(Fun, 1) -> Fun; +enc_func(Term) -> erlang:error({badarg, Term}). + abstract(T, L, _E) when is_integer(T) -> {integer,L,T}; abstract(T, L, _E) when is_float(T) -> {float,L,T}; abstract(T, L, _E) when is_atom(T) -> {atom,L,T}; abstract([], L, _E) -> {nil,L}; abstract(B, L, _E) when is_bitstring(B) -> {bin, L, [abstract_byte(Byte, L) || Byte <- bitstring_to_list(B)]}; -abstract([C|T], L, unicode=E) when ?UNICODE(C) -> - abstract_unicode_string(T, [C], L, E); -abstract([C|T], L, utf8=E) when ?UNICODE(C) -> - abstract_unicode_string(T, [C], L, E); -abstract([C|T], L, latin1=E) when is_integer(C), 0 =< C, C < 256 -> - abstract_string(T, [C], L, E); -abstract([H|T], L, E) -> +abstract([H|T], L, none=E) -> {cons,L,abstract(H, L, E),abstract(T, L, E)}; +abstract(List, L, E) when is_list(List) -> + abstract_list(List, [], L, E); abstract(Tuple, L, E) when is_tuple(Tuple) -> - {tuple,L,abstract_list(tuple_to_list(Tuple), L, E)}. - -abstract_string([C|T], String, L, E) when is_integer(C), 0 =< C, C < 256 -> - abstract_string(T, [C|String], L, E); -abstract_string([], String, L, _E) -> - {string, L, lists:reverse(String)}; -abstract_string(T, String, L, E) -> - not_string(String, abstract(T, L, E), L, E). - -abstract_unicode_string([C|T], String, L, E) when ?UNICODE(C) -> - abstract_unicode_string(T, [C|String], L, E); -abstract_unicode_string([], String, L, _E) -> + {tuple,L,abstract_tuple_list(tuple_to_list(Tuple), L, E)}. + +abstract_list([H|T], String, L, E) -> + case is_integer(H) andalso H >= 0 andalso E(H) of + true -> + abstract_list(T, [H|String], L, E); + false -> + AbstrList = {cons,L,abstract(H, L, E),abstract(T, L, E)}, + not_string(String, AbstrList, L, E) + end; +abstract_list([], String, L, _E) -> {string, L, lists:reverse(String)}; -abstract_unicode_string(T, String, L, E) -> +abstract_list(T, String, L, E) -> not_string(String, abstract(T, L, E), L, E). not_string([C|T], Result, L, E) -> @@ -979,9 +983,9 @@ not_string([C|T], Result, L, E) -> not_string([], Result, _L, _E) -> Result. -abstract_list([H|T], L, E) -> - [abstract(H, L, E)|abstract_list(T, L, E)]; -abstract_list([], _L, _E) -> +abstract_tuple_list([H|T], L, E) -> + [abstract(H, L, E)|abstract_tuple_list(T, L, E)]; +abstract_tuple_list([], _L, _E) -> []. abstract_byte(Byte, L) when is_integer(Byte) -> diff --git a/lib/stdlib/src/gen_event.erl b/lib/stdlib/src/gen_event.erl index 7629e88fbf..d39dd89d3a 100644 --- a/lib/stdlib/src/gen_event.erl +++ b/lib/stdlib/src/gen_event.erl @@ -40,6 +40,8 @@ system_continue/3, system_terminate/4, system_code_change/4, + system_get_state/1, + system_replace_state/2, format_status/2]). -export_type([handler/0, handler_args/0, add_handler_ret/0, @@ -229,24 +231,6 @@ wake_hib(Parent, ServerName, MSL, Debug) -> fetch_msg(Parent, ServerName, MSL, Debug, Hib) -> receive - {system, From, get_state} -> - States = [{Mod,Id,State} || #handler{module=Mod, id=Id, state=State} <- MSL], - sys:handle_system_msg(get_state, From, Parent, ?MODULE, Debug, - {States, [ServerName, MSL, Hib]}, Hib); - {system, From, {replace_state, StateFun}} -> - {NMSL, NStates} = - lists:unzip([begin - Cur = {Mod,Id,State}, - try - NState = {Mod,Id,NS} = StateFun(Cur), - {HS#handler{state=NS}, NState} - catch - _:_ -> - {HS, Cur} - end - end || #handler{module=Mod, id=Id, state=State}=HS <- MSL]), - sys:handle_system_msg(replace_state, From, Parent, ?MODULE, Debug, - {NStates, [ServerName, NMSL, Hib]}, Hib); {system, From, Req} -> sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, [ServerName, MSL, Hib],Hib); @@ -383,6 +367,23 @@ system_code_change([ServerName, MSL, Hib], Module, OldVsn, Extra) -> MSL), {ok, [ServerName, MSL1, Hib]}. +system_get_state([_ServerName, MSL, _Hib]) -> + {ok, [{Mod,Id,State} || #handler{module=Mod, id=Id, state=State} <- MSL]}. + +system_replace_state(StateFun, [ServerName, MSL, Hib]) -> + {NMSL, NStates} = + lists:unzip([begin + Cur = {Mod,Id,State}, + try + NState = {Mod,Id,NS} = StateFun(Cur), + {HS#handler{state=NS}, NState} + catch + _:_ -> + {HS, Cur} + end + end || #handler{module=Mod, id=Id, state=State}=HS <- MSL]), + {ok, NStates, [ServerName, NMSL, Hib]}. + %%----------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees %% them, not as the real erlang messages. Use trace for that. diff --git a/lib/stdlib/src/gen_fsm.erl b/lib/stdlib/src/gen_fsm.erl index e9654322f1..e914f7d0b2 100644 --- a/lib/stdlib/src/gen_fsm.erl +++ b/lib/stdlib/src/gen_fsm.erl @@ -118,6 +118,8 @@ system_continue/3, system_terminate/4, system_code_change/4, + system_get_state/1, + system_replace_state/2, format_status/2]). -import(error_logger, [format/2]). @@ -422,17 +424,6 @@ wake_hib(Parent, Name, StateName, StateData, Mod, Debug) -> decode_msg(Msg,Parent, Name, StateName, StateData, Mod, Time, Debug, Hib) -> case Msg of - {system, From, get_state} -> - Misc = [Name, StateName, StateData, Mod, Time], - sys:handle_system_msg(get_state, From, Parent, ?MODULE, Debug, - {{StateName, StateData}, Misc}, Hib); - {system, From, {replace_state, StateFun}} -> - State = {StateName, StateData}, - NState = {NStateName, NStateData} = try StateFun(State) - catch _:_ -> State end, - NMisc = [Name, NStateName, NStateData, Mod, Time], - sys:handle_system_msg(replace_state, From, Parent, ?MODULE, Debug, - {NState, NMisc}, Hib); {system, From, Req} -> sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, [Name, StateName, StateData, Mod, Time], Hib); @@ -467,6 +458,13 @@ system_code_change([Name, StateName, StateData, Mod, Time], Else -> Else end. +system_get_state([_Name, StateName, StateData, _Mod, _Time]) -> + {ok, {StateName, StateData}}. + +system_replace_state(StateFun, [Name, StateName, StateData, Mod, Time]) -> + Result = {NStateName, NStateData} = StateFun({StateName, StateData}), + {ok, Result, [Name, NStateName, NStateData, Mod, Time]}. + %%----------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees %% them, not as the real erlang messages. Use trace for that. diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index 5f14e48b0a..202a931fae 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -98,6 +98,8 @@ -export([system_continue/3, system_terminate/4, system_code_change/4, + system_get_state/1, + system_replace_state/2, format_status/2]). %% Internal exports @@ -372,13 +374,6 @@ wake_hib(Parent, Name, State, Mod, Debug) -> decode_msg(Msg, Parent, Name, State, Mod, Time, Debug, Hib) -> case Msg of - {system, From, get_state} -> - sys:handle_system_msg(get_state, From, Parent, ?MODULE, Debug, - {State, [Name, State, Mod, Time]}, Hib); - {system, From, {replace_state, StateFun}} -> - NState = try StateFun(State) catch _:_ -> State end, - sys:handle_system_msg(replace_state, From, Parent, ?MODULE, Debug, - {NState, [Name, NState, Mod, Time]}, Hib); {system, From, Req} -> sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, [Name, State, Mod, Time], Hib); @@ -687,6 +682,13 @@ system_code_change([Name, State, Mod, Time], _Module, OldVsn, Extra) -> Else -> Else end. +system_get_state([_Name, State, _Mod, _Time]) -> + {ok, State}. + +system_replace_state(StateFun, [Name, State, Mod, Time]) -> + NState = StateFun(State), + {ok, NState, [Name, NState, Mod, Time]}. + %%----------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees %% them, not as the real erlang messages. Use trace for that. diff --git a/lib/stdlib/src/sets.erl b/lib/stdlib/src/sets.erl index be4b600f25..167a676281 100644 --- a/lib/stdlib/src/sets.erl +++ b/lib/stdlib/src/sets.erl @@ -55,9 +55,8 @@ %%------------------------------------------------------------------------------ --type seg() :: tuple(). --type segs(E) :: tuple() - | E. % dummy +-type seg() :: tuple(). +-type segs(_Element) :: tuple(). %% Define a hash set. The default values are the standard ones. -record(set, diff --git a/lib/stdlib/src/sys.erl b/lib/stdlib/src/sys.erl index 04f8dfb61b..e25cc25f57 100644 --- a/lib/stdlib/src/sys.erl +++ b/lib/stdlib/src/sys.erl @@ -102,20 +102,31 @@ get_status(Name, Timeout) -> send_system_msg(Name, get_status, Timeout). -spec get_state(Name) -> State when Name :: name(), State :: term(). -get_state(Name) -> send_system_msg(Name, get_state). +get_state(Name) -> + case send_system_msg(Name, get_state) of + {error, Reason} -> error(Reason); + State -> State + end. -spec get_state(Name, Timeout) -> State when Name :: name(), Timeout :: timeout(), State :: term(). -get_state(Name, Timeout) -> send_system_msg(Name, get_state, Timeout). +get_state(Name, Timeout) -> + case send_system_msg(Name, get_state, Timeout) of + {error, Reason} -> error(Reason); + State -> State + end. -spec replace_state(Name, StateFun) -> NewState when Name :: name(), StateFun :: fun((State :: term()) -> NewState :: term()), NewState :: term(). replace_state(Name, StateFun) -> - send_system_msg(Name, {replace_state, StateFun}). + case send_system_msg(Name, {replace_state, StateFun}) of + {error, Reason} -> error(Reason); + State -> State + end. -spec replace_state(Name, StateFun, Timeout) -> NewState when Name :: name(), @@ -123,7 +134,10 @@ replace_state(Name, StateFun) -> Timeout :: timeout(), NewState :: term(). replace_state(Name, StateFun, Timeout) -> - send_system_msg(Name, {replace_state, StateFun}, Timeout). + case send_system_msg(Name, {replace_state, StateFun}, Timeout) of + {error, Reason} -> error(Reason); + State -> State + end. -spec change_code(Name, Module, OldVsn, Extra) -> 'ok' | {error, Reason} when Name :: name(), @@ -390,10 +404,11 @@ do_cmd(_, suspend, _Parent, _Mod, Debug, Misc) -> {suspended, ok, Debug, Misc}; do_cmd(_, resume, _Parent, _Mod, Debug, Misc) -> {running, ok, Debug, Misc}; -do_cmd(SysState, get_state, _Parent, _Mod, Debug, {State, Misc}) -> - {SysState, State, Debug, Misc}; -do_cmd(SysState, replace_state, _Parent, _Mod, Debug, {State, Misc}) -> - {SysState, State, Debug, Misc}; +do_cmd(SysState, get_state, _Parent, Mod, Debug, Misc) -> + {SysState, do_get_state(Mod, Misc), Debug, Misc}; +do_cmd(SysState, {replace_state, StateFun}, _Parent, Mod, Debug, Misc) -> + {Res, NMisc} = do_replace_state(StateFun, Mod, Misc), + {SysState, Res, Debug, NMisc}; do_cmd(SysState, get_status, Parent, Mod, Debug, Misc) -> Res = get_status(SysState, Parent, Mod, Debug, Misc), {SysState, Res, Debug, Misc}; @@ -407,6 +422,40 @@ do_cmd(suspended, {change_code, Module, Vsn, Extra}, _Parent, do_cmd(SysState, Other, _Parent, _Mod, Debug, Misc) -> {SysState, {error, {unknown_system_msg, Other}}, Debug, Misc}. +do_get_state(Mod, Misc) -> + case erlang:function_exported(Mod, system_get_state, 1) of + true -> + try + {ok, State} = Mod:system_get_state(Misc), + State + catch + Cl:Exc -> + {error, {callback_failed,{Mod,system_get_state},{Cl,Exc}}} + end; + false -> + Misc + end. + +do_replace_state(StateFun, Mod, Misc) -> + case erlang:function_exported(Mod, system_replace_state, 2) of + true -> + try + {ok, State, NMisc} = Mod:system_replace_state(StateFun, Misc), + {State, NMisc} + catch + Cl:Exc -> + {{error, {callback_failed,{Mod,system_replace_state},{Cl,Exc}}}, Misc} + end; + false -> + try + NMisc = StateFun(Misc), + {NMisc, NMisc} + catch + Cl:Exc -> + {{error, {callback_failed,StateFun,{Cl,Exc}}}, Misc} + end + end. + get_status(SysState, Parent, Mod, Debug, Misc) -> PDict = get(), FmtMisc = diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile index af82f22b21..39f6ce423a 100644 --- a/lib/stdlib/test/Makefile +++ b/lib/stdlib/test/Makefile @@ -73,6 +73,8 @@ MODULES= \ supervisor_SUITE \ supervisor_bridge_SUITE \ sys_SUITE \ + sys_sp1 \ + sys_sp2 \ tar_SUITE \ timer_SUITE \ timer_simple_SUITE \ diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index fbb3c2886a..bb14de333d 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -92,7 +92,7 @@ all() -> bif_clash, behaviour_basic, behaviour_multiple, otp_7550, otp_8051, format_warn, {group, on_load}, too_many_arguments, basic_errors, bin_syntax_errors, predef, - maps,maps_type]. + maps, maps_type]. groups() -> [{unused_vars_warn, [], @@ -3435,11 +3435,10 @@ maps_type(Config) when is_list(Config) -> t(M) -> M. ">>, [], - {errors,[{3,erl_lint,{redefine_type,{map,0}}}],[]}}], + {warnings,[{3,erl_lint,{new_var_arity_type,map}}]}}], [] = run(Config, Ts), ok. - run(Config, Tests) -> F = fun({N,P,Ws,E}, BadL) -> case catch run_test(Config, P, Ws) of diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index 447e159cd4..35067e8116 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,7 +21,7 @@ init_per_group/2,end_per_group/2]). -export([ error_1/1, error_2/1, iso88591/1, otp_7810/1, otp_10302/1, - otp_10990/1, otp_10992/1]). + otp_10990/1, otp_10992/1, otp_11807/1]). -import(lists, [nth/2,flatten/1]). -import(io_lib, [print/1]). @@ -60,7 +60,8 @@ end_per_testcase(_Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992]. + [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992, + otp_11807]. groups() -> [{error, [], [error_1, error_2]}]. @@ -1144,6 +1145,25 @@ otp_10992(Config) when is_list(Config) -> erl_parse:abstract([$A,42.0], [{encoding,utf8}]), ok. +otp_11807(doc) -> + "OTP-11807. Generalize erl_parse:abstract/2."; +otp_11807(suite) -> + []; +otp_11807(Config) when is_list(Config) -> + {cons,0,{integer,0,97},{cons,0,{integer,0,98},{nil,0}}} = + erl_parse:abstract("ab", [{encoding,none}]), + {cons,0,{integer,0,-1},{nil,0}} = + erl_parse:abstract([-1], [{encoding,latin1}]), + ASCII = fun(I) -> I >= 0 andalso I < 128 end, + {string,0,"xyz"} = erl_parse:abstract("xyz", [{encoding,ASCII}]), + {cons,0,{integer,0,228},{nil,0}} = + erl_parse:abstract([228], [{encoding,ASCII}]), + {cons,0,{integer,0,97},{atom,0,a}} = + erl_parse:abstract("a"++a, [{encoding,latin1}]), + {'EXIT', {{badarg,bad},_}} = % minor backward incompatibility + (catch erl_parse:abstract("string", [{encoding,bad}])), + ok. + test_string(String, Expected) -> {ok, Expected, _End} = erl_scan:string(String), test(String). diff --git a/lib/stdlib/test/gen_event_SUITE.erl b/lib/stdlib/test/gen_event_SUITE.erl index 5819ef3890..60a1ba8c60 100644 --- a/lib/stdlib/test/gen_event_SUITE.erl +++ b/lib/stdlib/test/gen_event_SUITE.erl @@ -974,6 +974,10 @@ get_state(Config) when is_list(Config) -> [{dummy1_h,false,State1},{dummy1_h,id,State2}] = lists:sort(Result1), Result2 = sys:get_state(Pid, 5000), [{dummy1_h,false,State1},{dummy1_h,id,State2}] = lists:sort(Result2), + ok = sys:suspend(Pid), + Result3 = sys:get_state(Pid), + [{dummy1_h,false,State1},{dummy1_h,id,State2}] = lists:sort(Result3), + ok = sys:resume(Pid), ok = gen_event:stop(Pid), ok. @@ -998,4 +1002,11 @@ replace_state(Config) when is_list(Config) -> Replace3 = fun(_) -> exit(fail) end, [{dummy1_h,false,NState2}] = sys:replace_state(Pid, Replace3), [{dummy1_h,false,NState2}] = sys:get_state(Pid), + %% verify state replaced if process sys suspended + NState3 = "replaced again and again", + Replace4 = fun({dummy1_h,false,_}=S) -> setelement(3,S,NState3) end, + ok = sys:suspend(Pid), + [{dummy1_h,false,NState3}] = sys:replace_state(Pid, Replace4), + ok = sys:resume(Pid), + [{dummy1_h,false,NState3}] = sys:get_state(Pid), ok. diff --git a/lib/stdlib/test/gen_fsm_SUITE.erl b/lib/stdlib/test/gen_fsm_SUITE.erl index fd15838b7d..8aeec07ae8 100644 --- a/lib/stdlib/test/gen_fsm_SUITE.erl +++ b/lib/stdlib/test/gen_fsm_SUITE.erl @@ -426,6 +426,14 @@ get_state(Config) when is_list(Config) -> {idle, State} = sys:get_state(gfsm), {idle, State} = sys:get_state(gfsm, 5000), stop_it(Pid2), + + %% check that get_state works when pid is sys suspended + {ok, Pid3} = gen_fsm:start(gen_fsm_SUITE, {state_data, State}, []), + {idle, State} = sys:get_state(Pid3), + ok = sys:suspend(Pid3), + {idle, State} = sys:get_state(Pid3, 5000), + ok = sys:resume(Pid3), + stop_it(Pid3), ok. replace_state(Config) when is_list(Config) -> @@ -442,8 +450,18 @@ replace_state(Config) when is_list(Config) -> {state0, NState2} = sys:get_state(Pid), %% verify no change in state if replace function crashes Replace3 = fun(_) -> error(fail) end, - {state0, NState2} = sys:replace_state(Pid, Replace3), + {'EXIT',{{callback_failed, + {gen_fsm,system_replace_state},{error,fail}},_}} = + (catch sys:replace_state(Pid, Replace3)), {state0, NState2} = sys:get_state(Pid), + %% verify state replaced if process sys suspended + ok = sys:suspend(Pid), + Suffix2 = " and again", + NState3 = NState2 ++ Suffix2, + Replace4 = fun({StateName, _}) -> {StateName, NState3} end, + {state0, NState3} = sys:replace_state(Pid, Replace4), + ok = sys:resume(Pid), + {state0, NState3} = sys:get_state(Pid, 5000), stop_it(Pid), ok. diff --git a/lib/stdlib/test/gen_server_SUITE.erl b/lib/stdlib/test/gen_server_SUITE.erl index a360a0809b..960e7f60e7 100644 --- a/lib/stdlib/test/gen_server_SUITE.erl +++ b/lib/stdlib/test/gen_server_SUITE.erl @@ -1049,6 +1049,9 @@ get_state(Config) when is_list(Config) -> {ok, Pid} = gen_server:start_link(?MODULE, {state,State}, []), State = sys:get_state(Pid), State = sys:get_state(Pid, 5000), + ok = sys:suspend(Pid), + State = sys:get_state(Pid), + ok = sys:resume(Pid), ok. %% Verify that sys:replace_state correctly replaces gen_server state @@ -1075,8 +1078,18 @@ replace_state(Config) when is_list(Config) -> NState2 = sys:get_state(Pid, 5000), %% verify no change in state if replace function crashes Replace3 = fun(_) -> throw(fail) end, - NState2 = sys:replace_state(Pid, Replace3), + {'EXIT',{{callback_failed, + {gen_server,system_replace_state},{throw,fail}},_}} = + (catch sys:replace_state(Pid, Replace3)), NState2 = sys:get_state(Pid, 5000), + %% verify state replaced if process sys suspended + ok = sys:suspend(Pid), + Suffix2 = " and again", + NState3 = NState2 ++ Suffix2, + Replace4 = fun(S) -> S ++ Suffix2 end, + NState3 = sys:replace_state(Pid, Replace4), + ok = sys:resume(Pid), + NState3 = sys:get_state(Pid, 5000), ok. %% Test that the time for a huge message queue is not diff --git a/lib/stdlib/test/sys_SUITE.erl b/lib/stdlib/test/sys_SUITE.erl index c06ba545e7..f38bc87ae5 100644 --- a/lib/stdlib/test/sys_SUITE.erl +++ b/lib/stdlib/test/sys_SUITE.erl @@ -19,7 +19,7 @@ -module(sys_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2,log/1,log_to_file/1, - stats/1,trace/1,suspend/1,install/1]). + stats/1,trace/1,suspend/1,install/1,special_process/1]). -export([handle_call/3,terminate/2,init/1]). -include_lib("test_server/include/test_server.hrl"). @@ -27,14 +27,12 @@ %% Doesn't look into change_code at all -%% Doesn't address writing your own process that understands -%% system messages at all. suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [log, log_to_file, stats, trace, suspend, install]. + [log, log_to_file, stats, trace, suspend, install, special_process]. groups() -> []. @@ -157,6 +155,84 @@ install(Config) when is_list(Config) -> ?line stop(), ok. +special_process(suite) -> []; +special_process(Config) when is_list(Config) -> + ok = spec_proc(sys_sp1), + ok = spec_proc(sys_sp2). + +spec_proc(Mod) -> + {ok,_} = Mod:start_link(100), + ok = sys:statistics(Mod,true), + ok = sys:trace(Mod,true), + 1 = Ch = Mod:alloc(), + Free = lists:seq(2,100), + Replace = case sys:get_state(Mod) of + {[Ch],Free} -> + fun({A,F}) -> + Free = F, + {A,[2,3,4]} + end; + {state,[Ch],Free} -> + fun({state,A,F}) -> + Free = F, + {state,A,[2,3,4]} + end + end, + case sys:replace_state(Mod, Replace) of + {[Ch],[2,3,4]} -> ok; + {state,[Ch],[2,3,4]} -> ok + end, + ok = Mod:free(Ch), + case sys:get_state(Mod) of + {[],[1,2,3,4]} -> ok; + {state,[],[1,2,3,4]} -> ok + end, + {ok,[{start_time,_}, + {current_time,_}, + {reductions,_}, + {messages_in,2}, + {messages_out,1}]} = sys:statistics(Mod,get), + ok = sys:statistics(Mod,false), + [] = sys:replace_state(Mod, fun(_) -> [] end), + process_flag(trap_exit,true), + ok = case catch sys:get_state(Mod) of + [] -> + ok; + {'EXIT',{{callback_failed, + {Mod,system_get_state},{throw,fail}},_}} -> + ok + end, + Mod:stop(), + WaitForUnregister = fun W() -> + case whereis(Mod) of + undefined -> ok; + _ -> timer:sleep(10), W() + end + end, + WaitForUnregister(), + {ok,_} = Mod:start_link(4), + ok = case catch sys:replace_state(Mod, fun(_) -> {} end) of + {} -> + ok; + {'EXIT',{{callback_failed, + {Mod,system_replace_state},{throw,fail}},_}} -> + ok + end, + Mod:stop(), + WaitForUnregister(), + {ok,_} = Mod:start_link(4), + StateFun = fun(_) -> error(fail) end, + ok = case catch sys:replace_state(Mod, StateFun) of + {} -> + ok; + {'EXIT',{{callback_failed, + {Mod,system_replace_state},{error,fail}},_}} -> + ok; + {'EXIT',{{callback_failed,StateFun,{error,fail}},_}} -> + ok + end, + Mod:stop(). + %%%%%%%%%%%%%%%%%%%% %% Dummy server diff --git a/lib/stdlib/test/sys_sp1.erl b/lib/stdlib/test/sys_sp1.erl new file mode 100644 index 0000000000..e84ffcfa12 --- /dev/null +++ b/lib/stdlib/test/sys_sp1.erl @@ -0,0 +1,114 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(sys_sp1). +-export([start_link/1, stop/0]). +-export([alloc/0, free/1]). +-export([init/1]). +-export([system_continue/3, system_terminate/4, + write_debug/3, + system_get_state/1, system_replace_state/2]). + +%% Implements the ch4 example from the Design Principles doc. Same as +%% sys_sp2 except this module exports system_get_state/1 and +%% system_replace_state/2 + +start_link(NumCh) -> + proc_lib:start_link(?MODULE, init, [[self(),NumCh]]). + +stop() -> + ?MODULE ! stop, + ok. + +alloc() -> + ?MODULE ! {self(), alloc}, + receive + {?MODULE, Res} -> + Res + end. + +free(Ch) -> + ?MODULE ! {free, Ch}, + ok. + +init([Parent,NumCh]) -> + register(?MODULE, self()), + Chs = channels(NumCh), + Deb = sys:debug_options([]), + proc_lib:init_ack(Parent, {ok, self()}), + loop(Chs, Parent, Deb). + +loop(Chs, Parent, Deb) -> + receive + {From, alloc} -> + Deb2 = sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, alloc, From}), + {Ch, Chs2} = alloc(Chs), + From ! {?MODULE, Ch}, + Deb3 = sys:handle_debug(Deb2, fun write_debug/3, + ?MODULE, {out, {?MODULE, Ch}, From}), + loop(Chs2, Parent, Deb3); + {free, Ch} -> + Deb2 = sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, {free, Ch}}), + Chs2 = free(Ch, Chs), + loop(Chs2, Parent, Deb2); + {system, From, Request} -> + sys:handle_system_msg(Request, From, Parent, + ?MODULE, Deb, Chs); + stop -> + sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, stop}), + ok + end. + +system_continue(Parent, Deb, Chs) -> + loop(Chs, Parent, Deb). + +system_terminate(Reason, _Parent, _Deb, _Chs) -> + exit(Reason). + +system_get_state([]) -> + throw(fail); +system_get_state(Chs) -> + {ok, Chs}. + +system_replace_state(_StateFun, {}) -> + throw(fail); +system_replace_state(StateFun, Chs) -> + NChs = StateFun(Chs), + {ok, NChs, NChs}. + +write_debug(Dev, Event, Name) -> + io:format(Dev, "~p event = ~p~n", [Name, Event]). + +channels(NumCh) -> + {_Allocated=[], _Free=lists:seq(1,NumCh)}. + +alloc({_, []}) -> + {error, "no channels available"}; +alloc({Allocated, [H|T]}) -> + {H, {[H|Allocated], T}}. + +free(Ch, {Alloc, Free}=Channels) -> + case lists:member(Ch, Alloc) of + true -> + {lists:delete(Ch, Alloc), [Ch|Free]}; + false -> + Channels + end. diff --git a/lib/stdlib/test/sys_sp2.erl b/lib/stdlib/test/sys_sp2.erl new file mode 100644 index 0000000000..56a5e4d071 --- /dev/null +++ b/lib/stdlib/test/sys_sp2.erl @@ -0,0 +1,107 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(sys_sp2). +-export([start_link/1, stop/0]). +-export([alloc/0, free/1]). +-export([init/1]). +-export([system_continue/3, system_terminate/4, + write_debug/3]). + +%% Implements the ch4 example from the Design Principles doc. Same as +%% sys_sp1 except this module does not export system_get_state/1 or +%% system_replace_state/2 + +start_link(NumCh) -> + proc_lib:start_link(?MODULE, init, [[self(),NumCh]]). + +stop() -> + ?MODULE ! stop, + ok. + +alloc() -> + ?MODULE ! {self(), alloc}, + receive + {?MODULE, Res} -> + Res + end. + +free(Ch) -> + ?MODULE ! {free, Ch}, + ok. + +%% can't use 2-tuple for state here as we do in sys_sp1, since the 2-tuple +%% is not compatible with the backward compatibility handling for +%% sys:get_state in sys.erl +-record(state, {alloc,free}). + +init([Parent,NumCh]) -> + register(?MODULE, self()), + Chs = channels(NumCh), + Deb = sys:debug_options([]), + proc_lib:init_ack(Parent, {ok, self()}), + loop(Chs, Parent, Deb). + +loop(Chs, Parent, Deb) -> + receive + {From, alloc} -> + Deb2 = sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, alloc, From}), + {Ch, Chs2} = alloc(Chs), + From ! {?MODULE, Ch}, + Deb3 = sys:handle_debug(Deb2, fun write_debug/3, + ?MODULE, {out, {?MODULE, Ch}, From}), + loop(Chs2, Parent, Deb3); + {free, Ch} -> + Deb2 = sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, {free, Ch}}), + Chs2 = free(Ch, Chs), + loop(Chs2, Parent, Deb2); + {system, From, Request} -> + sys:handle_system_msg(Request, From, Parent, + ?MODULE, Deb, Chs); + stop -> + sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, stop}), + ok + end. + +system_continue(Parent, Deb, Chs) -> + loop(Chs, Parent, Deb). + +system_terminate(Reason, _Parent, _Deb, _Chs) -> + exit(Reason). + +write_debug(Dev, Event, Name) -> + io:format(Dev, "~p event = ~p~n", [Name, Event]). + +channels(NumCh) -> + #state{alloc=[], free=lists:seq(1,NumCh)}. + +alloc(#state{free=[]}=Channels) -> + {{error, "no channels available"}, Channels}; +alloc(#state{alloc=Allocated, free=[H|T]}) -> + {H, #state{alloc=[H|Allocated], free=T}}. + +free(Ch, #state{alloc=Alloc, free=Free}=Channels) -> + case lists:member(Ch, Alloc) of + true -> + #state{alloc=lists:delete(Ch, Alloc), free=[Ch|Free]}; + false -> + Channels + end. @@ -1283,8 +1283,8 @@ do_patch_app () fi # Generate .rel, .script and .boot - to tmp dir - start_clean="{release, {\"OTP APN 181 01\",\"$otp_major_vsn\"}, {erts, \"$erts_vsn\"},\n [{kernel,\"$kernel_vsn\"},\n {stdlib,\"$stdlib_vsn\"}]}.\n" - start_sasl="{release, {\"OTP APN 181 01\",\"$otp_major_vsn\"}, {erts, \"$erts_vsn\"},\n [{kernel,\"$kernel_vsn\"},\n {stdlib,\"$stdlib_vsn\"},\n {sasl,\"$sasl_vsn\"}]}.\n" + start_clean="{release, {\"Erlang/OTP\",\"$otp_major_vsn\"}, {erts, \"$erts_vsn\"},\n [{kernel,\"$kernel_vsn\"},\n {stdlib,\"$stdlib_vsn\"}]}.\n" + start_sasl="{release, {\"Erlang/OTP\",\"$otp_major_vsn\"}, {erts, \"$erts_vsn\"},\n [{kernel,\"$kernel_vsn\"},\n {stdlib,\"$stdlib_vsn\"},\n {sasl,\"$sasl_vsn\"}]}.\n" tmp_dir="$target_dir/tmp"; if [ ! -d "$tmp_dir" ]; then diff --git a/system/doc/design_principles/spec_proc.xml b/system/doc/design_principles/spec_proc.xml index 8de7a5fe03..e4fb5fdca7 100644 --- a/system/doc/design_principles/spec_proc.xml +++ b/system/doc/design_principles/spec_proc.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -130,7 +130,8 @@ ok -export([alloc/0, free/1]). -export([init/1]). -export([system_continue/3, system_terminate/4, - write_debug/3]). + write_debug/3, + system_get_state/1, system_replace_state/2]). start_link() -> proc_lib:start_link(ch4, init, [self()]). @@ -156,15 +157,15 @@ init(Parent) -> loop(Chs, Parent, Deb) -> receive {From, alloc} -> - Deb2 = sys:handle_debug(Deb, {ch4, write_debug}, + Deb2 = sys:handle_debug(Deb, fun ch4:write_debug/3, ch4, {in, alloc, From}), {Ch, Chs2} = alloc(Chs), From ! {ch4, Ch}, - Deb3 = sys:handle_debug(Deb2, {ch4, write_debug}, + Deb3 = sys:handle_debug(Deb2, fun ch4:write_debug/3, ch4, {out, {ch4, Ch}, From}), loop(Chs2, Parent, Deb3); {free, Ch} -> - Deb2 = sys:handle_debug(Deb, {ch4, write_debug}, + Deb2 = sys:handle_debug(Deb, fun ch4:write_debug/3, ch4, {in, {free, Ch}}), Chs2 = free(Ch, Chs), loop(Chs2, Parent, Deb2); @@ -177,9 +178,16 @@ loop(Chs, Parent, Deb) -> system_continue(Parent, Deb, Chs) -> loop(Chs, Parent, Deb). -system_terminate(Reason, Parent, Deb, Chs) -> +system_terminate(Reason, _Parent, _Deb, _Chs) -> exit(Reason). +system_get_state(Chs) -> + {ok, Chs}. + +system_replace_state(StateFun, Chs) -> + NChs = StateFun(Chs), + {ok, NChs, NChs}. + write_debug(Dev, Event, Name) -> io:format(Dev, "~p event = ~p~n", [Name, Event]).</pre> <p>Example on how the simple debugging functions in <c>sys</c> can @@ -281,10 +289,10 @@ sys:handle_debug(Deb, Func, Info, Event) => Deb1</code> <p><c>Deb</c> is the debug structure.</p> </item> <item> - <p><c>Func</c> is a tuple <c>{Module, Name}</c> (or a fun) and - should specify a (user defined) function used to format + <p><c>Func</c> is a fun specifying + a (user defined) function used to format trace output. For each system event, the format function is - called as <c>Module:Name(Dev, Event, Info)</c>, where:</p> + called as <c>Func(Dev, Event, Info)</c>, where:</p> <list type="bulleted"> <item> <p><c>Dev</c> is the IO device to which the output should @@ -319,15 +327,15 @@ sys:handle_debug(Deb, Func, Info, Event) => Deb1</code> loop(Chs, Parent, Deb) -> receive {From, alloc} -> - Deb2 = sys:handle_debug(Deb, {ch4, write_debug}, + Deb2 = sys:handle_debug(Deb, fun ch4:write_debug/3, ch4, {in, alloc, From}), {Ch, Chs2} = alloc(Chs), From ! {ch4, Ch}, - Deb3 = sys:handle_debug(Deb2, {ch4, write_debug}, + Deb3 = sys:handle_debug(Deb2, fun ch4:write_debug/3, ch4, {out, {ch4, Ch}, From}), loop(Chs2, Parent, Deb3); {free, Ch} -> - Deb2 = sys:handle_debug(Deb, {ch4, write_debug}, + Deb2 = sys:handle_debug(Deb, fun ch4:write_debug/3, ch4, {in, {free, Ch}}), Chs2 = free(Ch, Chs), loop(Chs2, Parent, Deb2); @@ -366,8 +374,15 @@ Module:system_terminate(Reason, Parent, Deb, State)</code> <item><c>Module</c> is the name of the module.</item> <item><c>Deb</c> is the debug structure.</item> <item><c>State</c> is a term describing the internal state and - is passed to <c>system_continue</c>/<c>system_terminate</c>.</item> + is passed to <c>system_continue</c>/<c>system_terminate</c>/ + <c>system_get_state</c>/<c>system_replace_state</c>.</item> </list> + <p>If the process should return its state <c>handle_system_msg</c> will call:</p> + <code type="none"> +Module:system_get_state(State)</code> + <p>or if the process should replace its state using the fun <c>StateFun</c>:</p> + <code type="none"> +Module:system_replace_state(StateFun, State)</code> <p>In the example:</p> <code type="none"> loop(Chs, Parent, Deb) -> @@ -383,7 +398,15 @@ system_continue(Parent, Deb, Chs) -> loop(Chs, Parent, Deb). system_terminate(Reason, Parent, Deb, Chs) -> - exit(Reason).</code> + exit(Reason). + +system_get_state(Chs) -> + {ok, Chs, Chs}. + +system_replace_state(StateFun, Chs) -> + NChs = StateFun(Chs), + {ok, NChs, NChs}. +</code> <p>If the special process is set to trap exits, note that if the parent process terminates, the expected behavior is to terminate with the same reason:</p> @@ -408,11 +431,11 @@ loop(...) -> <section> <title>User-Defined Behaviours</title> - <p>To implement a user-defined behaviour, write code similar to + <p><marker id="behaviours"/>To implement a user-defined behaviour, write code similar to code for a special process but calling functions in a callback module for handling specific tasks.</p> <p>If it is desired that the compiler should warn for missing callback - functions, as it does for the OTP behaviours, add callback attributes in the + functions, as it does for the OTP behaviours, add <c>-callback</c> attributes in the behaviour module to describe the expected callbacks:</p> <code type="none"> -callback Name1(Arg1_1, Arg1_2, ..., Arg1_N1) -> Res1. @@ -422,15 +445,15 @@ loop(...) -> <p>where <c>NameX</c> are the names of the expected callbacks and <c>ArgX_Y</c>, <c>ResX</c> are types as they are described in Specifications for functions in <seealso marker="../reference_manual/typespec">Types and - Function Specifications</seealso>. The whole syntax of spec attributes is - supported by callback attributes.</p> + Function Specifications</seealso>. The whole syntax of <c>-spec</c> attribute is + supported by <c>-callback</c> attribute.</p> <p>Alternatively you may directly implement and export the function:</p> <code type="none"> behaviour_info(callbacks) -> - [{Name1,Arity1},...,{NameN,ArityN}].</code> - <p>where each <c>{Name,Arity}</c> specifies the name and arity of a callback + [{Name1, Arity1},...,{NameN, ArityN}].</code> + <p>where each <c>{Name, Arity}</c> specifies the name and arity of a callback function. This function is otherwise automatically generated by the compiler - using the callback attributes.</p> + using the <c>-callback</c> attributes.</p> <p>When the compiler encounters the module attribute <c>-behaviour(Behaviour).</c> in a module <c>Mod</c>, it will call <c>Behaviour:behaviour_info(callbacks)</c> and compare the result with the diff --git a/system/doc/reference_manual/modules.xml b/system/doc/reference_manual/modules.xml index 9e5f4de385..cd4c3a1b1b 100644 --- a/system/doc/reference_manual/modules.xml +++ b/system/doc/reference_manual/modules.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2013</year> + <year>2003</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -134,8 +134,21 @@ fact(0) -> % | standard behaviours <c>gen_server</c>, <c>gen_fsm</c>, <c>gen_event</c> or <c>supervisor</c>.</p> <p>The spelling <c>behavior</c> is also accepted.</p> - <p>Read more about behaviours and callback modules in OTP Design - Principles.</p> + <p>The callback functions of the module can be specified either + directly by the exported function <c>behaviour_info/1</c>:</p> + <pre> +behaviour_info(callbacks) -> Callbacks.</pre> + <p>or by a <c>-callback</c> attribute for each callback + function:</p> + <pre> +-callback Name(Arguments) -> Result.</pre> + <p>where <c>Arguments</c> is a list of zero or more arguments. + The <c>-callback</c> attribute is to be preferred since the + extra type information can be used by tools to produce + documentation or find discrepancies.</p> + <p>Read more about behaviours and callback modules in + <seealso marker="doc/design_principles:spec_proc#behaviours"> + OTP Design Principles</seealso>.</p> </section> <section> diff --git a/system/doc/system_principles/versions.xml b/system/doc/system_principles/versions.xml index 6bfa9d1eed..2bf0d18010 100644 --- a/system/doc/system_principles/versions.xml +++ b/system/doc/system_principles/versions.xml @@ -79,18 +79,18 @@ </section> <section><title>OTP Versions Table</title> - <p>The text file <c><OTP source root>/otp_versions.table</c> - part of the source code contains information about all OTP versions + <p>The text file <c><OTP source root>/otp_versions.table</c> that + is part of the source code contains information about all OTP versions from OTP 17.0 up to current OTP version. Each line contains information - about application versions part of a specific OTP version, and is on - the format:</p> + about application versions that are part of a specific OTP version, and + is on the format:</p> <pre> -<OtpVersion> : <ChangedAppVersions> # <UnchangedAppVersions> +<OtpVersion> : <ChangedAppVersions> # <UnchangedAppVersions> : </pre> <p><c><OtpVersion></c> is on the format <c>OTP-<VSN></c>, i.e., the same as the git tag used to identify the source. <c><ChangedAppVersions></c> and <c><UnchangedAppVersions></c> - are white space separated lists of application versions on the + are space separated lists of application versions on the format <c><application>-<vsn></c>. <c><ChangedAppVersions></c> corresponds to changed applications with new version numbers in this OTP version, and @@ -99,14 +99,15 @@ not at the same time. If <ChangedAppVersions> is empty, no changes has been made that change the build result of any application. This could for example be a pure bug fix of the build system. The order of lines - is undefined.</p> + is undefined. All white space characters in this file are either space + (character 32) or line-break (character 10).</p> <p>Using ordinary UNIX tools like <c>sed</c> and <c>grep</c> one can easily find answers to various questions like:</p> <taglist> <tag>Which OTP versions are <c>kernel-3.0</c> part of?</tag> - <item><p><c> $ grep 'kernel-3\.0' otp_versions.table</c></p></item> + <item><p><c> $ grep ' kernel-3\.0 ' otp_versions.table</c></p></item> <tag>In which OTP version was <c>kernel-3.0</c> introduced?</tag> - <item><p><c> $ sed 's/#.*//;/kernel-3\.0/!d' otp_versions.table</c></p></item> + <item><p><c> $ sed 's/#.*//;/ kernel-3\.0 /!d' otp_versions.table</c></p></item> </taglist> <p>The above commands give a bit more information than the exact answers, but adequate information when manually searching for answers to these diff --git a/xcomp/erl-xcomp-arm-android.conf b/xcomp/erl-xcomp-arm-android.conf new file mode 100644 index 0000000000..4fd1f270d1 --- /dev/null +++ b/xcomp/erl-xcomp-arm-android.conf @@ -0,0 +1,264 @@ +## -*-shell-script-*- +## +## %CopyrightBegin% +## +## Copyright Ericsson AB 2009-2010. All Rights Reserved. +## +## The contents of this file are subject to the Erlang Public License, +## Version 1.1, (the "License"); you may not use this file except in +## compliance with the License. You should have received a copy of the +## Erlang Public License along with this software. If not, it can be +## retrieved online at http://www.erlang.org/. +## +## Software distributed under the License is distributed on an "AS IS" +## basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +## the License for the specific language governing rights and limitations +## under the License. +## +## %CopyrightEnd% +## +## File: erl-xcomp.conf.template +## Author: +## +## ----------------------------------------------------------------------------- +## When cross compiling Erlang/OTP using `otp_build', copy this file and set +## the variables needed below. Then pass the path to the copy of this file as +## an argument to `otp_build' in the configure stage: +## `otp_build configure --xcomp-conf=<FILE>' +## ----------------------------------------------------------------------------- + +## Note that you cannot define arbitrary variables in a cross compilation +## configuration file. Only the ones listed below will be guaranteed to be +## visible throughout the whole execution of all `configure' scripts. Other +## variables needs to be defined as arguments to `configure' or exported in +## the environment. + +## -- Variables for `otp_build' Only ------------------------------------------- + +## Variables in this section are only used, when configuring Erlang/OTP for +## cross compilation using `$ERL_TOP/otp_build configure'. + +## *NOTE*! These variables currently have *no* effect if you configure using +## the `configure' script directly. + +# * `erl_xcomp_build' - The build system used. This value will be passed as +# `--build=$erl_xcomp_build' argument to the `configure' script. It does +# not have to be a full `CPU-VENDOR-OS' triplet, but can be. The full +# `CPU-VENDOR-OS' triplet will be created by +# `$ERL_TOP/erts/autoconf/config.sub $erl_xcomp_build'. If set to `guess', +# the build system will be guessed using +# `$ERL_TOP/erts/autoconf/config.guess'. +erl_xcomp_build=guess + +# * `erl_xcomp_host' - Cross host/target system to build for. This value will +# be passed as `--host=$erl_xcomp_host' argument to the `configure' script. +# It does not have to be a full `CPU-VENDOR-OS' triplet, but can be. The +# full `CPU-VENDOR-OS' triplet will be created by +# `$ERL_TOP/erts/autoconf/config.sub $erl_xcomp_host'. +erl_xcomp_host=arm-linux-androideabi + +# * `erl_xcomp_configure_flags' - Extra configure flags to pass to the +# `configure' script. +erl_xcomp_configure_flags="--disable-hipe --without-termcap" + + +## -- Cross Compiler and Other Tools ------------------------------------------- + +## +## +NDK_SYSROOT=$NDK_ROOT/platforms/$NDK_PLAT/arch-arm + +## If the cross compilation tools are prefixed by `<HOST>-' you probably do +## not need to set these variables (where `<HOST>' is what has been passed as +## `--host=<HOST>' argument to `configure'). + +## All variables in this section can also be used when native compiling. + +# * `CC' - C compiler. +CC="arm-linux-androideabi-gcc --sysroot=$NDK_SYSROOT" + +# * `CFLAGS' - C compiler flags. +CFLAGS="-static -march=armv7-a -msoft-float -mthumb" + +# * `STATIC_CFLAGS' - Static C compiler flags. +#STATIC_CFLAGS= + +# * `CFLAG_RUNTIME_LIBRARY_PATH' - This flag should set runtime library +# search path for the shared libraries. Note that this actually is a +# linker flag, but it needs to be passed via the compiler. +#CFLAG_RUNTIME_LIBRARY_PATH= + +# * `CPP' - C pre-processor. +CPP="arm-linux-androideabi-cpp --sysroot=$NDK_SYSROOT" + +# * `CPPFLAGS' - C pre-processor flags. +CPPFLAGS="-static -march=armv7-a -msoft-float -mthumb" + +# * `CXX' - C++ compiler. +CXX="arm-linux-androideabi-c++ --sysroot=$NDK_SYSROOT" + +# * `CXXFLAGS' - C++ compiler flags. +CXXFLAGS="-static -march=armv7-a -msoft-float -mthumb" + +# * `LD' - Linker. +#LD= + +# * `LDFLAGS' - Linker flags. +LDFLAGS="-static -march=armv7-a -msoft-float -mthumb" + +# * `LIBS' - Libraries. +#LIBS= + +## -- *D*ynamic *E*rlang *D*river Linking -- + +## *NOTE*! Either set all or none of the `DED_LD*' variables. + +# * `DED_LD' - Linker for Dynamically loaded Erlang Drivers. +#DED_LD= + +# * `DED_LDFLAGS' - Linker flags to use with `DED_LD'. +#DED_LDFLAGS= + +# * `DED_LD_FLAG_RUNTIME_LIBRARY_PATH' - This flag should set runtime library +# search path for shared libraries when linking with `DED_LD'. +#DED_LD_FLAG_RUNTIME_LIBRARY_PATH= + +## -- Large File Support -- + +## *NOTE*! Either set all or none of the `LFS_*' variables. + +# * `LFS_CFLAGS' - Large file support C compiler flags. +#LFS_CFLAGS= + +# * `LFS_LDFLAGS' - Large file support linker flags. +#LFS_LDFLAGS= + +# * `LFS_LIBS' - Large file support libraries. +#LFS_LIBS= + +## -- Other Tools -- + +# * `RANLIB' - `ranlib' archive index tool. +#RANLIB= + +# * `AR' - `ar' archiving tool. +#AR= + +# * `GETCONF' - `getconf' system configuration inspection tool. `getconf' is +# currently used for finding out large file support flags to use, and +# on Linux systems for finding out if we have an NPTL thread library or +# not. +#GETCONF= + +## -- Cross System Root Locations ---------------------------------------------- + +# * `erl_xcomp_sysroot' - The absolute path to the system root of the cross +# compilation environment. Currently, the `crypto', `odbc', `ssh' and +# `ssl' applications need the system root. These applications will be +# skipped if the system root has not been set. The system root might be +# needed for other things too. If this is the case and the system root +# has not been set, `configure' will fail and request you to set it. +erl_xcomp_sysroot="$NDK_SYSROOT" + + +# * `erl_xcomp_isysroot' - The absolute path to the system root for includes +# of the cross compilation environment. If not set, this value defaults +# to `$erl_xcomp_sysroot', i.e., only set this value if the include system +# root path is not the same as the system root path. +#erl_xcomp_isysroot= + +## -- Optional Feature, and Bug Tests ------------------------------------------ + +## These tests cannot (always) be done automatically when cross compiling. You +## usually do not need to set these variables. Only set these if you really +## know what you are doing. + +## Note that some of these values will override results of tests performed +## by `configure', and some will not be used until `configure' is sure that +## it cannot figure the result out. + +## The `configure' script will issue a warning when a default value is used. +## When a variable has been set, no warning will be issued. + +# * `erl_xcomp_after_morecore_hook' - `yes|no'. Defaults to `no'. If `yes', +# the target system must have a working `__after_morecore_hook' that can be +# used for tracking used `malloc()' implementations core memory usage. +# This is currently only used by unsupported features. +#erl_xcomp_after_morecore_hook= + +# * `erl_xcomp_bigendian' - `yes|no'. No default. If `yes', the target system +# must be big endian. If `no', little endian. This can often be +# automatically detected, but not always. If not automatically detected, +# `configure' will fail unless this variable is set. Since no default +# value is used, `configure' will try to figure this out automatically. +#erl_xcomp_bigendian= + +# * `erl_xcomp_clock_gettime_cpu_time' - `yes|no'. Defaults to `no'. If `yes', +# the target system must have a working `clock_gettime()' implementation +# that can be used for retrieving process CPU time. +#erl_xcomp_clock_gettime_cpu_time= + +# * `erl_xcomp_getaddrinfo' - `yes|no'. Defaults to `no'. If `yes', the target +# system must have a working `getaddrinfo()' implementation that can +# handle both IPv4 and IPv6. +#erl_xcomp_getaddrinfo= + +# * `erl_xcomp_gethrvtime_procfs_ioctl' - `yes|no'. Defaults to `no'. If `yes', +# the target system must have a working `gethrvtime()' implementation and +# is used with procfs `ioctl()'. +#erl_xcomp_gethrvtime_procfs_ioctl= + +# * `erl_xcomp_dlsym_brk_wrappers' - `yes|no'. Defaults to `no'. If `yes', the +# target system must have a working `dlsym(RTLD_NEXT, <S>)' implementation +# that can be used on `brk' and `sbrk' symbols used by the `malloc()' +# implementation in use, and by this track the `malloc()' implementations +# core memory usage. This is currently only used by unsupported features. +#erl_xcomp_dlsym_brk_wrappers= + +# * `erl_xcomp_kqueue' - `yes|no'. Defaults to `no'. If `yes', the target +# system must have a working `kqueue()' implementation that returns a file +# descriptor which can be used by `poll()' and/or `select()'. If `no' and +# the target system has not got `epoll()' or `/dev/poll', the kernel-poll +# feature will be disabled. +#erl_xcomp_kqueue= + +# * `erl_xcomp_linux_clock_gettime_correction' - `yes|no'. Defaults to `yes' on +# Linux; otherwise, `no'. If `yes', `clock_gettime(CLOCK_MONOTONIC, _)' on +# the target system must work. This variable is recommended to be set to +# `no' on Linux systems with kernel versions less than 2.6. +#erl_xcomp_linux_clock_gettime_correction= + +# * `erl_xcomp_linux_nptl' - `yes|no'. Defaults to `yes' on Linux; otherwise, +# `no'. If `yes', the target system must have NPTL (Native POSIX Thread +# Library). Older Linux systems have LinuxThreads instead of NPTL (Linux +# kernel versions typically less than 2.6). +#erl_xcomp_linux_nptl= + +# * `erl_xcomp_linux_usable_sigaltstack' - `yes|no'. Defaults to `yes' on Linux; +# otherwise, `no'. If `yes', `sigaltstack()' must be usable on the target +# system. `sigaltstack()' on Linux kernel versions less than 2.4 are +# broken. +#erl_xcomp_linux_usable_sigaltstack= + +# * `erl_xcomp_linux_usable_sigusrx' - `yes|no'. Defaults to `yes'. If `yes', +# the `SIGUSR1' and `SIGUSR2' signals must be usable by the ERTS. Old +# LinuxThreads thread libraries (Linux kernel versions typically less than +# 2.2) used these signals and made them unusable by the ERTS. +#erl_xcomp_linux_usable_sigusrx= + +# * `erl_xcomp_poll' - `yes|no'. Defaults to `no' on Darwin/MacOSX; otherwise, +# `yes'. If `yes', the target system must have a working `poll()' +# implementation that also can handle devices. If `no', `select()' will be +# used instead of `poll()'. +#erl_xcomp_poll= + +# * `erl_xcomp_putenv_copy' - `yes|no'. Defaults to `no'. If `yes', the target +# system must have a `putenv()' implementation that stores a copy of the +# key/value pair. +#erl_xcomp_putenv_copy= + +# * `erl_xcomp_reliable_fpe' - `yes|no'. Defaults to `no'. If `yes', the target +# system must have reliable floating point exceptions. +#erl_xcomp_reliable_fpe= + +## ----------------------------------------------------------------------------- |