diff options
307 files changed, 11709 insertions, 2114 deletions
diff --git a/HOWTO/INSTALL-ANDROID.md b/HOWTO/INSTALL-ANDROID.md new file mode 100644 index 0000000000..31698d4ce3 --- /dev/null +++ b/HOWTO/INSTALL-ANDROID.md @@ -0,0 +1,52 @@ +Cross Compiling Erlang/OTP - ANDROID +==================================== + +Introduction +------------ + +This document describes how to cross compile Erlang OTP to Android/Rasberry Pi platforms. + +### Download and Install Android NDK ### + +https://developer.android.com/tools/sdk/ndk/index.html + +### Define System Variables ### + +export NDK_ROOT=/usr/local/android +export NDK_PLAT=android-9 +export PATH=$NDK_ROOT/toolchains/arm-linux-androideabi-4.8/prebuilt/darwin-x86_64/bin:$PATH + +### Configure OTP ### + +./otp_build configure \ + --xcomp-conf=./xcomp/erl-xcomp-arm-android.conf \ + --without-ssl + +### Compile OTP ### + +make noboot [-j4] + +### Make Release ### + +./otp_build release -a /usr/local/otp_R16B03_arm + +### Target Deployment ### + +Make a tarball out of /usr/local/otp_R16B03_arm and copy it to target device +(e.g. Raspberry Pi). Extract it and install + +./Install /usr/local/otp_R16B03_arm + +Android SDK (adb tool) is used to deploy OTP/Erlang to target device for +evaluation purpose only. + +adb push /usr/local/otp_R16B03_arm /mnt/sdcard/otp_R16B03_arm +adb shell + +### Known Issues ### + + * native inet:gethostbyname/1 return {error, nxdomain} on Raspberry PI. Use dns resolver to by-pass the issue (see http://www.erlang.org/doc/apps/erts/inet_cfg.html) + +### References ### + + The port derives some solutions from https://code.google.com/p/erlang4android/ diff --git a/Makefile.in b/Makefile.in index bfaf749465..c667eb5f79 100644 --- a/Makefile.in +++ b/Makefile.in @@ -397,9 +397,9 @@ endif ERL_TOP=$(ERL_TOP) PATH=$(INST_PATH_PREFIX)"$${PATH}" \ $(MAKE) BUILD_ALL=1 TESTROOT="$(RELEASE_ROOT)" release ifeq ($(RELEASE_ROOT),) - $(INSTALL_DATA) "$(ERL_TOP)/OTP_VERSION" "$(OTP_DEFAULT_RELEASE_PATH)" + $(INSTALL_DATA) "$(ERL_TOP)/OTP_VERSION" "$(OTP_DEFAULT_RELEASE_PATH)/releases/@OTP_REL@" else - $(INSTALL_DATA) "$(ERL_TOP)/OTP_VERSION" "$(RELEASE_ROOT)" + $(INSTALL_DATA) "$(ERL_TOP)/OTP_VERSION" "$(RELEASE_ROOT)/releases/@OTP_REL@" endif # --------------------------------------------------------------- @@ -1038,9 +1038,9 @@ install.Install: install.otp_version: ifeq ($(ERLANG_LIBDIR),) - $(INSTALL_DATA) "$(ERL_TOP)/OTP_VERSION" "$(OTP_DEFAULT_RELEASE_PATH)" + $(INSTALL_DATA) "$(ERL_TOP)/OTP_VERSION" "$(OTP_DEFAULT_RELEASE_PATH)/releases/@OTP_REL@" else - $(INSTALL_DATA) "$(ERL_TOP)/OTP_VERSION" "$(ERLANG_LIBDIR)" + $(INSTALL_DATA) "$(ERL_TOP)/OTP_VERSION" "$(ERLANG_LIBDIR)/releases/@OTP_REL@" endif # diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex f5800105d3..b57eef57ab 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex f5800105d3..b57eef57ab 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/configure.in b/configure.in index f6cccb4f4b..be906dcb4e 100644 --- a/configure.in +++ b/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1998-2012. All Rights Reserved. +dnl Copyright Ericsson AB 1998-2014. All Rights Reserved. dnl dnl The contents of this file are subject to the Erlang Public License, dnl Version 1.1, (the "License"); you may not use this file except in @@ -209,7 +209,7 @@ AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support])) AC_ARG_ENABLE(halfword-emulator, AS_HELP_STRING([--enable-halfword-emulator], - [enable halfword emulator (only for 64bit builds)])) + [enable halfword emulator (only for 64bit builds). Note: Halfword emulator is marked as deprecated and scheduled for removal in future major release.])) AC_ARG_ENABLE(smp-support, AS_HELP_STRING([--enable-smp-support], [enable smp support]) diff --git a/erts/Makefile.in b/erts/Makefile.in index e3db37d3fd..47298cccba 100644 --- a/erts/Makefile.in +++ b/erts/Makefile.in @@ -48,6 +48,7 @@ debug opt clean: ( cd $$d && $(MAKE) $@ FLAVOR=$(FLAVOR) ) || exit $$? ; \ fi ; \ done + (cd preloaded/src && $(MAKE) ../ebin/erts.app) # ---------------------------------------------------------------------- # These are "convenience targets", provided as shortcuts for developers @@ -135,6 +136,10 @@ release: ( cd $$d && $(MAKE) $@ ) || exit $$? ; \ fi ; \ done + ( $(MAKE) -f "$(ERL_TOP)/make/otp_released_app.mk" \ + APP_PWD="$(ERL_TOP)/erts" APP_VSN=VSN APP=erts \ + TESTROOT="$(TESTROOT)" update) \ + || exit $$? .PHONY: release_docs release_docs: diff --git a/erts/configure.in b/erts/configure.in index 074882532f..208c294106 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. -*-m4-*- dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1997-2013. All Rights Reserved. +dnl Copyright Ericsson AB 1997-2014. All Rights Reserved. dnl dnl The contents of this file are subject to the Erlang Public License, dnl Version 1.1, (the "License"); you may not use this file except in @@ -144,7 +144,7 @@ AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support]), AC_ARG_ENABLE(halfword-emulator, AS_HELP_STRING([--enable-halfword-emulator], - [enable halfword emulator (only for 64bit builds)]), + [enable halfword emulator (only for 64bit builds). Note: Halfword emulator is marked as deprecated and scheduled for removal in future major release.]), [ case "$enableval" in no) enable_halfword_emualtor=no ;; *) enable_halfword_emulator=yes ;; @@ -816,6 +816,16 @@ if test "$enable_halfword_emulator" = "yes"; then [Define if building a halfword-heap 64bit emulator]) ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS halfword" AC_MSG_RESULT([yes]) + + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> $ERL_TOP/erts/CONF_INFO <<EOF + + The HALFWORD emulator has been enabled. + This is a DEPRECATED feature scheduled for removal + in a future major release. + +EOF else AC_MSG_ERROR(no; halfword emulator not supported on this architecture) fi @@ -1151,7 +1161,7 @@ fi AC_SUBST(ERTS_BUILD_SMP_EMU) -AC_CHECK_FUNCS([posix_fadvise]) +AC_CHECK_FUNCS([posix_fadvise closefrom]) AC_CHECK_HEADERS([linux/falloc.h]) dnl * Old glibcs have broken fallocate64(). Make sure not to use it. AC_CACHE_CHECK([whether fallocate() works],i_cv_fallocate_works,[ diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 4aa3033f40..9724a1345a 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -536,7 +536,7 @@ <tag><marker id="file_name_encoding"></marker><c><![CDATA[+fnl]]></c></tag> <item> <p>The VM works with file names as if they are encoded using the ISO-latin-1 encoding, disallowing Unicode characters with codepoints beyond 255.</p> - <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names. Note that this value also applies to command-line parameters and environment variables (see <seealso marker="stdlib:unicode_usage#unicode_in_environment_and_parameters">STDLIB User's Guide</seealso>).</p> </item> <tag><c><![CDATA[+fnu[{w|i|e}]]]></c></tag> <item> @@ -555,7 +555,7 @@ encountered. <c>w</c> is the default. Note that <c>file:read_link/1</c> will always return an error if the link points to an invalid file name.</p> - <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names. Note that this value also applies to command-line parameters and environment variables (see <seealso marker="stdlib:unicode_usage#unicode_in_environment_and_parameters">STDLIB User's Guide</seealso>).</p> </item> <tag><c><![CDATA[+fna[{w|i|e}]]]></c></tag> <item> @@ -572,7 +572,7 @@ settings cause the behavior of <c>+fnl</c> to be selected, then <c>w</c>, <c>i</c>, or <c>e</c> will not have any effect.</p> - <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names. Note that this value also applies to command-line parameters and environment variables (see <seealso marker="stdlib:unicode_usage#unicode_in_environment_and_parameters">STDLIB User's Guide</seealso>).</p> </item> <tag><c><![CDATA[+hms Size]]></c></tag> <item> diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index 8da1836da7..ad37813ac0 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -315,10 +315,13 @@ <c>ERL_DRV_EXTENDED_MINOR_VERSION</c> will be incremented when new features are added. The runtime system uses the minor version of the driver to determine what features to use. - The runtime system will refuse to load a driver if the major + The runtime system will normally refuse to load a driver if the major versions differ, or if the major versions are equal and the minor version used by the driver is greater than the one used - by the runtime system.</p> + by the runtime system. Old drivers with lower major versions + will however be allowed after a bump of the major version during + a transition period of two major releases. Such old drivers might + however fail if deprecated features are used.</p> <p>The emulator will refuse to load a driver that does not use the extended driver interface, to allow for 64-bit capable drivers, diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 8b19725c02..6b1f4cccf8 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -316,6 +316,32 @@ ok <p>The library initialization callbacks <c>load</c>, <c>reload</c> and <c>upgrade</c> are all thread-safe even for shared state data.</p> </item> + + <tag><marker id="version_management"/>Version Management</tag> + <item><p> + When a NIF library is built, information about NIF API version + is compiled into the library. When a NIF library is loaded the + runtime system verifies that the library is of a compatible version. + <c>erl_nif.h</c> defines <c>ERL_NIF_MAJOR_VERSION</c>, and + <c>ERL_NIF_MINOR_VERSION</c>. <c>ERL_NIF_MAJOR_VERSION</c> will be + incremented when NIF library incompatible changes are made to the + Erlang runtime system. Normally it will suffice to recompile the NIF + library when the <c>ERL_NIF_MAJOR_VERSION</c> has changed, but it + could, under rare circumstances, mean that NIF libraries have to + be slightly modified. If so, this will of course be documented. + <c>ERL_NIF_MINOR_VERSION</c> will be incremented when + new features are added. The runtime system uses the minor version + to determine what features to use. + </p><p> + The runtime system will normally refuse to load a NIF library if + the major versions differ, or if the major versions are equal and + the minor version used by the NIF library is greater than the one + used by the runtime system. Old NIF libraries with lower major + versions will however be allowed after a bump of the major version + during a transition period of two major releases. Such old NIF + libraries might however fail if deprecated features are used. + </p></item> + <tag>Dirty NIFs</tag> <item><p><marker id="dirty_nifs"/><em>Note that the dirty NIF functionality is experimental</em> and that you have to enable support for dirty diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index aeded7c719..b06d5aeb12 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -6121,8 +6121,8 @@ ok <c>erlang:system_info()</c> argument giving the exact OTP version. This since the exact OTP version in the general case is hard to determine. For more information see - <seealso marker="doc/installation_guide:otp_version">the - documentation of the OTP version in the installation + <seealso marker="doc/system_principles:versions">the + documentation of versions in the system principles guide</seealso>.</p> </item> <tag><marker id="system_info_port_parallelism"><c>port_parallelism</c></marker></tag> diff --git a/erts/doc/src/time_correction.xml b/erts/doc/src/time_correction.xml index d52cc7f3e2..7f7c28fc30 100644 --- a/erts/doc/src/time_correction.xml +++ b/erts/doc/src/time_correction.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="utf8" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>1999</year><year>2013</year> + <year>1999</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 2a9a0a5c2f..58e77ed1fa 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -378,7 +378,9 @@ LIBS += -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) endif # erts_internal_r +ifneq ($(TARGET),arm-unknown-linux-androideabi) LIBS += @LIBRT@ +endif LIBS += @LIBCARBON@ diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 49a34ab4ad..4e711c89e0 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -642,7 +642,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) erts_smp_atomic_inc_nob(&bp->count->acount); } - if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { + if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && erts_is_tracer_proc_valid(c_p)) { Eterm w; erts_trace_time_call(c_p, I, bp->time); w = (BeamInstr) *c_p->cp; @@ -730,7 +730,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && - IS_TRACED_FL(p, F_TRACE_CALLS)) { + IS_TRACED_FL(p, F_TRACE_CALLS) && + erts_is_tracer_proc_valid(p)) { BeamInstr *pc = (BeamInstr *)ep->code+3; erts_trace_time_call(p, pc, bp->time); } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 0cec9ea3ec..9634faff1d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1301,7 +1301,7 @@ void process_main(void) (Eterm)fptr[1], (Uint)fptr[2], NULL, fun_buf); } else { - erts_snprintf(fun_buf, sizeof(fun_buf), + erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), "<unknown/%p>", next); } } diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 3a987e213b..50548850eb 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -48,7 +48,8 @@ copy_object(Eterm obj, Process* to) if (DTRACE_ENABLED(copy_object)) { DTRACE_CHARBUF(proc_name, 64); - erts_snprintf(proc_name, sizeof(proc_name), "%T", to->common.id); + erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), + "%T", to->common.id); DTRACE2(copy_object, proc_name, size); } #endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 6ecf3f0722..ec07ddcd9c 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -851,9 +851,12 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) #ifdef USE_VM_PROBES *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(receiver_name), "%T", remote); + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), + "%T", remote); msize = size_object(message); if (token != NIL && token != am_have_dt_utag) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); @@ -908,9 +911,11 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) #ifdef USE_VM_PROBES *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(receiver_name), + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "{%T,%s}", remote_name, node_name); msize = size_object(message); if (token != NIL && token != am_have_dt_utag) { @@ -971,11 +976,14 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, #ifdef USE_VM_PROBES *node_name = *sender_name = *remote_name = '\0'; if (DTRACE_ENABLED(process_exit_signal_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(remote_name, sizeof(remote_name), + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(remote_name, sizeof(DTRACE_CHARBUF_NAME(remote_name)), "{%T,%s}", remote, node_name); - erts_snprintf(reason_str, sizeof(reason), "%T", reason); + erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), + "%T", reason); if (token != NIL && token != am_have_dt_utag) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); @@ -1797,8 +1805,9 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", cid); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", cid); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", dep->sysname); DTRACE3(dist_port_not_busy, erts_this_node_sysname, port_str, remote_str); @@ -1855,9 +1864,11 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) DTRACE_CHARBUF(remote_str, 64); DTRACE_CHARBUF(pid_str, 16); - erts_snprintf(port_str, sizeof(port_str), "%T", cid); - erts_snprintf(remote_str, sizeof(remote_str), "%T", dep->sysname); - erts_snprintf(pid_str, sizeof(pid_str), "%T", c_p->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", cid); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), + "%T", dep->sysname); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), + "%T", c_p->common.id); DTRACE4(dist_port_busy, erts_this_node_sysname, port_str, remote_str, pid_str); } @@ -1890,8 +1901,9 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE4(dist_output, erts_this_node_sysname, port_str, remote_str, size); @@ -1944,8 +1956,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE4(dist_outputv, erts_this_node_sysname, port_str, remote_str, size); @@ -2280,8 +2293,9 @@ erts_dist_port_not_busy(Port *prt) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE3(dist_port_not_busy, erts_this_node_sysname, port_str, remote_str); @@ -3246,10 +3260,10 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas DTRACE_CHARBUF(type_str, 12); DTRACE_CHARBUF(reason_str, 64); - erts_snprintf(what_str, sizeof(what_str), "%T", what); - erts_snprintf(node_str, sizeof(node_str), "%T", node); - erts_snprintf(type_str, sizeof(type_str), "%T", type); - erts_snprintf(reason_str, sizeof(reason_str), "%T", reason); + erts_snprintf(what_str, sizeof(DTRACE_CHARBUF_NAME(what_str)), "%T", what); + erts_snprintf(node_str, sizeof(DTRACE_CHARBUF_NAME(node_str)), "%T", node); + erts_snprintf(type_str, sizeof(DTRACE_CHARBUF_NAME(type_str)), "%T", type); + erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T", reason); DTRACE5(dist_monitor, erts_this_node_sysname, what_str, node_str, type_str, reason_str); } diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 942eaa47d0..d3109b9432 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -492,7 +492,7 @@ static TYPE * \ NAME##_alloc(void) \ { \ ErtsSchedulerData *esdp = erts_get_scheduler_data(); \ - if (!esdp) \ + if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp)) \ return NULL; \ return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__, \ (int) esdp->no - 1); \ diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index b3dc327704..decae6b2ca 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -292,7 +292,8 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) if (DTRACE_ENABLED(aio_pool_add)) { DTRACE_CHARBUF(port_str, 16); - erts_snprintf(port_str, sizeof(port_str), "%T", a->port); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", a->port); /* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */ len = -1; DTRACE2(aio_pool_add, port_str, len); @@ -327,7 +328,8 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, if (DTRACE_ENABLED(aio_pool_get)) { DTRACE_CHARBUF(port_str, 16); - erts_snprintf(port_str, sizeof(port_str), "%T", a->port); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", a->port); /* DTRACE TODO: Get the length from erts_thr_q_dequeue() ? */ len = -1; DTRACE2(aio_pool_get, port_str, len); diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 1728b200f7..56cd2ba04f 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1548,8 +1548,10 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) switch (dp->extended_marker) { case ERL_DRV_EXTENDED_MARKER: - if (ERL_DRV_EXTENDED_MAJOR_VERSION != dp->major_version - || ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version) { + if (dp->major_version < ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + || (ERL_DRV_EXTENDED_MAJOR_VERSION < dp->major_version + || (ERL_DRV_EXTENDED_MAJOR_VERSION == dp->major_version + && ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version))) { /* Incompatible driver version */ res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION; goto error; diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 77627a6897..afb33c1cdb 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -882,7 +882,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(p, process_str); - erts_snprintf(port_str, sizeof(port_str), "%T", port->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", port->common.id); DTRACE3(port_open, process_str, name_buf, port_str); } #endif diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 5517c26ba4..3ecb379326 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -136,6 +136,22 @@ typedef struct { #define ERL_DRV_EXTENDED_MINOR_VERSION 0 /* + * The emulator will refuse to load a driver with a major version + * lower than ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load + * may however fail if user have not removed use of deprecated + * symbols. + * + * The ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow + * loading of drivers built at least two major OTP releases + * ago. + * + * Bump of major version to 3 happened in OTP 17. That is, in + * OTP 19 we can increase ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + * to 3. + */ +#define ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2 + +/* * The emulator will refuse to load a driver with different major * version than the one used by the emulator. */ diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 6a9030fd99..0eb8117980 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -896,8 +896,10 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send)) { - erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->common.id); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), + "%T", receiver->common.id); } #endif if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 40860e141c..063dba056e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2049,8 +2049,10 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) { ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful"); } - else if (entry->major != ERL_NIF_MAJOR_VERSION - || entry->minor > ERL_NIF_MINOR_VERSION + else if (entry->major < ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + || (ERL_NIF_MAJOR_VERSION < entry->major + || (ERL_NIF_MAJOR_VERSION == entry->major + && ERL_NIF_MINOR_VERSION < entry->minor)) || (entry->major==2 && entry->minor == 5)) { /* experimental maps */ ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index c12ba4d554..5b93c2398e 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -46,6 +46,18 @@ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 6 +/* + * The emulator will refuse to load a nif-lib with a major version + * lower than ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load + * may however fail if user have not removed use of deprecated + * symbols. + * + * The ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow + * loading of nif-libs built at least two major OTP releases + * ago. + */ +#define ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2 + #include <stdlib.h> #ifdef SIZEOF_CHAR diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index fb6048b41f..31d9a1e26e 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -2035,9 +2035,9 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) DTRACE_CHARBUF(pid_str, 16); ErtsProcList* plp2 = plp; - erts_snprintf(port_str, sizeof(port_str), "%T", pp->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", pp->common.id); while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid); DTRACE2(process_port_unblocked, pid_str, port_str); } } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 37e1d07107..8bf0cc9491 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -11313,7 +11313,7 @@ send_exit_signal(Process *c_p, /* current process if and only dtrace_pid_str(from, sender_str); dtrace_proc_str(rp, receiver_str); - erts_snprintf(reason_buf, sizeof(reason_buf) - 1, "%T", reason); + erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason); DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf); } #endif diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 6978a5f11a..305058ceff 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -151,6 +151,11 @@ do { \ message dispatcher thread takes care of that). */ #define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \ do { (RES) = (TPID); } while(0) +int +erts_is_tracer_proc_valid(Process* p) +{ + return 1; +} #else #define ERTS_NULL_TRACER_REF NULL #define ERTS_TRACER_REF_TYPE Process * @@ -163,6 +168,20 @@ do { \ return; \ } \ } while (0) +int +erts_is_tracer_proc_valid(Process* p) +{ + Process* tracer; + + tracer = erts_proc_lookup(ERTS_TRACER_PROC(p)); + if (tracer && ERTS_TRACE_FLAGS(tracer) & F_TRACER) { + return 1; + } else { + ERTS_TRACER_PROC(p) = NIL; + ERTS_TRACE_FLAGS(p) = ~TRACEE_FLAGS; + return 0; + } +} #endif static Uint active_sched; diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 853c6cb0d8..4f2c70d6e7 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -39,6 +39,7 @@ void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp); void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp); void erts_set_system_monitor(Eterm monitor); Eterm erts_get_system_monitor(void); +int erts_is_tracer_proc_valid(Process* p); #ifdef ERTS_SMP void erts_check_my_tracer_proc(Process *); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index cd5060ebb3..a25716b9b4 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -2473,7 +2473,7 @@ set_port_connected(int bang_op, DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE); dtrace_pid_str(connect, process_str); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); dtrace_proc_str(rp, newprocess_str); DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str); } @@ -3591,9 +3591,9 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); DTRACE_CHARBUF(rreason_str, 64); - erts_snprintf(from_str, sizeof(from_str), "%T", from); + erts_snprintf(from_str, sizeof(DTRACE_CHARBUF_NAME(from_str)), "%T", from); dtrace_port_str(p, port_str); - erts_snprintf(rreason_str, sizeof(rreason_str), "%T", rreason); + erts_snprintf(rreason_str, sizeof(DTRACE_CHARBUF_NAME(rreason_str)), "%T", rreason); DTRACE4(port_exit, from_str, port_str, p->name, rreason_str); } #endif @@ -4660,7 +4660,7 @@ set_busy_port(ErlDrvPort dprt, int on) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_busy)) { - erts_snprintf(port_str, sizeof(port_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); DTRACE1(port_busy, port_str); } @@ -4673,7 +4673,7 @@ set_busy_port(ErlDrvPort dprt, int on) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_not_busy)) { - erts_snprintf(port_str, sizeof(port_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); DTRACE1(port_not_busy, port_str); } @@ -4725,9 +4725,9 @@ erts_port_resume_procs(Port *prt) DTRACE_CHARBUF(pid_str, 16); ErtsProcList* plp2 = plp; - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid); DTRACE2(process_port_unblocked, pid_str, port_str); } } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index e273056a2b..05f07e57b2 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -154,10 +154,14 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; /* In VC++, noreturn is a declspec that has to be before the types, * but in GNUC it is an att ribute to be placed between return type * and function name, hence __decl_noreturn <types> __noreturn <function name> + * + * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h */ #if __GNUC__ # define __decl_noreturn -# define __noreturn __attribute__((noreturn)) +# ifndef __noreturn +# define __noreturn __attribute__((noreturn)) +# endif #else # if defined(__WIN32__) && defined(_MSC_VER) # define __noreturn diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c index ef539f8f9b..1ef1602ec9 100644 --- a/erts/emulator/drivers/common/gzio.c +++ b/erts/emulator/drivers/common/gzio.c @@ -230,6 +230,7 @@ local ErtsGzFile gz_open (path, mode) errno = 0; #if defined(FILENAMES_16BIT) { + FILE* efile_wfopen(const WCHAR* name, const WCHAR* mode); WCHAR wfmode[80]; int i = 0; int j; @@ -237,7 +238,7 @@ local ErtsGzFile gz_open (path, mode) wfmode[i++] = (WCHAR) fmode[j]; } wfmode[i++] = L'\0'; - s->file = _wfopen((WCHAR *)path, wfmode); + s->file = efile_wfopen((WCHAR *)path, wfmode); if (s->file == NULL) { return s->destroy(s), (ErtsGzFile)Z_NULL; } diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c index 3fe5d282dc..3143e4511d 100644 --- a/erts/emulator/drivers/common/zlib_drv.c +++ b/erts/emulator/drivers/common/zlib_drv.c @@ -21,6 +21,9 @@ * ZLib interface for erlang * */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif #include <stdio.h> #include <zlib.h> #include <errno.h> diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 480ba23239..a321bb9641 100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2013. All Rights Reserved. + * Copyright Ericsson AB 1997-2014. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -29,12 +29,27 @@ #include <wchar.h> #include "erl_efile.h" +#define DBG_TRACE_MASK 0 +/* 1 = file name ops + * 2 = file descr ops + * 4 = errors + * 8 = path name conversion + */ +#if !DBG_TRACE_MASK +# define DBG_TRACE(M,S) +# define DBG_TRACE1(M,FMT,A) +# define DBG_TRACE2(M,FMT,A,B) +#else +# define DBG_TRACE(M,S) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: %s\r\n", __LINE__, (WCHAR*)(S)); }while(0) +# define DBG_TRACE1(M,FMT,A) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: " L##FMT L"\r\n", __LINE__, (A)); }while(0) +# define DBG_TRACE2(M,FMT,A,B) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: " L##FMT L"\r\n", __LINE__, (A), (B)); }while(0) +#endif + /* * Microsoft-specific function to map a WIN32 error code to a Posix errno. */ #define ISSLASH(a) ((a) == L'\\' || (a) == L'/') - #define ISDIR(st) (((st).st_mode&S_IFMT) == S_IFDIR) #define ISREG(st) (((st).st_mode&S_IFMT) == S_IFREG) @@ -69,10 +84,92 @@ static int check_error(int result, Efile_error* errInfo); static int set_error(Efile_error* errInfo); +static int set_os_errno(Efile_error* errInfo, DWORD os_errno); static int is_root_unc_name(const WCHAR *path); static int extract_root(WCHAR *name); static unsigned short dos_to_posix_mode(int attr, const WCHAR *name); + +struct wpath_tmp_buffer { + struct wpath_tmp_buffer* next; + WCHAR buffer[1]; +}; + +typedef struct { + Efile_error* errInfo; + struct wpath_tmp_buffer* buf_list; +}Efile_call_state; + +static void call_state_init(Efile_call_state* state, Efile_error* errInfo) +{ + state->errInfo = errInfo; + state->buf_list = NULL; +} +static WCHAR* wpath_tmp_alloc(Efile_call_state* state, size_t len) +{ + size_t sz = offsetof(struct wpath_tmp_buffer, buffer) + + (len+1)*sizeof(WCHAR); + struct wpath_tmp_buffer* p = driver_alloc(sz); + p->next = state->buf_list; + state->buf_list = p; + return p->buffer; +} +static void call_state_free(Efile_call_state* state) +{ + while(state->buf_list) { + struct wpath_tmp_buffer* next = state->buf_list->next; + driver_free(state->buf_list); + state->buf_list = next; + } +} +static WCHAR* get_cwd_wpath_tmp(Efile_call_state* state) +{ + WCHAR dummy; + DWORD size = GetCurrentDirectoryW(0, &dummy); + WCHAR* ret = NULL; + + if (size) { + ret = wpath_tmp_alloc(state, size); + if (!GetCurrentDirectoryW(size, ret)) { + ret = NULL; + } + } + return ret; +} +static WCHAR* get_full_wpath_tmp(Efile_call_state* state, + const WCHAR* file, + WCHAR** file_part, + DWORD extra) +{ + WCHAR dummy; + DWORD size = GetFullPathNameW(file, 0, &dummy, NULL); + WCHAR* ret = NULL; + + if (size) { + int ok; + ret = wpath_tmp_alloc(state, size + extra); + if (file_part) { + ok = (GetFullPathNameW(file, size, ret, file_part) != 0); + } + else { + ok = (_wfullpath(ret, file, size) != NULL); + } + if (!ok) { + ret = NULL; + } + } + return ret; +} + +static void ensure_wpath_max(Efile_call_state* state, WCHAR** pathp, size_t max); +static int do_rmdir(Efile_call_state*, char* name); +static int do_rename(Efile_call_state*, char* src, char* dst); +static int do_readdir(Efile_call_state*, char* name, EFILE_DIR_HANDLE*, char* buffer, size_t *size); +static int do_fileinfo(Efile_call_state*, Efile_info*, char* orig_name, int info_for_link); +static char* do_readlink(Efile_call_state*, char* name, char* buffer, size_t size); +static int do_altname(Efile_call_state*, char* orig_name, char* buffer, size_t size); + + static int errno_map(DWORD last_error) { switch (last_error) { @@ -154,6 +251,8 @@ static int errno_map(DWORD last_error) { return EAGAIN; case ERROR_CANT_RESOLVE_FILENAME: return EMLINK; + case ERROR_PRIVILEGE_NOT_HELD: + return EPERM; case ERROR_ARENA_TRASHED: case ERROR_INVALID_BLOCK: case ERROR_BAD_ENVIRONMENT: @@ -176,11 +275,23 @@ check_error(int result, Efile_error* errInfo) if (result < 0) { errInfo->posix_errno = errno; errInfo->os_errno = GetLastError(); + DBG_TRACE2(4, "ERROR os_error=%d errno=%d @@@@@@@@@@@@@@@@@@@@@@@@@@@@", + errInfo->os_errno, errInfo->posix_errno); return 0; } return 1; } +static void +save_last_error(Efile_error* errInfo) +{ + errInfo->posix_errno = errno; + errInfo->os_errno = GetLastError(); + DBG_TRACE2(4, "ERROR os_error=%d errno=%d $$$$$$$$$$$$$$$$$$$$$$$$$$$$$", + errInfo->os_errno, errInfo->posix_errno); +} + + /* * Fills the provided error information structure with information * with the error code given by GetLastError() and its corresponding @@ -192,7 +303,18 @@ check_error(int result, Efile_error* errInfo) static int set_error(Efile_error* errInfo) { - errInfo->posix_errno = errno_map(errInfo->os_errno = GetLastError()); + set_os_errno(errInfo, GetLastError()); + return 0; +} + + +static int +set_os_errno(Efile_error* errInfo, DWORD os_errno) +{ + errInfo->os_errno = os_errno; + errInfo->posix_errno = errno_map(os_errno); + DBG_TRACE2(4, "ERROR os_error=%d errno=%d ############################", + errInfo->os_errno, errInfo->posix_errno); return 0; } @@ -226,21 +348,151 @@ win_writev(Efile_error* errInfo, } +/* Check '*pathp' and convert it if needed to something that windows will accept. + * Typically use UNC path with \\?\ prefix if absolute path is longer than 260. + */ +static void ensure_wpath(Efile_call_state* state, WCHAR** pathp) +{ + ensure_wpath_max(state, pathp, MAX_PATH); +} + +static void ensure_wpath_max(Efile_call_state* state, WCHAR** pathp, size_t max) +{ + WCHAR* path = *pathp; + WCHAR* p; + size_t len = wcslen(path); + int unc_fixup = 0; + + if (path[0] == 0) { + DBG_TRACE(8, L"Let empty path pass through"); + return; + } + + DBG_TRACE1(8,"IN: %s", path); + + if (path[1] == L':' && ISSLASH(path[2])) { /* absolute path */ + if (len >= max) { + WCHAR *src, *dst; + + *pathp = wpath_tmp_alloc(state, 4+len+1); + dst = *pathp; + wcscpy(dst, L"\\\\?\\"); + for (src=path,dst+=4; *src; src++) { + if (*src == L'/') { + if (dst[-1] != L'\\') { + *dst++ = L'\\'; + } + /*else ignore redundant slashes */ + } + else + *dst++ = *src; + } + *dst = 0; + unc_fixup = 1; + } + } + else if (!(ISSLASH(path[0]) && ISSLASH(path[1]))) { /* relative path */ + DWORD cwdLen = GetCurrentDirectoryW(0, NULL); + DWORD absLen = cwdLen + 1 + len; + if (absLen >= max) { + WCHAR *fullPath = wpath_tmp_alloc(state, 4+4+absLen); + DWORD fullLen; + + fullLen = GetFullPathNameW(path, 4 + absLen, fullPath+4, NULL); + if (fullLen >= 4+absLen) { + *pathp = path; + DBG_TRACE2(8,"ensure_wpath FAILED absLen=%u %s", (int)absLen, path); + return; + } + /* GetFullPathNameW can return paths longer than MAX_PATH without the \\?\ prefix. + * At least seen on Windows 7. Go figure... + */ + if (fullLen >= max && wcsncmp(fullPath+4, L"\\\\?\\", 4) != 0) { + wcsncpy(fullPath, L"\\\\?\\", 4); + *pathp = fullPath; + } + else { + *pathp = fullPath + 4; + } + } + } + + if (unc_fixup) { + WCHAR* endp; + + p = *pathp; + len = wcslen(p); + endp = p + len; + if (len > 4) { + p += 4; + while (*p) { + if (p[0] == L'\\' && p[1] == L'.') { + if (p[2] == L'\\' || !p[2]) { /* single dot */ + wmemmove(p, p+2, (&endp[1] - &p[2])); + endp -= 2; + } + else if (p[2] == L'.' && (p[3] == L'\\' || !p[3])) { /* double dot */ + WCHAR* r; + for (r=p-1; *r == L'\\'; --r) + /*skip redundant slashes*/; + for (; *r != L'\\'; --r) + /*find start of prev directory*/; + if (r < *pathp + 6) + break; + wmemmove(r, p+3, (&endp[1] - &p[3])); + p = r; + } + else p += 3; + } + else ++p; + } + } + } + DBG_TRACE1(8,"OUT: %s", *pathp); +} int efile_mkdir(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of directory to create. */ { - return check_error(_wmkdir((WCHAR *) name), errInfo); + Efile_call_state state; + WCHAR* wname = (WCHAR*)name; + int ret; + + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ensure_wpath_max(&state, &wname, 248); /* Yes, 248 limit for normal paths */ + + ret = (int) CreateDirectoryW(wname, NULL); + if (!ret) + set_error(errInfo); + + call_state_free(&state); + return ret; } int efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of directory to delete. */ { + Efile_call_state state; + int ret; + + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ret = do_rmdir(&state, name); + call_state_free(&state); + return ret; +} + +static int do_rmdir(Efile_call_state* state, char* name) +{ OSVERSIONINFO os; DWORD attr; WCHAR *wname = (WCHAR *) name; + WCHAR *buffer = NULL; + + ensure_wpath(state, &wname); if (RemoveDirectoryW(wname) != FALSE) { return 1; @@ -270,10 +522,9 @@ efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ if (os.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) { HANDLE handle; WIN32_FIND_DATAW data; - WCHAR buffer[2*MAX_PATH]; - int len; + int len = wcslen(wname); - len = wcslen(wname); + buffer = wpath_tmp_alloc(state, len + 4); wcscpy(buffer, wname); if (buffer[0] && buffer[len-1] != L'\\' && buffer[len-1] != L'/') { wcscat(buffer, L"\\"); @@ -311,16 +562,30 @@ efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ } end: - return check_error(-1, errInfo); + save_last_error(state->errInfo); + return 0; } int efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of file to delete. */ { + Efile_call_state state; + int ret; + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ret = do_delete_file(&state, name); + call_state_free(&state); + return ret; +} + +static int do_delete_file(Efile_call_state* state, char* name) +{ DWORD attr; WCHAR *wname = (WCHAR *) name; + ensure_wpath(state, &wname); + if (DeleteFileW(wname) != FALSE) { return 1; } @@ -359,7 +624,7 @@ efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ errno = EACCES; } - return check_error(-1, errInfo); + return check_error(-1, state->errInfo); } /* @@ -393,14 +658,29 @@ efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ */ int -efile_rename(Efile_error* errInfo, /* Where to return error codes. */ - char* src, /* Original name. */ - char* dst) /* New name. */ +efile_rename(Efile_error* errInfo, char* src, char* dst) +{ + Efile_call_state state; + int ret; + DBG_TRACE(1, src); + call_state_init(&state, errInfo); + ret = do_rename(&state, src, dst); + call_state_free(&state); + return ret; +} + +static int +do_rename(Efile_call_state* state, + char* src, /* Original name. */ + char* dst) /* New name. */ { DWORD srcAttr, dstAttr; WCHAR *wsrc = (WCHAR *) src; WCHAR *wdst = (WCHAR *) dst; - + + ensure_wpath(state, &wsrc); + ensure_wpath(state, &wdst); + if (MoveFileW(wsrc, wdst) != FALSE) { return 1; } @@ -417,23 +697,27 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ if (errno == EBADF) { errno = EACCES; - return check_error(-1, errInfo); + return check_error(-1, state->errInfo); } if (errno == EACCES) { decode: if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) { - WCHAR srcPath[MAX_PATH], dstPath[MAX_PATH]; + WCHAR *srcPath, *dstPath; WCHAR *srcRest, *dstRest; int size; - size = GetFullPathNameW(wsrc, MAX_PATH, srcPath, &srcRest); - if ((size == 0) || (size > MAX_PATH)) { - return check_error(-1, errInfo); + srcPath = get_full_wpath_tmp(state, wsrc, &srcRest, 0); + if (!srcPath) { + save_last_error(state->errInfo); + return 0; } - size = GetFullPathNameW(wdst, MAX_PATH, dstPath, &dstRest); - if ((size == 0) || (size > MAX_PATH)) { - return check_error(-1, errInfo); + + dstPath = get_full_wpath_tmp(state, wdst, &dstRest, 0); + if (!dstPath) { + save_last_error(state->errInfo); + return 0; } + if (srcRest == NULL) { srcRest = srcPath + wcslen(srcPath); } @@ -538,14 +822,16 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ * put temp file back to old name. */ - WCHAR tempName[MAX_PATH]; - int result, size; + WCHAR *tempName; + int result; WCHAR *rest; - size = GetFullPathNameW(wdst, MAX_PATH, tempName, &rest); - if ((size == 0) || (size > MAX_PATH) || (rest == NULL)) { - return check_error(-1, errInfo); + tempName = get_full_wpath_tmp(state, wdst, &rest, 14); + if (!tempName || !rest) { + save_last_error(state->errInfo); + return 0; } + *rest = L'\0'; result = -1; if (GetTempFileNameW(tempName, L"erlr", 0, tempName) != 0) { @@ -578,7 +864,6 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ /* * Decode the EACCES to a more meaningful error. */ - goto decode; } } @@ -586,16 +871,20 @@ efile_rename(Efile_error* errInfo, /* Where to return error codes. */ } } } - return check_error(-1, errInfo); + return check_error(-1, state->errInfo); } int efile_chdir(Efile_error* errInfo, /* Where to return error codes. */ char* name) /* Name of directory to make current. */ -{ - int success = check_error(_wchdir((WCHAR *) name), errInfo); - if (!success && errInfo->posix_errno == EINVAL) - /* POSIXification of errno */ +{
+ /* We don't even try to handle long paths here
+ * as current working directory is always limited to MAX_PATH
+ * even if we use UNC paths and SetCurrentDirectoryW()
+ */
+ int success = check_error(_wchdir((WCHAR *) name), errInfo);
+ if (!success && errInfo->posix_errno == EINVAL)
+ /* POSIXification of errno */
errInfo->posix_errno = ENOENT; return success; } @@ -608,28 +897,45 @@ efile_getdcwd(Efile_error* errInfo, /* Where to return error codes. */ { WCHAR *wbuffer = (WCHAR *) buffer; size_t wbuffer_size = size / 2; - if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL) + DBG_TRACE(1, L"#getdcwd#"); + if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL) { return check_error(-1, errInfo); + } + DBG_TRACE1(8, "getdcwd OS=%s", wbuffer); + if (wcsncmp(wbuffer, L"\\\\?\\", 4) == 0) { + wmemmove(wbuffer, wbuffer+4, wcslen(wbuffer+4)+1); + } for ( ; *wbuffer; wbuffer++) if (*wbuffer == L'\\') *wbuffer = L'/'; + DBG_TRACE1(8, "getdcwd ERLANG=%s", (WCHAR*)buffer); return 1; } int -efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name, /* Name of directory to list */ - EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */ - char* buffer, /* Buffer to put one filename in */ - size_t *size) /* in-out size of buffer/size of filename excluding zero - termination in bytes*/ +efile_readdir(Efile_error* errInfo, char* name, EFILE_DIR_HANDLE* dir_handle, + char* buffer, size_t *size) +{ + Efile_call_state state; + int ret; + DBG_TRACE(dir_handle?2:1, name); + call_state_init(&state, errInfo); + ret = do_readdir(&state, name, dir_handle, buffer, size); + call_state_free(&state); + return ret; +} + +static int do_readdir(Efile_call_state* state, + char* name, /* Name of directory to list */ + EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */ + char* buffer, /* Buffer to put one filename in */ + size_t *size) /* in-out size of buffer/size of filename excluding zero + termination in bytes*/ { HANDLE dir; /* Handle to directory. */ - WCHAR wildcard[MAX_PATH]; /* Wildcard to search for. */ WIN32_FIND_DATAW findData; /* Data found by FindFirstFile() or FindNext(). */ /* Alignment is not honored, this works on x86 because of alignment fixup by processor. Not perfect, but faster than alinging by hand (really) */ - WCHAR *wname = (WCHAR *) name; WCHAR *wbuffer = (WCHAR *) buffer; /* @@ -637,13 +943,15 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ */ if (*dir_handle == NULL) { - int length = wcslen(wname); + WCHAR *wname = (WCHAR *) name; + WCHAR* wildcard; + int length; WCHAR* s; - if (length+3 >= MAX_PATH) { - errno = ENAMETOOLONG; - return check_error(-1, errInfo); - } + ensure_wpath_max(state, &wname, MAX_PATH-2); + length = wcslen(wname); + + wildcard = wpath_tmp_alloc(state, length+3); wcscpy(wildcard, wname); s = wildcard+length-1; @@ -653,8 +961,10 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ *++s = L'\0'; DEBUGF(("Reading %ws\n", wildcard)); dir = FindFirstFileW(wildcard, &findData); - if (dir == INVALID_HANDLE_VALUE) - return set_error(errInfo); + if (dir == INVALID_HANDLE_VALUE) { + set_error(state->errInfo); + return 0; + } *dir_handle = (EFILE_DIR_HANDLE) dir; if (!IS_DOT_OR_DOTDOT(findData.cFileName)) { @@ -664,7 +974,6 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ } } - /* * Retrieve the name of the next file using the directory handle. */ @@ -681,24 +990,36 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ } if (GetLastError() == ERROR_NO_MORE_FILES) { - FindClose(dir); - errInfo->posix_errno = errInfo->os_errno = 0; - return 0; + state->errInfo->posix_errno = state->errInfo->os_errno = 0; + } + else { + set_error(state->errInfo); } - - set_error(errInfo); FindClose(dir); return 0; } } int -efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ - char* name, /* Name of directory to open. */ - int flags, /* Flags to use for opening. */ - int* pfd, /* Where to store the file descriptor. */ - Sint64* pSize) /* Where to store the size of the file. */ +efile_openfile(Efile_error* errInfo, char* name, int flags, int* pfd, Sint64* pSize) { + Efile_call_state state; + int ret; + DBG_TRACE1(1, "openfile(%s)", name); + call_state_init(&state, errInfo); + ret = do_openfile(&state, name, flags, pfd, pSize); + call_state_free(&state); + return ret; +} + +static +int do_openfile(Efile_call_state* state, /* Where to return error codes. */ + char* name, /* Name of directory to open. */ + int flags, /* Flags to use for opening. */ + int* pfd, /* Where to store the file descriptor. */ + Sint64* pSize) /* Where to store the size of the file. */ +{ + Efile_error* errInfo = state->errInfo; BY_HANDLE_FILE_INFORMATION fileInfo; /* File information from a handle. */ HANDLE fd; /* Handle to open file. */ DWORD access; /* Access mode: GENERIC_READ, GENERIC_WRITE. */ @@ -735,6 +1056,7 @@ efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ if (flags & EFILE_MODE_EXCL) { crFlags = CREATE_NEW; } + ensure_wpath(state, &wname); fd = CreateFileW(wname, access, FILE_SHARE_FLAGS, NULL, crFlags, flagsAndAttrs, NULL); @@ -777,34 +1099,56 @@ efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ } int -efile_may_openfile(Efile_error* errInfo, char *name) { +efile_may_openfile(Efile_error* errInfo, char *name) +{ + Efile_call_state state; WCHAR *wname = (WCHAR *) name; DWORD attr; + int ret; + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ensure_wpath(&state, &wname); if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) { errno = ENOENT; - return check_error(-1, errInfo); + ret = check_error(-1, errInfo); } - - if (attr & FILE_ATTRIBUTE_DIRECTORY) { + else if (attr & FILE_ATTRIBUTE_DIRECTORY) { errno = EISDIR; - return check_error(-1, errInfo); + ret = check_error(-1, errInfo); } - return 1; + else ret = 1; + + call_state_free(&state); + return ret; } void efile_closefile(fd) int fd; /* File descriptor for file to close. */ { + DBG_TRACE(2, L""); CloseHandle((HANDLE) fd); } +FILE* efile_wfopen(const WCHAR* name, const WCHAR* mode) +{ + Efile_call_state state; + Efile_error dummy; + FILE* f; + call_state_init(&state, &dummy); + ensure_wpath(&state, (WCHAR**)&name); + f = _wfopen(name, mode); + call_state_free(&state); + return f; +} + int efile_fdatasync(errInfo, fd) Efile_error* errInfo; /* Where to return error codes. */ int fd; /* File descriptor for file to sync. */ { + DBG_TRACE(2, L""); /* Not available in Windows, just call regular fsync */ return efile_fsync(errInfo, fd); } @@ -814,6 +1158,7 @@ efile_fsync(errInfo, fd) Efile_error* errInfo; /* Where to return error codes. */ int fd; /* File descriptor for file to sync. */ { + DBG_TRACE(2, L""); if (!FlushFileBuffers((HANDLE) fd)) { return check_error(-1, errInfo); } @@ -824,64 +1169,87 @@ int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, char* orig_name, int info_for_link) { + Efile_call_state state; + int ret; + DBG_TRACE(1, L""); + call_state_init(&state, errInfo); + ret = do_fileinfo(&state, pInfo, orig_name, info_for_link); + call_state_free(&state); + return ret; +} + +static int +do_fileinfo(Efile_call_state* state, Efile_info* pInfo, + char* orig_name, int info_for_link) +{ + Efile_error* errInfo = state->errInfo; HANDLE findhandle; /* Handle returned by FindFirstFile(). */ WIN32_FIND_DATAW findbuf; /* Data return by FindFirstFile(). */ - WCHAR name[_MAX_PATH]; + WCHAR* name = NULL; + WCHAR* win_path; int name_len; - WCHAR *path; - WCHAR pathbuf[_MAX_PATH]; int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ - WCHAR *worig_name = (WCHAR *) orig_name; + WCHAR *worig_name = (WCHAR *) orig_name; + ensure_wpath(state, &worig_name); /* Don't allow wildcards to be interpreted by system */ - if (wcspbrk(worig_name, L"?*")) { - enoent: - errInfo->posix_errno = ENOENT; - errInfo->os_errno = ERROR_FILE_NOT_FOUND; - return 0; - } /* * Move the name to a buffer and make sure to remove a trailing * slash, because it causes FindFirstFile() to fail on Win95. */ - if ((name_len = wcslen(worig_name)) >= _MAX_PATH) { - goto enoent; - } else { - wcscpy(name, worig_name); - if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != L':') { - name[name_len-1] = L'\0'; - } + name_len = wcslen(worig_name); + + name = wpath_tmp_alloc(state, name_len+1); + wcscpy(name, worig_name); + if (name_len > 2 && ISSLASH(name[name_len-1]) && + name[name_len-2] != L':') { + name[name_len-1] = L'\0'; } - + + win_path = name; + if (wcsncmp(name, L"\\\\?\\", 4) == 0) { + win_path += 4; + } + + if (wcspbrk(win_path, L"?*")) { + enoent: + errInfo->posix_errno = ENOENT; + errInfo->os_errno = ERROR_FILE_NOT_FOUND; + return 0; + } + /* Try to get disk from name. If none, get current disk. */ - if (name[1] != L':') { + if (win_path[1] != L':') { + WCHAR* cwd_path = get_cwd_wpath_tmp(state); drive = 0; - if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) && - pathbuf[1] == L':') { - drive = towlower(pathbuf[0]) - L'a' + 1; + if (cwd_path[1] == L':') { + drive = towlower(cwd_path[0]) - L'a' + 1; } - } else if (*name && name[2] == L'\0') { + } else if (*win_path && win_path[2] == L'\0') { /* * X: and nothing more is an error. */ errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; - } else - drive = towlower(*name) - L'a' + 1; + } else { + drive = towlower(*win_path) - L'a' + 1; + } findhandle = FindFirstFileW(name, &findbuf); if (findhandle == INVALID_HANDLE_VALUE) { + WCHAR* path = NULL; + if (!(wcspbrk(name, L"./\\") && - (path = _wfullpath(pathbuf, name, _MAX_PATH)) && + (path = get_full_wpath_tmp(state, name, NULL, 0)) && /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ ((wcslen(path) == 3) || is_root_unc_name(path)) && (GetDriveTypeW(path) > 1) ) ) { + errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; @@ -908,13 +1276,11 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, /* * given that we know this is a symlink, we should be able to find its target */ - WCHAR target_name[_MAX_PATH]; - if (efile_readlink(errInfo, (char *) name, - (char *) target_name, - _MAX_PATH * sizeof(WCHAR)) == 1) { + WCHAR* target_name = (WCHAR*) do_readlink(state, (char *) name, NULL, 0); + if (target_name) { FindClose(findhandle); - return efile_fileinfo(errInfo, pInfo, - (char *) target_name, info_for_link); + return do_fileinfo(state, pInfo, + (char *) target_name, info_for_link); } } @@ -981,6 +1347,20 @@ efile_write_info(Efile_error* errInfo, Efile_info* pInfo, char* name) { + Efile_call_state state; + int ret; + call_state_init(&state, errInfo); + ret = do_write_info(&state, pInfo, name); + call_state_free(&state); + return ret; +} + +static int +do_write_info(Efile_call_state* state, + Efile_info* pInfo, + char* name) +{ + Efile_error* errInfo = state->errInfo; SYSTEMTIME timebuf; FILETIME ModifyFileTime; FILETIME AccessFileTime; @@ -990,6 +1370,10 @@ efile_write_info(Efile_error* errInfo, DWORD tempAttr; WCHAR *wname = (WCHAR *) name; + DBG_TRACE(1, name); + + ensure_wpath(state, &wname); + /* * Get the attributes for the file. */ @@ -1066,7 +1450,9 @@ char* buf; /* Buffer to write. */ size_t count; /* Number of bytes to write. */ Sint64 offset; /* where to write it */ { - int res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); + int res; + DBG_TRACE(2, L""); + res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); if (res) { return efile_write(errInfo, EFILE_MODE_WRITE, fd, buf, count); } else { @@ -1084,7 +1470,9 @@ char* buf; /* Buffer to read into. */ size_t count; /* Number of bytes to read. */ size_t* pBytesRead; /* Where to return number of bytes read. */ { - int res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); + int res; + DBG_TRACE(2, L""); + res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); if (res) { return efile_read(errInfo, EFILE_MODE_READ, fd, buf, count, pBytesRead); } else { @@ -1106,6 +1494,7 @@ size_t count; /* Number of bytes to write. */ OVERLAPPED overlapped; OVERLAPPED* pOverlapped = NULL; + DBG_TRACE(2, L""); if (flags & EFILE_MODE_APPEND) { memset(&overlapped, 0, sizeof(overlapped)); overlapped.Offset = 0xffffffff; @@ -1135,6 +1524,7 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ OVERLAPPED overlapped; OVERLAPPED* pOverlapped = NULL; + DBG_TRACE(2, L""); ASSERT(iovcnt >= 0); if (flags & EFILE_MODE_APPEND) { @@ -1171,6 +1561,8 @@ size_t count; /* Number of bytes to read. */ size_t* pBytesRead; /* Where to return number of bytes read. */ { DWORD nbytes = 0; + + DBG_TRACE(2, L""); if (!ReadFile((HANDLE) fd, buf, count, &nbytes, NULL)) return set_error(errInfo); @@ -1190,6 +1582,7 @@ Sint64* new_location; /* Resulting new location in file. */ { LARGE_INTEGER off, new_loc; + DBG_TRACE(2, L""); switch (origin) { case EFILE_SEEK_SET: origin = FILE_BEGIN; break; case EFILE_SEEK_CUR: origin = FILE_CURRENT; break; @@ -1221,6 +1614,7 @@ Efile_error* errInfo; /* Where to return error codes. */ int *fd; /* File descriptor for file to truncate. */ int flags; { + DBG_TRACE(2, L""); if (!SetEndOfFile((HANDLE) (*fd))) return set_error(errInfo); return 1; @@ -1373,9 +1767,24 @@ dos_to_posix_mode(int attr, const WCHAR *name) return uxmode; } + int efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) { + Efile_call_state state; + int ret; + DBG_TRACE(1, name); + call_state_init(&state, errInfo); + ret = !!do_readlink(&state, name, buffer, size); + call_state_free(&state); + return ret; +} + +/* If buffer==0, return buffer allocated by wpath_tmp_allocate +*/ +static char* +do_readlink(Efile_call_state* state, char* name, char* buffer, size_t size) +{ /* * load dll and see if we have CreateSymbolicLink at runtime: * (Vista only) @@ -1383,6 +1792,9 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) HINSTANCE hModule = NULL; WCHAR *wname = (WCHAR *) name; WCHAR *wbuffer = (WCHAR *) buffer; + DWORD wsize = size / sizeof(WCHAR); + char* ret = NULL; + if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { typedef DWORD (WINAPI * GETFINALPATHNAMEBYHANDLEPTR)( HANDLE hFile, @@ -1393,58 +1805,84 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) GETFINALPATHNAMEBYHANDLEPTR pGetFinalPathNameByHandle = (GETFINALPATHNAMEBYHANDLEPTR)GetProcAddress(hModule, "GetFinalPathNameByHandleW"); - if (pGetFinalPathNameByHandle == NULL) { - FreeLibrary(hModule); - } else { + if (pGetFinalPathNameByHandle != NULL) { + DWORD fileAttributes; + ensure_wpath(state, &wname); /* first check if file is a symlink; {error, einval} otherwise */ - DWORD fileAttributes = GetFileAttributesW(wname); + fileAttributes = GetFileAttributesW(wname); if ((fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - BOOLEAN success = 0; + DWORD success = 0; HANDLE h = CreateFileW(wname, GENERIC_READ, FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); int len; if(h != INVALID_HANDLE_VALUE) { - success = pGetFinalPathNameByHandle(h, wbuffer, size / sizeof(WCHAR),0); - /* GetFinalPathNameByHandle prepends path with "\\?\": */ - len = wcslen(wbuffer); - wmemmove(wbuffer,wbuffer+4,len-3); - if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' && - wbuffer[0] <= L'Z') { - wbuffer[0] = wbuffer[0] + L'a' - L'A'; + if (!wbuffer) { /* dynamic allocation */ + WCHAR dummy; + wsize = pGetFinalPathNameByHandle(h, &dummy, 0, 0); + if (wsize) { + wbuffer = wpath_tmp_alloc(state, wsize); + } } + if (wbuffer + && (success = pGetFinalPathNameByHandle(h, wbuffer, wsize, 0)) + && success < wsize) { + WCHAR* wp; + + /* GetFinalPathNameByHandle prepends path with "\\?\": */ + len = wcslen(wbuffer); + wmemmove(wbuffer,wbuffer+4,len-3); + if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' && + wbuffer[0] <= L'Z') { + wbuffer[0] = wbuffer[0] + L'a' - L'A'; + } - for ( ; *wbuffer; wbuffer++) - if (*wbuffer == L'\\') - *wbuffer = L'/'; + for (wp=wbuffer ; *wp; wp++) + if (*wp == L'\\') + *wp = L'/'; + } CloseHandle(h); - } - FreeLibrary(hModule); + } if (success) { - return 1; + ret = (char*) wbuffer; } else { - return set_error(errInfo); + set_error(state->errInfo); } } else { - FreeLibrary(hModule); errno = EINVAL; - return check_error(-1, errInfo); + save_last_error(state->errInfo); } + goto done; } } errno = ENOTSUP; - return check_error(-1, errInfo); + save_last_error(state->errInfo); + +done: + if (hModule) + FreeLibrary(hModule); + return ret; } int efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) { + Efile_call_state state; + int ret; + DBG_TRACE(1, orig_name); + call_state_init(&state, errInfo); + ret = do_altname(&state, orig_name, buffer, size); + call_state_free(&state); + return ret; +} + +static int +do_altname(Efile_call_state* state, char* orig_name, char* buffer, size_t size) +{ WIN32_FIND_DATAW wfd; HANDLE fh; - WCHAR name[_MAX_PATH+1]; + WCHAR* name; int name_len; - WCHAR* path; - WCHAR pathbuf[_MAX_PATH+1]; /* Unclear weather GetCurrentDirectory will access one char after - _MAX_PATH */ + WCHAR* full_path = NULL; WCHAR *worig_name = (WCHAR *) orig_name; WCHAR *wbuffer = (WCHAR *) buffer; int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ @@ -1453,8 +1891,8 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) if (wcspbrk(worig_name, L"?*")) { enoent: - errInfo->posix_errno = ENOENT; - errInfo->os_errno = ERROR_FILE_NOT_FOUND; + state->errInfo->posix_errno = ENOENT; + state->errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; } @@ -1462,24 +1900,23 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) * Move the name to a buffer and make sure to remove a trailing * slash, because it causes FindFirstFile() to fail on Win95. */ - - if ((name_len = wcslen(worig_name)) >= _MAX_PATH) { - goto enoent; - } else { - wcscpy(name, worig_name); - if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != L':') { - name[name_len-1] = L'\0'; - } + ensure_wpath(state, &worig_name); + name_len = wcslen(worig_name); + + name = wpath_tmp_alloc(state, name_len + 1); + wcscpy(name, worig_name); + if (name_len > 2 && ISSLASH(name[name_len-1]) && + name[name_len-2] != L':') { + name[name_len-1] = L'\0'; } /* Try to get disk from name. If none, get current disk. */ if (name[1] != L':') { + WCHAR* cwd_path = get_cwd_wpath_tmp(state); drive = 0; - if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) && - pathbuf[1] == L':') { - drive = towlower(pathbuf[0]) - L'a' + 1; + if (cwd_path[1] == L':') { + drive = towlower(cwd_path[0]) - L'a' + 1; } } else if (*name && name[2] == L'\0') { /* @@ -1491,13 +1928,15 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) } fh = FindFirstFileW(name,&wfd); if (fh == INVALID_HANDLE_VALUE) { + DWORD fff_error = GetLastError(); if (!(wcspbrk(name, L"./\\") && - (path = _wfullpath(pathbuf, name, _MAX_PATH)) && + (full_path = get_full_wpath_tmp(state, name, NULL, 0)) && /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ - ((wcslen(path) == 3) || is_root_unc_name(path)) && - (GetDriveTypeW(path) > 1) ) ) { - errno = errno_map(GetLastError()); - return check_error(-1, errInfo); + ((wcslen(full_path) == 3) || is_root_unc_name(full_path)) && + (GetDriveTypeW(full_path) > 1) ) ) { + + set_os_errno(state->errInfo, fff_error); + return 0; } /* * Root directories (such as C:\ or \\server\share\ are fabricated. @@ -1518,17 +1957,37 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) int efile_link(Efile_error* errInfo, char* old, char* new) { + Efile_call_state state; WCHAR *wold = (WCHAR *) old; WCHAR *wnew = (WCHAR *) new; + int ret; + DBG_TRACE(1, old); + call_state_init(&state, errInfo); + ensure_wpath(&state, &wold); + ensure_wpath(&state, &wnew); if(!CreateHardLinkW(wnew, wold, NULL)) { - return set_error(errInfo); + ret = set_error(errInfo); } - return 1; + else ret =1; + call_state_free(&state); + return ret; } int efile_symlink(Efile_error* errInfo, char* old, char* new) { + Efile_call_state state; + int ret; + DBG_TRACE2(1, "symlink(%s <- %s)", old, new); + call_state_init(&state, errInfo); + ret = do_symlink(&state, old, new); + call_state_free(&state); + return ret; +} + +static int +do_symlink(Efile_call_state* state, char* old, char* new) +{ /* * Load dll and see if we have CreateSymbolicLink at runtime: * (Vista only) @@ -1536,6 +1995,8 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) HINSTANCE hModule = NULL; WCHAR *wold = (WCHAR *) old; WCHAR *wnew = (WCHAR *) new; + + DBG_TRACE(1, old); if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { typedef BOOLEAN (WINAPI * CREATESYMBOLICLINKFUNCPTR) ( LPCWSTR lpSymlinkFileName, @@ -1547,6 +2008,9 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) "CreateSymbolicLinkW"); /* A for MBCS, W for UNICODE... char* above implies 'W'! */ if (pCreateSymbolicLink != NULL) { + ensure_wpath(state, &wold); + ensure_wpath(state, &wnew); + { DWORD attr = GetFileAttributesW(wold); int flag = (attr != INVALID_FILE_ATTRIBUTES && attr & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0; @@ -1557,19 +2021,21 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) if (success) { return 1; } else { - return set_error(errInfo); + return set_error(state->errInfo); } + } } else FreeLibrary(hModule); } errno = ENOTSUP; - return check_error(-1, errInfo); + return check_error(-1, state->errInfo); } int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length, int advise) { + DBG_TRACE(2, L""); /* posix_fadvise is not available on Windows, do nothing */ errno = ERROR_SUCCESS; return check_error(0, errInfo); @@ -1578,6 +2044,7 @@ efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, int efile_fallocate(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length) { + DBG_TRACE(2, L""); /* No file preallocation method available in Windows. */ errno = errno_map(ERROR_NOT_SUPPORTED); SetLastError(ERROR_NOT_SUPPORTED); diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index 7637049bc3..054911e822 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -182,3 +182,10 @@ BIF_RETTYPE hipe_bifs_debug_native_called_2(BIF_ALIST_2) BIF_RET(am_ok); } +/* Stub-BIF for LLVM: + * Reloads BP, SP (in llvm unwind label) */ + +BIF_RETTYPE hipe_bifs_llvm_fix_pinned_regs_0(BIF_ALIST_0) +{ + BIF_RET(am_ok); +} diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab index 45a395bf57..1b659cfa90 100644 --- a/erts/emulator/hipe/hipe_bif2.tab +++ b/erts/emulator/hipe/hipe_bif2.tab @@ -30,3 +30,4 @@ bif hipe_bifs:in_native/0 bif hipe_bifs:modeswitch_debug_on/0 bif hipe_bifs:modeswitch_debug_off/0 bif hipe_bifs:debug_native_called/2 +bif hipe_bifs:llvm_fix_pinned_regs/0 diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 245841a768..1db673e7f3 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -527,7 +527,8 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, /* fast track to stop_select callback */ stop_select_fn = prt->drv_ptr->stop_select; #ifdef USE_VM_PROBES - strncpy(name, prt->drv_ptr->name, sizeof(name)-1); + strncpy(name, prt->drv_ptr->name, + sizeof(DTRACE_CHARBUF_NAME(name))-1); name[sizeof(name)-1] = '\0'; #endif ret = 0; diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 7c6e4a2f37..94eb6b1547 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -54,6 +54,17 @@ void sys_sigrelease(int sig) #endif /* !SIG_SIGNAL */ #endif /* !SIG_SIGSET */ +#if defined(__ANDROID__) +int __system_properties_fd(void); +#endif /* __ANDROID__ */ + +#if defined(__ANDROID__) +#define SHELL "/system/bin/sh" +#else +#define SHELL "/bin/sh" +#endif /* __ANDROID__ */ + + int main(int argc, char *argv[]) { @@ -89,8 +100,23 @@ main(int argc, char *argv[]) if (sscanf(argv[CS_ARGV_FD_CR_IX], "%d:%d", &from, &to) != 2) return 1; + +#if defined(__ANDROID__) + for (i = from; i <= to; i++) { + if (i!=__system_properties_fd) + (void) close(i); + } +#else for (i = from; i <= to; i++) (void) close(i); +#endif /* __ANDROID__ */ + +#if defined(HAVE_CLOSEFROM) + closefrom(from); +#else + for (i = from; i <= to; i++) + (void) close(i); +#endif if (!(argv[CS_ARGV_WD_IX][0] == '.' && argv[CS_ARGV_WD_IX][1] == '\0') && chdir(argv[CS_ARGV_WD_IX]) < 0) @@ -116,7 +142,25 @@ main(int argc, char *argv[]) execv(argv[CS_ARGV_NO_OF_ARGS],&(argv[CS_ARGV_NO_OF_ARGS + 1])); } } else { - execl("/bin/sh", "sh", "-c", argv[CS_ARGV_CMD_IX], (char *) NULL); + execl(SHELL, "sh", "-c", argv[CS_ARGV_CMD_IX], (char *) NULL); } return 1; } + + + +#if defined(__ANDROID__) +int __system_properties_fd(void) +{ + int s, fd; + char *env; + + env = getenv("ANDROID_PROPERTY_WORKSPACE"); + if (!env) { + return -1; + } + fd = atoi(env); + return fd; +} +#endif /* __ANDROID__ */ + diff --git a/erts/emulator/sys/unix/erl_unix_sys_ddll.c b/erts/emulator/sys/unix/erl_unix_sys_ddll.c index 8760b58839..2659d623c7 100644 --- a/erts/emulator/sys/unix/erl_unix_sys_ddll.c +++ b/erts/emulator/sys/unix/erl_unix_sys_ddll.c @@ -123,6 +123,7 @@ int erts_sys_ddll_open(const char *full_name, void **handle, ErtsSysDdllError* e int erts_sys_ddll_open_noext(char *dlname, void **handle, ErtsSysDdllError* err) { +#if defined(HAVE_DLOPEN) int ret = ERL_DE_NO_ERROR; char *str; dlerror(); @@ -148,6 +149,9 @@ int erts_sys_ddll_open_noext(char *dlname, void **handle, ErtsSysDdllError* err) ret = ERL_DE_DYNAMIC_ERROR_OFFSET - find_errcode(str, err); } return ret; +#else + return ERL_DE_ERROR_NO_DDLL_FUNCTIONALITY; +#endif } /* diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 865cb50a56..c3d7440409 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -149,6 +149,13 @@ extern void erl_crash_dump(char* file, int line, char* fmt, ...); #define DIR_SEPARATOR_CHAR '/' +#if defined(__ANDROID__) +#define SHELL "/system/bin/sh" +#else +#define SHELL "/bin/sh" +#endif /* __ANDROID__ */ + + #if defined(DEBUG) #define ERL_BUILD_TYPE_MARKER ".debug" #elif defined(PURIFY) @@ -1596,7 +1603,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op } } } else { - execle("/bin/sh", "sh", "-c", cmd_line, (char *) NULL, new_environ); + execle(SHELL, "sh", "-c", cmd_line, (char *) NULL, new_environ); } child_error: _exit(1); @@ -1717,7 +1724,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op fcntl(i, F_SETFD, 1); qnx_spawn_options.flags = _SPAWN_SETSID; - if ((pid = spawnl(P_NOWAIT, "/bin/sh", "/bin/sh", "-c", cmd_line, + if ((pid = spawnl(P_NOWAIT, SHELL, SHELL, "-c", cmd_line, (char *) 0)) < 0) { erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); reset_qnx_spawn(); diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h index 8015e8f378..a78dbf64af 100644 --- a/erts/emulator/sys/win32/erl_win_sys.h +++ b/erts/emulator/sys/win32/erl_win_sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2012. All Rights Reserved. + * Copyright Ericsson AB 1997-2014. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -60,16 +60,18 @@ #include <windows.h> #undef WIN32_LEAN_AND_MEAN -/* - * Define MAXPATHLEN in terms of MAXPATH if available. - */ - -#ifndef MAXPATH -#define MAXPATH MAX_PATH -#endif /* MAXPATH */ #ifndef MAXPATHLEN -#define MAXPATHLEN MAXPATH +#define MAXPATHLEN 4096 +/* + erts-6.0 (OTP 17.0): + We now accept windows paths longer than 260 (MAX_PATH) by conversion to + UNC path format. In order to also return long paths from the driver we + increased MAXPATHLEN from 260 to larger (but arbitrary) value 4096. + It would of course be nicer to instead dynamically allocate large enough + tmp buffers when efile_drv needs to return really long paths, and do that + for unix as well. + */ #endif /* MAXPATHLEN */ /* diff --git a/erts/emulator/test/driver_SUITE_data/smaller_major_vsn_drv.c b/erts/emulator/test/driver_SUITE_data/smaller_major_vsn_drv.c index a1299fe807..6b9d4745ba 100644 --- a/erts/emulator/test/driver_SUITE_data/smaller_major_vsn_drv.c +++ b/erts/emulator/test/driver_SUITE_data/smaller_major_vsn_drv.c @@ -20,12 +20,12 @@ * Author: Rickard Green * * Description: Implementation of a driver with a smaller major - * driver version than the current system. + * driver version than allowed on load. */ #define VSN_MISMATCH_DRV_NAME_STR "smaller_major_vsn_drv" #define VSN_MISMATCH_DRV_NAME smaller_major_vsn_drv -#define VSN_MISMATCH_DRV_MAJOR_VSN_DIFF (-1) +#define VSN_MISMATCH_DRV_MAJOR_VSN_DIFF (ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD - ERL_DRV_EXTENDED_MAJOR_VERSION - 1) #define VSN_MISMATCH_DRV_MINOR_VSN_DIFF 0 #include "vsn_mismatch_drv_impl.c" diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index a854d3f05b..b2da6f58af 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -37,7 +37,7 @@ threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, is_checks/1, get_length/1, make_atom/1, make_string/1, reverse_list_test/1, - otp_9668/1, consume_timeslice/1, dirty_nif/1 + otp_9668/1, consume_timeslice/1, dirty_nif/1, dirty_nif_send/1 ]). -export([many_args_100/100]). @@ -64,7 +64,7 @@ all() -> resource_takeover, threading, send, send2, send3, send_threaded, neg, is_checks, get_length, make_atom, make_string,reverse_list_test, - otp_9668, consume_timeslice, dirty_nif + otp_9668, consume_timeslice, dirty_nif, dirty_nif_send ]. groups() -> @@ -1538,6 +1538,24 @@ dirty_nif(Config) when is_list(Config) -> {skipped,"No dirty scheduler support"} end. +dirty_nif_send(Config) when is_list(Config) -> + try erlang:system_info(dirty_cpu_schedulers) of + N when is_integer(N) -> + ensure_lib_loaded(Config), + Parent = self(), + Pid = spawn_link(fun() -> + Self = self(), + {ok, Self} = receive_any(), + Parent ! {ok, Self} + end), + {ok, Pid} = send_from_dirty_nif(Pid), + {ok, Pid} = receive_any(), + ok + catch + error:badarg -> + {skipped,"No dirty scheduler support"} + end. + next_msg(_Pid) -> receive M -> M @@ -1668,6 +1686,7 @@ type_sizes() -> ?nif_stub. otp_9668_nif(_) -> ?nif_stub. consume_timeslice_nif(_,_) -> ?nif_stub. call_dirty_nif(_,_,_) -> ?nif_stub. +send_from_dirty_nif(_) -> ?nif_stub. %% maps is_map_nif(_) -> ?nif_stub. diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 160f4843ad..955dc64189 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -1533,6 +1533,37 @@ static ERL_NIF_TERM call_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM return dirty_nif(env, argc, argv); } } + +static ERL_NIF_TERM dirty_sender(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM result; + ErlNifPid pid; + ErlNifEnv* menv; + int res; + + enif_get_local_pid(env, argv[0], &pid); + result = enif_make_tuple2(env, enif_make_atom(env, "ok"), enif_make_pid(env, &pid)); + menv = enif_alloc_env(); + res = enif_send(env, &pid, menv, result); + enif_free_env(menv); + if (!res) + /* Note the next line will crash, since dirty nifs can't return exceptions. + * This is intentional, since enif_send should not fail if the test succeeds. + */ + return enif_schedule_dirty_nif_finalizer(env, enif_make_badarg(env), enif_dirty_nif_finalizer); + else + return enif_schedule_dirty_nif_finalizer(env, result, enif_dirty_nif_finalizer); +} + +static ERL_NIF_TERM send_from_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM result; + ErlNifPid pid; + + if (!enif_get_local_pid(env, argv[0], &pid)) + return enif_make_badarg(env); + return enif_schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, dirty_sender, argc, argv); +} #endif static ERL_NIF_TERM is_map_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -1713,6 +1744,7 @@ static ErlNifFunc nif_funcs[] = {"consume_timeslice_nif", 2, consume_timeslice_nif}, #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT {"call_dirty_nif", 3, call_dirty_nif}, + {"send_from_dirty_nif", 1, send_from_dirty_nif}, #endif {"is_map_nif", 1, is_map_nif}, {"get_map_size_nif", 1, get_map_size_nif}, diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl index 5dfa87bbee..3036d2957b 100644 --- a/erts/emulator/test/trace_call_time_SUITE.erl +++ b/erts/emulator/test/trace_call_time_SUITE.erl @@ -33,7 +33,7 @@ %% Exported end user tests -export([seq/3, seq_r/3]). --export([loaded/1, a_function/1, a_called_function/1, dec/1, nif_dec/1]). +-export([loaded/1, a_function/1, a_called_function/1, dec/1, nif_dec/1, dead_tracer/1]). -define(US_ERROR, 10000). -define(R_ERROR, 0.8). @@ -89,7 +89,7 @@ all() -> true -> [not_run]; false -> [basic, on_and_off, info, pause_and_restart, scheduling, - combo, bif, nif, called_function] + combo, bif, nif, called_function, dead_tracer] end. groups() -> @@ -470,6 +470,92 @@ called_function(Config) when is_list(Config) -> ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), ok. +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +dead_tracer(Config) when is_list(Config) -> + Self = self(), + FirstTracer = tracer(), + StartTracing = fun() -> turn_on_tracing(Self) end, + tell_tracer(FirstTracer, StartTracing), + [1,2,3,4,5,6,7,8] = seq(1, 8, fun(I) -> I + 1 end), + Ref = erlang:monitor(process, FirstTracer), + FirstTracer ! quit, + receive + {'DOWN',Ref,process,FirstTracer,normal} -> + ok + end, + erlang:yield(), + + %% Collect and check that we only get call_time info for the current process. + Info1 = collect_all_info(), + [] = other_than_self(Info1), + io:format("~p\n", [Info1]), + + %% Note that we have not turned off tracing for the current process, + %% but that the tracer has terminated. No more call_time information should be recorded. + [1,2,3] = seq(1, 3, fun(I) -> I + 1 end), + [] = collect_all_info(), + + %% When we start a second tracer process, that tracer process must + %% not inherit the tracing flags and the dead tracer (even though + %% we used set_on_spawn). + SecondTracer = tracer(), + tell_tracer(SecondTracer, StartTracing), + Seq20 = lists:seq(1, 20), + Seq20 = seq(1, 20, fun(I) -> I + 1 end), + Info2 = collect_all_info(), + io:format("~p\n", [Info2]), + [] = other_than_self(Info2), + SecondTracer ! quit, + + ok. + +other_than_self(Info) -> + [{Pid,MFA} || {MFA,[{Pid,_,_,_}]} <- Info, + Pid =/= self()]. + +tell_tracer(Tracer, Fun) -> + Tracer ! {execute,self(),Fun}, + receive + {Tracer,executed} -> + ok + end. + +tracer() -> + spawn_link(fun Loop() -> + receive + quit -> + ok; + {execute,From,Fun} -> + Fun(), + From ! {self(),executed}, + Loop() + end + end). + +turn_on_tracing(Pid) -> + _ = erlang:trace(Pid, true, [call,set_on_spawn]), + _ = erlang:trace_pattern({?MODULE,'_','_'}, true, [call_time]), + _ = now(), + ok. + +collect_all_info() -> + collect_all_info([{?MODULE,F,A} || {F,A} <- module_info(functions)] ++ + erlang:system_info(snifs)). + +collect_all_info([MFA|T]) -> + CallTime = erlang:trace_info(MFA, call_time), + erlang:trace_pattern(MFA, restart, [call_time]), + case CallTime of + {call_time,false} -> + collect_all_info(T); + {call_time,[]} -> + collect_all_info(T); + {call_time,[_|_]=List} -> + [{MFA,List}|collect_all_info(T)] + end; +collect_all_info([]) -> []. + %%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% The Tests %%% @@ -478,7 +564,6 @@ called_function(Config) when is_list(Config) -> %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Local helpers - load_nif(Config) -> ?line Path = ?config(data_dir, Config), ?line ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0). @@ -602,8 +687,11 @@ collect(A, Ref) -> end. setup() -> + setup([]). + +setup(Opts) -> Pid = spawn_link(fun() -> loop() end), - ?line 1 = erlang:trace(Pid, true, [call]), + 1 = erlang:trace(Pid, true, [call|Opts]), Pid. execute(Pids, Mfa) when is_list(Pids) -> diff --git a/erts/epmd/src/epmd.c b/erts/epmd/src/epmd.c index 5d5c3a1c3c..1678d537d1 100644 --- a/erts/epmd/src/epmd.c +++ b/erts/epmd/src/epmd.c @@ -52,7 +52,7 @@ static int epmd_main(int, char **, int); int epmd_dbg(int level,int port) /* Utility to debug epmd... */ { - char* argv[MAX_DEBUG+2]; + char* argv[MAX_DEBUG+4]; char ibuff[100]; int argc = 0; diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in index 5c2cd8aded..cfd36af962 100644 --- a/erts/etc/common/Makefile.in +++ b/erts/etc/common/Makefile.in @@ -66,7 +66,9 @@ LIBS = @LIBS@ LDFLAGS = @LDFLAGS@ # For clock_gettime in heart +ifneq ($(TARGET),arm-unknown-linux-androideabi) RTLIBS = @LIBRT@ +endif ifeq ($(TARGET),win32) ifeq ($(TYPE),debug) diff --git a/erts/etc/win32/nsis/erlang.nsi b/erts/etc/win32/nsis/erlang.nsi index 162e634148..f4fd2b4cdb 100644 --- a/erts/etc/win32/nsis/erlang.nsi +++ b/erts/etc/win32/nsis/erlang.nsi @@ -93,7 +93,6 @@ SectionIn 1 RO skip_silent_mode: SetOutPath "$INSTDIR" - File "${TESTROOT}\OTP_VERSION" File "${TESTROOT}\Install.ini" File "${TESTROOT}\Install.exe" File /r "${TESTROOT}\releases" diff --git a/erts/etc/win32/nsis/erlang20.nsi b/erts/etc/win32/nsis/erlang20.nsi index 3ee33e8121..3333c4a9aa 100644 --- a/erts/etc/win32/nsis/erlang20.nsi +++ b/erts/etc/win32/nsis/erlang20.nsi @@ -144,7 +144,6 @@ Section "Development" SecErlangDev SectionIn 1 RO
SetOutPath "$INSTDIR"
- File "${TESTROOT}\OTP_VERSION"
File "${TESTROOT}\Install.ini"
File "${TESTROOT}\Install.exe"
SetOutPath "$INSTDIR\releases"
diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c index 7f27b5f29c..79784c5b84 100644 --- a/erts/lib_src/pthread/ethread.c +++ b/erts/lib_src/pthread/ethread.c @@ -541,7 +541,11 @@ int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset) return EINVAL; } #endif - return pthread_sigmask(how, set, oset); +#if defined(__ANDROID__) + return sigprocmask(how, set, oset); +#else + return pthread_sigmask(how, set, oset); +#endif } int ethr_sigwait(const sigset_t *set, int *sig) diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index fd3e8cb692..a15da3a421 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -35,7 +35,8 @@ {registered, []}, {applications, []}, {env, []}, - {mod, {erts, []}} + {mod, {erts, []}}, + {runtime_dependencies, ["stdlib-2.0", "kernel-3.0", "sasl-2.4"]} ]}. %% vim: ft=erlang diff --git a/erts/start_scripts/no_dot_erlang.rel.src b/erts/start_scripts/no_dot_erlang.rel.src index 03b64ebf1a..6208572c00 100644 --- a/erts/start_scripts/no_dot_erlang.rel.src +++ b/erts/start_scripts/no_dot_erlang.rel.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,6 +16,6 @@ %% %% %CopyrightEnd% %% -{release, {"OTP APN 181 01","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, +{release, {"Erlang/OTP","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, [{kernel,"%KERNEL_VSN%"}, {stdlib,"%STDLIB_VSN%"}]}. diff --git a/erts/start_scripts/start_all_example.rel.src b/erts/start_scripts/start_all_example.rel.src index 581eb2eb0b..2a1cabe7bb 100644 --- a/erts/start_scripts/start_all_example.rel.src +++ b/erts/start_scripts/start_all_example.rel.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,7 +16,7 @@ %% %% %CopyrightEnd% %% -{release, {"OTP APN 181 01","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, +{release, {"Erlang/OTP","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, [{kernel,"%KERNEL_VSN%"}, {stdlib,"%STDLIB_VSN%"}, {sasl, "%SASL_VSN%"}, diff --git a/erts/start_scripts/start_clean.rel.src b/erts/start_scripts/start_clean.rel.src index d2df422c51..e229721e36 100644 --- a/erts/start_scripts/start_clean.rel.src +++ b/erts/start_scripts/start_clean.rel.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,6 +16,6 @@ %% %% %CopyrightEnd% %% -{release, {"OTP APN 181 01","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, +{release, {"Erlang/OTP","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, [{kernel,"%KERNEL_VSN%"}, {stdlib,"%STDLIB_VSN%"}]}. diff --git a/erts/start_scripts/start_sasl.rel.src b/erts/start_scripts/start_sasl.rel.src index e521e8df91..e68a34af76 100644 --- a/erts/start_scripts/start_sasl.rel.src +++ b/erts/start_scripts/start_sasl.rel.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,7 +16,7 @@ %% %% %CopyrightEnd% %% -{release, {"OTP APN 181 01","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, +{release, {"Erlang/OTP","%SYS_VSN%"}, {erts, "%ERTS_VSN%"}, [{kernel,"%KERNEL_VSN%"}, {stdlib,"%STDLIB_VSN%"}, {sasl, "%SASL_VSN%"}]}. diff --git a/erts/test/otp_SUITE.erl b/erts/test/otp_SUITE.erl index 8e4a1a4b1c..1fb452501f 100644 --- a/erts/test/otp_SUITE.erl +++ b/erts/test/otp_SUITE.erl @@ -24,7 +24,7 @@ -export([undefined_functions/1,deprecated_not_in_obsolete/1, obsolete_but_not_deprecated/1,call_to_deprecated/1, call_to_size_1/1,strong_components/1, - erl_file_encoding/1,xml_file_encoding/1]). + erl_file_encoding/1,xml_file_encoding/1,runtime_dependencies/1]). -include_lib("test_server/include/test_server.hrl"). @@ -36,7 +36,8 @@ all() -> [undefined_functions, deprecated_not_in_obsolete, obsolete_but_not_deprecated, call_to_deprecated, call_to_size_1, strong_components, - erl_file_encoding, xml_file_encoding]. + erl_file_encoding, xml_file_encoding, + runtime_dependencies]. groups() -> []. @@ -380,6 +381,63 @@ is_bad_encoding(File) -> true end. +runtime_dependencies(Config) -> + %% Verify that (at least) OTP application runtime dependencies found + %% by xref are listed in the runtime_dependencies field of the .app file + %% of each application. + Server = ?config(xref_server, Config), + {ok, AE} = xref:q(Server, "AE"), + SAE = lists:keysort(1, AE), + {AppDep, AppDeps} = lists:foldl(fun ({App, App}, Acc) -> + Acc; + ({App, Dep}, {undefined, []}) -> + {{App, [Dep]}, []}; + ({App, Dep}, {{App, Deps}, AppDeps}) -> + {{App, [Dep|Deps]}, AppDeps}; + ({App, Dep}, {AppDep, AppDeps}) -> + {{App, [Dep]}, [AppDep | AppDeps]} + end, + {undefined, []}, + SAE), + [] = check_apps_deps([AppDep|AppDeps]), + ok. + +have_rdep(_App, [], _Dep) -> + false; +have_rdep(App, [RDep | RDeps], Dep) -> + [AppStr, _VsnStr] = string:tokens(RDep, "-"), + case Dep == list_to_atom(AppStr) of + true -> + io:format("~p -> ~s~n", [App, RDep]), + true; + false -> + have_rdep(App, RDeps, Dep) + end. + +check_app_deps(_App, _AppFile, _AFDeps, []) -> + []; +check_app_deps(App, AppFile, AFDeps, [XRDep | XRDeps]) -> + ResOtherDeps = check_app_deps(App, AppFile, AFDeps, XRDeps), + case have_rdep(App, AFDeps, XRDep) of + true -> + ResOtherDeps; + false -> + [{missing_runtime_dependency, AppFile, XRDep} | ResOtherDeps] + end. + +check_apps_deps([]) -> + []; +check_apps_deps([{App, Deps}|AppDeps]) -> + ResOtherApps = check_apps_deps(AppDeps), + AppFile = code:where_is_file(atom_to_list(App) ++ ".app"), + {ok,[{application, App, Info}]} = file:consult(AppFile), + case lists:keyfind(runtime_dependencies, 1, Info) of + {runtime_dependencies, RDeps} -> + check_app_deps(App, AppFile, RDeps, Deps) ++ ResOtherApps; + false -> + [{missing_runtime_dependencies_key, AppFile} | ResOtherApps] + end. + %%% %%% Common help functions. %%% diff --git a/lib/asn1/src/asn1.app.src b/lib/asn1/src/asn1.app.src index f2ee8deb75..02cbba0f10 100644 --- a/lib/asn1/src/asn1.app.src +++ b/lib/asn1/src/asn1.app.src @@ -10,5 +10,6 @@ asn1db ]}, {env, []}, - {applications, [kernel, stdlib]} + {applications, [kernel, stdlib]}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl index 9ec43197bf..8470e5a1b4 100644 --- a/lib/asn1/src/asn1ct.erl +++ b/lib/asn1/src/asn1ct.erl @@ -566,6 +566,8 @@ get_pos_of_def(#pobjectdef{pos=Pos}) -> Pos; get_pos_of_def(#pobjectsetdef{pos=Pos}) -> Pos; +get_pos_of_def(#'Externaltypereference'{pos=Pos}) -> + Pos; get_pos_of_def(#'Externalvaluereference'{pos=Pos}) -> Pos; get_pos_of_def(_) -> diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl index b9f2cb876a..e788aa5c6c 100644 --- a/lib/asn1/src/asn1ct_check.erl +++ b/lib/asn1/src/asn1ct_check.erl @@ -270,46 +270,30 @@ check_exports(S,Module = #module{}) -> end end. -check_imports(S,Module = #module{ }) -> - case Module#module.imports of - {imports,[]} -> - []; - {imports,ImportList} when is_list(ImportList) -> - check_imports2(S,ImportList,[]); - _ -> - [] - end. -check_imports2(_S,[],Acc) -> +check_imports(S, #module{imports={imports,Imports}}) -> + check_imports_1(S, Imports, []). + +check_imports_1(_S, [], Acc) -> Acc; -check_imports2(S,[#'SymbolsFromModule'{symbols=Imports,module=ModuleRef}|SFMs],Acc) -> - NameOfDef = - fun(#'Externaltypereference'{type=N}) -> N; - (#'Externalvaluereference'{value=N}) -> N - end, - Module = NameOfDef(ModuleRef), - Refs = [{M,R}||{{M,_},R} <- [{catch get_referenced_type(S,Ref),Ref}||Ref <- Imports]], - {Illegal,Other} = lists:splitwith(fun({error,_}) -> true;(_) -> false end, - Refs), - ChainedRefs = [R||{M,R} <- Other, M =/= Module], - IllegalRefs = [R||{error,R} <- Illegal] ++ - [R||{M,R} <- ChainedRefs, - ok =/= chained_import(S,Module,M,NameOfDef(R))], - ReportError = - fun(Ref) -> - NewS=S#state{type=Ref,tname=NameOfDef(Ref)}, - error({import,"imported undefined entity",NewS}) - end, - check_imports2(S,SFMs,[ReportError(Err)||Err <- IllegalRefs]++Acc). +check_imports_1(S, [#'SymbolsFromModule'{symbols=Imports,module=ModuleRef}|SFMs], Acc0) -> + Module = name_of_def(ModuleRef), + Refs0 = [{catch get_referenced_type(S, Ref),Ref} || Ref <- Imports], + Refs = [{M,R} || {{M,_},R} <- Refs0], + {Illegal,Other} = lists:splitwith(fun({error,_}) -> true; + (_) -> false + end, Refs), + ChainedRefs = [R || {M,R} <- Other, M =/= Module], + IllegalRefs = [R || {error,R} <- Illegal] ++ + [R || {M,R} <- ChainedRefs, + ok =/= chained_import(S, Module, M, name_of_def(R))], + Acc = [return_asn1_error(S, Ref, {undefined_import,name_of_def(Ref),Module}) || + Ref <- IllegalRefs] ++ Acc0, + check_imports_1(S, SFMs, Acc). chained_import(S,ImpMod,DefMod,Name) -> %% Name is a referenced structure that is not defined in ImpMod, %% but must be present in the Imports list of ImpMod. The chain of %% imports of Name must end in DefMod. - NameOfDef = - fun(#'Externaltypereference'{type=N}) -> N; - (#'Externalvaluereference'{value=N}) -> N; - (Other) -> Other - end, GetImports = fun(_M_) -> case asn1_db:dbget(_M_,'MODULE') of @@ -321,9 +305,9 @@ chained_import(S,ImpMod,DefMod,Name) -> FindNameInImports = fun([],N,_) -> {no_mod,N}; ([#'SymbolsFromModule'{symbols=Imports,module=ModuleRef}|SFMs],N,F) -> - case [NameOfDef(X)||X <- Imports, NameOfDef(X) =:= N] of + case [name_of_def(X) || X <- Imports, name_of_def(X) =:= N] of [] -> F(SFMs,N,F); - [N] -> {NameOfDef(ModuleRef),N} + [N] -> {name_of_def(ModuleRef),N} end end, case GetImports(ImpMod) of @@ -1567,13 +1551,13 @@ check_defaultfields(S, Fields, ClassFields) -> [] -> ok; [_|_]=Invalid -> - throw(asn1_error(S, T, {invalid_fields,Invalid,Obj})) + asn1_error(S, T, {invalid_fields,Invalid,Obj}) end, case ordsets:subtract(Mandatory, Present) of [] -> check_defaultfields_1(S, Fields, ClassFields, []); [_|_]=Missing -> - throw(asn1_error(S, T, {missing_mandatory_fields,Missing,Obj})) + asn1_error(S, T, {missing_mandatory_fields,Missing,Obj}) end. check_defaultfields_1(_S, [], _ClassFields, Acc) -> @@ -2614,7 +2598,7 @@ normalize_octetstring(S,Value,CType) -> normalize_octetstring(S,String,CType); _ -> Item = S#state.value, - throw(asn1_error(S, Item, illegal_octet_string_value)) + asn1_error(S, Item, illegal_octet_string_value) end. normalize_objectidentifier(S, Value) -> @@ -2645,7 +2629,7 @@ lookup_enum_value(S, Id, NNL) when is_atom(Id) -> {_,_}=Ret -> Ret; false -> - throw(asn1_error(S, S#state.value, {undefined,Id})) + asn1_error(S, S#state.value, {undefined,Id}) end. normalize_choice(S,{'CHOICE',{C,V}},CType,NameList) when is_atom(C) -> @@ -3084,7 +3068,7 @@ check_type(S=#state{recordtopname=TopName},Type,Ts) when is_record(Ts,type) -> merge_tags(Tag,?TAG_PRIMITIVE(?N_INTEGER))}; {'INTEGER',NamedNumberList} -> - TempNewDef#newt{type={'INTEGER',check_integer(S,NamedNumberList,Constr)}, + TempNewDef#newt{type={'INTEGER',check_integer(S,NamedNumberList)}, tag= merge_tags(Tag,?TAG_PRIMITIVE(?N_INTEGER))}; 'REAL' -> @@ -3092,8 +3076,7 @@ check_type(S=#state{recordtopname=TopName},Type,Ts) when is_record(Ts,type) -> TempNewDef#newt{tag=merge_tags(Tag,?TAG_PRIMITIVE(?N_REAL))}; {'BIT STRING',NamedNumberList} -> - NewL = check_bitstring(S,NamedNumberList,Constr), -%% erlang:display({asn1ct_check,NamedNumberList,NewL}), + NewL = check_bitstring(S, NamedNumberList), TempNewDef#newt{type={'BIT STRING',NewL}, tag= merge_tags(Tag,?TAG_PRIMITIVE(?N_BIT_STRING))}; @@ -4910,73 +4893,46 @@ imported1(Name, end; imported1(_Name,[]) -> false. - -check_integer(_S,[],_C) -> +%% Check the named number list for an INTEGER or a BIT STRING. +check_named_number_list(_S, []) -> []; -check_integer(S,NamedNumberList,_C) -> - case [X || X <- NamedNumberList, tuple_size(X) =:= 2] of - NamedNumberList -> - %% An already checked integer with NamedNumberList - NamedNumberList; - _ -> - case check_unique(NamedNumberList,2) of - [] -> - check_int(S,NamedNumberList,[]); - L when is_list(L) -> - error({type,{duplicates,L},S}), - unchanged - end +check_named_number_list(_S, [{_,_}|_]=NNL) -> + %% The named number list has already been checked. + NNL; +check_named_number_list(S, NNL0) -> + %% Check that the names are unique. + T = S#state.type, + case check_unique(NNL0, 2) of + [] -> + NNL1 = [{Id,resolve_valueref(S, Val)} || {'NamedNumber',Id,Val} <- NNL0], + NNL = lists:keysort(2, NNL1), + case check_unique(NNL, 2) of + [] -> + NNL; + [Val|_] -> + asn1_error(S, T, {value_reused,Val}) + end; + [H|_] -> + asn1_error(S, T, {namelist_redefinition,H}) end. - -check_int(S,[{'NamedNumber',Id,Num}|T],Acc) when is_integer(Num) -> - check_int(S,T,[{Id,Num}|Acc]); -check_int(S,[{'NamedNumber',Id,{identifier,_,Name}}|T],Acc) -> - Val = dbget_ex(S,S#state.mname,Name), - check_int(S,[{'NamedNumber',Id,Val#valuedef.value}|T],Acc); -check_int(S,[{'NamedNumber',Id,{'Externalvaluereference',_,Mod,Name}}|T],Acc) -> - Val = dbget_ex(S,Mod,Name), - check_int(S,[{'NamedNumber',Id,Val#valuedef.value}|T],Acc); -check_int(_S,[],Acc) -> - lists:keysort(2,Acc). +resolve_valueref(S, #'Externalvaluereference'{module=Mod,value=Name}) -> + dbget_ex(S, Mod, Name); +resolve_valueref(_, Val) when is_integer(Val) -> + Val. -check_real(_S,_Constr) -> - ok. +check_integer(S, NNL) -> + check_named_number_list(S, NNL). -check_bitstring(_S,[],_Constr) -> - []; -check_bitstring(S,NamedNumberList,_Constr) -> - case check_unique(NamedNumberList,2) of - [] -> - check_bitstr(S,NamedNumberList,[]); - L when is_list(L) -> - error({type,{duplicates,L},S}), - unchanged - end. +check_bitstring(S, NNL0) -> + NNL = check_named_number_list(S, NNL0), + _ = [asn1_error(S, S#state.type, {invalid_bit_number,Bit}) || + {_,Bit} <- NNL, Bit < 0], + NNL. -check_bitstr(S,[{'NamedNumber',Id,Num}|T],Acc)when is_integer(Num) -> - check_bitstr(S,T,[{Id,Num}|Acc]); -check_bitstr(S,[{'NamedNumber',Id,Name}|T],Acc) when is_atom(Name) -> -%%check_bitstr(S,[{'NamedNumber',Id,{identifier,_,Name}}|T],Acc) -> -%% io:format("asn1ct_check:check_bitstr/3 hej hop ~w~n",[Name]), - Val = dbget_ex(S,S#state.mname,Name), -%% io:format("asn1ct_check:check_bitstr/3: ~w~n",[Val]), - check_bitstr(S,[{'NamedNumber',Id,Val#valuedef.value}|T],Acc); -check_bitstr(S,[],Acc) -> - case check_unique(Acc,2) of - [] -> - lists:keysort(2,Acc); - L when is_list(L) -> - error({type,{duplicate_values,L},S}), - unchanged - end; -%% When a BIT STRING already is checked, for instance a COMPONENTS OF S -%% where S is a sequence that has a component that is a checked BS, the -%% NamedNumber list is a list of {atom(),integer()} elements. -check_bitstr(S,[El={Id,Num}|Rest],Acc) when is_atom(Id),is_integer(Num) -> - check_bitstr(S,Rest,[El|Acc]). - +check_real(_S,_Constr) -> + ok. %% Check INSTANCE OF %% check that DefinedObjectClass is of TYPE-IDENTIFIER class @@ -4987,20 +4943,16 @@ check_instance_of(S,DefinedObjectClass,Constraint) -> check_type_identifier(S,DefinedObjectClass), iof_associated_type(S,Constraint). - -check_type_identifier(_S,'TYPE-IDENTIFIER') -> - ok; -check_type_identifier(S,Eref=#'Externaltypereference'{}) -> - case get_referenced_type(S,Eref) of - {_,#classdef{name='TYPE-IDENTIFIER'}} -> ok; - {_,#classdef{typespec=NextEref}} - when is_record(NextEref,'Externaltypereference') -> - check_type_identifier(S,NextEref); +check_type_identifier(S, Eref=#'Externaltypereference'{type=Class}) -> + case get_referenced_type(S, Eref) of + {_,#classdef{name='TYPE-IDENTIFIER'}} -> + ok; + {_,#classdef{typespec=#'Externaltypereference'{}=NextEref}} -> + check_type_identifier(S, NextEref); {_,TD=#typedef{typespec=#type{def=#'Externaltypereference'{}}}} -> - check_type_identifier(S,(TD#typedef.typespec)#type.def); - Err -> - error({type,{"object set in type INSTANCE OF " - "not of class TYPE-IDENTIFIER",Eref,Err},S}) + check_type_identifier(S, (TD#typedef.typespec)#type.def); + _ -> + asn1_error(S, S#state.type, {illegal_instance_of,Class}) end. iof_associated_type(S,[]) -> @@ -5130,9 +5082,6 @@ check_enumerated(S,NamedNumberList,_Constr) -> %% the latter is returned if the ENUMERATION contains EXTENSIONMARK check_enum(S,[{'NamedNumber',Id,Num}|T],Acc1,Acc2,Root) when is_integer(Num) -> check_enum(S,T,[{Id,Num}|Acc1],Acc2,Root); -check_enum(S,[{'NamedNumber',Id,{identifier,_,Name}}|T],Acc1,Acc2,Root) -> - Val = dbget_ex(S,S#state.mname,Name), - check_enum(S,[{'NamedNumber',Id,Val#valuedef.value}|T],Acc1,Acc2,Root); check_enum(S,['EXTENSIONMARK'|T],Acc1,Acc2,_Root) -> NewAcc2 = lists:keysort(2,Acc1), NewList = enum_number(lists:reverse(Acc2),NewAcc2,0,[],[]), @@ -6748,7 +6697,7 @@ storeindb(#state{mname=Module}=S, [H|T], Errors) -> storeindb(S, T, Errors); Prev -> PrevLine = asn1ct:get_pos_of_def(Prev), - {error,Error} = asn1_error(S, H, {already_defined,Name,PrevLine}), + Error = return_asn1_error(S, H, {already_defined,Name,PrevLine}), storeindb(S, T, [Error|Errors]) end; storeindb(_, [], []) -> @@ -6795,22 +6744,37 @@ findtypes_and_values([],Tacc,Vacc,Pacc,Cacc,Oacc,OSacc) -> {lists:reverse(Tacc),lists:reverse(Vacc),lists:reverse(Pacc), lists:reverse(Cacc),lists:reverse(Oacc),lists:reverse(OSacc)}. -asn1_error(#state{mname=Where}, Item, Error) -> +return_asn1_error(#state{mname=Where}, Item, Error) -> Pos = asn1ct:get_pos_of_def(Item), - {error,{structured_error,{Where,Pos},?MODULE,Error}}. + {structured_error,{Where,Pos},?MODULE,Error}. + +asn1_error(S, Item, Error) -> + throw({error,return_asn1_error(S, Item, Error)}). format_error({already_defined,Name,PrevLine}) -> io_lib:format("the name ~p has already been defined at line ~p", [Name,PrevLine]); +format_error({illegal_instance_of,Class}) -> + io_lib:format("using INSTANCE OF on class '~s' is illegal, " + "because INSTANCE OF may only be used on the class TYPE-IDENTFIER", + [Class]); format_error(illegal_octet_string_value) -> "expecting a bstring or an hstring as value for an OCTET STRING"; format_error({invalid_fields,Fields,Obj}) -> io_lib:format("invalid ~s in ~p", [format_fields(Fields),Obj]); +format_error({invalid_bit_number,Bit}) -> + io_lib:format("the bit number '~p' is invalid", [Bit]); format_error({missing_mandatory_fields,Fields,Obj}) -> io_lib:format("missing mandatory ~s in ~p", [format_fields(Fields),Obj]); +format_error({namelist_redefinition,Name}) -> + io_lib:format("the name '~s' can not be redefined", [Name]); format_error({undefined,Name}) -> io_lib:format("'~s' is referenced, but is not defined", [Name]); +format_error({undefined_import,Ref,Module}) -> + io_lib:format("'~s' is not exported from ~s", [Ref,Module]); +format_error({value_reused,Val}) -> + io_lib:format("the value '~p' is used more than once", [Val]); format_error(Other) -> io_lib:format("~p", [Other]). @@ -6826,14 +6790,6 @@ error({export,Msg,#state{mname=Mname,type=Ref,tname=Typename}}) -> Pos = Ref#'Externaltypereference'.pos, io:format("asn1error:~p:~p:~p~n~p~n",[Pos,Mname,Typename,Msg]), {error,{export,Pos,Mname,Typename,Msg}}; -error({import,Msg,#state{mname=Mname,type=Ref,tname=Typename}}) -> - PosOfDef = - fun(#'Externaltypereference'{pos=P}) -> P; - (#'Externalvaluereference'{pos=P}) -> P - end, - Pos = PosOfDef(Ref), - io:format("asn1error:~p:~p:~p~n~p~n",[Pos,Mname,Typename,Msg]), - {error,{import,Pos,Mname,Typename,Msg}}; % error({type,{Msg1,Msg2},#state{mname=Mname,type=Type,tname=Typename}}) % when is_record(Type,typedef) -> % io:format("asn1error:~p:~p:~p ~p~n", @@ -7134,3 +7090,6 @@ check_fold(S, [H|T], Check) -> [Error|check_fold(S, T, Check)] end; check_fold(_, [], Check) when is_function(Check, 3) -> []. + +name_of_def(#'Externaltypereference'{type=N}) -> N; +name_of_def(#'Externalvaluereference'{value=N}) -> N. diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl index 4707e517b4..44b050e59d 100644 --- a/lib/asn1/src/asn1ct_gen.erl +++ b/lib/asn1/src/asn1ct_gen.erl @@ -1125,7 +1125,22 @@ pgen_info() -> open_hrl(OutFile,Module) -> File = lists:concat([OutFile,".hrl"]), _ = open_output_file(File), - gen_hrlhead(Module). + gen_hrlhead(Module), + Protector = hrl_protector(OutFile), + emit(["-ifndef(",Protector,").\n", + "-define(",Protector,", true).\n" + "\n"]). + +hrl_protector(OutFile) -> + BaseName = filename:basename(OutFile), + P = "_" ++ string:to_upper(BaseName) ++ "_HRL_", + [if + $A =< C, C =< $Z -> C; + $a =< C, C =< $a -> C; + $0 =< C, C =< $9 -> C; + true -> $_ + end || C <- P]. + %% EMIT functions ************************ %% *************************************** @@ -1232,6 +1247,8 @@ pgen_hrl(Erules,Module,TypeOrVal,Options,_Indent) -> 0 -> 0; Y -> + Protector = hrl_protector(get(outfile)), + emit(["-endif. %% ",Protector,"\n"]), close_output_file(), asn1ct:verbose("--~p--~n", [{generated,lists:concat([get(outfile),".hrl"])}], diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index d438300596..782217ed2d 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -813,10 +813,10 @@ testExport(Config) -> testImport(Config) -> test(Config, fun testImport/3). testImport(Config, Rule, Opts) -> - {error, _} = asn1ct:compile(filename:join(?config(data_dir, Config), - "ImportsFrom"), - [Rule, {outdir, ?config(priv_dir, Config)} - |Opts]). + Files = ["ImportsFrom","ImportsFrom2","ImportsFrom3"], + asn1_test_lib:compile_all(Files, Config, [Rule|Opts]), + 42 = 'ImportsFrom':i(), + ok. testMegaco(Config) -> test(Config, fun testMegaco/3). testMegaco(Config, Rule, Opts) -> diff --git a/lib/asn1/test/asn1_SUITE_data/INSTANCEOF.asn1 b/lib/asn1/test/asn1_SUITE_data/INSTANCEOF.asn1 index 8c4f3a8f7e..b4ea943040 100644 --- a/lib/asn1/test/asn1_SUITE_data/INSTANCEOF.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/INSTANCEOF.asn1 @@ -16,7 +16,9 @@ Names ::= SEQUENCE { thirdName [2] INSTANCE OF OTHER-NAME ({TI}) } -OTHER-NAME ::= TYPE-IDENTIFIER +OTHER-NAME ::= YET-ANOTHER-NAME + +YET-ANOTHER-NAME ::= TYPE-IDENTIFIER TI OTHER-NAME ::= {{INTEGER IDENTIFIED BY {2 4}} | {Seq IDENTIFIED BY {2 3 4}} | diff --git a/lib/asn1/test/asn1_SUITE_data/ImportsFrom.asn1 b/lib/asn1/test/asn1_SUITE_data/ImportsFrom.asn1 index 896a35d627..32b8f75dde 100644 --- a/lib/asn1/test/asn1_SUITE_data/ImportsFrom.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/ImportsFrom.asn1 @@ -1,16 +1,8 @@ -ImportsFrom DEFINITIONS ::= - +ImportsFrom DEFINITIONS AUTOMATIC TAGS ::= BEGIN -IMPORTS -Type1, Type2, Type3 -FROM RemoteFile1 objid -val1, val2, val3 -FROM RemoteFile2; - -objid OBJECT IDENTIFIER ::= {joint-iso-ccitt(2) remote-operations(4) notation(0)} - -LocalType ::= INTEGER +IMPORTS Int FROM ImportsFrom2; +i Int ::= 42 END diff --git a/lib/asn1/test/asn1_SUITE_data/ImportsFrom2.asn1 b/lib/asn1/test/asn1_SUITE_data/ImportsFrom2.asn1 new file mode 100644 index 0000000000..b0c29d24ae --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/ImportsFrom2.asn1 @@ -0,0 +1,7 @@ +ImportsFrom2 DEFINITIONS AUTOMATIC TAGS ::= +BEGIN +IMPORTS Int FROM ImportsFrom3; + +LocalDef ::= OCTET STRING + +END diff --git a/lib/asn1/test/asn1_SUITE_data/ImportsFrom3.asn1 b/lib/asn1/test/asn1_SUITE_data/ImportsFrom3.asn1 new file mode 100644 index 0000000000..ca27b20697 --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/ImportsFrom3.asn1 @@ -0,0 +1,4 @@ +ImportsFrom3 DEFINITIONS AUTOMATIC TAGS ::= +BEGIN + Int ::= INTEGER (0..63) +END diff --git a/lib/asn1/test/error_SUITE.erl b/lib/asn1/test/error_SUITE.erl index 930b44cea6..8a0414708d 100644 --- a/lib/asn1/test/error_SUITE.erl +++ b/lib/asn1/test/error_SUITE.erl @@ -19,7 +19,8 @@ -module(error_SUITE). -export([suite/0,all/0,groups/0, - already_defined/1,enumerated/1,objects/1,values/1]). + already_defined/1,bitstrings/1,enumerated/1, + imports/1,instance_of/1,integers/1,objects/1,values/1]). -include_lib("test_server/include/test_server.hrl"). @@ -31,7 +32,11 @@ all() -> groups() -> [{p,parallel(), [already_defined, + bitstrings, enumerated, + imports, + instance_of, + integers, objects, values]}]. @@ -70,6 +75,23 @@ already_defined(Config) -> } = run(P, Config), ok. +bitstrings(Config) -> + M = 'Bitstrings', + P = {M, + <<"Bitstrings DEFINITIONS AUTOMATIC TAGS ::= BEGIN\n" + " Bs1 ::= BIT STRING {a(1), a(1)}\n" + " Bs2 ::= BIT STRING {a(1), b(2), a(3)}\n" + " Bs3 ::= BIT STRING {x(1), y(1)}\n" + " Bs4 ::= BIT STRING {x(-1), y(0)}\n" + "END\n">>}, + {error, + [{structured_error,{M,2},asn1ct_check,{namelist_redefinition,a}}, + {structured_error,{M,3},asn1ct_check,{namelist_redefinition,a}}, + {structured_error,{M,4},asn1ct_check,{value_reused,1}}, + {structured_error,{M,5},asn1ct_check,{invalid_bit_number,-1}} + ]} = run(P, Config), + ok. + enumerated(Config) -> M = 'Enumerated', P = {M, @@ -98,6 +120,63 @@ enumerated(Config) -> } = run(P, Config), ok. +imports(Config) -> + Ext = 'ExternalModule', + ExtP = {Ext, + <<"ExternalModule DEFINITIONS AUTOMATIC TAGS ::= BEGIN\n" + "END\n">>}, + ok = run(ExtP, Config), + + M = 'Imports', + P = {M, + <<"Imports DEFINITIONS AUTOMATIC TAGS ::= BEGIN\n" + "IMPORTS NotDefined FROM ExternalModule\n" + "X FROM UndefinedModule objid\n" + "Y, Z FROM UndefinedModule2;\n" + "objid OBJECT IDENTIFIER ::= {joint-iso-ccitt(2) remote-operations(4)\n" + " notation(0)}\n" + "END\n">>}, + {error,[{structured_error,{M,2},asn1ct_check, + {undefined_import,'NotDefined','ExternalModule'}}, + {structured_error,{M,3},asn1ct_check,{undefined_import,'X','UndefinedModule'}}, + {structured_error,{M,4},asn1ct_check,{undefined_import,'Y','UndefinedModule2'}}, + {structured_error,{M,4},asn1ct_check,{undefined_import,'Z','UndefinedModule2'}} + ]} = run(P, Config), + ok. + +instance_of(Config) -> + M = 'InstanceOf', + P = {M, + <<"InstanceOf DEFINITIONS AUTOMATIC TAGS ::= BEGIN\n" + "XX ::= INSTANCE OF CL ({TI})\n" + "CL ::= CLASS {\n" + "&id INTEGER,\n" + "&Type\n" + "}\n" + "o1 CL ::= {&id 1, &Type OCTET STRING}\n" + "TI CL ::= { o1 }\n" + "END\n">>}, + {error, + [{structured_error,{M,2},asn1ct_check,{illegal_instance_of,'CL'}} + ]} = run(P, Config), + ok. + +integers(Config) -> + M = 'Integers', + P = {M, + <<"Integers DEFINITIONS AUTOMATIC TAGS ::= BEGIN\n" + " Int1 ::= INTEGER {a(1), a(1)}\n" + " Int2 ::= INTEGER {a(1), b(2), a(3)}\n" + " Int3 ::= INTEGER {x(1), y(1)}\n" + "END\n">>}, + {error, + [{structured_error,{M,2},asn1ct_check,{namelist_redefinition,a}}, + {structured_error,{M,3},asn1ct_check,{namelist_redefinition,a}}, + {structured_error,{M,4},asn1ct_check,{value_reused,1}} + ]} = run(P, Config), + ok. + + objects(Config) -> M = 'Objects', P = {M, diff --git a/lib/asn1/vsn.mk b/lib/asn1/vsn.mk index 153c64ebdd..1f16f31f6b 100644 --- a/lib/asn1/vsn.mk +++ b/lib/asn1/vsn.mk @@ -1,2 +1,2 @@ #next version number to use is 2.0 -ASN1_VSN = 2.0.4 +ASN1_VSN = 3.0 diff --git a/lib/common_test/src/common_test.app.src b/lib/common_test/src/common_test.app.src index 18c1dec784..e28751fb59 100644 --- a/lib/common_test/src/common_test.app.src +++ b/lib/common_test/src/common_test.app.src @@ -62,5 +62,10 @@ ct_master, ct_master_logs]}, {applications, [kernel,stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies,["xmerl-1.3.7","webtool-0.8.10","tools-2.6.14", + "test_server-3.7","stdlib-2.0","ssh-3.0.1", + "snmp-4.25.1","sasl-2.4","runtime_tools-1.8.14", + "kernel-3.0","inets-5.10","erts-6.0", + "debugger-4.0","crypto-3.3","compiler-5.0"]}]}. diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl index d733df27dc..cff02a46d9 100644 --- a/lib/common_test/src/ct_conn_log_h.erl +++ b/lib/common_test/src/ct_conn_log_h.erl @@ -204,13 +204,8 @@ pretty_head({{{Y,Mo,D},{H,Mi,S}},MicroS},ConnMod,Text0) -> micro2milli(MicroS)]). pretty_title(#conn_log{client=Client}=Info) -> - case actionstr(Info) of - {no_server,Action} -> - io_lib:format("= Client ~w ~s ",[Client,Action]); - Action -> - io_lib:format("= Client ~w ~s ~ts ",[Client,Action, - serverstr(Info)]) - end. + io_lib:format("= Client ~w ~s ~ts ", + [Client,actionstr(Info),serverstr(Info)]). actionstr(#conn_log{action=send}) -> "----->"; actionstr(#conn_log{action=cmd}) -> "----->"; diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 63bfea68c4..7d577462b0 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -249,8 +249,8 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) -> end end. -ct_suite_init(Suite, Func, PostInitHook, Config) when is_list(Config) -> - case ct_hooks:init_tc(Suite, Func, Config) of +ct_suite_init(Suite, FuncSpec, PostInitHook, Config) when is_list(Config) -> + case ct_hooks:init_tc(Suite, FuncSpec, Config) of NewConfig when is_list(NewConfig) -> PostInitHookResult = do_post_init_hook(PostInitHook, NewConfig), {ok, [PostInitHookResult ++ NewConfig]}; @@ -660,10 +660,7 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> ct_util:delete_testdata(comment), ct_util:delete_suite_data(last_saved_config), - FuncSpec = case group_or_func(Func,Args) of - {_,_GroupName,_} = Group -> Group; - _ -> Func - end, + FuncSpec = group_or_func(Func,Args), {Result1,FinalNotify} = case ct_hooks:end_tc( diff --git a/lib/common_test/src/ct_gen_conn.erl b/lib/common_test/src/ct_gen_conn.erl index 078d6b1a44..56082086f6 100644 --- a/lib/common_test/src/ct_gen_conn.erl +++ b/lib/common_test/src/ct_gen_conn.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2013. All Rights Reserved. +%% Copyright Ericsson AB 2003-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -307,7 +307,8 @@ call(Pid, Msg, Timeout) -> end. return({To,Ref},Result) -> - To ! {Ref, Result}. + To ! {Ref, Result}, + ok. init_gen(Parent,Opts) -> process_flag(trap_exit,true), @@ -344,7 +345,7 @@ loop(Opts) -> link(NewPid), put(conn_pid,NewPid), loop(Opts#gen_opts{conn_pid=NewPid, - cb_state=NewState}); + cb_state=NewState}); Error -> ct_util:unregister_connection(self()), log("Reconnect failed. Giving up!", diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index 2e667030a9..df4c98d9d1 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -64,11 +64,16 @@ terminate(Hooks) -> %% @doc Called as each test case is started. This includes all configuration %% tests. --spec init_tc(Mod :: atom(), Func :: atom(), Args :: list()) -> +-spec init_tc(Mod :: atom(), + FuncSpec :: atom() | + {ConfigFunc :: init_per_group | end_per_group, + GroupName :: atom(), + Properties :: list()}, + Args :: list()) -> NewConfig :: proplists:proplist() | - {skip, Reason :: term()} | - {auto_skip, Reason :: term()} | - {fail, Reason :: term()}. + {skip, Reason :: term()} | + {auto_skip, Reason :: term()} | + {fail, Reason :: term()}. init_tc(Mod, init_per_suite, Config) -> Info = try proplists:get_value(ct_hooks, Mod:suite(),[]) of @@ -82,8 +87,8 @@ init_tc(Mod, init_per_suite, Config) -> call(fun call_generic/3, Config ++ Info, [pre_init_per_suite, Mod]); init_tc(Mod, end_per_suite, Config) -> call(fun call_generic/3, Config, [pre_end_per_suite, Mod]); -init_tc(Mod, {init_per_group, GroupName, Opts}, Config) -> - maybe_start_locker(Mod, GroupName, Opts), +init_tc(Mod, {init_per_group, GroupName, Properties}, Config) -> + maybe_start_locker(Mod, GroupName, Properties), call(fun call_generic/3, Config, [pre_init_per_group, GroupName]); init_tc(_Mod, {end_per_group, GroupName, _}, Config) -> call(fun call_generic/3, Config, [pre_end_per_group, GroupName]); @@ -93,15 +98,18 @@ init_tc(_Mod, TC, Config) -> %% @doc Called as each test case is completed. This includes all configuration %% tests. -spec end_tc(Mod :: atom(), - Func :: atom(), + FuncSpec :: atom() | + {ConfigFunc :: init_per_group | end_per_group, + GroupName :: atom(), + Properties :: list()}, Args :: list(), Result :: term(), - Resturn :: term()) -> + Return :: term()) -> NewConfig :: proplists:proplist() | - {skip, Reason :: term()} | - {auto_skip, Reason :: term()} | - {fail, Reason :: term()} | - ok | '$ct_no_change'. + {skip, Reason :: term()} | + {auto_skip, Reason :: term()} | + {fail, Reason :: term()} | + ok | '$ct_no_change'. end_tc(Mod, init_per_suite, Config, _Result, Return) -> call(fun call_generic/3, Return, [post_init_per_suite, Mod, Config], @@ -112,10 +120,10 @@ end_tc(Mod, end_per_suite, Config, Result, _Return) -> end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config], '$ct_no_change'); -end_tc(Mod, {end_per_group, GroupName, Opts}, Config, Result, _Return) -> +end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) -> Res = call(fun call_generic/3, Result, [post_end_per_group, GroupName, Config], '$ct_no_change'), - maybe_stop_locker(Mod, GroupName,Opts), + maybe_stop_locker(Mod, GroupName, Properties), Res; end_tc(_Mod, TC, Config, Result, _Return) -> call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 35920ec1dc..6fc840745d 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -1,7 +1,7 @@ %%---------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2013. All Rights Reserved. +%% Copyright Ericsson AB 2012-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1334,7 +1334,7 @@ handle_data(NewData,#state{connection=Connection,buff=Buff} = State) -> %% first answer P=#pending{tref=TRef,caller=Caller} = lists:last(Pending), - timer:cancel(TRef), + _ = timer:cancel(TRef), Reason1 = {failed_to_parse_received_data,Reason}, ct_gen_conn:return(Caller,{error,Reason1}), lists:delete(P,Pending) @@ -1454,7 +1454,7 @@ decode({Tag,Attrs,_}=E, #state{connection=Connection,pending=Pending}=State) -> {noreply,State#state{hello_status = {error,Reason}}} end; #pending{tref=TRef,caller=Caller} -> - timer:cancel(TRef), + _ = timer:cancel(TRef), case decode_hello(E) of {ok,SessionId,Capabilities} -> ct_gen_conn:return(Caller,ok), @@ -1482,7 +1482,7 @@ decode({Tag,Attrs,_}=E, #state{connection=Connection,pending=Pending}=State) -> case [P || P = #pending{msg_id=undefined,op=undefined} <- Pending] of [#pending{tref=TRef, caller=Caller}] -> - timer:cancel(TRef), + _ = timer:cancel(TRef), ct_gen_conn:return(Caller,E), {noreply,State#state{pending=[]}}; _ -> @@ -1504,7 +1504,7 @@ get_msg_id(Attrs) -> decode_rpc_reply(MsgId,{_,Attrs,Content0}=E,#state{pending=Pending} = State) -> case lists:keytake(MsgId,#pending.msg_id,Pending) of {value, #pending{tref=TRef,op=Op,caller=Caller}, Pending1} -> - timer:cancel(TRef), + _ = timer:cancel(TRef), Content = forward_xmlns_attr(Attrs,Content0), {CallerReply,{ServerReply,State2}} = do_decode_rpc_reply(Op,Content,State#state{pending=Pending1}), @@ -1519,7 +1519,7 @@ decode_rpc_reply(MsgId,{_,Attrs,Content0}=E,#state{pending=Pending} = State) -> msg_id=undefined, op=undefined, caller=Caller}] -> - timer:cancel(TRef), + _ = timer:cancel(TRef), ct_gen_conn:return(Caller,E), {noreply,State#state{pending=[]}}; _ -> @@ -1862,10 +1862,7 @@ ssh_open(#options{host=Host,timeout=Timeout,port=Port,ssh=SshOpts,name=Name}) -> end; {error, Reason} -> ssh:close(CM), - {error,{ssh,could_not_open_channel,Reason}}; - Other -> - %% Bug in ssh?? got {closed,0} here once... - {error,{ssh,unexpected_from_session_channel,Other}} + {error,{ssh,could_not_open_channel,Reason}} end; {error,Reason} -> {error,{ssh,could_not_connect_to_server,Reason}} diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index b4d82a53cf..0a067b3a08 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -281,8 +281,16 @@ open(KeyOrName,ConnType,TargetMod,Extra) -> end, log(undefined,open,"Connecting to ~p(~p)", [KeyOrName,Addr1]), - ct_gen_conn:start(KeyOrName,full_addr(Addr1,ConnType), - {TargetMod,KeepAlive,Extra},?MODULE) + Reconnect = + case ct:get_config({telnet_settings,reconnection_attempts}) of + 0 -> false; + _ -> true + end, + ct_gen_conn:start(full_addr(Addr1,ConnType), + {TargetMod,KeepAlive,Extra}, + ?MODULE, [{name,KeyOrName}, + {reconnect,Reconnect}, + {old,true}]) end. %%%----------------------------------------------------------------- @@ -405,9 +413,11 @@ expect(Connection,Patterns) -> %%% Prompt = string() %%% Tag = term() %%% Opts = [Opt] -%%% Opt = {timeout,Timeout} | repeat | {repeat,N} | sequence | -%%% {halt,HaltPatterns} | ignore_prompt | no_prompt_check -%%% Timeout = integer() +%%% Opt = {idle_timeout,IdleTimeout} | {total_timeout,TotalTimeout} | +%%% repeat | {repeat,N} | sequence | {halt,HaltPatterns} | +%%% ignore_prompt | no_prompt_check +%%% IdleTimeout = infinity | integer() +%%% TotalTimeout = infinity | integer() %%% N = integer() %%% HaltPatterns = Patterns %%% MatchList = [Match] @@ -433,11 +443,16 @@ expect(Connection,Patterns) -> %%% will also include the matched <code>Tag</code>. Else, only %%% <code>RxMatch</code> is returned.</p> %%% -%%% <p>The <code>timeout</code> option indicates that the function +%%% <p>The <code>idle_timeout</code> option indicates that the function %%% shall return if the telnet client is idle (i.e. if no data is -%%% received) for more than <code>Timeout</code> milliseconds. Default +%%% received) for more than <code>IdleTimeout</code> milliseconds. Default %%% timeout is 10 seconds.</p> %%% +%%% <p>The <code>total_timeout</code> option sets a time limit for +%%% the complete expect operation. After <code>TotalTimeout</code> +%%% milliseconds, <code>{error,timeout}</code> is returned. The default +%%% value is <code>infinity</code> (i.e. no time limit).</p> +%%% %%% <p>The function will always return when a prompt is found, unless %%% any of the <code>ignore_prompt</code> or %%% <code>no_prompt_check</code> options are used, in which case it @@ -570,14 +585,14 @@ handle_msg({cmd,Cmd,Timeout},State) -> State#state.buffer, prompt, State#state.prx, - [{timeout,2000}]); + [{idle_timeout,2000}]); {ip,false} -> silent_teln_expect(State#state.name, State#state.teln_pid, State#state.buffer, prompt, State#state.prx, - [{timeout,200}]); + [{idle_timeout,200}]); {ip,true} -> ok end, @@ -601,11 +616,9 @@ handle_msg({cmd,Cmd,Timeout},State) -> end_gen_log(), {Return,State#state{buffer=NewBuffer,prompt=Prompt}}; handle_msg({send,Cmd},State) -> - log(State,send,"Cmd: ~p",[Cmd]), - + log(State,send,"Sending: ~p",[Cmd]), debug_cont_gen_log("Throwing Buffer:",[]), debug_log_lines(State#state.buffer), - case {State#state.type,State#state.prompt} of {ts,_} -> silent_teln_expect(State#state.name, @@ -613,14 +626,14 @@ handle_msg({send,Cmd},State) -> State#state.buffer, prompt, State#state.prx, - [{timeout,2000}]); + [{idle_timeout,2000}]); {ip,false} -> silent_teln_expect(State#state.name, State#state.teln_pid, State#state.buffer, prompt, State#state.prx, - [{timeout,200}]); + [{idle_timeout,200}]); {ip,true} -> ok end, @@ -783,66 +796,61 @@ log(#state{name=Name,teln_pid=TelnPid,host=Host,port=Port}, true -> Name end, Silent = get(silent), - case ct_util:get_testdata({cth_conn_log,?MODULE}) of - Result when Result /= undefined, Result /= silent, Silent /= true -> - {PrintHeader,PreBR} = if Action==undefined -> - {false,""}; - true -> - {true,"\n"} - end, - error_logger:info_report(#conn_log{header=PrintHeader, - client=self(), - conn_pid=TelnPid, - address={Host,Port}, - name=Name1, - action=Action, - module=?MODULE}, - {PreBR++String,Args}); - Result when Result /= undefined -> - ok; - _ when Action == open; Action == close; Action == reconnect; - Action == info; Action == error -> - ct_gen_conn:log(heading(Action,Name1),String,Args); - _ when ForcePrint == false -> - case ct_util:is_silenced(telnet) of - true -> - ok; - false -> - ct_gen_conn:cont_log(String,Args) + + if Action == general_io -> + case ct_util:get_testdata({cth_conn_log,?MODULE}) of + HookMode when HookMode /= undefined, HookMode /= silent, + Silent /= true -> + error_logger:info_report(#conn_log{header=false, + client=self(), + conn_pid=TelnPid, + address={Host,Port}, + name=Name1, + action=Action, + module=?MODULE}, + {String,Args}); + _ -> %% hook inactive or silence requested + ok end; - _ when ForcePrint == true -> - case ct_util:is_silenced(telnet) of - true -> - %% call log/3 now instead of cont_log/2 since - %% start_gen_log/1 will not have been previously called + + true -> + if Action == open; Action == close; Action == reconnect; + Action == info; Action == error -> ct_gen_conn:log(heading(Action,Name1),String,Args); - false -> - ct_gen_conn:cont_log(String,Args) + + ForcePrint == false -> + case ct_util:is_silenced(telnet) of + true -> + ok; + false -> + ct_gen_conn:cont_log(String,Args) + end; + + ForcePrint == true -> + case ct_util:is_silenced(telnet) of + true -> + %% call log/3 now instead of cont_log/2 since + %% start_gen_log/1 will not have been previously + %% called + ct_gen_conn:log(heading(Action,Name1),String,Args); + false -> + ct_gen_conn:cont_log(String,Args) + end end end. start_gen_log(Heading) -> - case ct_util:get_testdata({cth_conn_log,?MODULE}) of - undefined -> - %% check if output is suppressed - case ct_util:is_silenced(telnet) of - true -> ok; - false -> ct_gen_conn:start_log(Heading) - end; - _ -> - ok + %% check if output is suppressed + case ct_util:is_silenced(telnet) of + true -> ok; + false -> ct_gen_conn:start_log(Heading) end. end_gen_log() -> - case ct_util:get_testdata({cth_conn_log,?MODULE}) of - undefined -> - %% check if output is suppressed - case ct_util:is_silenced(telnet) of - true -> ok; - false -> ct_gen_conn:end_log() - end; - _ -> - ok + %% check if output is suppressed + case ct_util:is_silenced(telnet) of + true -> ok; + false -> ct_gen_conn:end_log() end. %%% @hidden @@ -879,7 +887,8 @@ teln_get_all_data(Pid,Prx,Data,Acc,LastLine) -> %% Expect options record -record(eo,{teln_pid, prx, - timeout, + idle_timeout, + total_timeout, haltpatterns=[], seq=false, repeat=false, @@ -921,11 +930,12 @@ teln_expect(Name,Pid,Data,Pattern0,Prx,Opts) -> Seq = get_seq(Opts), Pattern = convert_pattern(Pattern0,Seq), - Timeout = get_timeout(Opts), + {IdleTimeout,TotalTimeout} = get_timeouts(Opts), EO = #eo{teln_pid=Pid, prx=Prx, - timeout=Timeout, + idle_timeout=IdleTimeout, + total_timeout=TotalTimeout, seq=Seq, haltpatterns=HaltPatterns, prompt_check=PromptCheck}, @@ -964,11 +974,22 @@ rm_dupl([P|Ps],Acc) -> rm_dupl([],Acc) -> lists:reverse(Acc). -get_timeout(Opts) -> - case lists:keysearch(timeout,1,Opts) of - {value,{timeout,T}} -> T; - false -> ?DEFAULT_TIMEOUT - end. +get_timeouts(Opts) -> + {case lists:keysearch(idle_timeout,1,Opts) of + {value,{_,T}} -> + T; + false -> + %% this check is for backwards compatibility (pre CT v1.8) + case lists:keysearch(timeout,1,Opts) of + {value,{_,T}} -> T; + false -> ?DEFAULT_TIMEOUT + end + end, + case lists:keysearch(total_timeout,1,Opts) of + {value,{_,T}} -> T; + false -> infinity + end}. + get_repeat(Opts) -> case lists:keysearch(repeat,1,Opts) of {value,{repeat,N}} when is_integer(N) -> @@ -1010,7 +1031,8 @@ repeat_expect(Name,Pid,Data,Pattern,Acc,EO) -> {error,Reason} end. -teln_expect1(Name,Pid,Data,Pattern,Acc,EO) -> +teln_expect1(Name,Pid,Data,Pattern,Acc,EO=#eo{idle_timeout=IdleTO, + total_timeout=TotalTO}) -> ExpectFun = case EO#eo.seq of true -> fun() -> seq_expect(Name,Pid,Data,Pattern,Acc,EO) @@ -1027,12 +1049,12 @@ teln_expect1(Name,Pid,Data,Pattern,Acc,EO) -> NotFinished -> %% Get more data Fun = fun() -> get_data1(EO#eo.teln_pid) end, - case ct_gen_conn:do_within_time(Fun, EO#eo.timeout) of - {error,Reason} -> + case timer:tc(ct_gen_conn, do_within_time, [Fun, IdleTO]) of + {_,{error,Reason}} -> %% A timeout will occur when the telnet connection - %% is idle for EO#eo.timeout milliseconds. + %% is idle for EO#eo.idle_timeout milliseconds. {error,Reason}; - {ok,Data1} -> + {_,{ok,Data1}} when TotalTO == infinity -> case NotFinished of {nomatch,Rest} -> %% One expect @@ -1040,6 +1062,21 @@ teln_expect1(Name,Pid,Data,Pattern,Acc,EO) -> {continue,Patterns1,Acc1,Rest} -> %% Sequence teln_expect1(Name,Pid,Rest++Data1,Patterns1,Acc1,EO) + end; + {Elapsed,{ok,Data1}} -> + TVal = trunc(TotalTO - (Elapsed/1000)), + if TVal =< 0 -> + {error,timeout}; + true -> + EO1 = EO#eo{total_timeout = TVal}, + case NotFinished of + {nomatch,Rest} -> + %% One expect + teln_expect1(Name,Pid,Rest++Data1,Pattern,[],EO1); + {continue,Patterns1,Acc1,Rest} -> + %% Sequence + teln_expect1(Name,Pid,Rest++Data1,Patterns1,Acc1,EO1) + end end end end. diff --git a/lib/common_test/src/ct_telnet_client.erl b/lib/common_test/src/ct_telnet_client.erl index 2cbcba9c77..ce30dcb74b 100644 --- a/lib/common_test/src/ct_telnet_client.erl +++ b/lib/common_test/src/ct_telnet_client.erl @@ -32,7 +32,9 @@ -module(ct_telnet_client). --export([open/1, open/2, open/3, open/4, close/1]). +%% -define(debug, true). + +-export([open/2, open/3, open/4, open/5, close/1]). -export([send_data/2, get_data/1]). -define(TELNET_PORT, 23). @@ -64,20 +66,23 @@ -define(TERMINAL_TYPE, 24). -define(WINDOW_SIZE, 31). --record(state,{get_data, keep_alive=true}). +-record(state,{conn_name, get_data, keep_alive=true, log_pos=1}). -open(Server) -> - open(Server, ?TELNET_PORT, ?OPEN_TIMEOUT, true). +open(Server, ConnName) -> + open(Server, ?TELNET_PORT, ?OPEN_TIMEOUT, true, ConnName). -open(Server, Port) -> - open(Server, Port, ?OPEN_TIMEOUT, true). +open(Server, Port, ConnName) -> + open(Server, Port, ?OPEN_TIMEOUT, true, ConnName). -open(Server, Port, Timeout) -> - open(Server, Port, Timeout, true). +open(Server, Port, Timeout, ConnName) -> + open(Server, Port, Timeout, true, ConnName). -open(Server, Port, Timeout, KeepAlive) -> +open(Server, Port, Timeout, KeepAlive, ConnName) -> Self = self(), - Pid = spawn(fun() -> init(Self, Server, Port, Timeout, KeepAlive) end), + Pid = spawn(fun() -> + init(Self, Server, Port, Timeout, + KeepAlive, ConnName) + end), receive {open,Pid} -> {ok,Pid}; @@ -86,29 +91,34 @@ open(Server, Port, Timeout, KeepAlive) -> end. close(Pid) -> - Pid ! close. + Pid ! {close,self()}, + receive closed -> ok + after 5000 -> ok + end. send_data(Pid, Data) -> Pid ! {send_data, Data++"\n"}, ok. get_data(Pid) -> - Pid ! {get_data, self()}, + Pid ! {get_data,self()}, receive {data,Data} -> - {ok, Data} + {ok,Data} end. %%%----------------------------------------------------------------- %%% Internal functions -init(Parent, Server, Port, Timeout, KeepAlive) -> +init(Parent, Server, Port, Timeout, KeepAlive, ConnName) -> case gen_tcp:connect(Server, Port, [list,{packet,0}], Timeout) of {ok,Sock} -> - dbg("Connected to: ~p (port: ~w, keep_alive: ~w)\n", [Server,Port,KeepAlive]), - send([?IAC,?DO,?SUPPRESS_GO_AHEAD], Sock), + dbg("~p connected to: ~p (port: ~w, keep_alive: ~w)\n", + [ConnName,Server,Port,KeepAlive]), + send([?IAC,?DO,?SUPPRESS_GO_AHEAD], Sock, ConnName), Parent ! {open,self()}, - loop(#state{get_data=10, keep_alive=KeepAlive}, Sock, []), + loop(#state{conn_name=ConnName, get_data=10, keep_alive=KeepAlive}, + Sock, []), gen_tcp:close(Sock); Error -> Parent ! {Error,self()} @@ -118,6 +128,13 @@ loop(State, Sock, Acc) -> receive {tcp_closed,_} -> dbg("Connection closed\n", []), + Data = lists:reverse(lists:append(Acc)), + dbg("Printing queued messages: ~tp",[Data]), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + length(Data))]), receive {get_data,Pid} -> Pid ! closed @@ -125,11 +142,11 @@ loop(State, Sock, Acc) -> ok end; {tcp,_,Msg0} -> - dbg("tcp msg: ~p~n",[Msg0]), + dbg("tcp msg: ~tp~n",[Msg0]), Msg = check_msg(Sock,Msg0,[]), loop(State, Sock, [Msg | Acc]); {send_data,Data} -> - send(Data, Sock), + send(Data, Sock, State#state.conn_name), loop(State, Sock, Acc); {get_data,Pid} -> NewState = @@ -144,54 +161,100 @@ loop(State, Sock, Acc) -> end; _ -> Data = lists:reverse(lists:append(Acc)), - dbg("get_data ~p\n",[Data]), + Len = length(Data), + dbg("get_data ~tp\n",[Data]), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + Len)]), Pid ! {data,Data}, - State + State#state{log_pos = 1} end, loop(NewState, Sock, []); {get_data_delayed,Pid} -> NewState = case State of #state{keep_alive = true, get_data = 0} -> - if Acc == [] -> send([?IAC,?NOP], Sock); + if Acc == [] -> send([?IAC,?NOP], Sock, + State#state.conn_name); true -> ok end, State#state{get_data=10}; _ -> State end, - NewAcc = + {NewAcc,Pos} = case erlang:is_process_alive(Pid) of - true -> + true when Acc /= [] -> Data = lists:reverse(lists:append(Acc)), - dbg("get_data_delayed ~p\n",[Data]), + Len = length(Data), + dbg("get_data_delayed ~tp\n",[Data]), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + Len)]), Pid ! {data,Data}, - []; + {[],1}; + true when Acc == [] -> + dbg("get_data_delayed nodata\n",[]), + Pid ! {data,[]}, + {[],1}; false -> - Acc + {Acc,NewState#state.log_pos} end, - loop(NewState, Sock, NewAcc); - close -> + loop(NewState#state{log_pos=Pos}, Sock, NewAcc); + {close,Pid} -> dbg("Closing connection\n", []), + if Acc == [] -> + ok; + true -> + Data = lists:reverse(lists:append(Acc)), + dbg("Printing queued messages: ~tp",[Data]), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + length(Data))]) + end, gen_tcp:close(Sock), - ok + Pid ! closed after wait(State#state.keep_alive,?IDLE_TIMEOUT) -> - if - Acc == [] -> send([?IAC,?NOP], Sock); - true -> ok - end, - loop(State, Sock, Acc) + Data = lists:reverse(lists:append(Acc)), + case Data of + [] -> + send([?IAC,?NOP], Sock, State#state.conn_name), + loop(State, Sock, Acc); + _ when State#state.log_pos == length(Data)+1 -> + loop(State, Sock, Acc); + _ -> + dbg("Idle timeout, printing ~tp\n",[Data]), + Len = length(Data), + ct_telnet:log(State#state.conn_name, + general_io, "~ts", + [lists:sublist(Data, + State#state.log_pos, + Len)]), + loop(State#state{log_pos = Len+1}, Sock, Acc) + end end. wait(true, Time) -> Time; wait(false, _) -> infinity. -send(Data, Sock) -> +send(Data, Sock, ConnName) -> case Data of [?IAC|_] = Cmd -> cmd_dbg(Cmd); _ -> - dbg("Sending: ~p\n", [Data]) + dbg("Sending: ~tp\n", [Data]), + try io_lib:format("[~w] ~ts", [?MODULE,Data]) of + Str -> + ct_telnet:log(ConnName, general_io, Str, []) + catch + _:_ -> ok + end end, gen_tcp:send(Sock, Data), ok. diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl index a731c8054c..0e6c877c5d 100644 --- a/lib/common_test/src/cth_conn_log.erl +++ b/lib/common_test/src/cth_conn_log.erl @@ -100,7 +100,6 @@ get_log_opts(Opts) -> Hosts = proplists:get_value(hosts,Opts,[]), {LogType,Hosts}. - pre_init_per_testcase(TestCase,Config,CthState) -> Logs = lists:map( diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index 7ed2018bdf..bb12171ea7 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -79,6 +79,10 @@ init(Path, Opts) -> url_base = proplists:get_value(url_base,Opts), timer = now() }. +pre_init_per_suite(Suite,SkipOrFail,State) when is_tuple(SkipOrFail) -> + {SkipOrFail, init_tc(State#state{curr_suite = Suite, + curr_suite_ts = now()}, + SkipOrFail) }; pre_init_per_suite(Suite,Config,#state{ test_cases = [] } = State) -> TcLog = proplists:get_value(tc_logfile,Config), CurrLogDir = filename:dirname(TcLog), diff --git a/lib/common_test/src/unix_telnet.erl b/lib/common_test/src/unix_telnet.erl index e049c3bf39..b05386a5ab 100644 --- a/lib/common_test/src/unix_telnet.erl +++ b/lib/common_test/src/unix_telnet.erl @@ -109,7 +109,7 @@ connect(ConnName,Ip,Port,Timeout,KeepAlive,Extra) -> connect1(Name,Ip,Port,Timeout,KeepAlive,Username,Password) -> start_gen_log("unix_telnet connect"), Result = - case ct_telnet_client:open(Ip,Port,Timeout,KeepAlive) of + case ct_telnet_client:open(Ip,Port,Timeout,KeepAlive,Name) of {ok,Pid} -> case ct_telnet:silent_teln_expect(Name,Pid,[], [prompt],?prx,[]) of @@ -143,13 +143,13 @@ connect1(Name,Ip,Port,Timeout,KeepAlive,Username,Password) -> {ok,[{prompt,_OtherPrompt1},{prompt,_OtherPrompt2}],_} -> {ok,Pid}; Error -> - log(Name,error, + log(Name,conn_error, "Did not get expected prompt from ~p:~p\n~p\n", [Ip,Port,Error]), {error,Error} end; Error -> - log(Name,error, + log(Name,conn_error, "Could not open telnet connection to ~p:~p\n~p\n", [Ip,Port,Error]), Error diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index 7408cbe376..e90513f888 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -81,7 +81,8 @@ end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -suite() -> [{ct_hooks,[ts_install_cth]}]. +suite() -> [{timetrap,{seconds,60}}, + {ct_hooks,[ts_install_cth]}]. all() -> [ct_master_test]. diff --git a/lib/common_test/test/ct_telnet_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE.erl index acce4eca14..f5cff76fd1 100644 --- a/lib/common_test/test/ct_telnet_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE.erl @@ -72,19 +72,32 @@ init_per_suite(Config) -> end_per_suite(Config) -> ct_test_support:end_per_suite(Config). -init_per_testcase(TestCase, Config) when TestCase=/=unix_telnet-> +init_per_testcase(TestCase, Config) when TestCase /= unix_telnet -> + ct:pal("Testcase ~p starting!", [TestCase]), TS = telnet_server:start([{port,?erl_telnet_server_port}, {users,[{?erl_telnet_server_user, ?erl_telnet_server_pwd}]}]), ct_test_support:init_per_testcase(TestCase, [{telnet_server,TS}|Config]); init_per_testcase(TestCase, Config) -> - ct_test_support:init_per_testcase(TestCase, Config). - + ct:pal("Testcase ~p starting. Checking connection to telnet server...", + [TestCase]), + ct:require(testconn, {unix,[telnet]}), + case {os:type(),ct_telnet:open(testconn)} of + {_,{ok,Handle}} -> + ok = ct_telnet:close(Handle), + ct:pal("Connection ok, starting tests!", []), + ct_test_support:init_per_testcase(TestCase, Config); + {{unix,_},{error,Reason}} -> + ct:fail("No connection to telnet server! Reason: ~tp", [Reason]); + {_,{error,Reason}} -> + {skip,{no_access_to_telnet_server,Reason}} + end. + +end_per_testcase(TestCase, Config) when TestCase /= unix_telnet -> + ct:pal("Stopping the telnet_server now!", []), + telnet_server:stop(?config(telnet_server,Config)), + ct_test_support:end_per_testcase(TestCase, Config); end_per_testcase(TestCase, Config) -> - case ?config(telnet_server,Config) of - undefined -> ok; - TS -> telnet_server:stop(TS) - end, ct_test_support:end_per_testcase(TestCase, Config). @@ -179,7 +192,12 @@ telnet_config(_, LogType) -> {port, ?erl_telnet_server_port}, {username,?erl_telnet_server_user}, {password,?erl_telnet_server_pwd}, - {keep_alive,true}]} | + {keep_alive,true}]}, + {telnet_settings, [{connect_timeout,10000}, + {command_timeout,10000}, + {reconnection_attempts,0}, + {reconnection_interval,0}, + {keep_alive,true}]} | if LogType == legacy -> [{ct_conn_log,[]}]; true -> diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl index 8d142e85a8..0ee0525216 100644 --- a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl @@ -20,7 +20,9 @@ all() -> expect_repeat, expect_sequence, expect_error_prompt, - expect_error_timeout, + expect_error_timeout1, + expect_error_timeout2, + expect_error_timeout3, no_prompt_check, no_prompt_check_repeat, no_prompt_check_sequence, @@ -28,7 +30,9 @@ all() -> ignore_prompt, ignore_prompt_repeat, ignore_prompt_sequence, - ignore_prompt_timeout]. + ignore_prompt_timeout, + server_speaks, + server_disconnects]. groups() -> []. @@ -85,13 +89,34 @@ expect_error_prompt(_) -> %% Check that expect returns after idle timeout, and even if the %% expected pattern is received - as long as not newline or prompt is %% received it will not match. -expect_error_timeout(_) -> +expect_error_timeout1(_) -> {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_no_prompt xxx"), {error,timeout} = ct_telnet:expect(Handle, ["xxx"], [{timeout,1000}]), ok = ct_telnet:close(Handle), ok. +expect_error_timeout2(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "echo_no_prompt xxx"), + {error,timeout} = ct_telnet:expect(Handle, ["xxx"], [{idle_timeout,1000}, + {total_timeout,infinity}]), + ok = ct_telnet:close(Handle), + ok. + +%% Check that if server loops and pattern not matching, the operation +%% can be aborted +expect_error_timeout3(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "echo_loop 5000 xxx"), + {error,timeout} = ct_telnet:expect(Handle, ["yyy"], + [{idle_timeout,infinity}, + {total_timeout,3000}]), + ok = ct_telnet:send(Handle, "echo ayt"), + {ok,["ayt"]} = ct_telnet:expect(Handle, ["ayt"]), + ok = ct_telnet:close(Handle), + ok. + %% expect with ignore_prompt option should not return even if a prompt %% is found. The pattern after the prompt (here "> ") can be matched. ignore_prompt(_) -> @@ -188,3 +213,37 @@ no_prompt_check_timeout(_) -> {timeout,1000}]), ok = ct_telnet:close(Handle), ok. + +%% The server says things. Manually check that it gets printed correctly +%% in the general IO log. +server_speaks(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the first message\r\n"), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the second message\r\n"), + %% let ct_telnet_client get an idle timeout + timer:sleep(15000), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the third message\r\n"), + {ok,_} = ct_telnet:expect(Handle, ["the"], [no_prompt_check]), + {error,timeout} = ct_telnet:expect(Handle, ["the"], [no_prompt_check, + {timeout,1000}]), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the fourth message\r\n"), + %% give the server time to respond + timer:sleep(2000), + %% closing the connection should print last message in log + ok = ct_telnet:close(Handle), + ok. + +%% Let the server close the connection. Make sure buffered data gets printed +%% to the general IO log. +server_disconnects(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "disconnect_after 1500"), + %% wait until the get_data operation (triggered by send/2) times out + %% before sending the msg + timer:sleep(500), + ok = ct_telnet:send(Handle, "echo_no_prompt This is the message\r\n"), + %% when the server closes the connection, the last message should be + %% printed in the log + timer:sleep(3000), + _ = ct_telnet:close(Handle), + ok. diff --git a/lib/common_test/test/telnet_server.erl b/lib/common_test/test/telnet_server.erl index 1760100d8e..ae56787819 100644 --- a/lib/common_test/test/telnet_server.erl +++ b/lib/common_test/test/telnet_server.erl @@ -51,32 +51,51 @@ stop(Pid) -> init(Opts) -> Port = proplists:get_value(port,Opts), Users = proplists:get_value(users,Opts,[]), - {ok, LSock} = gen_tcp:listen(Port, [list, {packet, 0}, - {active, true}]), + {ok, LSock} = listen(5, Port, [list, {packet, 0}, + {active, true}, + {reuseaddr,true}]), State = #state{listen=LSock,users=Users}, accept(State), - ok = gen_tcp:close(LSock). + ok = gen_tcp:close(LSock), + dbg("telnet_server closed the listen socket ~p\n", [LSock]), + timer:sleep(1000), + ok. + +listen(0, _Port, _Opts) -> + {error,eaddrinuse}; +listen(Retries, Port, Opts) -> + case gen_tcp:listen(Port, Opts) of + {error,eaddrinuse} -> + dbg("Listen port not released, trying again..."), + timer:sleep(5000), + listen(Retries-1, Port, Opts); + Ok = {ok,_LSock} -> + Ok; + Error -> + exit(Error) + end. accept(#state{listen=LSock}=State) -> Server = self(), Acceptor = spawn_link(fun() -> do_accept(LSock,Server) end), receive {Acceptor,Sock} when is_port(Sock) -> + dbg("Connected to client on socket ~p\n", [Sock]), case init_client(State#state{client=Sock}) of stopped -> - io:format("[telnet_server] telnet_server stopped\n"), + dbg("telnet_server stopped\n"), ok; R -> - io:format("[telnet_server] connection to client" - "closed with reason ~p~n",[R]), + dbg("Connection to client " + "closed with reason ~p~n",[R]), accept(State) end; {Acceptor,closed} -> - io:format("[telnet_server] listen socket closed unexpectedly, " - "terminating telnet_server\n"), + dbg("Listen socket closed unexpectedly, " + "terminating telnet_server\n"), ok; stop -> - io:format("[telnet_server] telnet_server stopped\n"), + dbg("telnet_server stopped\n"), ok end. @@ -97,19 +116,21 @@ init_client(#state{client=Sock}=State) -> dbg("Server sending: ~p~n",["login: "]), R = case gen_tcp:send(Sock,"login: ") of ok -> - loop(State); + loop(State, 1); Error -> Error end, _ = gen_tcp:close(Sock), R. -loop(State) -> +loop(State, N) -> receive {tcp,_,Data} -> try handle_data(Data,State) of {ok,State1} -> - loop(State1) + loop(State1, N); + closed -> + closed catch throw:Error -> Error @@ -118,6 +139,11 @@ loop(State) -> closed; {tcp_error,_,Error} -> {error,tcp,Error}; + disconnect -> + Sock = State#state.client, + dbg("Server closing connection on socket ~p~n", [Sock]), + ok = gen_tcp:close(Sock), + closed; stop -> stopped end. @@ -130,10 +156,16 @@ handle_data(Data,State) -> case get_line(Data,[]) of {Line,Rest} -> WholeLine = lists:flatten(lists:reverse(State#state.buffer,Line)), - {ok,State1} = do_handle_data(WholeLine,State), - case Rest of - [] -> {ok,State1}; - _ -> handle_data(Rest,State1) + case do_handle_data(WholeLine,State) of + {ok,State1} -> + case Rest of + [] -> {ok,State1}; + _ -> handle_data(Rest,State1) + end; + {close,State1} -> + dbg("Server closing connection~n",[]), + gen_tcp:close(State1#state.client), + closed end; false -> {ok,State#state{buffer=[Data|State#state.buffer]}} @@ -163,22 +195,34 @@ do_handle_data(Data,#state{authorized=false}=State) -> check_user(Data,State); do_handle_data(Data,#state{authorized={user,_}}=State) -> check_pwd(Data,State); -do_handle_data("echo "++ Data,State) -> +do_handle_data("echo " ++ Data,State) -> send(Data++"\r\n> ",State), {ok,State}; -do_handle_data("echo_no_prompt "++ Data,State) -> +do_handle_data("echo_no_prompt " ++ Data,State) -> send(Data,State), {ok,State}; -do_handle_data("echo_ml "++ Data,State) -> +do_handle_data("echo_ml " ++ Data,State) -> Lines = string:tokens(Data," "), ReturnData = string:join(Lines,"\n"), send(ReturnData++"\r\n> ",State), {ok,State}; -do_handle_data("echo_ml_no_prompt "++ Data,State) -> +do_handle_data("echo_ml_no_prompt " ++ Data,State) -> Lines = string:tokens(Data," "), ReturnData = string:join(Lines,"\n"), send(ReturnData,State), {ok,State}; +do_handle_data("echo_loop " ++ Data,State) -> + [TStr|Lines] = string:tokens(Data," "), + ReturnData = string:join(Lines,"\n"), + send_loop(list_to_integer(TStr),ReturnData,State), + {ok,State}; +do_handle_data("disconnect_after " ++WaitStr,State) -> + Wait = list_to_integer(string:strip(WaitStr,right,$\n)), + dbg("Server will close connection in ~w ms...", [Wait]), + erlang:send_after(Wait,self(),disconnect), + {ok,State}; +do_handle_data("disconnect" ++_,State) -> + {close,State}; do_handle_data([],State) -> send("> ",State), {ok,State}; @@ -212,6 +256,20 @@ send(Data,State) -> throw({error,send,Error}) end. +send_loop(T,Data,State) -> + dbg("Server sending ~p in loop for ~w ms...~n",[Data,T]), + send_loop(now(),T,Data,State). + +send_loop(T0,T,Data,State) -> + ElapsedMS = trunc(timer:now_diff(now(),T0)/1000), + if ElapsedMS >= T -> + ok; + true -> + send(Data,State), + timer:sleep(500), + send_loop(T0,T,Data,State) + end. + get_line([$\r,$\n|Rest],Acc) -> {lists:reverse(Acc),Rest}; get_line([$\r,0|Rest],Acc) -> @@ -226,4 +284,4 @@ get_line([],_) -> dbg(_F) -> dbg(_F,[]). dbg(_F,_A) -> - io:format("[telnet_server] "++_F,_A). + io:format("[telnet_server] " ++ _F,_A). diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index 568405b110..f8a5aab686 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.7.4 +COMMON_TEST_VSN = 1.8 diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index ecc4b2c9b1..e400e4f185 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -132,16 +132,13 @@ ann_c_map_pair/4 ]). --export_type([c_binary/0, c_call/0, c_clause/0, c_cons/0, c_fun/0, c_literal/0, - c_module/0, c_tuple/0, c_values/0, c_var/0, cerl/0, var_name/0]). +-export_type([c_binary/0, c_bitstr/0, c_call/0, c_clause/0, c_cons/0, c_fun/0, + c_literal/0, c_map_pair/0, c_module/0, c_tuple/0, + c_values/0, c_var/0, cerl/0, var_name/0]). %% HiPE does not understand Maps %% (guard functions is_map/1 and map_size/1 in ann_c_map/3) -compile(no_native). -%% -%% needed by the include file below -- do not move -%% --type var_name() :: integer() | atom() | {atom(), integer()}. -include("core_parse.hrl"). @@ -176,6 +173,8 @@ | c_module() | c_primop() | c_receive() | c_seq() | c_try() | c_tuple() | c_values() | c_var(). +-type var_name() :: integer() | atom() | {atom(), integer()}. + %% ===================================================================== %% Representation (general) %% @@ -207,13 +206,15 @@ %% <td>call</td> %% <td>case</td> %% <td>catch</td> -%% </tr><tr> %% <td>clause</td> +%% </tr><tr> %% <td>cons</td> %% <td>fun</td> %% <td>let</td> %% <td>letrec</td> %% <td>literal</td> +%% <td>map</td> +%% <td>map_pair</td> %% <td>module</td> %% </tr><tr> %% <td>primop</td> @@ -264,10 +265,10 @@ %% @see subtrees/1 %% @see meta/1 --type ctype() :: 'alias' | 'apply' | 'binary' | 'bitrst' | 'call' | 'case' - | 'catch' | 'clause' | 'cons' | 'fun' | 'let' | 'letrec' - | 'literal' | 'map' | 'module' | 'primop' | 'receive' | 'seq' - | 'try' | 'tuple' | 'values' | 'var'. +-type ctype() :: 'alias' | 'apply' | 'binary' | 'bitrst' | 'call' | 'case' + | 'catch' | 'clause' | 'cons' | 'fun' | 'let' | 'letrec' + | 'literal' | 'map' | 'map_pair' | 'module' | 'primop' + | 'receive' | 'seq' | 'try' | 'tuple' | 'values' | 'var'. -spec type(cerl()) -> ctype(). @@ -4377,12 +4378,8 @@ meta_1(cons, Node) -> %% we get exactly one element, we generate a 'c_cons' call %% instead of 'make_list' to reconstruct the node. case split_list(Node) of - {[H], none} -> - meta_call(c_cons, [meta(H), meta(c_nil())]); {[H], Node1} -> meta_call(c_cons, [meta(H), meta(Node1)]); - {L, none} -> - meta_call(make_list, [make_list(meta_list(L))]); {L, Node1} -> meta_call(make_list, [make_list(meta_list(L)), meta(Node1)]) @@ -4469,8 +4466,6 @@ split_list(Node, L) -> case type(Node) of cons when A =:= [] -> split_list(cons_tl(Node), [cons_hd(Node) | L]); - nil when A =:= [] -> - {lists:reverse(L), none}; _ -> {lists:reverse(L), Node} end. diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index 8775c84698..8f68915f8e 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -67,4 +67,6 @@ ]}, {registered, []}, {applications, [kernel, stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","hipe-3.10.3","erts-6.0", + "crypto-3.3"]}]}. diff --git a/lib/compiler/src/core_parse.hrl b/lib/compiler/src/core_parse.hrl index 20f3a46991..4a00535360 100644 --- a/lib/compiler/src/core_parse.hrl +++ b/lib/compiler/src/core_parse.hrl @@ -34,7 +34,7 @@ -record(c_apply, {anno=[], op, % op :: Tree, args}). % args :: [Tree] --record(c_binary, {anno=[], segments}). % segments :: [#c_bitstr{}] +-record(c_binary, {anno=[], segments :: [cerl:c_bitstr()]}). -record(c_bitstr, {anno=[], val, % val :: Tree, size, % size :: Tree, @@ -70,6 +70,15 @@ -record(c_literal, {anno=[], val}). % val :: literal() +-record(c_map, {anno=[], + arg=#c_literal{val=#{}} :: cerl:c_var() | cerl:c_literal(), + es :: [cerl:c_map_pair()]}). + +-record(c_map_pair, {anno=[], + op :: #c_literal{val::'assoc'} | #c_literal{val::'exact'}, + key, + val}). + -record(c_module, {anno=[], name, % name :: Tree, exports, % exports :: [Tree], attrs, % attrs :: [#c_def{}], @@ -96,12 +105,3 @@ -record(c_values, {anno=[], es}). % es :: [Tree] -record(c_var, {anno=[], name :: cerl:var_name()}). - --record(c_map_pair, {anno=[], - op :: #c_literal{val::'assoc'} | #c_literal{val::'exact'}, - key, - val}). - --record(c_map, {anno=[], - arg=#c_literal{val=#{}} :: #c_var{} | #c_literal{}, - es :: [#c_map_pair{}]}). diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index 082809b8a0..04210ae243 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -2206,6 +2206,8 @@ lit_vars(Lit) -> lit_vars(Lit, []). lit_vars(#c_cons{hd=H,tl=T}, Vs) -> lit_vars(H, lit_vars(T, Vs)); lit_vars(#c_tuple{es=Es}, Vs) -> lit_list_vars(Es, Vs); +lit_vars(#c_map{arg=V,es=Es}, Vs) -> lit_vars(V, lit_list_vars(Es, Vs)); +lit_vars(#c_map_pair{key=K,val=V}, Vs) -> lit_vars(K, lit_vars(V, Vs)); lit_vars(#c_var{name=V}, Vs) -> add_element(V, Vs); lit_vars(_, Vs) -> Vs. %These are atomic diff --git a/lib/compiler/test/map_SUITE.erl b/lib/compiler/test/map_SUITE.erl index b7e27afef1..90eae6fb4f 100644 --- a/lib/compiler/test/map_SUITE.erl +++ b/lib/compiler/test/map_SUITE.erl @@ -43,7 +43,8 @@ %% errors in 17.0-rc1 t_update_values/1, - t_expand_map_update/1 + t_expand_map_update/1, + t_export/1 ]). suite() -> []. @@ -70,7 +71,8 @@ all() -> [ %% errors in 17.0-rc1 t_update_values, - t_expand_map_update + t_expand_map_update, + t_export ]. groups() -> []. @@ -285,6 +287,12 @@ t_expand_map_update(Config) when is_list(Config) -> #{<<"hello">> := <<"les gens">>} = M, ok. +t_export(Config) when is_list(Config) -> + Raclette = id(#{}), + case brie of brie -> Fromage = Raclette end, + Raclette = Fromage#{}, + ok. + check_val(#{val1:=V1, val2:=V2},V1,V2) -> ok. get_val(#{ "wazzup" := _, val := V}) -> V; diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index cbdf57f177..c0c3d56472 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 4.9.4 +COMPILER_VSN = 5.0 diff --git a/lib/cosEvent/src/cosEvent.app.src b/lib/cosEvent/src/cosEvent.app.src index c1cb9e0cc9..66b0d2e168 100644 --- a/lib/cosEvent/src/cosEvent.app.src +++ b/lib/cosEvent/src/cosEvent.app.src @@ -38,7 +38,8 @@ {registered, []}, {applications, [orber, stdlib, kernel]}, {env, []}, - {mod, {cosEventApp, []}} + {mod, {cosEventApp, []}}, + {runtime_dependencies, ["stdlib-2.0","orber-3.6.27","kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/cosEvent/vsn.mk b/lib/cosEvent/vsn.mk index 6745bee079..40bf1ba49d 100644 --- a/lib/cosEvent/vsn.mk +++ b/lib/cosEvent/vsn.mk @@ -1,3 +1,3 @@ -COSEVENT_VSN = 2.1.14 +COSEVENT_VSN = 2.1.15 diff --git a/lib/cosEventDomain/src/cosEventDomain.app.src b/lib/cosEventDomain/src/cosEventDomain.app.src index e4307e1f99..60114b6a91 100644 --- a/lib/cosEventDomain/src/cosEventDomain.app.src +++ b/lib/cosEventDomain/src/cosEventDomain.app.src @@ -27,5 +27,7 @@ {registered, []}, {applications, [orber, stdlib, kernel]}, {env, []}, - {mod, {cosEventDomainApp, []}} + {mod, {cosEventDomainApp, []}}, + {runtime_dependencies, ["stdlib-2.0","orber-3.6.27","kernel-3.0","erts-6.0", + "cosNotification-1.1.21"]} ]}. diff --git a/lib/cosEventDomain/vsn.mk b/lib/cosEventDomain/vsn.mk index e9cf92395a..6317ed3c22 100644 --- a/lib/cosEventDomain/vsn.mk +++ b/lib/cosEventDomain/vsn.mk @@ -1,3 +1,3 @@ -COSEVENTDOMAIN_VSN = 1.1.13 +COSEVENTDOMAIN_VSN = 1.1.14 diff --git a/lib/cosFileTransfer/src/cosFileTransfer.app.src b/lib/cosFileTransfer/src/cosFileTransfer.app.src index 31d94b6f0d..21226b0c6b 100644 --- a/lib/cosFileTransfer/src/cosFileTransfer.app.src +++ b/lib/cosFileTransfer/src/cosFileTransfer.app.src @@ -36,6 +36,8 @@ {registered, []}, {applications, [orber, stdlib, kernel]}, {env, []}, - {mod, {cosFileTransferApp, []}} + {mod, {cosFileTransferApp, []}}, + {runtime_dependencies, ["stdlib-2.0","ssl-5.3.4","orber-3.6.27","kernel-3.0", + "inets-5.10","erts-6.0","cosProperty-1.1.17"]} ]}. diff --git a/lib/cosFileTransfer/vsn.mk b/lib/cosFileTransfer/vsn.mk index cf33926334..f52a1bd800 100644 --- a/lib/cosFileTransfer/vsn.mk +++ b/lib/cosFileTransfer/vsn.mk @@ -1 +1 @@ -COSFILETRANSFER_VSN = 1.1.15 +COSFILETRANSFER_VSN = 1.1.16 diff --git a/lib/cosNotification/src/cosNotification.app.src b/lib/cosNotification/src/cosNotification.app.src index 04beac36e8..ad02eb4421 100644 --- a/lib/cosNotification/src/cosNotification.app.src +++ b/lib/cosNotification/src/cosNotification.app.src @@ -116,5 +116,7 @@ {registered, [cosNotificationSup, oe_cosNotificationFactory]}, {applications, [orber, stdlib, kernel]}, {env, []}, - {mod, {cosNotificationApp, []}} + {mod, {cosNotificationApp, []}}, + {runtime_dependencies, ["stdlib-2.0","orber-3.6.27","kernel-3.0","erts-6.0", + "cosTime-1.1.14","cosEvent-2.1.15"]} ]}. diff --git a/lib/cosNotification/vsn.mk b/lib/cosNotification/vsn.mk index ea59800164..28d6ec71bf 100644 --- a/lib/cosNotification/vsn.mk +++ b/lib/cosNotification/vsn.mk @@ -1,2 +1,2 @@ -COSNOTIFICATION_VSN = 1.1.20 +COSNOTIFICATION_VSN = 1.1.21 diff --git a/lib/cosProperty/src/cosProperty.app.src b/lib/cosProperty/src/cosProperty.app.src index 3099e904f7..b977bb5984 100644 --- a/lib/cosProperty/src/cosProperty.app.src +++ b/lib/cosProperty/src/cosProperty.app.src @@ -41,5 +41,7 @@ {registered, [oe_cosPropertySup]}, {applications, [orber, stdlib, kernel]}, {env, []}, - {mod, {cosProperty, []}} + {mod, {cosProperty, []}}, + {runtime_dependencies, ["stdlib-2.0","orber-3.6.27","mnesia-4.12", + "kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/cosProperty/vsn.mk b/lib/cosProperty/vsn.mk index ac7820216e..0f546a2da8 100644 --- a/lib/cosProperty/vsn.mk +++ b/lib/cosProperty/vsn.mk @@ -1,2 +1,2 @@ -COSPROPERTY_VSN = 1.1.16 +COSPROPERTY_VSN = 1.1.17 diff --git a/lib/cosTime/src/cosTime.app.src b/lib/cosTime/src/cosTime.app.src index 191ee5f3db..cd01de35cb 100644 --- a/lib/cosTime/src/cosTime.app.src +++ b/lib/cosTime/src/cosTime.app.src @@ -26,5 +26,7 @@ {registered, [oe_cosTimeSup, oe_cosTimerEventService]}, {applications, [orber, stdlib, kernel]}, {env, []}, - {mod, {cosTime, []}} + {mod, {cosTime, []}}, + {runtime_dependencies, ["stdlib-2.0","orber-3.6.27","kernel-3.0","erts-6.0", + "cosEvent-2.1.15"]} ]}. diff --git a/lib/cosTime/vsn.mk b/lib/cosTime/vsn.mk index 02cd669222..9e9e5c0250 100644 --- a/lib/cosTime/vsn.mk +++ b/lib/cosTime/vsn.mk @@ -1,2 +1,3 @@ -COSTIME_VSN = 1.1.13 +COSTIME_VSN = 1.1.14 + diff --git a/lib/cosTransactions/src/cosTransactions.app.src b/lib/cosTransactions/src/cosTransactions.app.src index 52769b1711..6b99915ad6 100644 --- a/lib/cosTransactions/src/cosTransactions.app.src +++ b/lib/cosTransactions/src/cosTransactions.app.src @@ -39,5 +39,6 @@ {registered, [cosTransactions_sup, oe_cosTransactionsFactory]}, {applications, [orber, stdlib, kernel]}, {env, []}, - {mod, {cosTransactions, []}} + {mod, {cosTransactions, []}}, + {runtime_dependencies, ["stdlib-2.0","orber-3.6.27","kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/cosTransactions/vsn.mk b/lib/cosTransactions/vsn.mk index 5414270a3d..7aed212523 100644 --- a/lib/cosTransactions/vsn.mk +++ b/lib/cosTransactions/vsn.mk @@ -1 +1 @@ -COSTRANSACTIONS_VSN = 1.2.13 +COSTRANSACTIONS_VSN = 1.2.14 diff --git a/lib/crypto/doc/src/crypto_app.xml b/lib/crypto/doc/src/crypto_app.xml index 6d26076c04..1d10773401 100644 --- a/lib/crypto/doc/src/crypto_app.xml +++ b/lib/crypto/doc/src/crypto_app.xml @@ -1,11 +1,11 @@ -<?xml version="1.0" encoding="iso-8859-1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE appref SYSTEM "appref.dtd"> <appref> <header> <copyright> <year>1999</year> - <year>2013</year> + <year>2014</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> diff --git a/lib/crypto/src/crypto.app.src b/lib/crypto/src/crypto.app.src index d3084ff336..823a27ee39 100644 --- a/lib/crypto/src/crypto.app.src +++ b/lib/crypto/src/crypto.app.src @@ -23,6 +23,7 @@ crypto_ec_curves]}, {registered, []}, {applications, [kernel, stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["erts-6.0","stdlib-2.0","kernel-3.0"]}]}. diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 98c071cf87..a2bd6f851a 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 3.2 +CRYPTO_VSN = 3.3 diff --git a/lib/debugger/src/dbg_iload.erl b/lib/debugger/src/dbg_iload.erl index 266cf239dd..ad05a7c529 100644 --- a/lib/debugger/src/dbg_iload.erl +++ b/lib/debugger/src/dbg_iload.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -42,18 +42,21 @@ load_mod(Mod, File, Binary, Db) -> Flag = process_flag(trap_exit, true), - Pid = spawn_link(fun () -> load_mod1(Mod, File, Binary, Db) end), + Pid = spawn_link(load_mod1(Mod, File, Binary, Db)), receive {'EXIT', Pid, What} -> process_flag(trap_exit, Flag), What end. --spec load_mod1(atom(), file:filename(), binary(), ets:tid()) -> no_return(). +-spec load_mod1(atom(), file:filename(), binary(), ets:tid()) -> + fun(() -> no_return()). load_mod1(Mod, File, Binary, Db) -> - store_module(Mod, File, Binary, Db), - exit({ok, Mod}). + fun() -> + store_module(Mod, File, Binary, Db), + exit({ok, Mod}) + end. %%==================================================================== %% Internal functions diff --git a/lib/debugger/src/debugger.app.src b/lib/debugger/src/debugger.app.src index 84fb98c94e..f102385d39 100644 --- a/lib/debugger/src/debugger.app.src +++ b/lib/debugger/src/debugger.app.src @@ -46,4 +46,6 @@ int ]}, {registered, [dbg_iserver, dbg_wx_mon, dbg_wx_winman]}, - {applications, [kernel, stdlib]}]}. + {applications, [kernel, stdlib]}, + {runtime_dependencies, ["wx-1.2","stdlib-2.0","kernel-3.0","erts-6.0", + "compiler-5.0"]}]}. diff --git a/lib/debugger/vsn.mk b/lib/debugger/vsn.mk index a245e26a55..cd107599e9 100644 --- a/lib/debugger/vsn.mk +++ b/lib/debugger/vsn.mk @@ -1 +1 @@ -DEBUGGER_VSN = 3.2.12 +DEBUGGER_VSN = 4.0 diff --git a/lib/dialyzer/src/dialyzer.app.src b/lib/dialyzer/src/dialyzer.app.src index 0d048b607e..1756800c4f 100644 --- a/lib/dialyzer/src/dialyzer.app.src +++ b/lib/dialyzer/src/dialyzer.app.src @@ -44,4 +44,7 @@ dialyzer_worker]}, {registered, []}, {applications, [compiler, gs, hipe, kernel, stdlib, wx]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["wx-1.2","syntax_tools-1.6.14","stdlib-2.0", + "kernel-3.0","hipe-3.10.3","erts-6.0", + "compiler-5.0"]}]}. diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index bb7e39dfda..cec94a49fd 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2013. All Rights Reserved. +%% Copyright Ericsson AB 2006-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -172,7 +172,7 @@ run(Opts) -> end, case dialyzer_cl:start(OptsRecord) of {?RET_DISCREPANCIES, Warnings} -> Warnings; - {?RET_NOTHING_SUSPICIOUS, []} -> [] + {?RET_NOTHING_SUSPICIOUS, _} -> [] end catch throw:{dialyzer_error, ErrorMsg} -> @@ -447,7 +447,6 @@ message_to_string({opaque_size, [SizeType, Size]}) -> message_to_string({opaque_call, [M, F, Args, Culprit, OpaqueType]}) -> io_lib:format("The call ~s:~s~s breaks the opaqueness of the term ~s :: ~s\n", [M, F, Args, Culprit, OpaqueType]); - %%----- Warnings for concurrency errors -------------------- message_to_string({race_condition, [M, F, Args, Reason]}) -> io_lib:format("The call ~w:~w~s ~s\n", [M, F, Args, Reason]); @@ -474,7 +473,14 @@ message_to_string({callback_missing, [B, F, A]}) -> io_lib:format("Undefined callback function ~w/~w (behaviour '~w')\n", [F, A, B]); message_to_string({callback_info_missing, [B]}) -> - io_lib:format("Callback info about the ~w behaviour is not available\n", [B]). + io_lib:format("Callback info about the ~w behaviour is not available\n", [B]); +%%----- Warnings for unknown functions, types, and behaviours ------------- +message_to_string({unknown_type, {M, F, A}}) -> + io_lib:format("Unknown type ~w:~w/~w", [M, F, A]); +message_to_string({unknown_function, {M, F, A}}) -> + io_lib:format("Unknown function ~w:~w/~w", [M, F, A]); +message_to_string({unknown_behaviour, B}) -> + io_lib:format("Unknown behaviour ~w", [B]). %%----------------------------------------------------------------------------- %% Auxiliary functions below @@ -557,4 +563,4 @@ form_position_string(ArgNs) -> ordinal(1) -> "1st"; ordinal(2) -> "2nd"; ordinal(3) -> "3rd"; -ordinal(N) when is_integer(N) -> io_lib:format("~wth",[N]). +ordinal(N) when is_integer(N) -> io_lib:format("~wth", [N]). diff --git a/lib/dialyzer/src/dialyzer.hrl b/lib/dialyzer/src/dialyzer.hrl index 105a174e31..9a25f86512 100644 --- a/lib/dialyzer/src/dialyzer.hrl +++ b/lib/dialyzer/src/dialyzer.hrl @@ -2,7 +2,7 @@ %%% %%% %CopyrightBegin% %%% -%%% Copyright Ericsson AB 2006-2012. All Rights Reserved. +%%% Copyright Ericsson AB 2006-2014. All Rights Reserved. %%% %%% The contents of this file are subject to the Erlang Public License, %%% Version 1.1, (the "License"); you may not use this file except in @@ -58,6 +58,7 @@ -define(WARN_RACE_CONDITION, warn_race_condition). -define(WARN_BEHAVIOUR, warn_behaviour). -define(WARN_UNDEFINED_CALLBACK, warn_undefined_callbacks). +-define(WARN_UNKNOWN, warn_unknown). %% %% The following type has double role: @@ -73,7 +74,7 @@ | ?WARN_CONTRACT_SUPERTYPE | ?WARN_CALLGRAPH | ?WARN_UNMATCHED_RETURN | ?WARN_RACE_CONDITION | ?WARN_BEHAVIOUR | ?WARN_CONTRACT_RANGE - | ?WARN_UNDEFINED_CALLBACK. + | ?WARN_UNDEFINED_CALLBACK | ?WARN_UNKNOWN. %% %% This is the representation of each warning as they will be returned @@ -88,12 +89,6 @@ -type dial_error() :: any(). %% XXX: underspecified %%-------------------------------------------------------------------- -%% THIS TYPE SHOULD ONE DAY DISAPPEAR -- IT DOES NOT BELONG HERE -%%-------------------------------------------------------------------- - --type ordset(T) :: [T] . %% XXX: temporarily - -%%-------------------------------------------------------------------- %% Basic types used either in the record definitions below or in other %% parts of the application %%-------------------------------------------------------------------- @@ -143,7 +138,7 @@ init_plts = [] :: [file:filename()], include_dirs = [] :: [file:filename()], output_plt = none :: 'none' | file:filename(), - legal_warnings = ordsets:new() :: ordset(dial_warn_tag()), + legal_warnings = ordsets:new() :: ordsets:ordset(dial_warn_tag()), report_mode = normal :: rep_mode(), erlang_mode = false :: boolean(), use_contracts = true :: boolean(), @@ -167,4 +162,4 @@ dialyzer_timing:end_stamp(Server), Var end). --define(timing(Server, Msg, Expr),?timing(Server, Msg, _T, Expr)). +-define(timing(Server, Msg, Expr), ?timing(Server, Msg, _T, Expr)). diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index 793efe4b50..e013d39a0e 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -658,7 +658,8 @@ return_value(State = #cl_state{erlang_mode = ErlangMode, mod_deps = ModDeps, output_plt = OutputPlt, plt_info = PltInfo, - stored_warnings = StoredWarnings}, + stored_warnings = StoredWarnings, + legal_warnings = LegalWarnings}, Plt) -> case OutputPlt =:= none of true -> ok; @@ -678,16 +679,33 @@ return_value(State = #cl_state{erlang_mode = ErlangMode, maybe_close_output_file(State), {RetValue, []}; true -> - {RetValue, process_warnings(StoredWarnings)} + Unknown = + case ordsets:is_element(?WARN_UNKNOWN, LegalWarnings) of + true -> + unknown_functions(State) ++ + unknown_types(State) ++ + unknown_behaviours(State); + false -> [] + end, + UnknownWarnings = + [{?WARN_UNKNOWN, {_Filename = "", _Line = 0}, W} || W <- Unknown], + AllWarnings = + UnknownWarnings ++ process_warnings(StoredWarnings), + {RetValue, AllWarnings} end. +unknown_functions(#cl_state{external_calls = Calls}) -> + [{unknown_function, MFA} || MFA <- Calls]. + print_ext_calls(#cl_state{report_mode = quiet}) -> ok; print_ext_calls(#cl_state{output = Output, external_calls = Calls, stored_warnings = Warnings, - output_format = Format}) -> - case Calls =:= [] of + output_format = Format, + legal_warnings = LegalWarnings}) -> + case not ordsets:is_element(?WARN_UNKNOWN, LegalWarnings) + orelse Calls =:= [] of true -> ok; false -> case Warnings =:= [] of @@ -710,14 +728,19 @@ do_print_ext_calls(Output, [{M,F,A}|T], Before) -> do_print_ext_calls(_, [], _) -> ok. +unknown_types(#cl_state{external_types = Types}) -> + [{unknown_type, MFA} || MFA <- Types]. + print_ext_types(#cl_state{report_mode = quiet}) -> ok; print_ext_types(#cl_state{output = Output, external_calls = Calls, external_types = Types, stored_warnings = Warnings, - output_format = Format}) -> - case Types =:= [] of + output_format = Format, + legal_warnings = LegalWarnings}) -> + case not ordsets:is_element(?WARN_UNKNOWN, LegalWarnings) + orelse Types =:= [] of true -> ok; false -> case Warnings =:= [] andalso Calls =:= [] of @@ -740,6 +763,15 @@ do_print_ext_types(Output, [{M,F,A}|T], Before) -> do_print_ext_types(_, [], _) -> ok. +unknown_behaviours(#cl_state{unknown_behaviours = DupBehaviours, + legal_warnings = LegalWarnings}) -> + case ordsets:is_element(?WARN_BEHAVIOUR, LegalWarnings) of + false -> []; + true -> + Behaviours = lists:usort(DupBehaviours), + [{unknown_behaviour, B} || B <- Behaviours] + end. + %%print_unknown_behaviours(#cl_state{report_mode = quiet}) -> %% ok; print_unknown_behaviours(#cl_state{output = Output, diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 46eaeaa303..283031eb9a 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -20,6 +20,8 @@ -module(dialyzer_contracts). +-compile(export_all). + -export([check_contract/2, check_contracts/4, contracts_without_fun/3, @@ -439,7 +441,8 @@ contract_from_form([], _RecDict, _FileLine, TypeAcc, FormAcc) -> {lists:reverse(TypeAcc), lists:reverse(FormAcc)}. process_constraints(Constrs, RecDict, ExpTypes, AllRecords) -> - Init = initialize_constraints(Constrs, RecDict, ExpTypes, AllRecords), + Init0 = initialize_constraints(Constrs, RecDict, ExpTypes, AllRecords), + Init = remove_cycles(Init0), constraints_fixpoint(Init, RecDict, ExpTypes, AllRecords). initialize_constraints(Constrs, RecDict, ExpTypes, AllRecords) -> @@ -479,12 +482,9 @@ constraints_fixpoint(OldVarDict, Constrs, RecDict, ExpTypes, AllRecords) -> constraints_fixpoint(NewVarDict, Constrs, RecDict, ExpTypes, AllRecords) end. --define(TYPE_LIMIT, 4). - final_form(Form, RecDict, ExpTypes, AllRecords, VarDict) -> T1 = erl_types:t_from_form(Form, RecDict, VarDict), - T2 = erl_types:t_solve_remote(T1, ExpTypes, AllRecords), - erl_types:t_limit(T2, ?TYPE_LIMIT). + erl_types:t_solve_remote(T1, ExpTypes, AllRecords). constraints_to_dict(Constrs, RecDict, ExpTypes, AllRecords, VarDict) -> Subtypes = @@ -499,6 +499,74 @@ constraints_to_subs([C|Rest], RecDict, ExpTypes, AllRecords, VarDict, Acc) -> NewAcc = [{subtype, T1, T2}|Acc], constraints_to_subs(Rest, RecDict, ExpTypes, AllRecords, VarDict, NewAcc). +%% Replaces variables with '_' when necessary to break up cycles among +%% the constraints. + +remove_cycles(Constrs0) -> + Uses = find_uses(Constrs0), + G = digraph:new(), + Vs0 = [V || {V, _} <- Uses] ++ [V || {_, V} <- Uses], + Vs = lists:usort(Vs0), + lists:foreach(fun(V) -> _ = digraph:add_vertex(G, V) end, Vs), + lists:foreach(fun({From, To}) -> + _ = digraph:add_edge(G, {From, To}, From, To, []) + end, Uses), + ok = remove_cycles(G, Vs), + ToRemove = ordsets:subtract(ordsets:from_list(Uses), + ordsets:from_list(digraph:edges(G))), + Constrs = remove_uses(ToRemove, Constrs0), + digraph:delete(G), + Constrs. + +find_uses([{Var, Form}|Constrs]) -> + UsedVars = form_vars(Form, []), + VarName = erl_types:t_var_name(Var), + [{VarName, UsedVar} || UsedVar <- UsedVars] ++ find_uses(Constrs); +find_uses([]) -> + []. + +form_vars({var, _, '_'}, Vs) -> Vs; +form_vars({var, _, V}, Vs) -> [V|Vs]; +form_vars(T, Vs) when is_tuple(T) -> + form_vars(tuple_to_list(T), Vs); +form_vars([E|Es], Vs) -> + form_vars(Es, form_vars(E, Vs)); +form_vars(_, Vs) -> Vs. + +remove_cycles(G, Vs) -> + NumberOfEdges = digraph:no_edges(G), + lists:foreach(fun(V) -> + case digraph:get_cycle(G, V) of + false -> true; + [V] -> digraph:del_edge(G, {V, V}); + [V, V1|_] -> digraph:del_edge(G, {V, V1}) + end + end, Vs), + case digraph:no_edges(G) =:= NumberOfEdges of + true -> ok; + false -> remove_cycles(G, Vs) + end. + +remove_uses([], Constrs) -> Constrs; +remove_uses([{Var, Use}|ToRemove], Constrs0) -> + Constrs = remove_uses(Var, Use, Constrs0), + remove_uses(ToRemove, Constrs). + +remove_uses(_Var, _Use, []) -> []; +remove_uses(Var, Use, [Constr|Constrs]) -> + {V, Form} = Constr, + case erl_types:t_var_name(V) =:= Var of + true -> [{V, remove_use(Form, Use)}|Constrs]; + false -> [Constr|remove_uses(Var, Use, Constrs)] + end. + +remove_use({var, L, V}, V) -> {var, L, '_'}; +remove_use(T, V) when is_tuple(T) -> + list_to_tuple(remove_use(tuple_to_list(T), V)); +remove_use([E|Es], V) -> + [remove_use(E, V)|remove_use(Es, V)]; +remove_use(T, _V) -> T. + %% Gets the most general domain of a list of domains of all %% the overloaded contracts diff --git a/lib/dialyzer/src/dialyzer_dep.erl b/lib/dialyzer/src/dialyzer_dep.erl index f1ac41ff04..e3ece144c9 100644 --- a/lib/dialyzer/src/dialyzer_dep.erl +++ b/lib/dialyzer/src/dialyzer_dep.erl @@ -55,11 +55,11 @@ %% %% Letrecs = a dict mapping var labels to their recursive definition. %% top-level letrecs are not included as they are handled -%% separatedly. +%% separately. %% -spec analyze(cerl:c_module()) -> - {dict:dict(), ordset('external' | label()), dict:dict(), dict:dict()}. + {dict:dict(), ordsets:ordset('external' | label()), dict:dict(), dict:dict()}. analyze(Tree) -> %% io:format("Handling ~w\n", [cerl:atom_val(cerl:module_name(Tree))]), diff --git a/lib/dialyzer/src/dialyzer_gui_wx.erl b/lib/dialyzer/src/dialyzer_gui_wx.erl index 08f31c1e13..7070fa240d 100644 --- a/lib/dialyzer/src/dialyzer_gui_wx.erl +++ b/lib/dialyzer/src/dialyzer_gui_wx.erl @@ -61,7 +61,7 @@ init_plt :: dialyzer_plt:plt(), dir_entry :: wx:wx_object(), file_box :: wx:wx_object(), - files_to_analyze :: ordset(string()), + files_to_analyze :: ordsets:ordset(string()), gui :: wx:wx_object(), log :: wx:wx_object(), menu :: menu(), diff --git a/lib/dialyzer/src/dialyzer_options.erl b/lib/dialyzer/src/dialyzer_options.erl index 06672e595f..a92b8b1958 100644 --- a/lib/dialyzer/src/dialyzer_options.erl +++ b/lib/dialyzer/src/dialyzer_options.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2012. All Rights Reserved. +%% Copyright Ericsson AB 2006-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -51,7 +51,8 @@ build(Opts) -> ?WARN_CONTRACT_TYPES, ?WARN_CONTRACT_SYNTAX, ?WARN_BEHAVIOUR, - ?WARN_UNDEFINED_CALLBACK], + ?WARN_UNDEFINED_CALLBACK, + ?WARN_UNKNOWN], DefaultWarns1 = ordsets:from_list(DefaultWarns), InitPlt = dialyzer_plt:get_default_plt(), DefaultOpts = #options{}, @@ -310,6 +311,8 @@ build_warnings([Opt|Opts], Warnings) -> ordsets:add_element(?WARN_CONTRACT_SUBTYPE, Warnings); underspecs -> ordsets:add_element(?WARN_CONTRACT_SUPERTYPE, Warnings); + no_unknown -> + ordsets:del_element(?WARN_UNKNOWN, Warnings); OtherAtom -> bad_option("Unknown dialyzer warning option", OtherAtom) end, diff --git a/lib/dialyzer/src/dialyzer_races.erl b/lib/dialyzer/src/dialyzer_races.erl index 48fcde8014..b1f849b16f 100644 --- a/lib/dialyzer/src/dialyzer_races.erl +++ b/lib/dialyzer/src/dialyzer_races.erl @@ -85,6 +85,12 @@ -type race_tag() :: 'whereis_register' | 'whereis_unregister' | 'ets_lookup_insert' | 'mnesia_dirty_read_write'. +%% The following type is similar to the dial_warning() type but has a +%% tag which is local to this module and is not propagated to outside +-type dial_race_warning() :: {race_warn_tag(), file_line(), {atom(), [term()]}}. +-type race_warn_tag() :: ?WARN_WHEREIS_REGISTER | ?WARN_WHEREIS_UNREGISTER + | ?WARN_ETS_LOOKUP_INSERT | ?WARN_MNESIA_DIRTY_READ_WRITE. + -record(beg_clause, {arg :: var_to_map1(), pats :: var_to_map1(), guard :: cerl:cerl()}). @@ -103,7 +109,7 @@ args :: args(), arg_types :: [erl_types:erl_type()], vars :: [core_vars()], - state :: _, %% XXX: recursive + state :: dialyzer_dataflow:state(), file_line :: file_line(), var_map :: dict:dict()}). -record(fun_call, {caller :: dialyzer_callgraph:mfa_or_funlbl(), @@ -141,7 +147,7 @@ race_tags = [] :: [#race_fun{}], %% true for fun types and warning mode race_analysis = false :: boolean(), - race_warnings = [] :: [dial_warning()]}). + race_warnings = [] :: [dial_race_warning()]}). %%% =========================================================================== %%% @@ -1763,7 +1769,7 @@ ets_list_args(MaybeList) -> catch _:_ -> [?no_label] end; false -> [ets_tuple_args(MaybeList)] - end. + end. ets_list_argtypes(ListStr) -> ListStr1 = string:strip(ListStr, left, $[), diff --git a/lib/dialyzer/test/opaque_SUITE_data/dialyzer_options b/lib/dialyzer/test/opaque_SUITE_data/dialyzer_options index 3ff26b87db..44a65f6e90 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/opaque_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, [{warnings, [no_unused, no_return]}]}. +{dialyzer_options, [{warnings, [no_unused, no_return, no_unknown]}]}. diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/modules/opaque_erl_scan.erl b/lib/dialyzer/test/opaque_SUITE_data/src/modules/opaque_erl_scan.erl index 9ecd4f92a1..24d0793a7c 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/modules/opaque_erl_scan.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/modules/opaque_erl_scan.erl @@ -1,4 +1,4 @@ -%% -*- coding: utf-8 -*- +%% %% %% %CopyrightBegin% %% diff --git a/lib/dialyzer/test/options1_SUITE_data/dialyzer_options b/lib/dialyzer/test/options1_SUITE_data/dialyzer_options index c612e77d3e..65d233ac0d 100644 --- a/lib/dialyzer/test/options1_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/options1_SUITE_data/dialyzer_options @@ -1,2 +1,2 @@ -{dialyzer_options, [{include_dirs, ["my_include"]}, {defines, [{'COMPILER_VSN', 42}]}, {warnings, [no_improper_lists]}]}. +{dialyzer_options, [{include_dirs, ["my_include"]}, {defines, [{'COMPILER_VSN', 42}]}, {warnings, [no_improper_lists, no_unknown]}]}. {time_limit, 30}. diff --git a/lib/dialyzer/test/r9c_SUITE_data/dialyzer_options b/lib/dialyzer/test/r9c_SUITE_data/dialyzer_options index e00e23bb66..ba0e6b1ad7 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/r9c_SUITE_data/dialyzer_options @@ -1,2 +1,2 @@ -{dialyzer_options, [{defines, [{vsn, 42}]}]}. +{dialyzer_options, [{defines, [{vsn, 42}]}, {warnings, [no_unknown]}]}. {time_limit, 20}. diff --git a/lib/dialyzer/test/race_SUITE_data/dialyzer_options b/lib/dialyzer/test/race_SUITE_data/dialyzer_options index 44e1720715..6992fc6c40 100644 --- a/lib/dialyzer/test/race_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/race_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, [{warnings, [race_conditions]}]}. +{dialyzer_options, [{warnings, [race_conditions, no_unknown]}]}. diff --git a/lib/dialyzer/test/small_SUITE_data/dialyzer_options b/lib/dialyzer/test/small_SUITE_data/dialyzer_options index 50991c9bc5..0d91699e4d 100644 --- a/lib/dialyzer/test/small_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/small_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, []}. +{dialyzer_options, [{warnings, [no_unknown]}]}. diff --git a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes index bfa33cd296..fbdd182358 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes +++ b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes @@ -1,27 +1,28 @@ contracts_with_subtypes.erl:106: The call contracts_with_subtypes:rec_arg({'a','b'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) contracts_with_subtypes.erl:107: The call contracts_with_subtypes:rec_arg({'b','a'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:108: The call contracts_with_subtypes:rec_arg({'a',{'b','a'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) contracts_with_subtypes.erl:109: The call contracts_with_subtypes:rec_arg({'b',{'a','b'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:110: The call contracts_with_subtypes:rec_arg({'a',{'b',{'a','b'}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:111: The call contracts_with_subtypes:rec_arg({'b',{'a',{'b','a'}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A} | {'b',B}), is_subtype(A,'a' | {'b',B}), is_subtype(B,'b' | {'a',A}) -contracts_with_subtypes.erl:142: The pattern 1 can never match the type string() -contracts_with_subtypes.erl:145: The pattern 'alpha' can never match the type {'ok',_} | {'ok',_,string()} -contracts_with_subtypes.erl:147: The pattern 42 can never match the type {'ok',_} | {'ok',_,string()} -contracts_with_subtypes.erl:163: The pattern 'alpha' can never match the type {'ok',_} -contracts_with_subtypes.erl:165: The pattern 42 can never match the type {'ok',_} -contracts_with_subtypes.erl:183: The pattern 'alpha' can never match the type {'ok',_} -contracts_with_subtypes.erl:185: The pattern 42 can never match the type {'ok',_} -contracts_with_subtypes.erl:202: The pattern 1 can never match the type string() -contracts_with_subtypes.erl:205: The pattern {'ok', _} can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:206: The pattern 'alpha' can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:207: The pattern {'ok', 42} can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:208: The pattern 42 can never match the type {'ok',_,string()} -contracts_with_subtypes.erl:234: Function flat_ets_new_t/0 has no local return -contracts_with_subtypes.erl:235: The call contracts_with_subtypes:flat_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,'set' | 'ordered_set' | 'bag' | 'duplicate_bag' | 'public' | 'protected' | 'private' | 'named_table' | {'keypos',integer()} | {'heir',pid(),term()} | {'heir','none'} | {'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed') +contracts_with_subtypes.erl:135: The call contracts_with_subtypes:rec2({'a','b'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:136: The call contracts_with_subtypes:rec2({'b','a'}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:137: The call contracts_with_subtypes:rec2({'a',{'b','a'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:138: The call contracts_with_subtypes:rec2({'b',{'a','b'}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,ab()) +contracts_with_subtypes.erl:171: The pattern 1 can never match the type string() +contracts_with_subtypes.erl:174: The pattern 'alpha' can never match the type {'ok',_} | {'ok',_,string()} +contracts_with_subtypes.erl:176: The pattern 42 can never match the type {'ok',_} | {'ok',_,string()} +contracts_with_subtypes.erl:192: The pattern 'alpha' can never match the type {'ok',_} +contracts_with_subtypes.erl:194: The pattern 42 can never match the type {'ok',_} +contracts_with_subtypes.erl:212: The pattern 'alpha' can never match the type {'ok',_} +contracts_with_subtypes.erl:214: The pattern 42 can never match the type {'ok',_} +contracts_with_subtypes.erl:231: The pattern 1 can never match the type string() +contracts_with_subtypes.erl:234: The pattern {'ok', _} can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:235: The pattern 'alpha' can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:236: The pattern {'ok', 42} can never match the type {'ok',_,string()} +contracts_with_subtypes.erl:237: The pattern 42 can never match the type {'ok',_,string()} contracts_with_subtypes.erl:23: Invalid type specification for function contracts_with_subtypes:extract2/0. The success typing is () -> 'something' -contracts_with_subtypes.erl:261: Function factored_ets_new_t/0 has no local return -contracts_with_subtypes.erl:262: The call contracts_with_subtypes:factored_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,Type | Access | 'named_table' | {'keypos',Pos} | {'heir',Pid::pid(),HeirData} | {'heir','none'} | Tweaks), is_subtype(Type,type()), is_subtype(Access,access()), is_subtype(Tweaks,{'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed'), is_subtype(Pos,pos_integer()), is_subtype(HeirData,term()) +contracts_with_subtypes.erl:263: Function flat_ets_new_t/0 has no local return +contracts_with_subtypes.erl:264: The call contracts_with_subtypes:flat_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,'set' | 'ordered_set' | 'bag' | 'duplicate_bag' | 'public' | 'protected' | 'private' | 'named_table' | {'keypos',integer()} | {'heir',pid(),term()} | {'heir','none'} | {'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed') +contracts_with_subtypes.erl:290: Function factored_ets_new_t/0 has no local return +contracts_with_subtypes.erl:291: The call contracts_with_subtypes:factored_ets_new(12,[]) breaks the contract (Name,Options) -> atom() when is_subtype(Name,atom()), is_subtype(Options,[Option]), is_subtype(Option,Type | Access | 'named_table' | {'keypos',Pos} | {'heir',Pid::pid(),HeirData} | {'heir','none'} | Tweaks), is_subtype(Type,type()), is_subtype(Access,access()), is_subtype(Tweaks,{'write_concurrency',boolean()} | {'read_concurrency',boolean()} | 'compressed'), is_subtype(Pos,pos_integer()), is_subtype(HeirData,term()) contracts_with_subtypes.erl:77: The call contracts_with_subtypes:foo1(5) breaks the contract (Arg1) -> Res when is_subtype(Arg1,atom()), is_subtype(Res,atom()) contracts_with_subtypes.erl:78: The call contracts_with_subtypes:foo2(5) breaks the contract (Arg1) -> Res when is_subtype(Arg1,Arg2), is_subtype(Arg2,atom()), is_subtype(Res,atom()) contracts_with_subtypes.erl:79: The call contracts_with_subtypes:foo3(5) breaks the contract (Arg1) -> Res when is_subtype(Arg2,atom()), is_subtype(Arg1,Arg2), is_subtype(Res,atom()) diff --git a/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 new file mode 100644 index 0000000000..9f5433a13d --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/contracts_with_subtypes2 @@ -0,0 +1,3 @@ + +contracts_with_subtypes2.erl:18: Function t/0 has no local return +contracts_with_subtypes2.erl:19: The call contracts_with_subtypes2:t({'a',{'b',{'c',{'d',{'e',{'g',3}}}}}}) breaks the contract (Arg) -> 'ok' when is_subtype(Arg,{'a',A}), is_subtype(A,{'b',B}), is_subtype(B,{'c',C}), is_subtype(C,{'d',D}), is_subtype(D,{'e',E}), is_subtype(E,{'f',_}) diff --git a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl index d72138d509..d7dfd9752e 100644 --- a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl +++ b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes.erl @@ -103,15 +103,44 @@ c(babb) -> rec_arg({b, {a, {b, b}}}); c(ababb) -> rec_arg({a, {b, {a, {b, b}}}}); c(babaa) -> rec_arg({b, {a, {b, {a, a}}}}). -w(ab) -> rec_arg({a, b}); -w(ba) -> rec_arg({b, a}); -w(aba) -> rec_arg({a, {b, a}}); -w(bab) -> rec_arg({b, {a, b}}); -w(abab) -> rec_arg({a, {b, {a, b}}}); -w(baba) -> rec_arg({b, {a, {b, a}}}); +w(ab) -> rec_arg({a, b}); % breaks the contract +w(ba) -> rec_arg({b, a}); % breaks the contract +w(aba) -> rec_arg({a, {b, a}}); % no longer breaks the contract +w(bab) -> rec_arg({b, {a, b}}); % breaks the contract +w(abab) -> rec_arg({a, {b, {a, b}}}); % no longer breaks the contract +w(baba) -> rec_arg({b, {a, {b, a}}}); % no longer breaks the contract w(ababa) -> rec_arg({a, {b, {a, {b, a}}}}); w(babab) -> rec_arg({b, {a, {b, {a, b}}}}). +%% For comparison: the same thing with types + +-type ab() :: {a, a()} | {b, b()}. +-type a() :: a | {b, b()}. +-type b() :: b | {a, a()}. + +-spec rec2(Arg) -> ok when + Arg :: ab(). + +rec2(X) -> get(X). + +d(aa) -> rec2({a, a}); +d(bb) -> rec2({b, b}); +d(abb) -> rec2({a, {b, b}}); +d(baa) -> rec2({b, {a, a}}); +d(abaa) -> rec2({a, {b, {a, a}}}); +d(babb) -> rec2({b, {a, {b, b}}}); +d(ababb) -> rec2({a, {b, {a, {b, b}}}}); +d(babaa) -> rec2({b, {a, {b, {a, a}}}}). + +q(ab) -> rec2({a, b}); % breaks the contract +q(ba) -> rec2({b, a}); % breaks the contract +q(aba) -> rec2({a, {b, a}}); % breaks the contract +q(bab) -> rec2({b, {a, b}}); % breaks the contract +q(abab) -> rec2({a, {b, {a, b}}}); +q(baba) -> rec2({b, {a, {b, a}}}); +q(ababa) -> rec2({a, {b, {a, {b, a}}}}); +q(babab) -> rec2({b, {a, {b, {a, b}}}}). + %=============================================================================== -type dublo(X) :: {X, X}. @@ -143,7 +172,7 @@ st(X) when is_atom(X) -> _Other -> ok end; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. @@ -161,7 +190,7 @@ dt(X) when is_atom(X) -> err2 -> ok; {ok, X} -> ok; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. @@ -181,7 +210,7 @@ dt2(X) when is_atom(X) -> err2 -> ok; {ok, X} -> ok; alpha -> bad; - {ok, 42} -> bad; + {ok, 42} -> ok; 42 -> bad end. diff --git a/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl new file mode 100644 index 0000000000..d2f945b284 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/contracts_with_subtypes2.erl @@ -0,0 +1,40 @@ +-module(contracts_with_subtypes2). + +-compile(export_all). + +-behaviour(supervisor). + +-spec t(Arg) -> ok when + Arg :: {a, A}, + A :: {b, B}, + B :: {c, C}, + C :: {d, D}, + D :: {e, E}, + E :: {f, _}. + +t(X) -> + get(X). + +t() -> + t({a, {b, {c, {d, {e, {g, 3}}}}}}). % breaks the contract + +%% This one should possibly result in warnings about unused variables. +-spec l() -> ok when + X :: Y, + Y :: X. + +l() -> + ok. + +%% This is the example from seq12547 (ticket OTP-11798). +%% There used to be a warning. + +-spec init(term()) -> Result when + Result :: {ok, {{supervisor:strategy(), + non_neg_integer(), + pos_integer()}, + [supervisor:child_spec()]}} + | ignore. + +init(_) -> + foo:bar(). diff --git a/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl b/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl new file mode 100644 index 0000000000..70059f73b6 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/maps_redef.erl @@ -0,0 +1,12 @@ +-module(maps_redef). + +-export([t/0]). + +%% OK in Erlang/OTP 17, at least. + +-type map() :: atom(). % redefine built-in type + +-spec t() -> map(). + +t() -> + a. % OK diff --git a/lib/dialyzer/test/underspecs_SUITE_data/dialyzer_options b/lib/dialyzer/test/underspecs_SUITE_data/dialyzer_options index f7197ac30f..6843119b9d 100644 --- a/lib/dialyzer/test/underspecs_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/underspecs_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, [{warnings, [underspecs]}]}. +{dialyzer_options, [{warnings, [underspecs, no_unknown]}]}. diff --git a/lib/dialyzer/test/user_SUITE_data/dialyzer_options b/lib/dialyzer/test/user_SUITE_data/dialyzer_options index 513ed7752b..d20ecd389f 100644 --- a/lib/dialyzer/test/user_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/user_SUITE_data/dialyzer_options @@ -1,2 +1,2 @@ -{dialyzer_options, []}. +{dialyzer_options, [{warnings, [no_unknown]}]}. {time_limit, 3}.
\ No newline at end of file diff --git a/lib/diameter/src/diameter.app.src b/lib/diameter/src/diameter.app.src index ceefb9b398..509de9e595 100644 --- a/lib/diameter/src/diameter.app.src +++ b/lib/diameter/src/diameter.app.src @@ -24,5 +24,7 @@ {registered, [%REGISTERED%]}, {applications, [stdlib, kernel]}, {env, []}, - {mod, {diameter_app, []}} + {mod, {diameter_app, []}}, + {runtime_dependencies, ["syntax_tools-1.6.14","stdlib-2.0","ssl-5.3.4", + "runtime_tools-1.8.14","kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/diameter/test/diameter_codec_test.erl b/lib/diameter/test/diameter_codec_test.erl index 0b4568a9e5..90536dcf2b 100644 --- a/lib/diameter/test/diameter_codec_test.erl +++ b/lib/diameter/test/diameter_codec_test.erl @@ -1,8 +1,7 @@ -%% coding: utf-8 %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2013. All Rights Reserved. +%% Copyright Ericsson AB 2010-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/edoc/src/edoc.app.src b/lib/edoc/src/edoc.app.src index 0c8d5b85f8..9e1155d3e8 100644 --- a/lib/edoc/src/edoc.app.src +++ b/lib/edoc/src/edoc.app.src @@ -22,4 +22,6 @@ otpsgml_layout]}, {registered,[]}, {applications, [compiler,kernel,stdlib,syntax_tools]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["xmerl-1.3.7","syntax_tools-1.6.14","stdlib-2.0", + "kernel-3.0","inets-5.10","erts-6.0"]}]}. diff --git a/lib/edoc/vsn.mk b/lib/edoc/vsn.mk index 2fcc97e406..0172aac48b 100644 --- a/lib/edoc/vsn.mk +++ b/lib/edoc/vsn.mk @@ -1 +1 @@ -EDOC_VSN = 0.7.12.1 +EDOC_VSN = 0.7.13 diff --git a/lib/eldap/src/Makefile b/lib/eldap/src/Makefile index ebb7967e11..2e1110ec2c 100644 --- a/lib/eldap/src/Makefile +++ b/lib/eldap/src/Makefile @@ -88,7 +88,7 @@ $(TARGET_FILES): $(HRL_FILES) # Special Build Targets # ---------------------------------------------------- $(ASN1_HRL): ../asn1/$(ASN1_FILES) - $(asn_verbose)$(ERLC) -o $(EBIN) -bber $(ERL_COMPILE_FLAGS) ../asn1/ELDAPv3.asn1 + $(asn_verbose)$(ERLC) -o $(EBIN) +legacy_erlang_types -bber $(ERL_COMPILE_FLAGS) ../asn1/ELDAPv3.asn1 # ---------------------------------------------------- # Release Target diff --git a/lib/eldap/src/eldap.app.src b/lib/eldap/src/eldap.app.src index 8215328910..03a7d7c562 100644 --- a/lib/eldap/src/eldap.app.src +++ b/lib/eldap/src/eldap.app.src @@ -4,5 +4,7 @@ {modules, [eldap, 'ELDAPv3']}, {registered, []}, {applications, [kernel, stdlib]}, - {env, []} + {env, []}, + {runtime_dependencies, ["stdlib-2.0","ssl-5.3.4","kernel-3.0","erts-6.0", + "asn1-3.0"]} ]}. diff --git a/lib/eldap/test/eldap_misc_SUITE.erl b/lib/eldap/test/eldap_misc_SUITE.erl new file mode 100644 index 0000000000..ca810ee33c --- /dev/null +++ b/lib/eldap/test/eldap_misc_SUITE.erl @@ -0,0 +1,51 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012-2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(eldap_misc_SUITE). + +-compile(export_all). %% Use this only in test suites... + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eldap/include/eldap.hrl"). +-include_lib("eldap/ebin/ELDAPv3.hrl"). + +all() -> + [ + encode, + decode + ]. + + +encode(_Config) -> + {ok,Bin} = 'ELDAPv3':encode('AddRequest', #'AddRequest'{entry="hejHopp" ,attributes=[]} ), + Expected = <<104,11,4,7,104,101,106,72,111,112,112,48,0>>, + Expected = Bin. + +decode(_Config) -> + {ok,Res} = 'ELDAPv3':decode('AddRequest', <<104,11,4,7,104,101,106,72,111,112,112,48,0>>), + ct:log("Res = ~p", [Res]), + Expected = #'AddRequest'{entry = "hejHopp",attributes = []}, + case Res of + Expected -> ok; + #'AddRequest'{entry= <<"hejHopp">>, attributes=[]} -> + {fail, "decoded to (correct) binary!!"}; + _ -> + {fail, "Bad decode"} + end. + diff --git a/lib/erl_docgen/src/erl_docgen.app.src b/lib/erl_docgen/src/erl_docgen.app.src index daad172106..e2830b2692 100644 --- a/lib/erl_docgen/src/erl_docgen.app.src +++ b/lib/erl_docgen/src/erl_docgen.app.src @@ -8,7 +8,7 @@ }, {registered,[]}, {applications, [kernel,stdlib]}, - {env, [] - } + {env, []}, + {runtime_dependencies, ["xmerl-1.3.7","stdlib-2.0","edoc-0.7.13","erts-6.0"]} ] }. diff --git a/lib/erl_docgen/vsn.mk b/lib/erl_docgen/vsn.mk index cda8671cfd..0f89922275 100644 --- a/lib/erl_docgen/vsn.mk +++ b/lib/erl_docgen/vsn.mk @@ -1 +1 @@ -ERL_DOCGEN_VSN = 0.3.4.1 +ERL_DOCGEN_VSN = 0.3.5 diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index 3175d1bdfd..2e8418d61e 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -1166,7 +1166,11 @@ static unsigned int gen_challenge(void) uname(&s.name); s.cpu = clock(); s.pid = getpid(); +#ifndef __ANDROID__ s.hid = gethostid(); +#else + s.hid = 0; +#endif s.uid = getuid(); s.gid = getgid(); diff --git a/lib/erl_interface/src/connect/ei_resolve.c b/lib/erl_interface/src/connect/ei_resolve.c index 74dcba61a7..cffcac801c 100644 --- a/lib/erl_interface/src/connect/ei_resolve.c +++ b/lib/erl_interface/src/connect/ei_resolve.c @@ -642,7 +642,7 @@ struct hostent *ei_gethostbyname_r(const char *name, #ifndef HAVE_GETHOSTBYNAME_R return my_gethostbyname_r(name,hostp,buffer,buflen,h_errnop); #else -#if (defined(__GLIBC__) || (__FreeBSD_version >= 602000) || defined(__DragonFly__)) +#if (defined(__GLIBC__) || (__FreeBSD_version >= 602000) || defined(__DragonFly__) || defined(__ANDROID__)) struct hostent *result; gethostbyname_r(name, hostp, buffer, buflen, &result, h_errnop); diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index f386ce09a8..8731283265 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1 +1,2 @@ -EI_VSN = 3.7.15 +EI_VSN = 3.7.16 +ERL_INTERFACE_VSN = $(EI_VSN) diff --git a/lib/et/src/et.app.src b/lib/et/src/et.app.src index f7189a4197..c26d9320d8 100644 --- a/lib/et/src/et.app.src +++ b/lib/et/src/et.app.src @@ -31,5 +31,7 @@ ]}, {registered, [et_collector]}, {applications, [stdlib, kernel]}, - {env, []} + {env, []}, + {runtime_dependencies, ["wx-1.2","stdlib-2.0","runtime_tools-1.8.14", + "kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/et/vsn.mk b/lib/et/vsn.mk index 282991aa49..a47be678ca 100644 --- a/lib/et/vsn.mk +++ b/lib/et/vsn.mk @@ -1 +1 @@ -ET_VSN = 1.4.4.5 +ET_VSN = 1.5 diff --git a/lib/eunit/src/eunit.app.src b/lib/eunit/src/eunit.app.src index 5e16dfa2ce..7a3978e200 100644 --- a/lib/eunit/src/eunit.app.src +++ b/lib/eunit/src/eunit.app.src @@ -18,4 +18,5 @@ eunit_tty]}, {registered,[]}, {applications, [kernel,stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/eunit/vsn.mk b/lib/eunit/vsn.mk index 8f816b3b94..f04c0536fe 100644 --- a/lib/eunit/vsn.mk +++ b/lib/eunit/vsn.mk @@ -1 +1 @@ -EUNIT_VSN = 2.2.6 +EUNIT_VSN = 2.2.7 diff --git a/lib/gs/src/gs.app.src b/lib/gs/src/gs.app.src index c83c9b54d7..c6f88e5144 100644 --- a/lib/gs/src/gs.app.src +++ b/lib/gs/src/gs.app.src @@ -10,4 +10,5 @@ gstk_window,tcl2erl,tool_file_dialog,tool_utils, gs_packer,gse]}, {registered, [gs_frontend]}, - {applications, [kernel, stdlib]}]}. + {applications, [kernel, stdlib]}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/gs/vsn.mk b/lib/gs/vsn.mk index 5c18153c34..96786b300c 100644 --- a/lib/gs/vsn.mk +++ b/lib/gs/vsn.mk @@ -1,2 +1,2 @@ -GS_VSN = 1.5.15.2 +GS_VSN = 1.5.16 diff --git a/lib/hipe/Makefile b/lib/hipe/Makefile index a9e24f4d17..46cbc33ae2 100644 --- a/lib/hipe/Makefile +++ b/lib/hipe/Makefile @@ -22,7 +22,7 @@ include $(ERL_TOP)/make/target.mk include $(ERL_TOP)/make/$(TARGET)/otp.mk ifdef HIPE_ENABLED -HIPE_SUBDIRS = regalloc sparc ppc x86 amd64 arm opt tools +HIPE_SUBDIRS = regalloc sparc ppc x86 amd64 arm opt tools llvm else HIPE_SUBDIRS = endif diff --git a/lib/hipe/arm/hipe_arm_assemble.erl b/lib/hipe/arm/hipe_arm_assemble.erl index 2af786994e..e9de96a927 100644 --- a/lib/hipe/arm/hipe_arm_assemble.erl +++ b/lib/hipe/arm/hipe_arm_assemble.erl @@ -44,8 +44,8 @@ assemble(CompiledCode, Closures, Exports, Options) -> print("Total num bytes=~w\n", [CodeSize], Options), %% SC = hipe_pack_constants:slim_constmap(ConstMap), - DataRelocs = mk_data_relocs(RefsFromConsts, LabelMap), - SSE = slim_sorted_exportmap(ExportMap,Closures,Exports), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap,Closures,Exports), SlimRefs = hipe_pack_constants:slim_refs(AccRefs), Bin = term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, ConstAlign, ConstSize, @@ -320,7 +320,7 @@ do_pseudo_li(I, MFA, ConstMap, Address, PrevImms, PendImms) -> Atom when is_atom(Atom) -> {load_atom, Atom}; {Label,constant} -> - ConstNo = find_const({MFA,Label}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,Label}, ConstMap), {load_address, {constant,ConstNo}}; {Label,closure} -> {load_address, {closure,Label}}; @@ -518,37 +518,6 @@ fix_pc_refs(I, InsnAddress, FunAddress, LabelMap) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -mk_data_relocs(RefsFromConsts, LabelMap) -> - lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). - -mk_data_relocs([{MFA,Labels} | Rest], LabelMap, Acc) -> - Map = [case Label of - {L,Pos} -> - Offset = find({MFA,L}, LabelMap), - {Pos,Offset}; - {sorted,Base,OrderedLabels} -> - {sorted, Base, [begin - Offset = find({MFA,L}, LabelMap), - {Order, Offset} - end - || {L,Order} <- OrderedLabels]} - end - || Label <- Labels], - %% msg("Map: ~w Map\n",[Map]), - mk_data_relocs(Rest, LabelMap, [Map,Acc]); -mk_data_relocs([],_,Acc) -> Acc. - -find({_MFA,_L} = MFAL, LabelMap) -> - gb_trees:get(MFAL, LabelMap). - -slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> - IsClosure = lists:member({M,F,A}, Closures), - IsExported = is_exported(F, A, Exports), - [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; -slim_sorted_exportmap([],_,_) -> []. - -is_exported(F, A, Exports) -> lists:member({F,A}, Exports). - %%% %%% Assembly listing support (pp_asm option). %%% @@ -594,17 +563,6 @@ fill_spaces(N) when N > 0 -> fill_spaces(0) -> []. -%%% -%%% Lookup a constant in a ConstMap. -%%% - -find_const({MFA,Label},[{pcm_entry,MFA,Label,ConstNo,_,_,_}|_]) -> - ConstNo; -find_const(N,[_|R]) -> - find_const(N,R); -find_const(C,[]) -> - ?EXIT({constant_not_found,C}). - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index aa69b57fa2..5938d94e65 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -502,9 +502,9 @@ t_contains_opaque(?int_range(_From, _To), _Opaques) -> false; t_contains_opaque(?int_set(_Set), _Opaques) -> false; t_contains_opaque(?list(Type, Tail, _), Opaques) -> t_contains_opaque(Type, Opaques) orelse t_contains_opaque(Tail, Opaques); -t_contains_opaque(?map(Pairs), Opaques) -> - list_contains_opaque([V||{_,V}<-Pairs], Opaques) orelse - list_contains_opaque([K||{K,_}<-Pairs], Opaques); +t_contains_opaque(?map(_) = Map, Opaques) -> + list_contains_opaque(map_values(Map), Opaques) orelse + list_contains_opaque(map_keys(Map), Opaques); t_contains_opaque(?matchstate(_P, _Slots), _Opaques) -> false; t_contains_opaque(?nil, _Opaques) -> false; t_contains_opaque(?number(_Set, _Tag), _Opaques) -> false; @@ -2093,6 +2093,8 @@ t_has_var(?tuple(Elements, _, _)) -> t_has_var_list(Elements); t_has_var(?tuple_set(_) = T) -> t_has_var_list(t_tuple_subtypes(T)); +t_has_var(?map(_)= Map) -> + t_has_var_list(map_keys(Map)) orelse t_has_var_list(map_values(Map)); t_has_var(?opaque(Set)) -> %% Assume variables in 'args' are also present i 'struct' t_has_var_list([O#opaque.struct || O <- set_to_list(Set)]); @@ -2120,21 +2122,28 @@ t_collect_vars(?function(Domain, Range), Acc) -> t_collect_vars(?list(Contents, Termination, _), Acc) -> ordsets:union(t_collect_vars(Contents, Acc), t_collect_vars(Termination, [])); t_collect_vars(?product(Types), Acc) -> - lists:foldl(fun(T, TmpAcc) -> t_collect_vars(T, TmpAcc) end, Acc, Types); + t_collect_vars_list(Types, Acc); t_collect_vars(?tuple(?any, ?any, ?any), Acc) -> Acc; t_collect_vars(?tuple(Types, _, _), Acc) -> - lists:foldl(fun(T, TmpAcc) -> t_collect_vars(T, TmpAcc) end, Acc, Types); + t_collect_vars_list(Types, Acc); t_collect_vars(?tuple_set(_) = TS, Acc) -> - lists:foldl(fun(T, TmpAcc) -> t_collect_vars(T, TmpAcc) end, Acc, - t_tuple_subtypes(TS)); + t_collect_vars_list(t_tuple_subtypes(TS), Acc); +t_collect_vars(?map(_) = Map, Acc0) -> + Acc = t_collect_vars_list(map_keys(Map), Acc0), + t_collect_vars_list(map_values(Map), Acc); t_collect_vars(?opaque(Set), Acc) -> %% Assume variables in 'args' are also present i 'struct' - lists:foldl(fun(T, TmpAcc) -> t_collect_vars(T, TmpAcc) end, Acc, - [O#opaque.struct || O <- set_to_list(Set)]); + t_collect_vars_list([O#opaque.struct || O <- set_to_list(Set)], Acc); +t_collect_vars(?union(List), Acc) -> + t_collect_vars_list(List, Acc); t_collect_vars(_, Acc) -> Acc. +t_collect_vars_list([T|Ts], Acc0) -> + Acc = t_collect_vars(T, Acc0), + t_collect_vars_list(Ts, Acc); +t_collect_vars_list([], Acc) -> Acc. %%============================================================================= %% @@ -3081,6 +3090,9 @@ t_subst_dict(?tuple(Elements, _Arity, _Tag), Dict) -> t_tuple([t_subst_dict(E, Dict) || E <- Elements]); t_subst_dict(?tuple_set(_) = TS, Dict) -> t_sup([t_subst_dict(T, Dict) || T <- t_tuple_subtypes(TS)]); +t_subst_dict(?map(Pairs), Dict) -> + ?map([{t_subst_dict(K, Dict), t_subst_dict(V, Dict)} || + {K, V} <- Pairs]); t_subst_dict(?opaque(Es), Dict) -> List = [Opaque#opaque{args = [t_subst_dict(Arg, Dict) || Arg <- Args], struct = t_subst_dict(S, Dict)} || @@ -3130,6 +3142,9 @@ t_subst_aux(?tuple(Elements, _Arity, _Tag), VarMap) -> t_tuple([t_subst_aux(E, VarMap) || E <- Elements]); t_subst_aux(?tuple_set(_) = TS, VarMap) -> t_sup([t_subst_aux(T, VarMap) || T <- t_tuple_subtypes(TS)]); +t_subst_aux(?map(Pairs), VarMap) -> + ?map([{t_subst_aux(K, VarMap), t_subst_aux(V, VarMap)} || + {K, V} <- Pairs]); t_subst_aux(?opaque(Es), VarMap) -> List = [Opaque#opaque{args = [t_subst_aux(Arg, VarMap) || Arg <- Args], struct = t_subst_aux(S, VarMap)} || @@ -3705,7 +3720,7 @@ t_unopaque(T) -> t_unopaque(?opaque(_) = T, Opaques) -> case Opaques =:= 'universe' orelse is_opaque_type(T, Opaques) of true -> t_unopaque(t_opaque_structure(T), Opaques); - false -> T % XXX: needs revision for parametric opaque data types + false -> T end; t_unopaque(?list(ElemT, Termination, Sz), Opaques) -> ?list(t_unopaque(ElemT, Opaques), t_unopaque(Termination, Opaques), Sz); @@ -3725,11 +3740,12 @@ t_unopaque(?union([A,B,F,I,L,N,T,M,O,R,Map]), Opaques) -> UL = t_unopaque(L, Opaques), UT = t_unopaque(T, Opaques), UF = t_unopaque(F, Opaques), + UMap = t_unopaque(Map, Opaques), {OF,UO} = case t_unopaque(O, Opaques) of ?opaque(_) = O1 -> {O1, []}; Type -> {?none, [Type]} end, - t_sup([?union([A,B,UF,I,UL,N,UT,M,OF,R,Map])|UO]); + t_sup([?union([A,B,UF,I,UL,N,UT,M,OF,R,UMap])|UO]); t_unopaque(T, _) -> T. @@ -4236,8 +4252,8 @@ t_from_form({type, _L, list, []}, _TypeNames, _RecDict, _VarDict) -> t_from_form({type, _L, list, [Type]}, TypeNames, RecDict, VarDict) -> {T, R} = t_from_form(Type, TypeNames, RecDict, VarDict), {t_list(T), R}; -t_from_form({type, _L, map, _}, _TypeNames, _RecDict, _VarDict) -> - {t_map([]), []}; +t_from_form({type, _L, map, _}, TypeNames, RecDict, VarDict) -> + builtin_type(map, t_map([]), TypeNames, RecDict, VarDict); t_from_form({type, _L, mfa, []}, _TypeNames, _RecDict, _VarDict) -> {t_mfa(), []}; t_from_form({type, _L, module, []}, _TypeNames, _RecDict, _VarDict) -> @@ -4700,6 +4716,12 @@ is_same_type_name2(gb_trees, gb_tree, [], gb_trees, tree, [_, _]) -> true; is_same_type_name2(gb_trees, tree, [_, _], gb_trees, gb_tree, []) -> true; is_same_type_name2(_, _, _, _, _, _) -> false. +map_keys(?map(Pairs)) -> + [K || {K, _} <- Pairs]. + +map_values(?map(Pairs)) -> + [V || {_, V} <- Pairs]. + %% ----------------------------------- %% Set %% diff --git a/lib/hipe/llvm/Makefile b/lib/hipe/llvm/Makefile new file mode 100644 index 0000000000..92f378924a --- /dev/null +++ b/lib/hipe/llvm/Makefile @@ -0,0 +1,109 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2001-2014. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +ifndef EBIN +EBIN = ../ebin +endif + +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- +include ../vsn.mk +VSN=$(HIPE_VSN) + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- +RELSYSDIR = $(RELEASE_PATH)/lib/hipe-$(VSN) + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- +ifdef HIPE_ENABLED +HIPE_MODULES = hipe_rtl_to_llvm \ + hipe_llvm \ + elf_format \ + hipe_llvm_main \ + hipe_llvm_merge \ + hipe_llvm_liveness +else +HIPE_MODULES = +endif + +MODULES = $(HIPE_MODULES) + +HRL_FILES= elf_format.hrl elf32_format.hrl elf64_format.hrl \ + hipe_llvm_arch.hrl +ERL_FILES= $(MODULES:%=%.erl) +TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) + +# APP_FILE= +# App_SRC= $(APP_FILE).src +# APP_TARGET= $(EBIN)/$(APP_FILE) +# +# APPUP_FILE= +# APPUP_SRC= $(APPUP_FILE).src +# APPUP_TARGET= $(EBIN)/$(APPUP_FILE) + +# ---------------------------------------------------- +# FLAGS: Please keep +inline below +# ---------------------------------------------------- + +include ../native.mk + +ERL_COMPILE_FLAGS += +inline #+warn_missing_spec + +# if in 32 bit backend define BIT32 symbol +ARCH = $(shell echo $(TARGET) | sed 's/^\(x86_64\)-.*/64bit/') +ifneq ($(ARCH), 64bit) +ERL_COMPILE_FLAGS += -DBIT32 +endif + +# ---------------------------------------------------- +# Targets +# ---------------------------------------------------- + +debug opt: $(TARGET_FILES) + +docs: + +clean: + rm -f $(TARGET_FILES) + rm -f core erl_crash.dump + +# ---------------------------------------------------- +# Special Build Targets +# ---------------------------------------------------- + + +# ---------------------------------------------------- +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +release_spec: opt + $(INSTALL_DIR) $(RELSYSDIR)/llvm + $(INSTALL_DATA) $(ERL_FILES) $(HRL_FILES) $(RELSYSDIR)/llvm + $(INSTALL_DIR) $(RELSYSDIR)/ebin + $(INSTALL_DATA) $(TARGET_FILES) $(RELSYSDIR)/ebin + +release_docs_spec: diff --git a/lib/hipe/llvm/elf32_format.hrl b/lib/hipe/llvm/elf32_format.hrl new file mode 100644 index 0000000000..af1d95bf5b --- /dev/null +++ b/lib/hipe/llvm/elf32_format.hrl @@ -0,0 +1,59 @@ +%% -*- erlang-indent-level: 2 -*- + +%%% @copyright 2011-2014 Yiannis Tsiouris <[email protected]>, +%%% Chris Stavrakakis <[email protected]> +%%% @author Yiannis Tsiouris <[email protected]> +%%% [http://www.softlab.ntua.gr/~gtsiour/] + +%%% @doc This header file contains very very useful macros for handling +%%% various segments of an ELF-32 formated object file, such as sizes, +%%% offsets and predefined constants. For further information about +%%% each field take a quick look at +%%% "[http://www.sco.com/developers/gabi/latest/contents.html]" +%%% that contain the current HP/Intel definition of the ELF object +%%% file format. + +%%------------------------------------------------------------------------------ +%% ELF-32 Data Types (in bytes) +%%------------------------------------------------------------------------------ +-define(ELF_ADDR_SIZE, 4). +-define(ELF_OFF_SIZE, 4). +-define(ELF_HALF_SIZE, 2). +-define(ELF_WORD_SIZE, 4). +-define(ELF_SWORD_SIZE, 4). +-define(ELF_XWORD_SIZE, ?ELF_WORD_SIZE). % for compatibility +-define(ELF_SXWORD_SIZE, ?ELF_WORD_SIZE). +-define(ELF_UNSIGNED_CHAR_SIZE, 1). + +%%------------------------------------------------------------------------------ +%% ELF-32 Symbol Table Entries +%%------------------------------------------------------------------------------ +%% Precomputed offset for Symbol Table entries in SymTab binary (needed because +%% of the different offsets in 32 and 64 bit formats). +-define(ST_NAME_OFFSET, 0). +-define(ST_VALUE_OFFSET, (?ST_NAME_OFFSET + ?ST_NAME_SIZE) ). +-define(ST_SIZE_OFFSET, (?ST_VALUE_OFFSET + ?ST_VALUE_SIZE) ). +-define(ST_INFO_OFFSET, (?ST_SIZE_OFFSET + ?ST_SIZE_SIZE) ). +-define(ST_OTHER_OFFSET, (?ST_INFO_OFFSET + ?ST_INFO_SIZE) ). +-define(ST_SHNDX_OFFSET, (?ST_OTHER_OFFSET + ?ST_OTHER_SIZE) ). + +%%------------------------------------------------------------------------------ +%% ELF-64 Relocation Entries +%%------------------------------------------------------------------------------ +%% Useful macros to extract information from r_info field +-define(ELF_R_SYM(I), (I bsr 8) ). +-define(ELF_R_TYPE(I), (I band 16#ff) ). +-define(ELF_R_INFO(S, T), ((S bsl 8) + (T band 16#ff)) ). + +%%------------------------------------------------------------------------------ +%% ELF-64 Program Header Table +%%------------------------------------------------------------------------------ +%% Offsets of various fields in a Program Header Table entry binary. +-define(P_TYPE_OFFSET, 0). +-define(P_OFFSET_OFFSET, (?P_FLAGS_OFFSET + ?P_FLAGS_SIZE) ). +-define(P_VADDR_OFFSET, (?P_OFFSET_OFFSET + ?P_OFFSET_SIZE) ). +-define(P_PADDR_OFFSET, (?P_VADDR_OFFSET + ?P_VADDR_SIZE) ). +-define(P_FILESZ_OFFSET, (?P_PVADDR_OFFSET + ?P_PVADDR_SIZE) ). +-define(P_MEMSZ_OFFSET, (?P_FILESZ_OFFSET + ?P_FILESZ_SIZE) ). +-define(P_FLAGS_OFFSET, (?P_TYPE_OFFSET + ?P_TYPE_SIZE) ). +-define(P_ALIGN_OFFSET, (?P_MEMSZ_OFFSET + ?P_MEMSZ_SIZE) ). diff --git a/lib/hipe/llvm/elf64_format.hrl b/lib/hipe/llvm/elf64_format.hrl new file mode 100644 index 0000000000..794746ffdc --- /dev/null +++ b/lib/hipe/llvm/elf64_format.hrl @@ -0,0 +1,58 @@ +%% -*- erlang-indent-level: 2 -*- + +%%% @copyright 2011-2014 Yiannis Tsiouris <[email protected]>, +%%% Chris Stavrakakis <[email protected]> +%%% @author Yiannis Tsiouris <[email protected]> +%%% [http://www.softlab.ntua.gr/~gtsiour/] + +%%% @doc This header file contains very very useful macros for handling +%%% various segments of an ELF-64 formated object file, such as sizes, +%%% offsets and predefined constants. For further information about +%%% each field take a quick look at +%%% "[http://downloads.openwatcom.org/ftp/devel/docs/elf-64-gen.pdf]" +%%% that contain the current HP/Intel definition of the ELF object +%%% file format. + +%%------------------------------------------------------------------------------ +%% ELF-64 Data Types (in bytes) +%%------------------------------------------------------------------------------ +-define(ELF_ADDR_SIZE, 8). +-define(ELF_OFF_SIZE, 8). +-define(ELF_HALF_SIZE, 2). +-define(ELF_WORD_SIZE, 4). +-define(ELF_SWORD_SIZE, 4). +-define(ELF_XWORD_SIZE, 8). +-define(ELF_SXWORD_SIZE, 8). +-define(ELF_UNSIGNED_CHAR_SIZE, 1). + +%%------------------------------------------------------------------------------ +%% ELF-64 Symbol Table Entries +%%------------------------------------------------------------------------------ +%% Precomputed offset for Symbol Table entries in SymTab binary +-define(ST_NAME_OFFSET, 0). +-define(ST_INFO_OFFSET, (?ST_NAME_OFFSET + ?ST_NAME_SIZE) ). +-define(ST_OTHER_OFFSET, (?ST_INFO_OFFSET + ?ST_INFO_SIZE) ). +-define(ST_SHNDX_OFFSET, (?ST_OTHER_OFFSET + ?ST_OTHER_SIZE) ). +-define(ST_VALUE_OFFSET, (?ST_SHNDX_OFFSET + ?ST_SHNDX_SIZE) ). +-define(ST_SIZE_OFFSET, (?ST_VALUE_OFFSET + ?ST_VALUE_SIZE) ). + +%%------------------------------------------------------------------------------ +%% ELF-64 Relocation Entries +%%------------------------------------------------------------------------------ +%% Useful macros to extract information from r_info field +-define(ELF_R_SYM(I), (I bsr 32) ). +-define(ELF_R_TYPE(I), (I band 16#ffffffff) ). +-define(ELF_R_INFO(S, T), ((S bsl 32) + (T band 16#ffffffff)) ). + +%%------------------------------------------------------------------------------ +%% ELF-64 Program Header Table +%%------------------------------------------------------------------------------ +%% Offsets of various fields in a Program Header Table entry binary. +-define(P_TYPE_OFFSET, 0). +-define(P_FLAGS_OFFSET, (?P_TYPE_OFFSET + ?P_TYPE_SIZE) ). +-define(P_OFFSET_OFFSET, (?P_FLAGS_OFFSET + ?P_FLAGS_SIZE) ). +-define(P_VADDR_OFFSET, (?P_OFFSET_OFFSET + ?P_OFFSET_SIZE) ). +-define(P_PADDR_OFFSET, (?P_VADDR_OFFSET + ?P_VADDR_SIZE) ). +-define(P_FILESZ_OFFSET, (?P_PVADDR_OFFSET + ?P_PVADDR_SIZE) ). +-define(P_MEMSZ_OFFSET, (?P_FILESZ_OFFSET + ?P_FILESZ_SIZE) ). +-define(P_ALIGN_OFFSET, (?P_MEMSZ_OFFSET + ?P_MEMSZ_SIZE) ). diff --git a/lib/hipe/llvm/elf_format.erl b/lib/hipe/llvm/elf_format.erl new file mode 100644 index 0000000000..260da9b5e6 --- /dev/null +++ b/lib/hipe/llvm/elf_format.erl @@ -0,0 +1,790 @@ +%% -*- erlang-indent-level: 2 -*- + +%%% @copyright 2011-2014 Yiannis Tsiouris <[email protected]>, +%%% Chris Stavrakakis <[email protected]>, +%%% Kostis Sagonas <[email protected]> +%%% @author Yiannis Tsiouris <[email protected]> +%%% [http://www.softlab.ntua.gr/~gtsiour/] + +%%% @doc This module contains functions for extracting various pieces of +%%% information from an ELF formated Object file. To fully understand +%%% the ELF format and the use of these functions please read +%%% "[http://www.linuxjournal.com/article/1060?page=0,0]" carefully. + +-module(elf_format). + +-export([get_tab_entries/1, + %% Relocations + get_rodata_relocs/1, + get_text_relocs/1, + extract_rela/2, + get_rela_addends/1, + %% Note + extract_note/2, + %% Executable code + extract_text/1, + %% GCC Exception Table + get_exn_handlers/1, + %% Misc. + set_architecture_flag/1, + is64bit/0 + ]). + +-include("elf_format.hrl"). + +%%------------------------------------------------------------------------------ +%% Types +%%------------------------------------------------------------------------------ + +-type elf() :: binary(). + +-type lp() :: non_neg_integer(). % landing pad +-type num() :: non_neg_integer(). +-type index() :: non_neg_integer(). +-type offset() :: non_neg_integer(). +-type size() :: non_neg_integer(). +-type start() :: non_neg_integer(). + +-type info() :: index(). +-type nameoff() :: offset(). +-type valueoff() :: offset(). + +-type name() :: string(). +-type name_size() :: {name(), size()}. +-type name_sizes() :: [name_size()]. + +%%------------------------------------------------------------------------------ +%% Abstract Data Types and Accessors for ELF Structures. +%%------------------------------------------------------------------------------ + +%% File header +-record(elf_ehdr, {ident, % ELF identification + type, % Object file type + machine, % Machine Type + version, % Object file version + entry, % Entry point address + phoff, % Program header offset + shoff :: offset(), % Section header offset + flags, % Processor-specific flags + ehsize :: size(), % ELF header size + phentsize :: size(), % Size of program header entry + phnum :: num(), % Number of program header entries + shentsize :: size(), % Size of section header entry + shnum :: num(), % Number of section header entries + shstrndx :: index() % Section name string table index + }). +-type elf_ehdr() :: #elf_ehdr{}. + +-record(elf_ehdr_ident, {class, % File class + data, % Data encoding + version, % File version + osabi, % OS/ABI identification + abiversion, % ABI version + pad, % Start of padding bytes + nident % Size of e_ident[] + }). +%% -type elf_ehdr_ident() :: #elf_ehdr_ident{}. + +%% Section header entries +-record(elf_shdr, {name, % Section name + type, % Section type + flags, % Section attributes + addr, % Virtual address in memory + offset :: offset(), % Offset in file + size :: size(), % Size of section + link, % Link to other section + info, % Miscellaneous information + addralign, % Address align boundary + entsize % Size of entries, if section has table + }). +%% -type elf_shdr() :: #elf_shdr{}. + +%% Symbol table entries +-record(elf_sym, {name :: nameoff(), % Symbol name + info, % Type and Binding attributes + other, % Reserved + shndx, % Section table index + value :: valueoff(), % Symbol value + size :: size() % Size of object + }). +-type elf_sym() :: #elf_sym{}. + +%% Relocations +-record(elf_rel, {r_offset :: offset(), % Address of reference + r_info :: info() % Symbol index and type of relocation + }). +-type elf_rel() :: #elf_rel{}. + +-record(elf_rela, {r_offset :: offset(), % Address of reference + r_info :: info(), % Symbol index and type of relocation + r_addend :: offset() % Constant part of expression + }). +-type elf_rela() :: #elf_rela{}. + +%% %% Program header table +%% -record(elf_phdr, {type, % Type of segment +%% flags, % Segment attributes +%% offset, % Offset in file +%% vaddr, % Virtual address in memory +%% paddr, % Reserved +%% filesz, % Size of segment in file +%% memsz, % Size of segment in memory +%% align % Alignment of segment +%% }). + +%% %% GCC exception table +%% -record(elf_gccexntab, {lpbenc, % Landing pad base encoding +%% lpbase, % Landing pad base +%% ttenc, % Type table encoding +%% ttoff, % Type table offset +%% csenc, % Call-site table encoding +%% cstabsize, % Call-site table size +%% cstab :: cstab() % Call-site table +%% }). +%% -type elf_gccexntab() :: #elf_gccexntab{}. + +-record(elf_gccexntab_callsite, {start :: start(), % Call-site start + size :: size(), % Call-site size + lp :: lp(), % Call-site landing pad + % (exception handler) + onaction % On action (e.g. cleanup) + }). +%% -type elf_gccexntab_callsite() :: #elf_gccexntab_callsite{}. + +%%------------------------------------------------------------------------------ +%% Accessor Functions +%%------------------------------------------------------------------------------ + +%% File header +%% -spec mk_ehdr(...) -> elf_ehrd(). +mk_ehdr(Ident, Type, Machine, Version, Entry, Phoff, Shoff, Flags, Ehsize, + Phentsize, Phnum, Shentsize, Shnum, Shstrndx) -> + #elf_ehdr{ident = Ident, type = Type, machine = Machine, version = Version, + entry = Entry, phoff = Phoff, shoff = Shoff, flags = Flags, + ehsize = Ehsize, phentsize = Phentsize, phnum = Phnum, + shentsize = Shentsize, shnum = Shnum, shstrndx = Shstrndx}. + +%% -spec ehdr_shoff(elf_ehdr()) -> offset(). +%% ehdr_shoff(#elf_ehdr{shoff = Offset}) -> Offset. +%% +%% -spec ehdr_shentsize(elf_ehdr()) -> size(). +%% ehdr_shentsize(#elf_ehdr{shentsize = Size}) -> Size. +%% +%% -spec ehdr_shnum(elf_ehdr()) -> num(). +%% ehdr_shnum(#elf_ehdr{shnum = Num}) -> Num. +%% +%% -spec ehdr_shstrndx(elf_ehdr()) -> index(). +%% ehdr_shstrndx(#elf_ehdr{shstrndx = Index}) -> Index. + + +%%-spec mk_ehdr_ident(...) -> elf_ehdr_ident(). +mk_ehdr_ident(Class, Data, Version, OsABI, AbiVersion, Pad, Nident) -> + #elf_ehdr_ident{class = Class, data = Data, version = Version, osabi = OsABI, + abiversion = AbiVersion, pad = Pad, nident = Nident}. + +%%%------------------------- +%%% Section header entries +%%%------------------------- +mk_shdr(Name, Type, Flags, Addr, Offset, Size, Link, Info, AddrAlign, EntSize) -> + #elf_shdr{name = Name, type = Type, flags = Flags, addr = Addr, + offset = Offset, size = Size, link = Link, info = Info, + addralign = AddrAlign, entsize = EntSize}. + +%% -spec shdr_offset(elf_shdr()) -> offset(). +%% shdr_offset(#elf_shdr{offset = Offset}) -> Offset. +%% +%% -spec shdr_size(elf_shdr()) -> size(). +%% shdr_size(#elf_shdr{size = Size}) -> Size. + +%%%------------------------- +%%% Symbol Table Entries +%%%------------------------- +mk_sym(Name, Info, Other, Shndx, Value, Size) -> + #elf_sym{name = Name, info = Info, other = Other, + shndx = Shndx, value = Value, size = Size}. + +-spec sym_name(elf_sym()) -> nameoff(). +sym_name(#elf_sym{name = Name}) -> Name. + +%% -spec sym_value(elf_sym()) -> valueoff(). +%% sym_value(#elf_sym{value = Value}) -> Value. +%% +%% -spec sym_size(elf_sym()) -> size(). +%% sym_size(#elf_sym{size = Size}) -> Size. + +%%%------------------------- +%%% Relocations +%%%------------------------- +-spec mk_rel(offset(), info()) -> elf_rel(). +mk_rel(Offset, Info) -> + #elf_rel{r_offset = Offset, r_info = Info}. + +%% The following two functions capitalize on the fact that the two kinds of +%% relocation records (for 32- and 64-bit architectures have similar structure. + +-spec r_offset(elf_rel() | elf_rela()) -> offset(). +r_offset(#elf_rel{r_offset = Offset}) -> Offset; +r_offset(#elf_rela{r_offset = Offset}) -> Offset. + +-spec r_info(elf_rel() | elf_rela()) -> info(). +r_info(#elf_rel{r_info = Info}) -> Info; +r_info(#elf_rela{r_info = Info}) -> Info. + +-spec mk_rela(offset(), info(), offset()) -> elf_rela(). +mk_rela(Offset, Info, Addend) -> + #elf_rela{r_offset = Offset, r_info = Info, r_addend = Addend}. + +-spec rela_addend(elf_rela()) -> offset(). +rela_addend(#elf_rela{r_addend = Addend}) -> Addend. + +%% %%%------------------------- +%% %%% GCC exception table +%% %%%------------------------- +%% -type cstab() :: [elf_gccexntab_callsite()]. +%% +%% mk_gccexntab(LPbenc, LPbase, TTenc, TToff, CSenc, CStabsize, CStab) -> +%% #elf_gccexntab{lpbenc = LPbenc, lpbase = LPbase, ttenc = TTenc, +%% ttoff = TToff, csenc = CSenc, cstabsize = CStabsize, +%% cstab = CStab}. +%% +%% -spec gccexntab_cstab(elf_gccexntab()) -> cstab(). +%% gccexntab_cstab(#elf_gccexntab{cstab = CSTab}) -> CSTab. + +mk_gccexntab_callsite(Start, Size, LP, Action) -> + #elf_gccexntab_callsite{start = Start, size=Size, lp=LP, onaction=Action}. + +%% -spec gccexntab_callsite_start(elf_gccexntab_callsite()) -> start(). +%% gccexntab_callsite_start(#elf_gccexntab_callsite{start = Start}) -> Start. +%% +%% -spec gccexntab_callsite_size(elf_gccexntab_callsite()) -> size(). +%% gccexntab_callsite_size(#elf_gccexntab_callsite{size = Size}) -> Size. +%% +%% -spec gccexntab_callsite_lp(elf_gccexntab_callsite()) -> lp(). +%% gccexntab_callsite_lp(#elf_gccexntab_callsite{lp = LP}) -> LP. + +%%------------------------------------------------------------------------------ +%% Functions to manipulate the ELF File Header +%%------------------------------------------------------------------------------ + +%% @doc Extracts the File Header from an ELF formatted object file. Also sets +%% the ELF class variable in the process dictionary (used by many functions +%% in this and hipe_llvm_main modules). +-spec extract_header(elf()) -> elf_ehdr(). +extract_header(Elf) -> + Ehdr_bin = get_binary_segment(Elf, 0, ?ELF_EHDR_SIZE), + << %% Structural pattern matching on fields. + Ident_bin:?E_IDENT_SIZE/binary, + Type:?bits(?E_TYPE_SIZE)/integer-little, + Machine:?bits(?E_MACHINE_SIZE)/integer-little, + Version:?bits(?E_VERSION_SIZE)/integer-little, + Entry:?bits(?E_ENTRY_SIZE)/integer-little, + Phoff:?bits(?E_PHOFF_SIZE)/integer-little, + Shoff:?bits(?E_SHOFF_SIZE)/integer-little, + Flags:?bits(?E_FLAGS_SIZE)/integer-little, + Ehsize:?bits(?E_EHSIZE_SIZE)/integer-little, + Phentsize:?bits(?E_PHENTSIZE_SIZE)/integer-little, + Phnum:?bits(?E_PHNUM_SIZE)/integer-little, + Shentsize:?bits(?E_SHENTSIZE_SIZE)/integer-little, + Shnum:?bits(?E_SHENTSIZE_SIZE)/integer-little, + Shstrndx:?bits(?E_SHSTRNDX_SIZE)/integer-little + >> = Ehdr_bin, + <<16#7f, $E, $L, $F, Class, Data, Version, Osabi, Abiversion, + Pad:6/binary, Nident + >> = Ident_bin, + Ident = mk_ehdr_ident(Class, Data, Version, Osabi, + Abiversion, Pad, Nident), + mk_ehdr(Ident, Type, Machine, Version, Entry, Phoff, Shoff, Flags, + Ehsize, Phentsize, Phnum, Shentsize, Shnum, Shstrndx). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Section Header Entries +%%------------------------------------------------------------------------------ + +%% @doc Extracts the Section Header Table from an ELF formated Object File. +extract_shdrtab(Elf) -> + %% Extract File Header to get info about Section Header Offset (in bytes), + %% Entry Size (in bytes) and Number of entries + #elf_ehdr{shoff = ShOff, shentsize = ShEntsize, shnum = ShNum} = + extract_header(Elf), + %% Get actual Section header table (binary) + ShdrBin = get_binary_segment(Elf, ShOff, ShNum * ShEntsize), + get_shdrtab_entries(ShdrBin, []). + +get_shdrtab_entries(<<>>, Acc) -> + lists:reverse(Acc); +get_shdrtab_entries(ShdrBin, Acc) -> + <<%% Structural pattern matching on fields. + Name:?bits(?SH_NAME_SIZE)/integer-little, + Type:?bits(?SH_TYPE_SIZE)/integer-little, + Flags:?bits(?SH_FLAGS_SIZE)/integer-little, + Addr:?bits(?SH_ADDR_SIZE)/integer-little, + Offset:?bits(?SH_OFFSET_SIZE)/integer-little, + Size:?bits(?SH_SIZE_SIZE)/integer-little, + Link:?bits(?SH_LINK_SIZE)/integer-little, + Info:?bits(?SH_INFO_SIZE)/integer-little, + Addralign:?bits(?SH_ADDRALIGN_SIZE)/integer-little, + Entsize:?bits(?SH_ENTSIZE_SIZE)/integer-little, + MoreShdrE/binary + >> = ShdrBin, + ShdrE = mk_shdr(Name, Type, Flags, Addr, Offset, + Size, Link, Info, Addralign, Entsize), + get_shdrtab_entries(MoreShdrE, [ShdrE | Acc]). + +%% @doc Extracts a specific Entry of a Section Header Table. This function +%% takes as argument the Section Header Table (`SHdrTab') and the entry's +%% serial number (`EntryNum') and returns the entry (`shdr'). +get_shdrtab_entry(SHdrTab, EntryNum) -> + lists:nth(EntryNum + 1, SHdrTab). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Section Header String Table +%%------------------------------------------------------------------------------ + +%% @doc Extracts the Section Header String Table. This section is not a known +%% ELF Object File section. It is just a "hidden" table storing the +%% names of all sections that exist in current object file. +-spec extract_shstrtab(elf()) -> [name()]. +extract_shstrtab(Elf) -> + %% Extract Section Name String Table Index + #elf_ehdr{shstrndx = ShStrNdx} = extract_header(Elf), + ShHdrTab = extract_shdrtab(Elf), + %% Extract Section header entry and get actual Section-header String Table + #elf_shdr{offset = ShStrOffset, size = ShStrSize} = + get_shdrtab_entry(ShHdrTab, ShStrNdx), + case get_binary_segment(Elf, ShStrOffset, ShStrSize) of + <<>> -> %% Segment empty + []; + ShStrTab -> %% Convert to string table + [Name || {Name, _Size} <- get_names(ShStrTab)] + end. + +%%------------------------------------------------------------------------------ + +-spec get_tab_entries(elf()) -> [{name(), valueoff(), size()}]. +get_tab_entries(Elf) -> + SymTab = extract_symtab(Elf), + Ts = [{Name, Value, Size div ?ELF_XWORD_SIZE} + || #elf_sym{name = Name, value = Value, size = Size} <- SymTab, + Name =/= 0], + {NameIndices, ValueOffs, Sizes} = lists:unzip3(Ts), + %% Find the names of the symbols. + %% Get string table entries ([{Name, Offset in strtab section}]). Keep only + %% relevant entries: + StrTab = extract_strtab(Elf), + Relevant = [get_strtab_entry(StrTab, Off) || Off <- NameIndices], + %% Zip back to {Name, ValueOff, Size} + lists:zip3(Relevant, ValueOffs, Sizes). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Symbol Table +%%------------------------------------------------------------------------------ + +%% @doc Function that extracts Symbol Table from an ELF Object file. +extract_symtab(Elf) -> + Symtab_bin = extract_segment_by_name(Elf, ?SYMTAB), + get_symtab_entries(Symtab_bin, []). + +get_symtab_entries(<<>>, Acc) -> + lists:reverse(Acc); +get_symtab_entries(Symtab_bin, Acc) -> + <<SymE_bin:?ELF_SYM_SIZE/binary, MoreSymE/binary>> = Symtab_bin, + case is64bit() of + true -> + <<%% Structural pattern matching on fields. + Name:?bits(?ST_NAME_SIZE)/integer-little, + Info:?bits(?ST_INFO_SIZE)/integer-little, + Other:?bits(?ST_OTHER_SIZE)/integer-little, + Shndx:?bits(?ST_SHNDX_SIZE)/integer-little, + Value:?bits(?ST_VALUE_SIZE)/integer-little, + Size:?bits(?ST_SIZE_SIZE)/integer-little + >> = SymE_bin; + false -> + << %% Same fields in different order: + Name:?bits(?ST_NAME_SIZE)/integer-little, + Value:?bits(?ST_VALUE_SIZE)/integer-little, + Size:?bits(?ST_SIZE_SIZE)/integer-little, + Info:?bits(?ST_INFO_SIZE)/integer-little, + Other:?bits(?ST_OTHER_SIZE)/integer-little, + Shndx:?bits(?ST_SHNDX_SIZE)/integer-little + >> = SymE_bin + end, + SymE = mk_sym(Name, Info, Other, Shndx, Value, Size), + get_symtab_entries(MoreSymE, [SymE | Acc]). + +%% @doc Extracts a specific entry from the Symbol Table (as binary). +%% This function takes as arguments the Symbol Table (`SymTab') +%% and the entry's serial number and returns that entry (`sym'). +get_symtab_entry(SymTab, EntryNum) -> + lists:nth(EntryNum + 1, SymTab). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate String Table +%%------------------------------------------------------------------------------ + +%% @doc Extracts String Table from an ELF formated Object File. +-spec extract_strtab(elf()) -> [{string(), offset()}]. +extract_strtab(Elf) -> + Strtab_bin = extract_segment_by_name(Elf, ?STRTAB), + NamesSizes = get_names(Strtab_bin), + make_offsets(NamesSizes). + +%% @doc Returns the name of the symbol at the given offset. The string table +%% contains entries of the form {Name, Offset}. If no such offset exists +%% returns the empty string (`""'). +%% XXX: There might be a bug here because of the "compact" saving the ELF +%% format uses: e.g. only stores ".rela.text" for ".rela.text" and ".text". +get_strtab_entry(Strtab, Offset) -> + case lists:keyfind(Offset, 2, Strtab) of + {Name, Offset} -> Name; + false -> "" + end. + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Relocations +%%------------------------------------------------------------------------------ + +%% @doc This function gets as argument an ELF binary file and returns a list +%% with all .rela.rodata labels (i.e. constants and literals in code) +%% or an empty list if no ".rela.rodata" section exists in code. +-spec get_rodata_relocs(elf()) -> [offset()]. +get_rodata_relocs(Elf) -> + case is64bit() of + true -> + %% Only care about the addends (== offsets): + get_rela_addends(extract_rela(Elf, ?RODATA)); + false -> + %% Find offsets hardcoded in ".rodata" entry + %%XXX: Treat all 0s as padding and skip them! + [SkipPadding || SkipPadding <- extract_rodata(Elf), SkipPadding =/= 0] + end. + +-spec get_rela_addends([elf_rela()]) -> [offset()]. +get_rela_addends(RelaEntries) -> + [rela_addend(E) || E <- RelaEntries]. + +%% @doc Extract a list of the form `[{SymbolName, Offset}]' with all relocatable +%% symbols and their offsets in the code from the ".text" section. +-spec get_text_relocs(elf()) -> [{name(), offset()}]. +get_text_relocs(Elf) -> + %% Only care about the symbol table index and the offset: + NameOffsetTemp = [{?ELF_R_SYM(r_info(E)), r_offset(E)} + || E <- extract_rela(Elf, ?TEXT)], + {NameIndices, ActualOffsets} = lists:unzip(NameOffsetTemp), + %% Find the names of the symbols: + %% + %% Get those symbol table entries that are related to Text relocs: + Symtab = extract_symtab(Elf), + SymtabEs = [get_symtab_entry(Symtab, Index) || Index <- NameIndices], + %XXX: not zero-indexed! + %% Symbol table entries contain the offset of the name of the symbol in + %% String Table: + SymtabEs2 = [sym_name(E) || E <- SymtabEs], %XXX: Do we need to sort SymtabE? + %% Get string table entries ([{Name, Offset in strtab section}]). Keep only + %% relevant entries: + Strtab = extract_strtab(Elf), + Relevant = [get_strtab_entry(Strtab, Off) || Off <- SymtabEs2], + %% Zip back with actual offsets: + lists:zip(Relevant, ActualOffsets). + +%% @doc Extract the Relocations segment for section `Name' (that is passed +%% as second argument) from an ELF formated Object file binary. +-spec extract_rela(elf(), name()) -> [elf_rel() | elf_rela()]. +extract_rela(Elf, Name) -> + SegName = + case is64bit() of + true -> ?RELA(Name); % ELF-64 uses ".rela" + false -> ?REL(Name) % ...while ELF-32 uses ".rel" + end, + Rela_bin = extract_segment_by_name(Elf, SegName), + get_rela_entries(Rela_bin, []). + +get_rela_entries(<<>>, Acc) -> + lists:reverse(Acc); +get_rela_entries(Bin, Acc) -> + E = case is64bit() of + true -> + <<%% Structural pattern matching on fields of a Rela Entry. + Offset:?bits(?R_OFFSET_SIZE)/integer-little, + Info:?bits(?R_INFO_SIZE)/integer-little, + Addend:?bits(?R_ADDEND_SIZE)/integer-little, + Rest/binary + >> = Bin, + mk_rela(Offset, Info, Addend); + false -> + <<%% Structural pattern matching on fields of a Rel Entry. + Offset:?bits(?R_OFFSET_SIZE)/integer-little, + Info:?bits(?R_INFO_SIZE)/integer-little, + Rest/binary + >> = Bin, + mk_rel(Offset, Info) + end, + get_rela_entries(Rest, [E | Acc]). + +%% %% @doc Extract the `EntryNum' (serial number) Relocation Entry. +%% get_rela_entry(Rela, EntryNum) -> +%% lists:nth(EntryNum + 1, Rela). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Executable Code segment +%%------------------------------------------------------------------------------ + +%% @doc This function gets as arguments an ELF formated binary file and +%% returns the Executable Code (".text" segment) or an empty binary if it +%% is not found. +-spec extract_text(elf()) -> binary(). +extract_text(Elf) -> + extract_segment_by_name(Elf, ?TEXT). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Note Section +%%------------------------------------------------------------------------------ + +%% @doc Extract specific Note Section from an ELF Object file. The function +%% takes as first argument the object file (`Elf') and the `Name' of the +%% wanted Note Section (<b>without</b> the ".note" prefix!). It returns +%% the specified binary segment or an empty binary if no such section +%% exists. +-spec extract_note(elf(), string()) -> binary(). +extract_note(Elf, Name) -> + extract_segment_by_name(Elf, ?NOTE(Name)). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate GCC Exception Table segment +%%------------------------------------------------------------------------------ + +%% A description for the C++ exception table formats can be found at Exception +%% Handling Tables (http://www.codesourcery.com/cxx-abi/exceptions.pdf). + +%% A list with `{Start, End, HandlerOffset}' for all call sites in the code +-spec get_exn_handlers(elf()) -> [{start(), start(), lp()}]. +get_exn_handlers(Elf) -> + CallSites = extract_gccexntab_callsites(Elf), + [{Start, Start + Size, LP} + || #elf_gccexntab_callsite{start = Start, size = Size, lp = LP} <- CallSites]. + +%% @doc This function gets as argument an ELF binary file and returns +%% the table (list) of call sites which is stored in GCC +%% Exception Table (".gcc_except_table") section. +%% It returns an empty list if the Exception Table is not found. +%% XXX: Assumes there is *no* Action Record Table. +extract_gccexntab_callsites(Elf) -> + case extract_segment_by_name(Elf, ?GCC_EXN_TAB) of + <<>> -> + []; + ExnTab -> + %% First byte of LSDA is Landing Pad base encoding. + <<LBenc:8, More/binary>> = ExnTab, + %% Second byte is the Landing Pad base (if its encoding is not + %% DW_EH_PE_omit) (optional). + {_LPBase, LSDACont} = + case LBenc =:= ?DW_EH_PE_omit of + true -> % No landing pad base byte. (-1 denotes that) + {-1, More}; + false -> % Landing pad base. + <<Base:8, More2/binary>> = More, + {Base, More2} + end, + %% Next byte of LSDA is the encoding of the Type Table. + <<TTenc:8, More3/binary>> = LSDACont, + %% Next byte is the Types Table offset encoded in U-LEB128 (optional). + {_TTOff, LSDACont2} = + case TTenc =:= ?DW_EH_PE_omit of + true -> % There is no Types Table pointer. (-1 denotes that) + {-1, More3}; + false -> % The byte offset from this field to the start of the Types + % Table used for exception matching. + leb128_decode(More3) + end, + %% Next byte of LSDA is the encoding of the fields in the Call-site Table. + <<_CSenc:8, More4/binary>> = LSDACont2, + %% Sixth byte is the size (in bytes) of the Call-site Table encoded in + %% U-LEB128. + {_CSTabSize, CSTab} = leb128_decode(More4), + %% Extract all call site information + get_gccexntab_callsites(CSTab, []) + end. + +get_gccexntab_callsites(<<>>, Acc) -> + lists:reverse(Acc); +get_gccexntab_callsites(CSTab, Acc) -> + %% We are only interested in the Landing Pad of every entry. + <<Start:32/integer-little, Size:32/integer-little, + LP:32/integer-little, OnAction:8, More/binary + >> = CSTab, + GccCS = mk_gccexntab_callsite(Start, Size, LP, OnAction), + get_gccexntab_callsites(More, [GccCS | Acc]). + +%%------------------------------------------------------------------------------ +%% Functions to manipulate Read-only Data (.rodata) +%%------------------------------------------------------------------------------ +extract_rodata(Elf) -> + Rodata_bin = extract_segment_by_name(Elf, ?RODATA), + get_rodata_entries(Rodata_bin, []). + +get_rodata_entries(<<>>, Acc) -> + lists:reverse(Acc); +get_rodata_entries(Rodata_bin, Acc) -> + <<Num:?bits(?ELF_ADDR_SIZE)/integer-little, More/binary>> = Rodata_bin, + get_rodata_entries(More, [Num | Acc]). + +%%------------------------------------------------------------------------------ +%% Helper functions +%%------------------------------------------------------------------------------ + +%% @doc Returns the binary segment starting at `Offset' with length `Size' +%% (bytes) from a binary file. If `Offset' is bigger than the byte size of +%% the binary, an empty binary (`<<>>') is returned. +-spec get_binary_segment(binary(), offset(), size()) -> binary(). +get_binary_segment(Bin, Offset, _Size) when Offset > byte_size(Bin) -> + <<>>; +get_binary_segment(Bin, Offset, Size) -> + <<_Hdr:Offset/binary, BinSeg:Size/binary, _More/binary>> = Bin, + BinSeg. + +%% @doc This function gets as arguments an ELF formated binary object and +%% a string with the segments' name and returns the specified segment or +%% an empty binary (`<<>>') if there exists no segment with that name. +%% There are handy macros defined in elf_format.hrl for all Standard +%% Section Names. +-spec extract_segment_by_name(elf(), string()) -> binary(). +extract_segment_by_name(Elf, SectionName) -> + %% Extract Section Header Table and Section Header String Table from binary + SHdrTable = extract_shdrtab(Elf), + Names = extract_shstrtab(Elf), + %% Zip to a list of (Name,ShdrE) + [_Zero | ShdrEs] = lists:keysort(2, SHdrTable), % Skip first entry (zeros). + L = lists:zip(Names, ShdrEs), + %% Find Section Header Table entry by name + case lists:keyfind(SectionName, 1, L) of + {SectionName, ShdrE} -> %% Note: Same name. + #elf_shdr{offset = Offset, size = Size} = ShdrE, + get_binary_segment(Elf, Offset, Size); + false -> %% Not found. + <<>> + end. + +%% @doc Extracts a list of strings with (zero-separated) names from a binary. +%% Returns tuples of `{Name, Size}'. +%% XXX: Skip trailing 0. +-spec get_names(<<_:8,_:_*8>>) -> name_sizes(). +get_names(<<0, Bin/binary>>) -> + NamesSizes = get_names(Bin, []), + fix_names(NamesSizes, []). + +get_names(<<>>, Acc) -> + lists:reverse(Acc); +get_names(Bin, Acc) -> + {Name, MoreNames} = bin_get_string(Bin), + get_names(MoreNames, [{Name, length(Name)} | Acc]). + +%% @doc Fix names: +%% e.g. If ".rela.text" exists, ".text" does not. Same goes for +%% ".rel.text". In that way, the Section Header String Table is more +%% compact. Add ".text" just *before* the corresponding rela-field, +%% etc. +-spec fix_names(name_sizes(), name_sizes()) -> name_sizes(). +fix_names([], Acc) -> + lists:reverse(Acc); +fix_names([{Name, Size}=T | Names], Acc) -> + case is64bit() of + true -> + case string:str(Name, ".rela") =:= 1 of + true -> %% Name starts with ".rela": + Section = string:substr(Name, 6), + fix_names(Names, [{Section, Size - 5} + | [T | Acc]]); % XXX: Is order ok? (".text" + % always before ".rela.text") + false -> %% Name does not start with ".rela": + fix_names(Names, [T | Acc]) + end; + false -> + case string:str(Name, ".rel") =:= 1 of + true -> %% Name starts with ".rel": + Section = string:substr(Name, 5), + fix_names(Names, [{Section, Size - 4} + | [T | Acc]]); % XXX: Is order ok? (".text" + % always before ".rela.text") + false -> %% Name does not start with ".rel": + fix_names(Names, [T | Acc]) + end + end. + + +%% @doc A function that byte-reverses a binary. This might be needed because of +%% little (fucking!) endianess. +-spec bin_reverse(binary()) -> binary(). +bin_reverse(Bin) when is_binary(Bin) -> + bin_reverse(Bin, <<>>). + +-spec bin_reverse(binary(), binary()) -> binary(). +bin_reverse(<<>>, Acc) -> + Acc; +bin_reverse(<<Head, More/binary>>, Acc) -> + bin_reverse(More, <<Head, Acc/binary>>). + +%% @doc A function that extracts a null-terminated string from a binary. It +%% returns the found string along with the rest of the binary. +-spec bin_get_string(binary()) -> {string(), binary()}. +bin_get_string(Bin) -> + bin_get_string(Bin, <<>>). + +bin_get_string(<<>>, BinAcc) -> + Bin = bin_reverse(BinAcc), % little endian! + {binary_to_list(Bin), <<>>}; +bin_get_string(<<0, MoreBin/binary>>, BinAcc) -> + Bin = bin_reverse(BinAcc), % little endian! + {binary_to_list(Bin), MoreBin}; +bin_get_string(<<Letter, Tail/binary>>, BinAcc) -> + bin_get_string(Tail, <<Letter, BinAcc/binary>>). + +%% @doc +make_offsets(NamesSizes) -> + {Names, Sizes} = lists:unzip(NamesSizes), + Offsets = make_offsets_from_sizes(Sizes, 1, []), + lists:zip(Names, Offsets). + +make_offsets_from_sizes([], _, Acc) -> + lists:reverse(Acc); +make_offsets_from_sizes([Size | Sizes], Cur, Acc) -> + make_offsets_from_sizes(Sizes, Size+Cur+1, [Cur | Acc]). % For the "."! + +%% @doc Little-Endian Base 128 (LEB128) Decoder +%% This function extracts the <b>first</b> LEB128-encoded integer in a +%% binary and returns that integer along with the remaining binary. This is +%% done because a LEB128 number has variable bit-size and that is a way of +%% extracting only one number in a binary and continuing parsing the binary +%% for other kind of data (e.g. different encoding). +%% FIXME: Only decodes unsigned data! +-spec leb128_decode(binary()) -> {integer(), binary()}. +leb128_decode(LebNum) -> + leb128_decode(LebNum, 0, <<>>). + +-spec leb128_decode(binary(), integer(), binary()) -> {integer(), binary()}. +leb128_decode(LebNum, NoOfBits, Acc) -> + <<Sentinel:1/bits, NextBundle:7/bits, MoreLebNums/bits>> = LebNum, + case Sentinel of + <<1:1>> -> % more bytes to follow + leb128_decode(MoreLebNums, NoOfBits+7, <<NextBundle:7/bits, Acc/bits>>); + <<0:1>> -> % byte bundle stop + Size = NoOfBits+7, + <<Num:Size/integer>> = <<NextBundle:7/bits, Acc/bits>>, + {Num, MoreLebNums} + end. + +%% @doc Extract ELF Class from ELF header and export symbol to process +%% dictionary. +-spec set_architecture_flag(elf()) -> 'ok'. +set_architecture_flag(Elf) -> + %% Extract information about ELF Class from ELF Header + <<16#7f, $E, $L, $F, EI_Class, _MoreHeader/binary>> + = get_binary_segment(Elf, 0, ?ELF_EHDR_SIZE), + put(elf_class, EI_Class), + ok. + +%% @doc Read from object file header if the file class is ELF32 or ELF64. +-spec is64bit() -> boolean(). +is64bit() -> + case get(elf_class) of + ?ELFCLASS64 -> true; + ?ELFCLASS32 -> false + end. diff --git a/lib/hipe/llvm/elf_format.hrl b/lib/hipe/llvm/elf_format.hrl new file mode 100644 index 0000000000..78592e6e2a --- /dev/null +++ b/lib/hipe/llvm/elf_format.hrl @@ -0,0 +1,488 @@ +%% -*- erlang-indent-level: 2 -*- + +%%% @copyright 2011-2014 Yiannis Tsiouris <[email protected]>, +%%% Chris Stavrakakis <[email protected]> +%%% @author Yiannis Tsiouris <[email protected]> +%%% [http://www.softlab.ntua.gr/~gtsiour/] + +%%------------------------------------------------------------------------------ +%% +%% ELF Header File +%% +%%------------------------------------------------------------------------------ + +-ifdef(BIT32). +-include("elf32_format.hrl"). % ELF32-specific definitions. +-else. +-include("elf64_format.hrl"). % ELF64-specific definitions. +-endif. + +%%------------------------------------------------------------------------------ +%% ELF Data Types (in bytes) +%%------------------------------------------------------------------------------ +%%XXX: Included in either elf32_format or elf64_format. + +%%------------------------------------------------------------------------------ +%% ELF File Header +%%------------------------------------------------------------------------------ +-define(ELF_EHDR_SIZE, (?E_IDENT_SIZE + ?E_TYPE_SIZE + ?E_MACHINE_SIZE + +?E_VERSION_SIZE + ?E_ENTRY_SIZE + ?E_PHOFF_SIZE + +?E_SHOFF_SIZE + ?E_FLAGS_SIZE + ?E_EHSIZE_SIZE + +?E_PHENTSIZE_SIZE + ?E_PHNUM_SIZE + ?E_SHENTSIZE_SIZE + +?E_SHNUM_SIZE + ?E_SHSTRNDX_SIZE) ). + +-define(E_IDENT_SIZE, (16 * ?ELF_UNSIGNED_CHAR_SIZE) ). +-define(E_TYPE_SIZE, ?ELF_HALF_SIZE). +-define(E_MACHINE_SIZE, ?ELF_HALF_SIZE). +-define(E_VERSION_SIZE, ?ELF_WORD_SIZE). +-define(E_ENTRY_SIZE, ?ELF_ADDR_SIZE). +-define(E_PHOFF_SIZE, ?ELF_OFF_SIZE). +-define(E_SHOFF_SIZE, ?ELF_OFF_SIZE). +-define(E_FLAGS_SIZE, ?ELF_WORD_SIZE). +-define(E_EHSIZE_SIZE, ?ELF_HALF_SIZE). +-define(E_PHENTSIZE_SIZE, ?ELF_HALF_SIZE). +-define(E_PHNUM_SIZE, ?ELF_HALF_SIZE). +-define(E_SHENTSIZE_SIZE, ?ELF_HALF_SIZE). +-define(E_SHNUM_SIZE, ?ELF_HALF_SIZE). +-define(E_SHSTRNDX_SIZE, ?ELF_HALF_SIZE). + +%% Useful arithmetics for computing byte offsets for various File Header +%% entries from a File Header (erlang) binary +-define(E_IDENT_OFFSET, 0). +-define(E_TYPE_OFFSET, (?E_IDENT_OFFSET + ?E_IDENT_SIZE) ). +-define(E_MACHINE_OFFSET, (?E_TYPE_OFFSET + ?E_TYPE_SIZE) ). +-define(E_VERSION_OFFSET, (?E_MACHINE_OFFSET + ?E_MACHINE_SIZE) ). +-define(E_ENTRY_OFFSET, (?E_VERSION_OFFSET + ?E_VERSION_SIZE) ). +-define(E_PHOFF_OFFSET, (?E_ENTRY_OFFSET + ?E_ENTRY_SIZE) ). +-define(E_SHOFF_OFFSET, (?E_PHOFF_OFFSET + ?E_PHOFF_SIZE) ). +-define(E_FLAGS_OFFSET, (?E_SHOFF_OFFSET + ?E_SHOFF_SIZE) ). +-define(E_EHSIZE_OFFSET, (?E_FLAGS_OFFSET + ?E_FLAGS_SIZE) ). +-define(E_PHENTSIZE_OFFSET, (?E_EHSIZE_OFFSET + ?E_EHSIZE_SIZE) ). +-define(E_PHNUM_OFFSET, (?E_PHENTSIZE_OFFSET + ?E_PHENTSIZE_SIZE) ). +-define(E_SHENTSIZE_OFFSET, (?E_PHNUM_OFFSET + ?E_PHNUM_SIZE) ). +-define(E_SHNUM_OFFSET, (?E_SHENTSIZE_OFFSET + ?E_SHENTSIZE_SIZE) ). +-define(E_SHSTRNDX_OFFSET, (?E_SHNUM_OFFSET + ?E_SHNUM_SIZE) ). + +%% Name aliases of File Header fields information used in get_header_field +%% function of elf64_format module. +-define(E_IDENT, {?E_IDENT_OFFSET, ?E_IDENT_SIZE}). +-define(E_TYPE, {?E_TYPE_OFFSET, ?E_TYPE_SIZE}). +-define(E_MACHINE, {?E_MACHINE_OFFSET, ?E_MACHINE_SIZE}). +-define(E_VERSION, {?E_VERSION_OFFSET, ?E_VERSION_SIZE}) +-define(E_ENTRY, {?E_ENTRY_OFFSET, ?E_ENTRY_SIZE}). +-define(E_PHOFF, {?E_PHOFF_OFFSET, ?E_PHOFF_SIZE}). +-define(E_SHOFF, {?E_SHOFF_OFFSET, ?E_SHOFF_SIZE}). +-define(E_FLAGS, {?E_FLAGS_OFFSET, ?E_FLAGS_SIZE}). +-define(E_EHSIZE, {?E_EHSIZE_OFFSET, ?E_EHSIZE_SIZE}). +-define(E_PHENTSIZE, {?E_PHENTSIZE_OFFSET, ?E_PHENTSIZE_SIZE}). +-define(E_PHNUM, {?E_PHNUM_OFFSET, ?E_PHNUM_SIZE}). +-define(E_SHENTSIZE, {?E_SHENTSIZE_OFFSET, ?E_SHENTSIZE_SIZE}). +-define(E_SHNUM, {?E_SHNUM_OFFSET, ?E_SHNUM_SIZE}). +-define(E_SHSTRNDX, {?E_SHSTRNDX_OFFSET, ?E_SHSTRNDX_SIZE}). + +%% ELF Identification (e_ident) +-define(EI_MAG0, 0). +-define(EI_MAG1, 1). +-define(EI_MAG2, 2). +-define(EI_MAG3, 3). +-define(EI_CLASS, 4). +-define(EI_DATA, 5). +-define(EI_VERSION, 6). +-define(EI_OSABI, 7). +-define(EI_ABIVERSION, 8). +-define(EI_PAD, 9). +-define(EI_NIDENT, 16). + +%% Object File Classes (e_ident[EI_CLASS]) +-define(ELFCLASSNONE, 0). +-define(ELFCLASS32, 1). +-define(ELFCLASS64, 2). + +%% Data Encodings (e_ident[EI_DATA]) +-define(ELFDATA2LSB, 1). +-define(ELFDATA2MSB, 2). + +%% Operating System and ABI Identifiers (e_ident[EI_OSABI]) +-define(ELFOSABI_SYSV, 0). +-define(ELFOSABI_HPUX, 1). +-define(ELFOSABI_STANDALONE, 255). + +%% Object File Types (e_type) +-define(ET_NONE, 0). +-define(ET_REL, 1). +-define(ET_EXEC, 2). +-define(ET_DYN, 3). +-define(ET_CORE, 4). +-define(ET_LOOS, 16#FE00). +-define(ET_HIOS, 16#FEFF). +-define(ET_LOPROC, 16#FF00). +-define(ET_HIPROC, 16#FFFF). + +%%------------------------------------------------------------------------------ +%% ELF Section Header +%%------------------------------------------------------------------------------ +-define(ELF_SHDRENTRY_SIZE, (?SH_NAME_SIZE + ?SH_TYPE_SIZE + ?SH_FLAGS_SIZE + +?SH_ADDR_SIZE + ?SH_OFFSET_SIZE + ?SH_SIZE_SIZE + +?SH_LINK_SIZE + ?SH_INFO_SIZE + +?SH_ADDRALIGN_SIZE + ?SH_ENTSIZE_SIZE) ). + +-define(SH_NAME_SIZE, ?ELF_WORD_SIZE). +-define(SH_TYPE_SIZE, ?ELF_WORD_SIZE). +-define(SH_FLAGS_SIZE, ?ELF_XWORD_SIZE). +-define(SH_ADDR_SIZE, ?ELF_ADDR_SIZE). +-define(SH_OFFSET_SIZE, ?ELF_OFF_SIZE). +-define(SH_SIZE_SIZE, ?ELF_XWORD_SIZE). +-define(SH_LINK_SIZE, ?ELF_WORD_SIZE). +-define(SH_INFO_SIZE, ?ELF_WORD_SIZE). +-define(SH_ADDRALIGN_SIZE, ?ELF_XWORD_SIZE). +-define(SH_ENTSIZE_SIZE, ?ELF_XWORD_SIZE). + +%% Useful arithmetics for computing byte offsets for various fields from a +%% Section Header Entry (erlang) binary +-define(SH_NAME_OFFSET, 0). +-define(SH_TYPE_OFFSET, (?SH_NAME_OFFSET + ?SH_NAME_SIZE) ). +-define(SH_FLAGS_OFFSET, (?SH_TYPE_OFFSET + ?SH_TYPE_SIZE) ). +-define(SH_ADDR_OFFSET, (?SH_FLAGS_OFFSET + ?SH_FLAGS_SIZE) ). +-define(SH_OFFSET_OFFSET, (?SH_ADDR_OFFSET + ?SH_ADDR_SIZE) ). +-define(SH_SIZE_OFFSET, (?SH_OFFSET_OFFSET + ?SH_OFFSET_SIZE) ). +-define(SH_LINK_OFFSET, (?SH_SIZE_OFFSET + ?SH_SIZE_SIZE) ). +-define(SH_INFO_OFFSET, (?SH_LINK_OFFSET + ?SH_LINK_SIZE) ). +-define(SH_ADDRALIGN_OFFSET, (?SH_INFO_OFFSET + ?SH_INFO_SIZE) ). +-define(SH_ENTSIZE_OFFSET, (?SH_ADDRALIGN_OFFSET + ?SH_ADDRALIGN_SIZE) ). + +%% Name aliases of Section Header Table entry information used in +%% get_shdrtab_entry function of elf64_format module. +-define(SH_NAME, {?SH_NAME_OFFSET, ?SH_NAME_SIZE}). +-define(SH_TYPE, {?SH_TYPE_OFFSET, ?SH_TYPE_SIZE}). +-define(SH_FLAGS, {?SH_FLAGS_OFFSET, ?SH_FLAGS_SIZE}). +-define(SH_ADDR, {?SH_ADDR_OFFSET, ?SH_ADDR_SIZE}). +-define(SH_OFFSET, {?SH_OFFSET_OFFSET, ?SH_OFFSET_SIZE}). +-define(SH_SIZE, {?SH_SIZE_OFFSET, ?SH_SIZE_SIZE}). +-define(SH_LINK, {?SH_LINK_OFFSET, ?SH_LINK_SIZE}). +-define(SH_INFO, {?SH_INFO_OFFSET, ?SH_INFO_SIZE}). +-define(SH_ADDRALIGN, {?SH_ADDRALIGN_OFFSET, ?SH_ADDRALIGN_SIZE}). +-define(SH_ENTSIZE, {?SH_ENTSIZE_OFFSET, ?SH_ENTSIZE_SIZE}). + +%% Section Indices +-define(SHN_UNDEF, 0). +-define(SHN_LOPROC, 16#FF00). +-define(SHN_HIPROC, 16#FF1F). +-define(SHN_LOOS, 16#FF20). +-define(SHN_HIOS, 16#FF3F). +-define(SHN_ABS, 16#FFF1). +-define(SHN_COMMON, 16#FFF2). + +%% Section Types (sh_type) +-define(SHT_NULL, 0). +-define(SHT_PROGBITS, 1). +-define(SHT_SYMTAB, 2). +-define(SHT_STRTAB, 3). +-define(SHT_RELA, 4). +-define(SHT_HASH, 5). +-define(SHT_DYNAMIC, 6). +-define(SHT_NOTE, 7). +-define(SHT_NOBITS, 8). +-define(SHT_REL, 9). +-define(SHT_SHLIB, 10). +-define(SHT_DYNSYM, 11). +-define(SHT_LOOS, 16#60000000). +-define(SHT_HIOS, 16#6FFFFFFF). +-define(SHT_LOPROC, 16#70000000). +-define(SHT_HIPROC, 16#7FFFFFFF). + +%% Section Attributes (sh_flags) +-define(SHF_WRITE, 16#1). +-define(SHF_ALLOC, 16#2). +-define(SHF_EXECINSTR, 16#4). +-define(SHF_MASKOS, 16#0F000000). +-define(SHF_MASKPROC, 16#F0000000). + +%% +%% Standard Section names for Code and Data +%% +-define(BSS, ".bss"). +-define(DATA, ".data"). +-define(INTERP, ".interp"). +-define(RODATA, ".rodata"). +-define(TEXT, ".text"). +%% Other Standard Section names +-define(COMMENT, ".comment"). +-define(DYNAMIC, ".dynamic"). +-define(DYNSTR, ".dynstr"). +-define(GOT, ".got"). +-define(HASH, ".hash"). +-define(NOTE(Name), (".note" ++ Name)). +-define(PLT, ".plt"). +-define(REL(Name), (".rel" ++ Name) ). +-define(RELA(Name), (".rela" ++ Name) ). +-define(SHSTRTAB, ".shstrtab"). +-define(STRTAB, ".strtab"). +-define(SYMTAB, ".symtab"). +-define(GCC_EXN_TAB, ".gcc_except_table"). + +%%------------------------------------------------------------------------------ +%% ELF Symbol Table Entries +%%------------------------------------------------------------------------------ +-define(ELF_SYM_SIZE, (?ST_NAME_SIZE + ?ST_INFO_SIZE + ?ST_OTHER_SIZE + +?ST_SHNDX_SIZE + ?ST_VALUE_SIZE + ?ST_SIZE_SIZE) ). + +-define(ST_NAME_SIZE, ?ELF_WORD_SIZE). +-define(ST_INFO_SIZE, ?ELF_UNSIGNED_CHAR_SIZE). +-define(ST_OTHER_SIZE, ?ELF_UNSIGNED_CHAR_SIZE). +-define(ST_SHNDX_SIZE, ?ELF_HALF_SIZE). +-define(ST_VALUE_SIZE, ?ELF_ADDR_SIZE). +-define(ST_SIZE_SIZE, ?ELF_XWORD_SIZE). + +%% Precomputed offset for Symbol Table entries in SymTab binary +%%XXX: Included in either elf32_format or elf64_format. + +%% Name aliases for Symbol Table entry information +-define(ST_NAME, {?ST_NAME_OFFSET, ?ST_NAME_SIZE}). +-define(ST_INFO, {?ST_INFO_OFFSET, ?ST_INFO_SIZE}). +-define(ST_OTHER, {?ST_OTHER_OFFSET, ?ST_OTHER_SIZE}). +-define(ST_SHNDX, {?ST_SHNDX_OFFSET, ?ST_SHNDX_SIZE}). +-define(ST_VALUE, {?ST_VALUE_OFFSET, ?ST_VALUE_SIZE}). +-define(ST_SIZE, {?ST_SIZE_OFFSET, ?ST_SIZE_SIZE}). + +%% Macros to extract information from st_type +-define(ELF_ST_BIND(I), (I bsr 4) ). +-define(ELF_ST_TYPE(I), (I band 16#f) ). +-define(ELF_ST_INFO(B,T), (B bsl 4 + T band 16#f) ). + +%% Symbol Bindings +-define(STB_LOCAL, 0). +-define(STB_GLOBAL, 1). +-define(STB_WEAK, 2). +-define(STB_LOOS, 10). +-define(STB_HIOS, 12). +-define(STB_LOPROC, 13). +-define(STB_HIPROC, 15). + +%% Symbol Types +-define(STT_NOTYPE, 0). +-define(STT_OBJECT, 1). +-define(STT_FUNC, 2). +-define(STT_SECTION, 3). +-define(STT_FILE, 4). +-define(STT_LOOS, 10). +-define(STT_HIOS, 12). +-define(STT_LOPROC, 13). +-define(STT_HIPROC, 15). + +%%------------------------------------------------------------------------------ +%% ELF Relocation Entries +%%------------------------------------------------------------------------------ +-define(ELF_REL_SIZE, (?R_OFFSET_SIZE + ?R_INFO_SIZE) ). +-define(ELF_RELA_SIZE, (?R_OFFSET_SIZE + ?R_INFO_SIZE + ?R_ADDEND_SIZE) ). + +-define(R_OFFSET_SIZE, ?ELF_ADDR_SIZE). +-define(R_INFO_SIZE, ?ELF_XWORD_SIZE). +-define(R_ADDEND_SIZE, ?ELF_SXWORD_SIZE). + +%% Arithmetics for computing byte offsets in a Relocation entry binary +-define(R_OFFSET_OFFSET, 0). +-define(R_INFO_OFFSET, (?R_OFFSET_OFFSET + ?R_OFFSET_SIZE) ). +-define(R_ADDEND_OFFSET, (?R_INFO_OFFSET + ?R_INFO_SIZE) ). + +%% Name aliases for Relocation field information +-define(R_OFFSET, {?R_OFFSET_OFFSET, ?R_OFFSET_SIZE}). +-define(R_INFO, {?R_INFO_OFFSET, ?R_INFO_SIZE}). +-define(R_ADDEND, {?R_ADDEND_OFFSET, ?R_ADDEND_SIZE}). + +%% Useful macros to extract information from r_info field +%%XXX: Included in either elf32_format or elf64_format. + +%%------------------------------------------------------------------------------ +%% ELF Program Header Table +%%------------------------------------------------------------------------------ +-define(ELF_PHDR_SIZE, (?P_TYPE_SIZE + ?P_FLAGS_SIZE + ?P_OFFSET_SIZE + +?P_VADDR_SIZE + ?P_PADDR_SIZE + ?P_FILESZ_SIZE + +?P_MEMSZ_SIZE + ?P_ALIGN_SIZE) ). + +-define(P_TYPE_SIZE, ?ELF_WORD_SIZE). +-define(P_FLAGS_SIZE, ?ELF_WORD_SIZE). +-define(P_OFFSET_SIZE, ?ELF_OFF_SIZE). +-define(P_VADDR_SIZE, ?ELF_ADDR_SIZE). +-define(P_PADDR_SIZE, ?ELF_ADDR_SIZE). +-define(P_FILESZ_SIZE, ?ELF_XWORD_SIZE). +-define(P_MEMSZ_SIZE, ?ELF_XWORD_SIZE). +-define(P_ALIGN_SIZE, ?ELF_XWORD_SIZE). + +%% Offsets of various fields in a Program Header Table entry binary. +%%XXX: Included in either elf32_format or elf64_format. + +%% Name aliases for each Program Header Table entry field information. +-define(P_TYPE, {?P_TYPE_OFFSET, ?P_TYPE_SIZE} ). +-define(P_FLAGS, {?P_FLAGS_OFFSET, ?P_FLAGS_SIZE} ). +-define(P_OFFSET, {?P_OFFSET_OFFSET, ?P_OFFSET_SIZE} ). +-define(P_VADDR, {?P_VADDR_OFFSET, ?P_VADDR_SIZE} ). +-define(P_PADDR, {?P_PADDR_OFFSET, ?P_PADDR_SIZE} ). +-define(P_FILESZ, {?P_FILESZ_OFFSET, ?P_FILESZ_SIZE} ). +-define(P_MEMSZ, {?P_MEMSZ_OFFSET, ?P_MEMSZ_SIZE} ). +-define(P_ALIGN, {?P_ALIGN_OFFSET, ?P_ALIGN_SIZE} ). + +%% Segment Types (p_type) +-define(PT_NULL, 0). +-define(PT_LOAD, 1). +-define(PT_DYNAMIC, 2). +-define(PT_INTERP, 3). +-define(PT_NOTE, 4). +-define(PT_SHLIB, 5). +-define(PT_PHDR, 6). +-define(PT_LOOS, 16#60000000). +-define(PT_HIOS, 16#6FFFFFFF). +-define(PT_LOPROC, 16#70000000). +-define(PT_HIPROC, 16#7FFFFFFF). + +%% Segment Attributes (p_flags) +-define(PF_X, 16#1). +-define(PF_W, 16#2). +-define(PF_R, 16#4). +-define(PF_MASKOS, 16#00FF0000). +-define(PF_MASKPROC, 16#FF000000). + +%%------------------------------------------------------------------------------ +%% ELF Dynamic Table +%%------------------------------------------------------------------------------ +-define(ELF_DYN_SIZE, (?D_TAG_SIZE + ?D_VAL_PTR_SIZE) ). + +-define(D_TAG_SIZE, ?ELF_SXWORD_SIZE). +-define(D_VAL_PTR_SIZE, ?ELF_ADDR_SIZE). + +%% Offsets of each field in Dynamic Table entry in binary +-define(D_TAG_OFFSET, 0). +-define(D_VAL_PTR_OFFSET, (?D_TAG_OFFSET + ?D_TAG_SIZE)). + +%% Name aliases for each field of a Dynamic Table entry information +-define(D_TAG, {?D_TAG_OFFSET, ?D_TAG_SIZE} ). +-define(D_VAL_PTR, {?D_VAL_PTR_OFFSET, ?D_VAL_PTR_SIZE} ). + +%% Dynamic Table Entries +-define(DT_NULL, 0). +-define(DT_NEEDED, 1). +-define(DT_PLTRELSZ, 2). +-define(DT_PLTGOT, 3). +-define(DT_HASH, 4). +-define(DT_STRTAB, 5). +-define(DT_SYMTAB, 6). +-define(DT_RELA, 7). +-define(DT_RELASZ, 8). +-define(DT_RELAENT, 9). +-define(DT_STRSZ, 10). +-define(DT_SYMENT, 11). +-define(DT_INIT, 12). +-define(DT_FINI, 13). +-define(DT_SONAME, 14). +-define(DT_RPATH, 15). +-define(DT_SYMBOLIC, 16). +-define(DT_REL, 17). +-define(DT_RELSZ, 18). +-define(DT_RELENT, 19). +-define(DT_PLTREL, 20). +-define(DT_DEBUG, 21). +-define(DT_TEXTREL, 22). +-define(DT_JMPREL, 23). +-define(DT_BIND_NOW, 24). +-define(DT_INIT_ARRAY, 25). +-define(DT_FINI_ARRAY, 26). +-define(DT_INIT_ARRAYSZ, 27). +-define(DT_FINI_ARRAYSZ, 28). +-define(DT_LOOS, 16#60000000). +-define(DT_HIOS, 16#6FFFFFFF). +-define(DT_LOPROC, 16#700000000). +-define(DT_HIPROC, 16#7FFFFFFFF). + +%%------------------------------------------------------------------------------ +%% ELF GCC Exception Table +%%------------------------------------------------------------------------------ + +%% The DWARF Exception Header Encoding is used to describe the type of data used +%% in the .eh_frame_hdr (and .gcc_except_table) section. The upper 4 bits +%% indicate how the value is to be applied. The lower 4 bits indicate the format +%% of the data. + +%% DWARF Exception Header value format +-define(DW_EH_PE_omit, 16#ff). % No value is present. +-define(DW_EH_PE_uleb128, 16#01). % Unsigned value encoded using LEB128. +-define(DW_EH_PE_udata2, 16#02). % A 2 bytes unsigned value. +-define(DW_EH_PE_udata4, 16#03). % A 4 bytes unsigned value. +-define(DW_EH_PE_udata8, 16#04). % An 8 bytes unsigned value. +-define(DW_EH_PE_sleb128, 16#09). % Signed value encoded using LEB128. +-define(DW_EH_PE_sdata2, 16#0a). % A 2 bytes signed value. +-define(DW_EH_PE_sdata4, 16#0b). % A 4 bytes signed value. +-define(DW_EH_PE_sdata8, 16#0c). % An 8 bytes signed value. + +%% DWARF Exception Header application +-define(DW_EH_PE_absptr, 16#00). % Value is used with no modification. +-define(DW_EH_PE_pcrel, 16#10). % Value is relative to the current PC. +-define(DW_EH_PE_datarel, 16#30). % Value is relative to the beginning of the + % section. + +%%------------------------------------------------------------------------------ +%% ELF Read-only data (constants, literlas etc.) +%%------------------------------------------------------------------------------ +-define(RO_ENTRY_SIZE, 8). + +%%------------------------------------------------------------------------------ +%% Custom Note section: ".note.gc" for Erlang GC +%%------------------------------------------------------------------------------ + +%% The structure of this section is the following: +%% +%% .short <n> # number of safe points in code +%% +%% .long .L<label1> # safe point address | +%% .long .L<label2> # safe point address |-> safe point addrs +%% ..... | +%% .long .L<label3> # safe point address | +%% +%% .short <n> # stack frame size (in words) |-> fixed-size part +%% .short <n> # stack arity | +%% .short <n> # number of live roots that follow | +%% +%% .short <n> # live root's stack index | +%% ..... |-> live root indices +%% .short <n> # >> | + +%% The name of the custom Note Section +-define(NOTE_ERLGC_NAME, ".gc"). + +%% The first word of a Note Section for Erlang GC (".note.gc") is always the +%% number of safepoints in code. +-define(SP_COUNT, {?SP_COUNT_OFFSET, ?SP_COUNT_SIZE}). +-define(SP_COUNT_SIZE, ?ELF_HALF_SIZE). +-define(SP_COUNT_OFFSET, 0). %(always the first entry in sdesc) + +%% The fixed-size part of a safe point (SP) entry consists of 4 words: the SP +%% address (offset in code), the stack frame size of the function (where the SP +%% is located), the stack arity of the function (the registered values are *not* +%% counted), the number of live roots in the specific SP. +-define(SP_FIXED, {?SP_FIXED_OFF, ?SP_FIXED_SIZE}). +-define(SP_FIXED_OFF, 0). +%%XXX: Exclude SP_ADDR_SIZE from SP_FIXED_SIZE in lew of new GC layout +-define(SP_FIXED_SIZE, (?SP_STKFRAME_SIZE + ?SP_STKARITY_SIZE + + ?SP_LIVEROOTCNT_SIZE)). + +-define(SP_ADDR_SIZE, ?ELF_WORD_SIZE). +-define(SP_STKFRAME_SIZE, ?ELF_HALF_SIZE). +-define(SP_STKARITY_SIZE, ?ELF_HALF_SIZE). +-define(SP_LIVEROOTCNT_SIZE, ?ELF_HALF_SIZE). + +%%XXX: SP_STKFRAME is the first piece of information in the new GC layout +-define(SP_STKFRAME_OFFSET, 0). +-define(SP_STKARITY_OFFSET, (?SP_STKFRAME_OFFSET + ?SP_STKFRAME_SIZE) ). +-define(SP_LIVEROOTCNT_OFFSET, (?SP_STKARITY_OFFSET + ?SP_STKARITY_SIZE) ). + +%% Name aliases for safepoint fields. +-define(SP_STKFRAME, {?SP_STKFRAME_OFFSET, ?SP_STKFRAME_SIZE}). +-define(SP_STKARITY, {?SP_STKARITY_OFFSET, ?SP_STKARITY_SIZE}). +-define(SP_LIVEROOTCNT, {?SP_LIVEROOTCNT_OFFSET, ?SP_LIVEROOTCNT_SIZE}). + +%% After the fixed-size part a variable-size part exists. This part holds the +%% stack frame index of every live root in the specific SP. +-define(LR_STKINDEX_SIZE, ?ELF_HALF_SIZE). + +%%------------------------------------------------------------------------------ +%% Misc. +%%------------------------------------------------------------------------------ +-define(bits(Bytes), ((Bytes) bsl 3)). diff --git a/lib/hipe/llvm/hipe_llvm.erl b/lib/hipe/llvm/hipe_llvm.erl new file mode 100644 index 0000000000..5e33731a2b --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm.erl @@ -0,0 +1,1131 @@ +%% -*- erlang-indent-level: 2 -*- + +-module(hipe_llvm). + +-export([ + mk_ret/1, + ret_ret_list/1, + + mk_br/1, + br_dst/1, + + mk_br_cond/3, + mk_br_cond/4, + br_cond_cond/1, + br_cond_true_label/1, + br_cond_false_label/1, + br_cond_meta/1, + + mk_indirectbr/3, + indirectbr_type/1, + indirectbr_address/1, + indirectbr_label_list/1, + + mk_switch/4, + switch_type/1, + switch_value/1, + switch_default_label/1, + switch_value_label_list/1, + + mk_invoke/9, + invoke_dst/1, + invoke_cconv/1, + invoke_ret_attrs/1, + invoke_type/1, + invoke_fnptrval/1, + invoke_arglist/1, + invoke_fn_attrs/1, + invoke_to_label/1, + invoke_unwind_label/1, + + mk_operation/6, + operation_dst/1, + operation_op/1, + operation_type/1, + operation_src1/1, + operation_src2/1, + operation_options/1, + + mk_extractvalue/5, + extractvalue_dst/1, + extractvalue_type/1, + extractvalue_val/1, + extractvalue_idx/1, + extractvalue_idxs/1, + + mk_insertvalue/7, + insertvalue_dst/1, + insertvalue_val_type/1, + insertvalue_val/1, + insertvalue_elem_type/1, + insertvalue_elem/1, + insertvalue_idx/1, + insertvalue_idxs/1, + + mk_alloca/4, + alloca_dst/1, + alloca_type/1, + alloca_num/1, + alloca_align/1, + + mk_load/6, + load_dst/1, + load_p_type/1, + load_pointer/1, + load_alignment/1, + load_nontemporal/1, + load_volatile/1, + + mk_store/7, + store_type/1, + store_value/1, + store_p_type/1, + store_pointer/1, + store_alignment/1, + store_nontemporal/1, + store_volatile/1, + + mk_getelementptr/5, + getelementptr_dst/1, + getelementptr_p_type/1, + getelementptr_value/1, + getelementptr_typed_idxs/1, + getelementptr_inbounds/1, + + mk_conversion/5, + conversion_dst/1, + conversion_op/1, + conversion_src_type/1, + conversion_src/1, + conversion_dst_type/1, + + mk_sitofp/4, + sitofp_dst/1, + sitofp_src_type/1, + sitofp_src/1, + sitofp_dst_type/1, + + mk_ptrtoint/4, + ptrtoint_dst/1, + ptrtoint_src_type/1, + ptrtoint_src/1, + ptrtoint_dst_type/1, + + mk_inttoptr/4, + inttoptr_dst/1, + inttoptr_src_type/1, + inttoptr_src/1, + inttoptr_dst_type/1, + + mk_icmp/5, + icmp_dst/1, + icmp_cond/1, + icmp_type/1, + icmp_src1/1, + icmp_src2/1, + + mk_fcmp/5, + fcmp_dst/1, + fcmp_cond/1, + fcmp_type/1, + fcmp_src1/1, + fcmp_src2/1, + + mk_phi/3, + phi_dst/1, + phi_type/1, + phi_value_label_list/1, + + mk_select/6, + select_dst/1, + select_cond/1, + select_typ1/1, + select_val1/1, + select_typ2/1, + select_val2/1, + + mk_call/8, + call_dst/1, + call_is_tail/1, + call_cconv/1, + call_ret_attrs/1, + call_type/1, + call_fnptrval/1, + call_arglist/1, + call_fn_attrs/1, + + mk_fun_def/10, + fun_def_linkage/1, + fun_def_visibility/1, + fun_def_cconv/1, + fun_def_ret_attrs/1, + fun_def_type/1, + fun_def_name/1, + fun_def_arglist/1, + fun_def_fn_attrs/1, + fun_def_align/1, + fun_def_body/1, + + mk_fun_decl/8, + fun_decl_linkage/1, + fun_decl_visibility/1, + fun_decl_cconv/1, + fun_decl_ret_attrs/1, + fun_decl_type/1, + fun_decl_name/1, + fun_decl_arglist/1, + fun_decl_align/1, + + mk_landingpad/0, + + mk_comment/1, + comment_text/1, + + mk_label/1, + label_label/1, + is_label/1, + + mk_const_decl/4, + const_decl_dst/1, + const_decl_decl_type/1, + const_decl_type/1, + const_decl_value/1, + + mk_asm/1, + asm_instruction/1, + + mk_adj_stack/3, + adj_stack_offset/1, + adj_stack_register/1, + adj_stack_type/1, + + mk_branch_meta/3, + branch_meta_id/1, + branch_meta_true_weight/1, + branch_meta_false_weight/1 + ]). + +-export([ + mk_void/0, + + mk_label_type/0, + + mk_int/1, + int_width/1, + + mk_double/0, + + mk_pointer/1, + pointer_type/1, + + mk_array/2, + array_size/1, + array_type/1, + + mk_vector/2, + vector_size/1, + vector_type/1, + + mk_struct/1, + struct_type_list/1, + + mk_fun/2, + function_ret_type/1, + function_arg_type_list/1 + ]). + +-export([pp_ins_list/2, pp_ins/2]). + + +%%----------------------------------------------------------------------------- +%% Abstract Data Types for LLVM Assembly. +%%----------------------------------------------------------------------------- + +%% Terminator Instructions +-record(llvm_ret, {ret_list=[]}). +-type llvm_ret() :: #llvm_ret{}. + +-record(llvm_br, {dst}). +-type llvm_br() :: #llvm_br{}. + +-record(llvm_br_cond, {'cond', true_label, false_label, meta=[]}). +-type llvm_br_cond() :: #llvm_br_cond{}. + +-record(llvm_indirectbr, {type, address, label_list}). +-type llvm_indirectbr() :: #llvm_indirectbr{}. + +-record(llvm_switch, {type, value, default_label, value_label_list=[]}). +-type llvm_switch() :: #llvm_switch{}. + +-record(llvm_invoke, {dst, cconv=[], ret_attrs=[], type, fnptrval, arglist=[], + fn_attrs=[], to_label, unwind_label}). +-type llvm_invoke() :: #llvm_invoke{}. + +%% Binary Operations +-record(llvm_operation, {dst, op, type, src1, src2, options=[]}). +-type llvm_operation() :: #llvm_operation{}. + +%% Aggregate Operations +-record(llvm_extractvalue, {dst, type, val, idx, idxs=[]}). +-type llvm_extractvalue() :: #llvm_extractvalue{}. + +-record(llvm_insertvalue, {dst, val_type, val, elem_type, elem, idx, idxs=[]}). +-type llvm_insertvalue() :: #llvm_insertvalue{}. + +%% Memory Access and Addressing Operations +-record(llvm_alloca, {dst, type, num=[], align=[]}). +-type llvm_alloca() :: #llvm_alloca{}. + +-record(llvm_load, {dst, p_type, pointer, alignment=[], nontemporal=[], + volatile=false}). +-type llvm_load() :: #llvm_load{}. + +-record(llvm_store, {type, value, p_type, pointer, alignment=[], + nontemporal=[], volatile=false}). +-type llvm_store() :: #llvm_store{}. + +-record(llvm_getelementptr, {dst, p_type, value, typed_idxs, inbounds}). +-type llvm_getelementptr() :: #llvm_getelementptr{}. + +%% Conversion Operations +-record(llvm_conversion, {dst, op, src_type, src, dst_type}). +-type llvm_conversion() :: #llvm_conversion{}. + +-record(llvm_sitofp, {dst, src_type, src, dst_type}). +-type llvm_sitofp() :: #llvm_sitofp{}. + +-record(llvm_ptrtoint, {dst, src_type, src, dst_type}). +-type llvm_ptrtoint() :: #llvm_ptrtoint{}. + +-record(llvm_inttoptr, {dst, src_type, src, dst_type}). +-type llvm_inttoptr() :: #llvm_inttoptr{}. + +%% Other Operations +-record(llvm_icmp, {dst, 'cond', type, src1, src2}). +-type llvm_icmp() :: #llvm_icmp{}. + +-record(llvm_fcmp, {dst, 'cond', type, src1, src2}). +-type llvm_fcmp() :: #llvm_fcmp{}. + +-record(llvm_phi, {dst, type, value_label_list}). +-type llvm_phi() :: #llvm_phi{}. + +-record(llvm_select, {dst, 'cond', typ1, val1, typ2, val2}). +-type llvm_select() :: #llvm_select{}. + +-record(llvm_call, {dst=[], is_tail = false, cconv = [], ret_attrs = [], type, + fnptrval, arglist = [], fn_attrs = []}). +-type llvm_call() :: #llvm_call{}. + +-record(llvm_fun_def, {linkage=[], visibility=[], cconv=[], ret_attrs=[], + type, 'name', arglist=[], fn_attrs=[], align=[], body=[]}). +-type llvm_fun_def() :: #llvm_fun_def{}. + +-record(llvm_fun_decl, {linkage=[], visibility=[], cconv=[], ret_attrs=[], + type, 'name', arglist=[], align=[]}). +-type llvm_fun_decl() :: #llvm_fun_decl{}. + +-record(llvm_landingpad, {}). +-type llvm_landingpad() :: #llvm_landingpad{}. + +-record(llvm_comment, {text}). +-type llvm_comment() :: #llvm_comment{}. + +-record(llvm_label, {label}). +-type llvm_label() :: #llvm_label{}. + +-record(llvm_const_decl, {dst, decl_type, type, value}). +-type llvm_const_decl() :: #llvm_const_decl{}. + +-record(llvm_asm, {instruction}). +-type llvm_asm() :: #llvm_asm{}. + +-record(llvm_adj_stack, {offset, 'register', type}). +-type llvm_adj_stack() :: #llvm_adj_stack{}. + +-record(llvm_branch_meta, {id, true_weight, false_weight}). +-type llvm_branch_meta() :: #llvm_branch_meta{}. + +%% A type for any LLVM instruction +-type llvm_instr() :: llvm_ret() | llvm_br() | llvm_br_cond() + | llvm_indirectbr() | llvm_switch() | llvm_invoke() + | llvm_operation() | llvm_extractvalue() + | llvm_insertvalue() | llvm_alloca() | llvm_load() + | llvm_store() | llvm_getelementptr() | llvm_conversion() + | llvm_sitofp() | llvm_ptrtoint() | llvm_inttoptr() + | llvm_icmp() | llvm_fcmp() | llvm_phi() | llvm_select() + | llvm_call() | llvm_fun_def() | llvm_fun_decl() + | llvm_landingpad() | llvm_comment() | llvm_label() + | llvm_const_decl() | llvm_asm() | llvm_adj_stack() + | llvm_branch_meta(). + +%% Types +-record(llvm_void, {}). +%-type llvm_void() :: #llvm_void{}. + +-record(llvm_label_type, {}). +%-type llvm_label_type() :: #llvm_label_type{}. + +-record(llvm_int, {width}). +%-type llvm_int() :: #llvm_int{}. + +-record(llvm_float, {}). +%-type llvm_float() :: #llvm_float{}. + +-record(llvm_double, {}). +%-type llvm_double() :: #llvm_double{}. + +-record(llvm_fp80, {}). +%-type llvm_fp80() :: #llvm_fp80{}. + +-record(llvm_fp128, {}). +%-type llvm_fp128() :: #llvm_fp128{}. + +-record(llvm_ppc_fp128, {}). +%-type llvm_ppc_fp128() :: #llvm_ppc_fp128{}. + +-record(llvm_pointer, {type}). +%-type llvm_pointer() :: #llvm_pointer{}. + +-record(llvm_vector, {'size', type}). +%-type llvm_vector() :: #llvm_vector{}. + +-record(llvm_struct, {type_list}). +%-type llvm_struct() :: #llvm_struct{}. + +-record(llvm_array, {'size', type}). +%-type llvm_array() :: #llvm_array{}. + +-record(llvm_fun, {ret_type, arg_type_list}). +%-type llvm_fun() :: #llvm_fun{}. + +%%----------------------------------------------------------------------------- +%% Accessor Functions +%%----------------------------------------------------------------------------- + +%% ret +mk_ret(Ret_list) -> #llvm_ret{ret_list=Ret_list}. +ret_ret_list(#llvm_ret{ret_list=Ret_list}) -> Ret_list. + +%% br +mk_br(Dst) -> #llvm_br{dst=Dst}. +br_dst(#llvm_br{dst=Dst}) -> Dst. + +%% br_cond +mk_br_cond(Cond, True_label, False_label) -> + #llvm_br_cond{'cond'=Cond, true_label=True_label, false_label=False_label}. +mk_br_cond(Cond, True_label, False_label, Metadata) -> + #llvm_br_cond{'cond'=Cond, true_label=True_label, false_label=False_label, + meta=Metadata}. +br_cond_cond(#llvm_br_cond{'cond'=Cond}) -> Cond. +br_cond_true_label(#llvm_br_cond{true_label=True_label}) -> True_label. +br_cond_false_label(#llvm_br_cond{false_label=False_label}) -> + False_label. +br_cond_meta(#llvm_br_cond{meta=Metadata}) -> Metadata. + +%% indirectbr +mk_indirectbr(Type, Address, Label_list) -> #llvm_indirectbr{type=Type, address=Address, label_list=Label_list}. +indirectbr_type(#llvm_indirectbr{type=Type}) -> Type. +indirectbr_address(#llvm_indirectbr{address=Address}) -> Address. +indirectbr_label_list(#llvm_indirectbr{label_list=Label_list}) -> Label_list. + +%% invoke +mk_invoke(Dst, Cconv, Ret_attrs, Type, Fnptrval, Arglist, Fn_attrs, To_label, Unwind_label) -> + #llvm_invoke{dst=Dst, cconv=Cconv, ret_attrs=Ret_attrs, type=Type, + fnptrval=Fnptrval, arglist=Arglist, fn_attrs=Fn_attrs, to_label=To_label, + unwind_label=Unwind_label}. +invoke_dst(#llvm_invoke{dst=Dst}) -> Dst. +invoke_cconv(#llvm_invoke{cconv=Cconv}) -> Cconv. +invoke_ret_attrs(#llvm_invoke{ret_attrs=Ret_attrs}) -> Ret_attrs. +invoke_type(#llvm_invoke{type=Type}) -> Type. +invoke_fnptrval(#llvm_invoke{fnptrval=Fnptrval}) -> Fnptrval. +invoke_arglist(#llvm_invoke{arglist=Arglist}) -> Arglist. +invoke_fn_attrs(#llvm_invoke{fn_attrs=Fn_attrs}) -> Fn_attrs. +invoke_to_label(#llvm_invoke{to_label=To_label}) -> To_label. +invoke_unwind_label(#llvm_invoke{unwind_label=Unwind_label}) -> Unwind_label. + +%% switch +mk_switch(Type, Value, Default_label, Value_label_list) -> + #llvm_switch{type=Type, value=Value, default_label=Default_label, + value_label_list=Value_label_list}. +switch_type(#llvm_switch{type=Type}) -> Type. +switch_value(#llvm_switch{value=Value}) -> Value. +switch_default_label(#llvm_switch{default_label=Default_label}) -> + Default_label. +switch_value_label_list(#llvm_switch{value_label_list=Value_label_list}) -> + Value_label_list. + +%% operation +mk_operation(Dst, Op, Type, Src1, Src2, Options) -> + #llvm_operation{dst=Dst, op=Op, type=Type, src1=Src1, src2=Src2, + options=Options}. +operation_dst(#llvm_operation{dst=Dst}) -> Dst. +operation_op(#llvm_operation{op=Op}) -> Op. +operation_type(#llvm_operation{type=Type}) -> Type. +operation_src1(#llvm_operation{src1=Src1}) -> Src1. +operation_src2(#llvm_operation{src2=Src2}) -> Src2. +operation_options(#llvm_operation{options=Options}) -> Options. + +%% extractvalue +mk_extractvalue(Dst, Type, Val, Idx, Idxs) -> + #llvm_extractvalue{dst=Dst,type=Type,val=Val,idx=Idx,idxs=Idxs}. +extractvalue_dst(#llvm_extractvalue{dst=Dst}) -> Dst. +extractvalue_type(#llvm_extractvalue{type=Type}) -> Type. +extractvalue_val(#llvm_extractvalue{val=Val}) -> Val. +extractvalue_idx(#llvm_extractvalue{idx=Idx}) -> Idx. +extractvalue_idxs(#llvm_extractvalue{idxs=Idxs}) -> Idxs. + +%% insertvalue +mk_insertvalue(Dst, Val_type, Val, Elem_type, Elem, Idx, Idxs) -> + #llvm_insertvalue{dst=Dst, val_type=Val_type, val=Val, elem_type=Elem_type, + elem=Elem, idx=Idx, idxs=Idxs}. +insertvalue_dst(#llvm_insertvalue{dst=Dst}) -> Dst. +insertvalue_val_type(#llvm_insertvalue{val_type=Val_type}) -> Val_type. +insertvalue_val(#llvm_insertvalue{val=Val}) -> Val. +insertvalue_elem_type(#llvm_insertvalue{elem_type=Elem_type}) -> Elem_type. +insertvalue_elem(#llvm_insertvalue{elem=Elem}) -> Elem. +insertvalue_idx(#llvm_insertvalue{idx=Idx}) -> Idx. +insertvalue_idxs(#llvm_insertvalue{idxs=Idxs}) -> Idxs. + +%% alloca +mk_alloca(Dst, Type, Num, Align) -> + #llvm_alloca{dst=Dst, type=Type, num=Num, align=Align}. +alloca_dst(#llvm_alloca{dst=Dst}) -> Dst. +alloca_type(#llvm_alloca{type=Type}) -> Type. +alloca_num(#llvm_alloca{num=Num}) -> Num. +alloca_align(#llvm_alloca{align=Align}) -> Align. + +%% load +mk_load(Dst, Type, Pointer, Alignment, Nontemporal, Volatile) -> + #llvm_load{dst=Dst, p_type=Type, pointer=Pointer, alignment=Alignment, + nontemporal=Nontemporal, volatile=Volatile}. +load_dst(#llvm_load{dst=Dst}) -> Dst. +load_p_type(#llvm_load{p_type=Type}) -> Type. +load_pointer(#llvm_load{pointer=Pointer}) -> Pointer. +load_alignment(#llvm_load{alignment=Alignment}) -> Alignment. +load_nontemporal(#llvm_load{nontemporal=Nontemporal}) -> Nontemporal. +load_volatile(#llvm_load{volatile=Volatile}) -> Volatile. + +%% store +mk_store(Type, Value, P_Type, Pointer, Alignment, Nontemporal, Volatile) -> + #llvm_store{type=Type, value=Value, p_type=P_Type, pointer=Pointer, alignment=Alignment, + nontemporal=Nontemporal, volatile=Volatile}. +store_type(#llvm_store{type=Type}) -> Type. +store_value(#llvm_store{value=Value}) -> Value. +store_p_type(#llvm_store{p_type=P_Type}) -> P_Type. +store_pointer(#llvm_store{pointer=Pointer}) -> Pointer. +store_alignment(#llvm_store{alignment=Alignment}) -> Alignment. +store_nontemporal(#llvm_store{nontemporal=Nontemporal}) -> Nontemporal. +store_volatile(#llvm_store{volatile=Volatile}) -> Volatile. + +%% getelementptr +mk_getelementptr(Dst, P_Type, Value, Typed_Idxs, Inbounds) -> + #llvm_getelementptr{dst=Dst,p_type=P_Type, value=Value, + typed_idxs=Typed_Idxs, inbounds=Inbounds}. +getelementptr_dst(#llvm_getelementptr{dst=Dst}) -> Dst. +getelementptr_p_type(#llvm_getelementptr{p_type=P_Type}) -> P_Type. +getelementptr_value(#llvm_getelementptr{value=Value}) -> Value. +getelementptr_typed_idxs(#llvm_getelementptr{typed_idxs=Typed_Idxs}) -> Typed_Idxs. +getelementptr_inbounds(#llvm_getelementptr{inbounds=Inbounds}) -> Inbounds. + +%% conversion +mk_conversion(Dst, Op, Src_type, Src, Dst_type) -> + #llvm_conversion{dst=Dst, op=Op, src_type=Src_type, src=Src, dst_type=Dst_type}. +conversion_dst(#llvm_conversion{dst=Dst}) -> Dst. +conversion_op(#llvm_conversion{op=Op}) -> Op. +conversion_src_type(#llvm_conversion{src_type=Src_type}) -> Src_type. +conversion_src(#llvm_conversion{src=Src}) -> Src. +conversion_dst_type(#llvm_conversion{dst_type=Dst_type}) -> Dst_type. + +%% sitofp +mk_sitofp(Dst, Src_type, Src, Dst_type) -> + #llvm_sitofp{dst=Dst, src_type=Src_type, src=Src, dst_type=Dst_type}. +sitofp_dst(#llvm_sitofp{dst=Dst}) -> Dst. +sitofp_src_type(#llvm_sitofp{src_type=Src_type}) -> Src_type. +sitofp_src(#llvm_sitofp{src=Src}) -> Src. +sitofp_dst_type(#llvm_sitofp{dst_type=Dst_type}) -> Dst_type. + +%% ptrtoint +mk_ptrtoint(Dst, Src_Type, Src, Dst_Type) -> + #llvm_ptrtoint{dst=Dst, src_type=Src_Type, src=Src, dst_type=Dst_Type}. +ptrtoint_dst(#llvm_ptrtoint{dst=Dst}) -> Dst. +ptrtoint_src_type(#llvm_ptrtoint{src_type=Src_Type}) -> Src_Type. +ptrtoint_src(#llvm_ptrtoint{src=Src}) -> Src. +ptrtoint_dst_type(#llvm_ptrtoint{dst_type=Dst_Type}) -> Dst_Type . + +%% inttoptr +mk_inttoptr(Dst, Src_Type, Src, Dst_Type) -> + #llvm_inttoptr{dst=Dst, src_type=Src_Type, src=Src, dst_type=Dst_Type}. +inttoptr_dst(#llvm_inttoptr{dst=Dst}) -> Dst. +inttoptr_src_type(#llvm_inttoptr{src_type=Src_Type}) -> Src_Type. +inttoptr_src(#llvm_inttoptr{src=Src}) -> Src. +inttoptr_dst_type(#llvm_inttoptr{dst_type=Dst_Type}) -> Dst_Type . + +%% icmp +mk_icmp(Dst, Cond, Type, Src1, Src2) -> + #llvm_icmp{dst=Dst,'cond'=Cond,type=Type,src1=Src1,src2=Src2}. +icmp_dst(#llvm_icmp{dst=Dst}) -> Dst. +icmp_cond(#llvm_icmp{'cond'=Cond}) -> Cond. +icmp_type(#llvm_icmp{type=Type}) -> Type. +icmp_src1(#llvm_icmp{src1=Src1}) -> Src1. +icmp_src2(#llvm_icmp{src2=Src2}) -> Src2. + +%% fcmp +mk_fcmp(Dst, Cond, Type, Src1, Src2) -> + #llvm_fcmp{dst=Dst,'cond'=Cond,type=Type,src1=Src1,src2=Src2}. +fcmp_dst(#llvm_fcmp{dst=Dst}) -> Dst. +fcmp_cond(#llvm_fcmp{'cond'=Cond}) -> Cond. +fcmp_type(#llvm_fcmp{type=Type}) -> Type. +fcmp_src1(#llvm_fcmp{src1=Src1}) -> Src1. +fcmp_src2(#llvm_fcmp{src2=Src2}) -> Src2. + +%% phi +mk_phi(Dst, Type, Value_label_list) -> + #llvm_phi{dst=Dst, type=Type,value_label_list=Value_label_list}. +phi_dst(#llvm_phi{dst=Dst}) -> Dst. +phi_type(#llvm_phi{type=Type}) -> Type. +phi_value_label_list(#llvm_phi{value_label_list=Value_label_list}) -> + Value_label_list. + +%% select +mk_select(Dst, Cond, Typ1, Val1, Typ2, Val2) -> + #llvm_select{dst=Dst, 'cond'=Cond, typ1=Typ1, val1=Val1, typ2=Typ2, val2=Val2}. +select_dst(#llvm_select{dst=Dst}) -> Dst. +select_cond(#llvm_select{'cond'=Cond}) -> Cond. +select_typ1(#llvm_select{typ1=Typ1}) -> Typ1. +select_val1(#llvm_select{val1=Val1}) -> Val1. +select_typ2(#llvm_select{typ2=Typ2}) -> Typ2. +select_val2(#llvm_select{val2=Val2}) -> Val2. + +%% call +mk_call(Dst, Is_tail, Cconv, Ret_attrs, Type, Fnptrval, Arglist, Fn_attrs) -> + #llvm_call{dst=Dst, is_tail=Is_tail, cconv=Cconv, ret_attrs=Ret_attrs, + type=Type, fnptrval=Fnptrval, arglist=Arglist, fn_attrs=Fn_attrs}. +call_dst(#llvm_call{dst=Dst}) -> Dst. +call_is_tail(#llvm_call{is_tail=Is_tail}) -> Is_tail. +call_cconv(#llvm_call{cconv=Cconv}) -> Cconv. +call_ret_attrs(#llvm_call{ret_attrs=Ret_attrs}) -> Ret_attrs. +call_type(#llvm_call{type=Type}) -> Type. +call_fnptrval(#llvm_call{fnptrval=Fnptrval}) -> Fnptrval. +call_arglist(#llvm_call{arglist=Arglist}) -> Arglist. +call_fn_attrs(#llvm_call{fn_attrs=Fn_attrs}) -> Fn_attrs. + +%% fun_def +mk_fun_def(Linkage, Visibility, Cconv, Ret_attrs, Type, Name, Arglist, + Fn_attrs, Align, Body) -> + #llvm_fun_def{ + linkage=Linkage, + visibility=Visibility, + cconv=Cconv, + ret_attrs=Ret_attrs, + type=Type, + 'name'=Name, + arglist=Arglist, + fn_attrs=Fn_attrs, + align=Align, + body=Body + }. + +fun_def_linkage(#llvm_fun_def{linkage=Linkage}) -> Linkage. +fun_def_visibility(#llvm_fun_def{visibility=Visibility}) -> Visibility. +fun_def_cconv(#llvm_fun_def{cconv=Cconv}) -> Cconv . +fun_def_ret_attrs(#llvm_fun_def{ret_attrs=Ret_attrs}) -> Ret_attrs. +fun_def_type(#llvm_fun_def{type=Type}) -> Type. +fun_def_name(#llvm_fun_def{'name'=Name}) -> Name. +fun_def_arglist(#llvm_fun_def{arglist=Arglist}) -> Arglist. +fun_def_fn_attrs(#llvm_fun_def{fn_attrs=Fn_attrs}) -> Fn_attrs. +fun_def_align(#llvm_fun_def{align=Align}) -> Align. +fun_def_body(#llvm_fun_def{body=Body}) -> Body. + +%% fun_decl +mk_fun_decl(Linkage, Visibility, Cconv, Ret_attrs, Type, Name, Arglist, Align)-> + #llvm_fun_decl{ + linkage=Linkage, + visibility=Visibility, + cconv=Cconv, + ret_attrs=Ret_attrs, + type=Type, + 'name'=Name, + arglist=Arglist, + align=Align + }. + +fun_decl_linkage(#llvm_fun_decl{linkage=Linkage}) -> Linkage. +fun_decl_visibility(#llvm_fun_decl{visibility=Visibility}) -> Visibility. +fun_decl_cconv(#llvm_fun_decl{cconv=Cconv}) -> Cconv . +fun_decl_ret_attrs(#llvm_fun_decl{ret_attrs=Ret_attrs}) -> Ret_attrs. +fun_decl_type(#llvm_fun_decl{type=Type}) -> Type. +fun_decl_name(#llvm_fun_decl{'name'=Name}) -> Name. +fun_decl_arglist(#llvm_fun_decl{arglist=Arglist}) -> Arglist. +fun_decl_align(#llvm_fun_decl{align=Align}) -> Align. + +%% landingpad +mk_landingpad() -> #llvm_landingpad{}. + +%% comment +mk_comment(Text) -> #llvm_comment{text=Text}. +comment_text(#llvm_comment{text=Text}) -> Text. + +%% label +mk_label(Label) -> #llvm_label{label=Label}. +label_label(#llvm_label{label=Label}) -> Label. + +-spec is_label(llvm_instr()) -> boolean(). +is_label(#llvm_label{}) -> true; +is_label(#llvm_ret{}) -> false; +is_label(#llvm_br{}) -> false; +is_label(#llvm_br_cond{}) -> false; +is_label(#llvm_indirectbr{}) -> false; +is_label(#llvm_switch{}) -> false; +is_label(#llvm_invoke{}) -> false; +is_label(#llvm_operation{}) -> false; +is_label(#llvm_extractvalue{}) -> false; +is_label(#llvm_insertvalue{}) -> false; +is_label(#llvm_alloca{}) -> false; +is_label(#llvm_load{}) -> false; +is_label(#llvm_store{}) -> false; +is_label(#llvm_getelementptr{}) -> false; +is_label(#llvm_conversion{}) -> false; +is_label(#llvm_sitofp{}) -> false; +is_label(#llvm_ptrtoint{}) -> false; +is_label(#llvm_inttoptr{}) -> false; +is_label(#llvm_icmp{}) -> false; +is_label(#llvm_fcmp{}) -> false; +is_label(#llvm_phi{}) -> false; +is_label(#llvm_select{}) -> false; +is_label(#llvm_call{}) -> false; +is_label(#llvm_fun_def{}) -> false; +is_label(#llvm_fun_decl{}) -> false; +is_label(#llvm_landingpad{}) -> false; +is_label(#llvm_comment{}) -> false; +is_label(#llvm_const_decl{}) -> false; +is_label(#llvm_asm{}) -> false; +is_label(#llvm_adj_stack{}) -> false; +is_label(#llvm_branch_meta{}) -> false. + +%% const_decl +mk_const_decl(Dst, Decl_type, Type, Value) -> + #llvm_const_decl{dst=Dst, decl_type=Decl_type, type=Type, value=Value}. +const_decl_dst(#llvm_const_decl{dst=Dst}) -> Dst. +const_decl_decl_type(#llvm_const_decl{decl_type=Decl_type}) -> Decl_type. +const_decl_type(#llvm_const_decl{type=Type}) -> Type. +const_decl_value(#llvm_const_decl{value=Value}) -> Value. + +%% asm +mk_asm(Instruction) -> #llvm_asm{instruction=Instruction}. +asm_instruction(#llvm_asm{instruction=Instruction}) -> Instruction. + +%% adj_stack +mk_adj_stack(Offset, Register, Type) -> + #llvm_adj_stack{offset=Offset, 'register'=Register, type=Type}. +adj_stack_offset(#llvm_adj_stack{offset=Offset}) -> Offset. +adj_stack_register(#llvm_adj_stack{'register'=Register}) -> Register. +adj_stack_type(#llvm_adj_stack{type=Type}) -> Type. + +%% branch meta-data +mk_branch_meta(Id, True_weight, False_weight) -> + #llvm_branch_meta{id=Id, true_weight=True_weight, false_weight=False_weight}. +branch_meta_id(#llvm_branch_meta{id=Id}) -> Id. +branch_meta_true_weight(#llvm_branch_meta{true_weight=True_weight}) -> + True_weight. +branch_meta_false_weight(#llvm_branch_meta{false_weight=False_weight}) -> + False_weight. + +%% types +mk_void() -> #llvm_void{}. + +mk_label_type() -> #llvm_label_type{}. + +mk_int(Width) -> #llvm_int{width=Width}. +int_width(#llvm_int{width=Width}) -> Width. + +mk_double() -> #llvm_double{}. + +mk_pointer(Type) -> #llvm_pointer{type=Type}. +pointer_type(#llvm_pointer{type=Type}) -> Type. + +mk_array(Size, Type) -> #llvm_array{'size'=Size, type=Type}. +array_size(#llvm_array{'size'=Size}) -> Size. +array_type(#llvm_array{type=Type}) -> Type. + +mk_vector(Size, Type) -> #llvm_vector{'size'=Size, type=Type}. +vector_size(#llvm_vector{'size'=Size}) -> Size. +vector_type(#llvm_vector{type=Type}) -> Type. + +mk_struct(Type_list) -> #llvm_struct{type_list=Type_list}. +struct_type_list(#llvm_struct{type_list=Type_list}) -> Type_list. + +mk_fun(Ret_type, Arg_type_list) -> + #llvm_fun{ret_type=Ret_type, arg_type_list=Arg_type_list}. +function_ret_type(#llvm_fun{ret_type=Ret_type}) -> Ret_type. +function_arg_type_list(#llvm_fun{arg_type_list=Arg_type_list}) -> + Arg_type_list. + +%%---------------------------------------------------------------------------- +%% Pretty-printer Functions +%%---------------------------------------------------------------------------- + +%% @doc Pretty-print a list of LLVM instructions to a Device. +pp_ins_list(_Dev, []) -> ok; +pp_ins_list(Dev, [I|Is]) -> + pp_ins(Dev, I), + pp_ins_list(Dev, Is). + +pp_ins(Dev, I) -> + case indent(I) of + true -> write(Dev, " "); + false -> ok + end, + case I of + #llvm_ret{} -> + write(Dev, "ret "), + case ret_ret_list(I) of + [] -> write(Dev, "void"); + List -> pp_args(Dev, List) + end, + write(Dev, "\n"); + #llvm_br{} -> + write(Dev, ["br label ", br_dst(I), "\n"]); + #llvm_switch{} -> + write(Dev, "switch "), + pp_type(Dev, switch_type(I)), + write(Dev, [" ", switch_value(I), ", label ", switch_default_label(I), + " \n [\n"]), + pp_switch_value_label_list(Dev, switch_type(I), + switch_value_label_list(I)), + write(Dev, " ]\n"); + #llvm_invoke{} -> + write(Dev, [invoke_dst(I), " = invoke ", invoke_cconv(I), " "]), + pp_options(Dev, invoke_ret_attrs(I)), + pp_type(Dev, invoke_type(I)), + write(Dev, [" ", invoke_fnptrval(I), "("]), + pp_args(Dev, invoke_arglist(I)), + write(Dev, ") "), + pp_options(Dev, invoke_fn_attrs(I)), + write(Dev, [" to label ", invoke_to_label(I)," unwind label ", + invoke_unwind_label(I), " \n"]); + #llvm_br_cond{} -> + write(Dev, ["br i1 ", br_cond_cond(I), ", label ", br_cond_true_label(I), + ", label ", br_cond_false_label(I)]), + case br_cond_meta(I) of + [] -> ok; + Metadata -> + write(Dev, [", !prof !", Metadata]) + end, + write(Dev, "\n"); + #llvm_indirectbr{} -> + write(Dev, "indirectbr "), + pp_type(Dev, indirectbr_type(I)), + write(Dev, [" ", indirectbr_address(I), ", [ "]), + pp_args(Dev, indirectbr_label_list(I)), + write(Dev, " ]\n"); + #llvm_operation{} -> + write(Dev, [operation_dst(I), " = ", atom_to_list(operation_op(I)), " "]), + case op_has_options(operation_op(I)) of + true -> pp_options(Dev, operation_options(I)); + false -> ok + end, + pp_type(Dev, operation_type(I)), + write(Dev, [" ", operation_src1(I), ", ", operation_src2(I), "\n"]); + #llvm_extractvalue{} -> + write(Dev, [extractvalue_dst(I), " = extractvalue "]), + pp_type(Dev, extractvalue_type(I)), + %% TODO Print idxs + write(Dev, [" ", extractvalue_val(I), ", ", extractvalue_idx(I), "\n"]); + #llvm_insertvalue{} -> + write(Dev, [insertvalue_dst(I), " = insertvalue "]), + pp_type(Dev, insertvalue_val_type(I)), + write(Dev, [" ", insertvalue_val(I), ", "]), + pp_type(Dev, insertvalue_elem_type(I)), + %%TODO Print idxs + write(Dev, [" ", insertvalue_elem(I), ", ", insertvalue_idx(I), "\n"]); + #llvm_alloca{} -> + write(Dev, [alloca_dst(I), " = alloca "]), + pp_type(Dev, alloca_type(I)), + case alloca_num(I) of + [] -> ok; + Num -> + write(Dev, ", "), + pp_type(Dev, alloca_type(I)), + write(Dev, [" ", Num, " "]) + end, + case alloca_align(I) of + [] -> ok; + Align -> write(Dev, [",align ", Align]) + end, + write(Dev, "\n"); + #llvm_load{} -> + write(Dev, [load_dst(I), " = "]), + write(Dev, "load "), + case load_volatile(I) of + true -> write(Dev, "volatile "); + false -> ok + end, + pp_type(Dev, load_p_type(I)), + write(Dev, [" ", load_pointer(I), " "]), + case load_alignment(I) of + [] -> ok; + Al -> write(Dev, [", align ", Al, " "]) + end, + case load_nontemporal(I) of + [] -> ok; + In -> write(Dev, [", !nontemporal !", In]) + end, + write(Dev, "\n"); + #llvm_store{} -> + write(Dev, "store "), + case store_volatile(I) of + true -> write(Dev, "volatile "); + false -> ok + end, + pp_type(Dev, store_type(I)), + write(Dev, [" ", store_value(I), ", "]), + pp_type(Dev, store_p_type(I)), + write(Dev, [" ", store_pointer(I), " "]), + case store_alignment(I) of + [] -> ok; + Al -> write(Dev, [", align ", Al, " "]) + end, + case store_nontemporal(I) of + [] -> ok; + In -> write(Dev, [", !nontemporal !", In]) + end, + write(Dev, "\n"); + #llvm_getelementptr{} -> + write(Dev, [getelementptr_dst(I), " = getelementptr "]), + case getelementptr_inbounds(I) of + true -> write(Dev, "inbounds "); + false -> ok + end, + pp_type(Dev, getelementptr_p_type(I)), + write(Dev, [" ", getelementptr_value(I)]), + pp_typed_idxs(Dev, getelementptr_typed_idxs(I)), + write(Dev, "\n"); + #llvm_conversion{} -> + write(Dev, [conversion_dst(I), " = ", atom_to_list(conversion_op(I)), " "]), + pp_type(Dev, conversion_src_type(I)), + write(Dev, [" ", conversion_src(I), " to "]), + pp_type(Dev, conversion_dst_type(I)), + write(Dev, "\n"); + #llvm_icmp{} -> + write(Dev, [icmp_dst(I), " = icmp ", atom_to_list(icmp_cond(I)), " "]), + pp_type(Dev, icmp_type(I)), + write(Dev, [" ", icmp_src1(I), ", ", icmp_src2(I), "\n"]); + #llvm_fcmp{} -> + write(Dev, [fcmp_dst(I), " = fcmp ", atom_to_list(fcmp_cond(I)), " "]), + pp_type(Dev, fcmp_type(I)), + write(Dev, [" ", fcmp_src1(I), ", ", fcmp_src2(I), "\n"]); + #llvm_phi{} -> + write(Dev, [phi_dst(I), " = phi "]), + pp_type(Dev, phi_type(I)), + pp_phi_value_labels(Dev, phi_value_label_list(I)), + write(Dev, "\n"); + #llvm_select{} -> + write(Dev, [select_dst(I), " = select i1 ", select_cond(I), ", "]), + pp_type(Dev, select_typ1(I)), + write(Dev, [" ", select_val1(I), ", "]), + pp_type(Dev, select_typ2(I)), + write(Dev, [" ", select_val2(I), "\n"]); + #llvm_call{} -> + case call_dst(I) of + [] -> ok; + Dst -> write(Dev, [Dst, " = "]) + end, + case call_is_tail(I) of + true -> write(Dev, "tail "); + false -> ok + end, + write(Dev, ["call ", call_cconv(I), " "]), + pp_options(Dev, call_ret_attrs(I)), + pp_type(Dev, call_type(I)), + write(Dev, [" ", call_fnptrval(I), "("]), + pp_args(Dev, call_arglist(I)), + write(Dev, ") "), + pp_options(Dev, call_fn_attrs(I)), + write(Dev, "\n"); + #llvm_fun_def{} -> + write(Dev, "define "), + pp_options(Dev, fun_def_linkage(I)), + pp_options(Dev, fun_def_visibility(I)), + case fun_def_cconv(I) of + [] -> ok; + Cc -> write(Dev, [Cc, " "]) + end, + pp_options(Dev, fun_def_ret_attrs(I)), + write(Dev, " "), + pp_type(Dev, fun_def_type(I)), + write(Dev, [" @", fun_def_name(I), "("]), + pp_args(Dev, fun_def_arglist(I)), + write(Dev, ") "), + pp_options(Dev, fun_def_fn_attrs(I)), + case fun_def_align(I) of + [] -> ok; + N -> write(Dev, ["align ", N]) + end, + write(Dev, "{\n"), + pp_ins_list(Dev, fun_def_body(I)), + write(Dev, "}\n"); + #llvm_fun_decl{} -> + write(Dev, "declare "), + pp_options(Dev, fun_decl_linkage(I)), + pp_options(Dev, fun_decl_visibility(I)), + case fun_decl_cconv(I) of + [] -> ok; + Cc -> write(Dev, [Cc, " "]) + end, + pp_options(Dev, fun_decl_ret_attrs(I)), + pp_type(Dev, fun_decl_type(I)), + write(Dev, [" ", fun_decl_name(I), "("]), + pp_type_list(Dev, fun_decl_arglist(I)), + write(Dev, ") "), + case fun_decl_align(I) of + [] -> ok; + N -> write(Dev, ["align ", N]) + end, + write(Dev, "\n"); + #llvm_comment{} -> + write(Dev, ["; ", atom_to_list(comment_text(I)), "\n"]); + #llvm_label{} -> + write(Dev, [label_label(I), ":\n"]); + #llvm_const_decl{} -> + write(Dev, [const_decl_dst(I), " = ", const_decl_decl_type(I), " "]), + pp_type(Dev, const_decl_type(I)), + write(Dev, [" ", const_decl_value(I), "\n"]); + #llvm_landingpad{} -> + write(Dev, "landingpad { i8*, i32 } personality i32 (i32, i64, i8*,i8*)* + @__gcc_personality_v0 cleanup\n"); + #llvm_asm{} -> + write(Dev, [asm_instruction(I), "\n"]); + #llvm_adj_stack{} -> + write(Dev, ["call void asm sideeffect \"sub $0, ", + adj_stack_register(I), "\", \"r\"("]), + pp_type(Dev, adj_stack_type(I)), + write(Dev, [" ", adj_stack_offset(I),")\n"]); + #llvm_branch_meta{} -> + write(Dev, ["!", branch_meta_id(I), " = metadata !{metadata !\"branch_weights\", + i32 ", branch_meta_true_weight(I), ", i32 ", + branch_meta_false_weight(I), "}\n"]); + Other -> + exit({?MODULE, pp_ins, {"Unknown LLVM instruction", Other}}) + end. + +%% @doc Pretty-print a list of types +pp_type_list(_Dev, []) -> ok; +pp_type_list(Dev, [T]) -> + pp_type(Dev, T); +pp_type_list(Dev, [T|Ts]) -> + pp_type(Dev, T), + write(Dev, ", "), + pp_type_list(Dev, Ts). + +pp_type(Dev, Type) -> + case Type of + #llvm_void{} -> + write(Dev, "void"); + #llvm_label_type{} -> + write(Dev, "label"); + %% Integer + #llvm_int{} -> + write(Dev, ["i", integer_to_list(int_width(Type))]); + %% Float + #llvm_float{} -> + write(Dev, "float"); + #llvm_double{} -> + write(Dev, "double"); + #llvm_fp80{} -> + write(Dev, "x86_fp80"); + #llvm_fp128{} -> + write(Dev, "fp128"); + #llvm_ppc_fp128{} -> + write(Dev, "ppc_fp128"); + %% Pointer + #llvm_pointer{} -> + pp_type(Dev, pointer_type(Type)), + write(Dev, "*"); + %% Function + #llvm_fun{} -> + pp_type(Dev, function_ret_type(Type)), + write(Dev, " ("), + pp_type_list(Dev, function_arg_type_list(Type)), + write(Dev, ")"); + %% Aggregate + #llvm_array{} -> + write(Dev, ["[", integer_to_list(array_size(Type)), " x "]), + pp_type(Dev, array_type(Type)), + write(Dev, "]"); + #llvm_struct{} -> + write(Dev, "{"), + pp_type_list(Dev, struct_type_list(Type)), + write(Dev, "}"); + #llvm_vector{} -> + write(Dev, ["{", integer_to_list(vector_size(Type)), " x "]), + pp_type(Dev, vector_type(Type)), + write(Dev, "}") + end. + +%% @doc Pretty-print a list of typed arguments +pp_args(_Dev, []) -> ok; +pp_args(Dev, [{Type, Arg} | []]) -> + pp_type(Dev, Type), + write(Dev, [" ", Arg]); +pp_args(Dev, [{Type, Arg} | Args]) -> + pp_type(Dev, Type), + write(Dev, [" ", Arg, ", "]), + pp_args(Dev, Args). + +%% @doc Pretty-print a list of options +pp_options(_Dev, []) -> ok; +pp_options(Dev, [O|Os]) -> + write(Dev, [atom_to_list(O), " "]), + pp_options(Dev, Os). + +%% @doc Pretty-print a list of phi value-labels +pp_phi_value_labels(_Dev, []) -> ok; +pp_phi_value_labels(Dev, [{Value, Label}|[]]) -> + write(Dev, ["[ ", Value, ", ", Label, " ]"]); +pp_phi_value_labels(Dev,[{Value, Label}|VL]) -> + write(Dev, ["[ ", Value, ", ", Label, " ], "]), + pp_phi_value_labels(Dev, VL). + +%% @doc Pretty-print a list of typed indexes +pp_typed_idxs(_Dev, []) -> ok; +pp_typed_idxs(Dev, [{Type, Id} | Tids]) -> + write(Dev, ", "), + pp_type(Dev, Type), + write(Dev, [" ", Id]), + pp_typed_idxs(Dev, Tids). + +%% @doc Pretty-print a switch label list +pp_switch_value_label_list(_Dev, _Type, []) -> ok; +pp_switch_value_label_list(Dev, Type, [{Value, Label} | VLs]) -> + write(Dev, " "), + pp_type(Dev, Type), + write(Dev, [" ", Value, ", label ", Label, "\n"]), + pp_switch_value_label_list(Dev, Type, VLs). + +%%---------------------------------------------------------------------------- +%% Auxiliary Functions +%%---------------------------------------------------------------------------- + +%% @doc Returns if an instruction needs to be intended +indent(I) -> + case I of + #llvm_label{} -> false; + #llvm_fun_def{} -> false; + #llvm_fun_decl{} -> false; + #llvm_const_decl{} -> false; + #llvm_branch_meta{} -> false; + _ -> true + end. + +op_has_options(Op) -> + case Op of + 'and' -> false; + 'or' -> false; + 'xor' -> false; + _ -> true + end. + +%% @doc Abstracts actual writing to file operations +write(Dev, Msg) -> + ok = file:write(Dev, Msg). diff --git a/lib/hipe/llvm/hipe_llvm_arch.hrl b/lib/hipe/llvm/hipe_llvm_arch.hrl new file mode 100644 index 0000000000..689a5a52ea --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm_arch.hrl @@ -0,0 +1,11 @@ +-ifdef(BIT32). +-define(NR_PINNED_REGS, 2). +-define(NR_ARG_REGS, 3). +-define(ARCH_REGISTERS, hipe_x86_registers). +-define(FLOAT_OFFSET, 2). +-else. +-define(NR_PINNED_REGS, 2). +-define(NR_ARG_REGS, 4). +-define(ARCH_REGISTERS, hipe_amd64_registers). +-define(FLOAT_OFFSET, 6). +-endif. diff --git a/lib/hipe/llvm/hipe_llvm_liveness.erl b/lib/hipe/llvm/hipe_llvm_liveness.erl new file mode 100644 index 0000000000..d1c90ed4c9 --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm_liveness.erl @@ -0,0 +1,112 @@ +-module(hipe_llvm_liveness). + +-export([analyze/1]). + +%% @doc Find gc roots and explicitly mark when they go out of scope, based +%% on the liveness analyzis performed by the hipe_rtl_liveness:analyze/1. +analyze(RtlCfg) -> + Liveness = hipe_rtl_liveness:analyze(RtlCfg), + Roots = find_roots(RtlCfg, Liveness), + %% erlang:display(Roots), + NewRtlCfg = mark_dead_roots(RtlCfg, Liveness, Roots), + {NewRtlCfg, Roots}. + +%% @doc Determine which are the GC Roots.Possible roots are all +%% RTL variables (rtl_var). However, since safe points are function calls, we +%% consider as possible GC roots only RTL variables that are live around +%% function calls. +find_roots(Cfg, Liveness) -> + Labels = hipe_rtl_cfg:postorder(Cfg), + Roots = find_roots_bb(Labels, Cfg, Liveness, []), + lists:usort(lists:flatten(Roots)). + +find_roots_bb([], _Cfg, _Liveness, RootAcc) -> + RootAcc; +find_roots_bb([L|Ls], Cfg, Liveness, RootAcc) -> + Block = hipe_rtl_cfg:bb(Cfg, L), + BlockCode = hipe_bb:code(Block), + LiveIn = ordsets:from_list(strip(hipe_rtl_liveness:livein(Liveness, L))), + LiveOut = ordsets:from_list(strip(hipe_rtl_liveness:liveout(Liveness, L))), + Roots = do_find_roots_bb(BlockCode, L, LiveOut, LiveIn, []), + find_roots_bb(Ls, Cfg, Liveness, Roots++RootAcc). + +%% For each call inside a BB the GC roots are those RTL variables that +%% are live before and after the call. +%% --> Live Before Call: These are the RTL variables that belong to the +%% LiveIn list or are initialized inside the BB before the call +%% --> Live After Call: These are the RTL variables that belong to the +%% LiveOut list or are used after the call inside the BB (they die +%% inside the BB and so do not belong to the LiveOut list) +do_find_roots_bb([], _Label, _LiveOut, _LiveBefore, RootAcc) -> + RootAcc; +do_find_roots_bb([I|Is], L, LiveOut, LiveBefore, RootAcc) -> + case hipe_rtl:is_call(I) of + true -> + %% Used inside the BB after the call + UsedAfterCall_ = strip(lists:flatten([hipe_rtl:uses(V) || V <- Is])), + UsedAfterCall = ordsets:from_list(UsedAfterCall_), + LiveAfter = ordsets:union(UsedAfterCall, LiveOut), + %% The Actual Roots + Roots = ordsets:intersection(LiveBefore, LiveAfter), + %% The result of the instruction + Defines = ordsets:from_list(strip(hipe_rtl:defines(I))), + LiveBefore1 = ordsets:union(LiveBefore, Defines), + do_find_roots_bb(Is, L, LiveOut, LiveBefore1, [Roots|RootAcc]); + false -> + %% The result of the instruction + Defines = ordsets:from_list(strip(hipe_rtl:defines(I))), + LiveBefore1 = ordsets:union(LiveBefore, Defines), + do_find_roots_bb(Is, L, LiveOut, LiveBefore1, RootAcc) + end. + +%% @doc This function is responsible for marking when GC Roots, which can be +%% only RTL variables go out of scope (dead). This pass is needed for the LLVM +%% back end because the LLVM framework forces us to explicit mark when gc roots +%% are no longer live. +mark_dead_roots(CFG, Liveness, Roots) -> + Labels = hipe_rtl_cfg:postorder(CFG), + mark_dead_bb(Labels, CFG, Liveness, Roots). + +mark_dead_bb([], Cfg, _Liveness, _Roots) -> + Cfg; +mark_dead_bb([L|Ls], Cfg, Liveness, Roots) -> + Block = hipe_rtl_cfg:bb(Cfg, L), + BlockCode = hipe_bb:code(Block), + LiveOut = ordsets:from_list(strip(hipe_rtl_liveness:liveout(Liveness, L))), + NewBlockCode = do_mark_dead_bb(BlockCode, LiveOut, Roots, []), + %% Update the CFG + NewBB = hipe_bb:code_update(Block, NewBlockCode), + NewCFG = hipe_rtl_cfg:bb_add(Cfg, L, NewBB), + mark_dead_bb(Ls, NewCFG, Liveness, Roots). + +do_mark_dead_bb([], _LiveOut, _Roots, NewBlockCode) -> + lists:reverse(NewBlockCode); +do_mark_dead_bb([I|Is], LiveOut ,Roots, NewBlockCode) -> + Uses = ordsets:from_list(strip(hipe_rtl:uses(I))), + %% GC roots that are used in this instruction + RootsUsed = ordsets:intersection(Roots, Uses), + UsedAfter_ = strip(lists:flatten([hipe_rtl:uses(V) || V <- Is])), + UsedAfter = ordsets:from_list(UsedAfter_), + %% GC roots that are live after this instruction + LiveAfter = ordsets:union(LiveOut, UsedAfter), + %% GC roots that their last use is in this instruction + DeadRoots = ordsets:subtract(RootsUsed, LiveAfter), + %% Recreate the RTL variable from the corresponding Index + OldVars = [hipe_rtl:mk_var(V1) || V1 <- DeadRoots], + %% Mark the RTL variable as DEAD (last use) + NewVars = [kill_var(V2) || V2 <- OldVars], + %% Create a list with the substitution of the old vars with the new + %% ones which are marked with the dead keyword + Subtitution = lists:zip(OldVars, NewVars), + NewI = case Subtitution of + [] -> I; + _ -> hipe_rtl:subst_uses_llvm(Subtitution, I) + end, + do_mark_dead_bb(Is, LiveOut, Roots, [NewI|NewBlockCode]). + +%% Update the liveness of a var,in order to mark that this is the last use. +kill_var(Var) -> hipe_rtl:var_liveness_update(Var, dead). + +%% We are only interested for rtl_vars, since only rtl_vars are possible gc +%% roots. +strip(L) -> [Y || {rtl_var, Y, _} <- L]. diff --git a/lib/hipe/llvm/hipe_llvm_main.erl b/lib/hipe/llvm/hipe_llvm_main.erl new file mode 100644 index 0000000000..e911fb89c9 --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm_main.erl @@ -0,0 +1,514 @@ +%% -*- erlang-indent-level: 2 -*- +-module(hipe_llvm_main). + +-export([rtl_to_native/4]). + +-include("../../kernel/src/hipe_ext_format.hrl"). +-include("hipe_llvm_arch.hrl"). +-include("elf_format.hrl"). + +%% @doc Translation of RTL to a loadable object. This function takes the RTL +%% code and calls hipe_rtl_to_llvm:translate/2 to translate the RTL code to +%% LLVM code. After this, LLVM asm is printed to a file and the LLVM tool +%% chain is invoked in order to produce an object file. +rtl_to_native(MFA, RTL, Roots, Options) -> + %% Compile to LLVM and get Instruction List (along with infos) + {LLVMCode, RelocsDict, ConstTab} = + hipe_rtl_to_llvm:translate(RTL, Roots), + %% Fix function name to an acceptable LLVM identifier (needed for closures) + {_Module, Fun, Arity} = hipe_rtl_to_llvm:fix_mfa_name(MFA), + %% Write LLVM Assembly to intermediate file (on disk) + {ok, Dir, ObjectFile} = + compile_with_llvm(Fun, Arity, LLVMCode, Options, false), + %% + %% Extract information from object file + %% + ObjBin = open_object_file(ObjectFile), + %% Read and set the ELF class + elf_format:set_architecture_flag(ObjBin), + %% Get labels info (for switches and jump tables) + Labels = elf_format:get_rodata_relocs(ObjBin), + {Switches, Closures} = get_tables(ObjBin), + %% Associate Labels with Switches and Closures with stack args + {SwitchInfos, ExposedClosures} = + correlate_labels(Switches ++ Closures, Labels), + %% SwitchInfos: [{"table_50", [Labels]}] + %% ExposedClosures: [{"table_closures", [Labels]}] + + %% Labelmap contains the offsets of the labels in the code that are + %% used for switch's jump tables + LabelMap = create_labelmap(MFA, SwitchInfos, RelocsDict), + %% Get relocation info + TextRelocs = elf_format:get_text_relocs(ObjBin), + %% AccRefs contains the offsets of all references to relocatable symbols in + %% the code: + AccRefs = fix_relocations(TextRelocs, RelocsDict, MFA), + %% Get stack descriptors + SDescs = get_sdescs(ObjBin), + %% FixedSDescs are the stack descriptors after correcting calls that have + %% arguments in the stack + FixedSDescs = + fix_stack_descriptors(RelocsDict, AccRefs, SDescs, ExposedClosures), + Refs = AccRefs ++ FixedSDescs, + %% Get binary code from object file + BinCode = elf_format:extract_text(ObjBin), + %% Remove temp files (if needed) + ok = remove_temp_folder(Dir, Options), + %% Return the code together with information that will be used in the + %% hipe_llvm_merge module to produce the final binary that will be loaded + %% by the hipe unified loader. + {MFA, BinCode, byte_size(BinCode), ConstTab, Refs, LabelMap}. + +%%------------------------------------------------------------------------------ +%% LLVM tool chain +%%------------------------------------------------------------------------------ + +%% @doc Compile function FunName/Arity to LLVM. Return Dir (in order to remove +%% it if we do not want to store temporary files) and ObjectFile name that +%% is created by the LLVM tools. +compile_with_llvm(FunName, Arity, LLVMCode, Options, UseBuffer) -> + Filename = atom_to_list(FunName) ++ "_" ++ integer_to_list(Arity), + %% Save temp files in a unique folder + Dir = unique_folder(FunName, Arity, Options), + ok = file:make_dir(Dir), + %% Print LLVM assembly to file + OpenOpts = [append, raw] ++ + case UseBuffer of + %% true -> [delayed_write]; % Use delayed_write! + false -> [] + end, + {ok, File_llvm} = file:open(Dir ++ Filename ++ ".ll", OpenOpts), + hipe_llvm:pp_ins_list(File_llvm, LLVMCode), + %% delayed_write can cause file:close not to do a close, hence the two calls + ok = file:close(File_llvm), + __ = file:close(File_llvm), + %% Invoke LLVM compiler tools to produce an object file + llvm_opt(Dir, Filename, Options), + llvm_llc(Dir, Filename, Options), + compile(Dir, Filename, "gcc"), %%FIXME: use llc -filetype=obj and skip this! + {ok, Dir, Dir ++ Filename ++ ".o"}. + +%% @doc Invoke opt tool to optimize the bitcode (_name.ll -> _name.bc). +llvm_opt(Dir, Filename, Options) -> + Source = Dir ++ Filename ++ ".ll", + Dest = Dir ++ Filename ++ ".bc", + OptLevel = trans_optlev_flag(opt, Options), + OptFlags = [OptLevel, "-mem2reg", "-strip"], + Command = "opt " ++ fix_opts(OptFlags) ++ " " ++ Source ++ " -o " ++ Dest, + %% io:format("OPT: ~s~n", [Command]), + case os:cmd(Command) of + "" -> ok; + Error -> exit({?MODULE, opt, Error}) + end. + +%% @doc Invoke llc tool to compile the bitcode to object file +%% (_name.bc -> _name.o). +llvm_llc(Dir, Filename, Options) -> + Source = Dir ++ Filename ++ ".bc", + OptLevel = trans_optlev_flag(llc, Options), + Align = find_stack_alignment(), + LlcFlags = [OptLevel, "-code-model=medium", "-stack-alignment=" ++ Align + , "-tailcallopt", "-filetype=asm"], %%FIXME + Command = "llc " ++ fix_opts(LlcFlags) ++ " " ++ Source, + %% io:format("LLC: ~s~n", [Command]), + case os:cmd(Command) of + "" -> ok; + Error -> exit({?MODULE, llc, Error}) + end. + +%% @doc Invoke the compiler tool ("gcc", "llvmc", etc.) to generate an object +%% file from native assembly. +compile(Dir, Fun_Name, Compiler) -> + Source = Dir ++ Fun_Name ++ ".s", + Dest = Dir ++ Fun_Name ++ ".o", + Command = Compiler ++ " -c " ++ Source ++ " -o " ++ Dest, + %% io:format("~s: ~s~n", [Compiler, Command]), + case os:cmd(Command) of + "" -> ok; + Error -> exit({?MODULE, cc, Error}) + end. + +find_stack_alignment() -> + case get(hipe_target_arch) of + x86 -> "4"; + amd64 -> "8"; + _ -> exit({?MODULE, find_stack_alignment, "Unimplemented architecture"}) + end. + +%% @doc Join options. +fix_opts(Opts) -> + string:join(Opts, " "). + +%% @doc Translate optimization-level flag (default is "O2"). +trans_optlev_flag(Tool, Options) -> + Flag = case Tool of + opt -> llvm_opt; + llc -> llvm_llc + end, + case proplists:get_value(Flag, Options) of + o0 -> ""; % "-O0" does not exist in opt tool + o1 -> "-O1"; + o2 -> "-O2"; + o3 -> "-O3"; + undefined -> "-O2" + end. + +%%------------------------------------------------------------------------------ +%% Functions to manage Relocations +%%------------------------------------------------------------------------------ + +%% @doc Get switch table and closure table. +get_tables(Elf) -> + %% Search Symbol Table for an entry with name prefixed with "table_": + Triples = elf_format:get_tab_entries(Elf), + Switches = [T || T={"table_" ++ _, _, _} <- Triples], + Closures = [T || T={"table_closures" ++ _, _, _} <- Switches], + {Switches, Closures}. + +%% @doc This function associates symbols who point to some table of labels with +%% the corresponding offsets of the labels in the code. These tables can +%% either be jump tables for switches or a table which contains the labels +%% of blocks that contain closure calls with more than ?NR_ARG_REGS. +correlate_labels([], _L) -> {[], []}; +correlate_labels(Tables, Labels) -> + %% Sort "Tables" based on "ValueOffsets" + OffsetSortedTb = lists:ukeysort(2, Tables), + %% Unzip offset-sorted list of "Switches" + {Names, _Offsets, TablesSizeList} = lists:unzip3(OffsetSortedTb), + %% Associate switch names with labels + L = split_list(Labels, TablesSizeList), + %% Zip back! (to [{SwitchName, Values}]) + NamesValues = lists:zip(Names, L), + case lists:keytake("table_closures", 1, NamesValues) of + false -> %% No closures in the code, no closure table + {NamesValues, []}; + {value, ClosureTableNV, SwitchesNV} -> + {SwitchesNV, ClosureTableNV} + end. + +%% @doc Create a gb_tree which contains information about the labels that used +%% for switch's jump tables. The keys of the gb_tree are of the form +%% {MFA, Label} and the values are the actual Offsets. +create_labelmap(MFA, SwitchInfos, RelocsDict) -> + create_labelmap(MFA, SwitchInfos, RelocsDict, gb_trees:empty()). + +create_labelmap(_, [], _, LabelMap) -> LabelMap; +create_labelmap(MFA, [{Name, Offsets} | Rest], RelocsDict, LabelMap) -> + case dict:fetch(Name, RelocsDict) of + {switch, {_TableType, LabelList, _NrLabels, _SortOrder}, _JTabLab} -> + KVDict = lists:ukeysort(1, lists:zip(LabelList, Offsets)), + NewLabelMap = insert_to_labelmap(KVDict, LabelMap), + create_labelmap(MFA, Rest, RelocsDict, NewLabelMap); + _ -> + exit({?MODULE, create_labelmap, "Not a jump table!"}) + end. + +%% @doc Insert a list of [{Key,Value}] to a LabelMap (gb_tree). +insert_to_labelmap([], LabelMap) -> LabelMap; +insert_to_labelmap([{Key, Value}|Rest], LabelMap) -> + case gb_trees:lookup(Key, LabelMap) of + none -> + insert_to_labelmap(Rest, gb_trees:insert(Key, Value, LabelMap)); + {value, Value} -> %% Exists with the *exact* same Value. + insert_to_labelmap(Rest, LabelMap) + end. + +%% @doc Correlate object file relocation symbols with info from translation to +%% llvm code. +fix_relocations(Relocs, RelocsDict, MFA) -> + fix_relocs(Relocs, RelocsDict, MFA, []). + +fix_relocs([], _, _, RelocAcc) -> RelocAcc; +fix_relocs([{Name, Offset}|Rs], RelocsDict, {ModName,_,_}=MFA, RelocAcc) -> + case dict:fetch(Name, RelocsDict) of + {atom, AtomName} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?LOAD_ATOM, Offset, AtomName}|RelocAcc]); + {constant, Label} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?LOAD_ADDRESS, Offset, {constant, Label}}|RelocAcc]); + {switch, _, JTabLab} -> %% Treat switch exactly as constant + fix_relocs(Rs, RelocsDict, MFA, + [{?LOAD_ADDRESS, Offset, {constant, JTabLab}}|RelocAcc]); + {closure, _}=Closure -> + fix_relocs(Rs, RelocsDict, MFA, + [{?LOAD_ADDRESS, Offset, Closure}|RelocAcc]); + {call, {bif, BifName, _}} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?CALL_LOCAL, Offset, BifName}|RelocAcc]); + %% MFA calls to functions in the same module are of type 3, while all + %% other MFA calls are of type 2. + {call, {ModName,_F,_A}=CallMFA} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?CALL_LOCAL, Offset, CallMFA}|RelocAcc]); + {call, CallMFA} -> + fix_relocs(Rs, RelocsDict, MFA, + [{?CALL_REMOTE, Offset, CallMFA}|RelocAcc]); + Other -> + exit({?MODULE, fix_relocs, + {"Relocation not in relocation dictionary", Other}}) + end. + +%%------------------------------------------------------------------------------ +%% Functions to manage Stack Descriptors +%%------------------------------------------------------------------------------ + +%% @doc This function takes an ELF Object File binary and returns a proper sdesc +%% list for Erlang/OTP System's loader. The return value should be of the +%% form: +%% { +%% 4, Safepoint Address, +%% {ExnLabel OR [], FrameSize, StackArity, {Liveroot stack frame indexes}}, +%% } +get_sdescs(Elf) -> + case elf_format:extract_note(Elf, ?NOTE_ERLGC_NAME) of + <<>> -> % Object file has no ".note.gc" section! + []; + NoteGC_bin -> + %% Get safe point addresses (stored in ".rela.note.gc" section): + RelaNoteGC = elf_format:extract_rela(Elf, ?NOTE(?NOTE_ERLGC_NAME)), + SPCount = length(RelaNoteGC), + T = SPCount * ?SP_ADDR_SIZE, + %% Pattern match fields of ".note.gc": + <<SPCount:(?bits(?SP_COUNT_SIZE))/integer-little, % Sanity check! + SPAddrs:T/binary, % NOTE: In 64bit they are relocs! + StkFrameSize:(?bits(?SP_STKFRAME_SIZE))/integer-little, + StkArity:(?bits(?SP_STKARITY_SIZE))/integer-little, + _LiveRootCount:(?bits(?SP_LIVEROOTCNT_SIZE))/integer-little, % Skip + Roots/binary>> = NoteGC_bin, + LiveRoots = get_liveroots(Roots, []), + %% Extract information about the safe point addresses: + SPOffs = + case elf_format:is64bit() of + true -> %% Find offsets in ".rela.note.gc": + elf_format:get_rela_addends(RelaNoteGC); + false -> %% Find offsets in SPAddrs (in ".note.gc"): + get_spoffs(SPAddrs, []) + end, + %% Extract Exception Handler labels: + ExnHandlers = elf_format:get_exn_handlers(Elf), + %% Combine ExnHandlers and Safe point addresses (return addresses): + ExnAndSPOffs = combine_ras_and_exns(ExnHandlers, SPOffs, []), + create_sdesc_list(ExnAndSPOffs, StkFrameSize, StkArity, LiveRoots, []) + end. + +%% @doc Extracts a bunch of integers (live roots) from a binary. Returns a tuple +%% as need for stack descriptors. +get_liveroots(<<>>, Acc) -> + list_to_tuple(Acc); +get_liveroots(<<Root:?bits(?LR_STKINDEX_SIZE)/integer-little, + MoreRoots/binary>>, Acc) -> + get_liveroots(MoreRoots, [Root | Acc]). + +%% @doc Extracts a bunch of integers (safepoint offsets) from a binary. Returns +%% a tuple as need for stack descriptors. +get_spoffs(<<>>, Acc) -> + lists:reverse(Acc); +get_spoffs(<<SPOff:?bits(?SP_ADDR_SIZE)/integer-little, More/binary>>, Acc) -> + get_spoffs(More, [SPOff | Acc]). + +combine_ras_and_exns(_, [], Acc) -> + lists:reverse(Acc); +combine_ras_and_exns(ExnHandlers, [RA | MoreRAs], Acc) -> + %% FIXME: do something better than O(n^2) by taking advantage of the property + %% ||ExnHandlers|| <= ||RAs|| + Handler = find_exn_handler(RA, ExnHandlers), + combine_ras_and_exns(ExnHandlers, MoreRAs, [{Handler, RA} | Acc]). + +find_exn_handler(_, []) -> + []; +find_exn_handler(RA, [{Start, End, Handler} | MoreExnHandlers]) -> + case (RA >= Start andalso RA =< End) of + true -> + Handler; + false -> + find_exn_handler(RA, MoreExnHandlers) + end. + +create_sdesc_list([], _, _, _, Acc) -> + lists:reverse(Acc); +create_sdesc_list([{ExnLbl, SPOff} | MoreExnAndSPOffs], + StkFrameSize, StkArity, LiveRoots, Acc) -> + Hdlr = case ExnLbl of + 0 -> []; + N -> N + end, + create_sdesc_list(MoreExnAndSPOffs, StkFrameSize, StkArity, LiveRoots, + [{?SDESC, SPOff, {Hdlr, StkFrameSize, StkArity, LiveRoots}} + | Acc]). + +%% @doc This function is responsible for correcting the stack descriptors of +%% the calls that are found in the code and have more than NR_ARG_REGS +%% (thus, some of their arguments are passed to the stack). Because of the +%% Reserved Call Frame feature that the LLVM uses, the stack descriptors +%% are not correct since at the point of call the frame size is reduced +%% proportionally to the number of arguments that are passed on the stack. +%% Also the offsets of the roots need to be re-adjusted. +fix_stack_descriptors(_, _, [], _) -> + []; +fix_stack_descriptors(RelocsDict, Relocs, SDescs, ExposedClosures) -> + %% NamedCalls are MFA and BIF calls that need fix + NamedCalls = calls_with_stack_args(RelocsDict), + NamedCallsOffs = calls_offsets_arity(Relocs, NamedCalls), + ExposedClosures1 = + case dict:is_key("table_closures", RelocsDict) of + true -> %% A Table with closures exists + {table_closures, ArityList} = dict:fetch("table_closures", RelocsDict), + case ExposedClosures of + {_, Offsets} -> + lists:zip(Offsets, ArityList); + _ -> + exit({?MODULE, fix_stack_descriptors, + {"Wrong exposed closures", ExposedClosures}}) + end; + false -> + [] + end, + ClosuresOffs = closures_offsets_arity(ExposedClosures1, SDescs), + fix_sdescs(NamedCallsOffs ++ ClosuresOffs, SDescs). + +%% @doc This function takes as argument the relocation dictionary as produced by +%% the translation of RTL code to LLVM and finds the names of the calls +%% (MFA and BIF calls) that have more than NR_ARG_REGS. +calls_with_stack_args(Dict) -> + calls_with_stack_args(dict:to_list(Dict), []). + +calls_with_stack_args([], Calls) -> Calls; +calls_with_stack_args([ {_Name, {call, {M, F, A}}} | Rest], Calls) + when A > ?NR_ARG_REGS -> + Call = + case M of + bif -> {F,A}; + _ -> {M,F,A} + end, + calls_with_stack_args(Rest, [Call|Calls]); +calls_with_stack_args([_|Rest], Calls) -> + calls_with_stack_args(Rest, Calls). + +%% @doc This function extracts the stack arity and the offset in the code of +%% the named calls (MFAs, BIFs) that have stack arguments. +calls_offsets_arity(AccRefs, CallsWithStackArgs) -> + calls_offsets_arity(AccRefs, CallsWithStackArgs, []). + +calls_offsets_arity([], _, Acc) -> Acc; +calls_offsets_arity([{Type, Offset, Term} | Rest], CallsWithStackArgs, Acc) + when Type =:= ?CALL_REMOTE orelse Type =:= ?CALL_LOCAL -> + case lists:member(Term, CallsWithStackArgs) of + true -> + Arity = + case Term of + {_M, _F, A} -> A; + {_F, A} -> A + end, + calls_offsets_arity(Rest, CallsWithStackArgs, + [{Offset + 4, Arity - ?NR_ARG_REGS} | Acc]); + false -> + calls_offsets_arity(Rest, CallsWithStackArgs, Acc) + end; +calls_offsets_arity([_|Rest], CallsWithStackArgs, Acc) -> + calls_offsets_arity(Rest, CallsWithStackArgs, Acc). + +%% @doc This function extracts the stack arity and the offsets of closures that +%% have stack arity. The Closures argument represents the +%% hipe_bifs:llvm_exposure_closure/0 calls in the code. The actual closure +%% is the next call in the code, so the offset of the next call must be +%% calculated from the stack descriptors. +closures_offsets_arity([], _) -> + []; +closures_offsets_arity(ExposedClosures, SDescs) -> + Offsets = [Offset || {_, Offset, _} <- SDescs], + %% Offsets and closures must be sorted in order for find_offsets/3 to work + SortedOffsets = lists:sort(Offsets), + SortedExposedClosures = lists:keysort(1, ExposedClosures), + find_offsets(SortedExposedClosures, SortedOffsets, []). + +find_offsets([], _, Acc) -> Acc; +find_offsets([{Off,Arity}|Rest], Offsets, Acc) -> + [I | RestOffsets] = lists:dropwhile(fun (Y) -> Y<Off end, Offsets), + find_offsets(Rest, RestOffsets, [{I, Arity}|Acc]). + +%% The functions below correct the arity of calls, that are identified +%% by offset, in the stack descriptors. +fix_sdescs([], SDescs) -> SDescs; +fix_sdescs([{Offset, Arity} | Rest], SDescs) -> + case lists:keyfind(Offset, 2, SDescs) of + false -> + fix_sdescs(Rest, SDescs); + {?SDESC, Offset, SDesc} -> + {ExnHandler, FrameSize, StkArity, Roots} = SDesc, + DecRoot = fun(X) -> X-Arity end, + NewRootsList = lists:map(DecRoot, tuple_to_list(Roots)), + NewSDesc = + case length(NewRootsList) > 0 andalso hd(NewRootsList) >= 0 of + true -> + {?SDESC, Offset, {ExnHandler, FrameSize-Arity, StkArity, + list_to_tuple(NewRootsList)}}; + false -> + {?SDESC, Offset, {ExnHandler, FrameSize, StkArity, Roots}} + end, + RestSDescs = lists:keydelete(Offset, 2, SDescs), + fix_sdescs(Rest, [NewSDesc | RestSDescs]) + end. + + +%%------------------------------------------------------------------------------ +%% Miscellaneous functions +%%------------------------------------------------------------------------------ + +%% @doc A function that opens a file as binary. The function takes as argument +%% the name of the file and returns an Erlang binary. +-spec open_object_file(string()) -> binary(). +open_object_file(ObjFile) -> + case file:read_file(ObjFile) of + {ok, Binary} -> + Binary; + {error, Reason} -> + exit({?MODULE, open_file, Reason}) + end. + +remove_temp_folder(Dir, Options) -> + case proplists:get_bool(llvm_save_temps, Options) of + true -> ok; + false -> spawn(fun () -> "" = os:cmd("rm -rf " ++ Dir) end), ok + end. + +unique_id(FunName, Arity) -> + integer_to_list(erlang:phash2({FunName, Arity, now()})). + +unique_folder(FunName, Arity, Options) -> + DirName = "llvm_" ++ unique_id(FunName, Arity) ++ "/", + Dir = + case proplists:get_bool(llvm_save_temps, Options) of + true -> %% Store folder in current directory + DirName; + false -> %% Temporarily store folder in tempfs (/dev/shm/) + "/dev/shm/" ++ DirName + end, + %% Make sure it does not exist + case dir_exists(Dir) of + true -> %% Dir already exists! Generate again. + unique_folder(FunName, Arity, Options); + false -> + Dir + end. + +%% @doc Function that checks that a given Filename is an existing Directory +%% Name (from http://rosettacode.org/wiki/Ensure_that_a_file_exists#Erlang) +dir_exists(Filename) -> + {Flag, Info} = file:read_file_info(Filename), + (Flag =:= ok) andalso (element(3, Info) =:= directory). + +%% @doc Function that takes as arguments a list of integers and a list with +%% numbers indicating how many items should each tuple have and splits +%% the original list to a list of lists of integers (with the specified +%% number of elements), i.e. [ [...], [...] ]. +-spec split_list([integer()], [integer()]) -> [ [integer()] ]. +split_list(List, ElemsPerTuple) -> + split_list(List, ElemsPerTuple, []). + +-spec split_list([integer()], [integer()], [ [integer()] ]) -> [ [integer()] ]. +split_list([], [], Acc) -> + lists:reverse(Acc); +split_list(List, [NumOfElems | MoreNums], Acc) -> + {L1, L2} = lists:split(NumOfElems, List), + split_list(L2, MoreNums, [ L1 | Acc]). diff --git a/lib/hipe/llvm/hipe_llvm_merge.erl b/lib/hipe/llvm/hipe_llvm_merge.erl new file mode 100644 index 0000000000..3ababfc21a --- /dev/null +++ b/lib/hipe/llvm/hipe_llvm_merge.erl @@ -0,0 +1,114 @@ +%%% -*- erlang-indent-level: 2 -*- +-module(hipe_llvm_merge). + +-export([finalize/3]). + +-include("hipe_llvm_arch.hrl"). +-include("../../kernel/src/hipe_ext_format.hrl"). +-include("../rtl/hipe_literals.hrl"). +-include("../main/hipe.hrl"). + +finalize(CompiledCode, Closures, Exports) -> + CompiledCode1 = [CodePack || {_, CodePack} <- CompiledCode], + Code = [{MFA, [], ConstTab} + || {MFA, _, _ , ConstTab, _, _} <- CompiledCode1], + {ConstAlign, ConstSize, ConstMap, RefsFromConsts} = + hipe_pack_constants:pack_constants(Code, ?ARCH_REGISTERS:alignment()), + %% Compute total code size separately as a sanity check for alignment + CodeSize = compute_code_size(CompiledCode1, 0), + %% io:format("Code Size (pre-computed): ~w~n", [CodeSize]), + {CodeBinary, ExportMap} = merge_mfas(CompiledCode1, 0, <<>>, []), + %% io:format("Code Size (post-computed): ~w~n", [byte_size(CodeBinary)]), + ?VERBOSE_ASSERT(CodeSize =:= byte_size(CodeBinary)), + AccRefs = merge_refs(CompiledCode1, ConstMap, 0, []), + %% Bring CompiledCode to a combine_label_maps-acceptable form. + LabelMap = combine_label_maps(CompiledCode1, 0, gb_trees:empty()), + SC = hipe_pack_constants:slim_constmap(ConstMap), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap, Closures, Exports), + SlimRefs = hipe_pack_constants:slim_refs(AccRefs), + term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, + ConstAlign, ConstSize, + SC, % ConstMap + DataRelocs, % LabelMap + SSE, % ExportMap + CodeSize, CodeBinary, SlimRefs, + 0,[] % ColdCodeSize, SlimColdRefs + ]). + +%% Copied from hipe_x86_assemble.erl +nr_pad_bytes(Address) -> + (4 - (Address rem 4)) rem 4. % XXX: 16 or 32 instead? + +align_entry(Address) -> + Address + nr_pad_bytes(Address). + +compute_code_size([{_MFA, _BinaryCode, CodeSize, _, _, _}|Code], Size) -> + compute_code_size(Code, align_entry(Size+CodeSize)); +compute_code_size([], Size) -> Size. + +combine_label_maps([{MFA, _, CodeSize, _, _, LabelMap}|Code], Address, CLM) -> + NewCLM = merge_label_map(gb_trees:to_list(LabelMap), MFA, Address, CLM), + combine_label_maps(Code, align_entry(Address+CodeSize), NewCLM); +combine_label_maps([], _Address, CLM) -> CLM. + +merge_label_map([{Label,Offset}|Rest], MFA, Address, CLM) -> + NewCLM = gb_trees:insert({MFA,Label}, Address+Offset, CLM), + merge_label_map(Rest, MFA, Address, NewCLM); +merge_label_map([], _MFA, _Address, CLM) -> CLM. + +%% @doc Merge the MFAs' binary code to one continuous binary and compute the +%% size of this binary. At the same time create an exportmap in a form +%% of {Address, M, F, A}. +%% XXX: Is alignment correct/optimal for X86/AMD64? +merge_mfas([{{M,F,A}, CodeBinary, CodeSize, _, _, _}|Code], + Address, AccCode, AccExportMap) -> + ?VERBOSE_ASSERT(CodeSize =:= byte_size(CodeBinary)), + {Address1, Code1} = + case nr_pad_bytes(Address + CodeSize) of + 0 -> %% Retains alignment: + {Address + CodeSize, CodeBinary}; + NrPadBytes -> %% Needs padding! + Padding = list_to_binary(lists:duplicate(NrPadBytes, 0)), + {Address + CodeSize + NrPadBytes, % =:= align_entry(Address+CodeSize) + <<CodeBinary/binary, Padding/binary>>} + end, + ?VERBOSE_ASSERT(Address1 =:= + align_entry(Address + CodeSize)), %XXX: Should address be aligned? + AccCode1 = <<AccCode/binary, Code1/binary>>, + merge_mfas(Code, Address1, AccCode1, [{Address, M, F, A}|AccExportMap]); +merge_mfas([], _Address, AccCode, AccExportMap) -> + {AccCode, AccExportMap}. + +%% @doc Merge the references of relocatable symbols in the binary code. The +%% offsets must be updated because of the merging of the code binaries! +merge_refs([], _ConstMap, _Addr, AccRefs) -> AccRefs; +merge_refs([{MFA, _, CodeSize, _, Refs, _}|Rest], ConstMap, Address, AccRefs) -> + %% Important!: The hipe_pack_constants:pack_constants/2 function assignes + %% unique numbers to constants (ConstNo). This numbers are used from now on, + %% instead of labels that were used before. So, in order to be compatible, we + %% must change all the constant labels in the Refs to the corresponding + %% ConstNo, that can be found in the ConstMap (#pcm_entry{}). + UpdatedRefs = [update_ref(label_to_constno(Ref, MFA, ConstMap), Address) + || Ref <- Refs], + merge_refs(Rest, ConstMap, align_entry(Address+CodeSize), + UpdatedRefs++AccRefs). + +label_to_constno({Type, Offset, {constant, Label}}, MFA, ConstMap) -> + ConstNo = hipe_pack_constants:find_const({MFA, Label}, ConstMap), + {Type, Offset, {constant, ConstNo}}; +label_to_constno(Other, _MFA, _ConstMap) -> + Other. + +%% @doc Update offset to a reference. In case of stack descriptors we must check +%% if there exists an exception handler, because it must also be updated. +update_ref({?SDESC, Offset, SDesc}, CodeAddr) -> + NewRefAddr = Offset+CodeAddr, + case SDesc of + {[], _, _, _} -> % No handler; only update offset + {?SDESC, NewRefAddr, SDesc}; + {ExnHandler, FrameSize, StackArity, Roots} -> % Update exception handler + {?SDESC, NewRefAddr, {ExnHandler+CodeAddr, FrameSize, StackArity, Roots}} + end; +update_ref({Type, Offset, Term}, CodeAddr) -> + {Type, Offset+CodeAddr, Term}. diff --git a/lib/hipe/llvm/hipe_rtl_to_llvm.erl b/lib/hipe/llvm/hipe_rtl_to_llvm.erl new file mode 100644 index 0000000000..ba76e1d815 --- /dev/null +++ b/lib/hipe/llvm/hipe_rtl_to_llvm.erl @@ -0,0 +1,1612 @@ +%% -*- erlang-indent-level: 2 -*- + +-module(hipe_rtl_to_llvm). +-author("Chris Stavrakakis, Yiannis Tsiouris"). + +-export([translate/2]). % the main function of this module +-export([fix_mfa_name/1]). % a help function used in hipe_llvm_main + +-include("../rtl/hipe_rtl.hrl"). +-include("../rtl/hipe_literals.hrl"). +-include("hipe_llvm_arch.hrl"). + +-define(WORD_WIDTH, (?bytes_to_bits(hipe_rtl_arch:word_size()))). +-define(BRANCH_META_TAKEN, "0"). +-define(BRANCH_META_NOT_TAKEN, "1"). + +%%------------------------------------------------------------------------------ +%% @doc Main function for translating an RTL function to LLVM Assembly. Takes as +%% input the RTL code and the variable indexes of possible garbage +%% collection roots and returns the corresponing LLVM, a dictionary with +%% all the relocations in the code and a hipe_consttab() with informaton +%% about data. +%%------------------------------------------------------------------------------ +translate(RTL, Roots) -> + Fun = hipe_rtl:rtl_fun(RTL), + Params = hipe_rtl:rtl_params(RTL), + Data = hipe_rtl:rtl_data(RTL), + Code = hipe_rtl:rtl_code(RTL), + %% Init unique symbol generator and initialize the label counter to the last + %% RTL label. + hipe_gensym:init(llvm), + {_, MaxLabel} = hipe_rtl:rtl_label_range(RTL), + put({llvm,label_count}, MaxLabel + 1), + %% Put first label of RTL code in process dictionary + find_code_entry_label(Code), + %% Initialize relocations symbol dictionary + Relocs = dict:new(), + %% Print RTL to file + %% {ok, File_rtl} = file:open("rtl_" ++integer_to_list(random:uniform(2000)) + %% ++ ".rtl", [write]), + %% hipe_rtl:pp(File_rtl, RTL), + %% file:close(File_rtl), + + %% Pass on RTL code to handle exception handling and identify labels of Fail + %% Blocks + {Code1, FailLabels} = fix_code(Code), + %% Allocate stack slots for each virtual register and declare gc roots + AllocaStackCode = alloca_stack(Code1, Params, Roots), + %% Translate Code + {LLVM_Code1, Relocs1, NewData} = + translate_instr_list(Code1, [], Relocs, Data), + %% Create LLVM code to declare relocation symbols as external symbols along + %% with local variables in order to use them as just any other variable + {FinalRelocs, ExternalDecl, LocalVars} = + handle_relocations(Relocs1, Data, Fun), + %% Pass on LLVM code in order to create Fail blocks and a landingpad + %% instruction to each one + LLVM_Code2 = add_landingpads(LLVM_Code1, FailLabels), + %% Create LLVM Code for the compiled function + LLVM_Code3 = create_function_definition(Fun, Params, LLVM_Code2, + AllocaStackCode ++ LocalVars), + %% Final Code = CompiledFunction + External Declarations + FinalLLVMCode = [LLVM_Code3 | ExternalDecl], + {FinalLLVMCode, FinalRelocs, NewData}. + +find_code_entry_label([]) -> + exit({?MODULE, find_code_entry_label, "Empty code"}); +find_code_entry_label([I|_]) -> + case hipe_rtl:is_label(I) of + true -> + put(first_label, hipe_rtl:label_name(I)); + false -> + exit({?MODULE, find_code_entry_label, "First instruction is not a label"}) + end. + +%% @doc Create a stack slot for each virtual register. The stack slots +%% that correspond to possible garbage collection roots must be +%% marked as such. +alloca_stack(Code, Params, Roots) -> + %% Find all assigned virtual registers + Destinations = collect_destinations(Code), + %% Declare virtual registers, and declare garbage collection roots + do_alloca_stack(Destinations++Params, Params, Roots). + +collect_destinations(Code) -> + lists:usort(lists:flatmap(fun insn_dst/1, Code)). + +do_alloca_stack(Destinations, Params, Roots) -> + do_alloca_stack(Destinations, Params, Roots, []). + +do_alloca_stack([], _, _, Acc) -> + Acc; +do_alloca_stack([D|Ds], Params, Roots, Acc) -> + {Name, _I} = trans_dst(D), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + case hipe_rtl:is_var(D) of + true -> + Num = hipe_rtl:var_index(D), + I1 = hipe_llvm:mk_alloca(Name, WordTy, [], []), + case lists:member(Num, Roots) of + true -> %% Variable is a possible Root + T1 = mk_temp(), + BYTE_TYPE_PP = hipe_llvm:mk_pointer(ByteTyPtr), + I2 = + hipe_llvm:mk_conversion(T1, bitcast, WordTyPtr, Name, BYTE_TYPE_PP), + GcRootArgs = [{BYTE_TYPE_PP, T1}, {ByteTyPtr, "@gc_metadata"}], + I3 = hipe_llvm:mk_call([], false, [], [], hipe_llvm:mk_void(), + "@llvm.gcroot", GcRootArgs, []), + I4 = case lists:member(D, Params) of + false -> + hipe_llvm:mk_store(WordTy, "-5", WordTyPtr, Name, + [], [], false); + true -> [] + end, + do_alloca_stack(Ds, Params, Roots, [I1, I2, I3, I4 | Acc]); + false -> + do_alloca_stack(Ds, Params, Roots, [I1|Acc]) + end; + false -> + case hipe_rtl:is_reg(D) andalso isPrecoloured(D) of + true -> %% Precoloured registers are mapped to "special" stack slots + do_alloca_stack(Ds, Params, Roots, Acc); + false -> + I1 = case hipe_rtl:is_fpreg(D) of + true -> + FloatTy = hipe_llvm:mk_double(), + hipe_llvm:mk_alloca(Name, FloatTy, [], []); + false -> hipe_llvm:mk_alloca(Name, WordTy, [], []) + end, + do_alloca_stack(Ds, Params, Roots, [I1|Acc]) + end + end. + +%%------------------------------------------------------------------------------ +%% @doc Translation of the linearized RTL Code. Each RTL instruction is +%% translated to a list of LLVM Assembly instructions. The relocation +%% dictionary is updated when needed. +%%------------------------------------------------------------------------------ +translate_instr_list([], Acc, Relocs, Data) -> + {lists:reverse(lists:flatten(Acc)), Relocs, Data}; +translate_instr_list([I | Is], Acc, Relocs, Data) -> + {Acc1, NewRelocs, NewData} = translate_instr(I, Relocs, Data), + translate_instr_list(Is, [Acc1 | Acc], NewRelocs, NewData). + +translate_instr(I, Relocs, Data) -> + case I of + #alu{} -> + {I2, Relocs2} = trans_alu(I, Relocs), + {I2, Relocs2, Data}; + #alub{} -> + {I2, Relocs2} = trans_alub(I, Relocs), + {I2, Relocs2, Data}; + #branch{} -> + {I2, Relocs2} = trans_branch(I, Relocs), + {I2, Relocs2, Data}; + #call{} -> + {I2, Relocs2} = + case hipe_rtl:call_fun(I) of + %% In AMD64 this instruction does nothing! + %% TODO: chech use of fwait in other architectures! + fwait -> + {[], Relocs}; + _ -> + trans_call(I, Relocs) + end, + {I2, Relocs2, Data}; + #comment{} -> + {I2, Relocs2} = trans_comment(I, Relocs), + {I2, Relocs2, Data}; + #enter{} -> + {I2, Relocs2} = trans_enter(I, Relocs), + {I2, Relocs2, Data}; + #fconv{} -> + {I2, Relocs2} = trans_fconv(I, Relocs), + {I2, Relocs2, Data}; + #fload{} -> + {I2, Relocs2} = trans_fload(I, Relocs), + {I2, Relocs2, Data}; + #fmove{} -> + {I2, Relocs2} = trans_fmove(I, Relocs), + {I2, Relocs2, Data}; + #fp{} -> + {I2, Relocs2} = trans_fp(I, Relocs), + {I2, Relocs2, Data}; + #fp_unop{} -> + {I2, Relocs2} = trans_fp_unop(I, Relocs), + {I2, Relocs2, Data}; + #fstore{} -> + {I2, Relocs2} = trans_fstore(I, Relocs), + {I2, Relocs2, Data}; + #goto{} -> + {I2, Relocs2} = trans_goto(I, Relocs), + {I2, Relocs2, Data}; + #label{} -> + {I2, Relocs2} = trans_label(I, Relocs), + {I2, Relocs2, Data}; + #load{} -> + {I2, Relocs2} = trans_load(I, Relocs), + {I2, Relocs2, Data}; + #load_address{} -> + {I2, Relocs2} = trans_load_address(I, Relocs), + {I2, Relocs2, Data}; + #load_atom{} -> + {I2, Relocs2} = trans_load_atom(I, Relocs), + {I2, Relocs2, Data}; + #move{} -> + {I2, Relocs2} = trans_move(I, Relocs), + {I2, Relocs2, Data}; + #return{} -> + {I2, Relocs2} = trans_return(I, Relocs), + {I2, Relocs2, Data}; + #store{} -> + {I2, Relocs2} = trans_store(I, Relocs), + {I2, Relocs2, Data}; + #switch{} -> %% Only switch instruction updates Data + {I2, Relocs2, NewData} = trans_switch(I, Relocs, Data), + {I2, Relocs2, NewData}; + Other -> + exit({?MODULE, translate_instr, {"Unknown RTL instruction", Other}}) + end. + +%% +%% alu +%% +trans_alu(I, Relocs) -> + RtlDst = hipe_rtl:alu_dst(I), + TmpDst = mk_temp(), + {Src1, I1} = trans_src(hipe_rtl:alu_src1(I)), + {Src2, I2} = trans_src(hipe_rtl:alu_src2(I)), + Op = trans_op(hipe_rtl:alu_op(I)), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I3 = hipe_llvm:mk_operation(TmpDst, Op, WordTy, Src1, Src2, []), + I4 = store_stack_dst(TmpDst, RtlDst), + {[I4, I3, I2, I1], Relocs}. + +%% +%% alub +%% +trans_alub(I, Relocs) -> + case hipe_rtl:alub_cond(I) of + Op when Op =:= overflow orelse Op =:= not_overflow -> + trans_alub_overflow(I, signed, Relocs); + ltu -> %% ltu means unsigned overflow + trans_alub_overflow(I, unsigned, Relocs); + _ -> + trans_alub_no_overflow(I, Relocs) + end. + +trans_alub_overflow(I, Sign, Relocs) -> + {Src1, I1} = trans_src(hipe_rtl:alub_src1(I)), + {Src2, I2} = trans_src(hipe_rtl:alub_src2(I)), + RtlDst = hipe_rtl:alub_dst(I), + TmpDst = mk_temp(), + Name = trans_alub_op(I, Sign), + NewRelocs = relocs_store(Name, {call, {llvm, Name, 2}}, Relocs), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + ReturnType = hipe_llvm:mk_struct([WordTy, hipe_llvm:mk_int(1)]), + T1 = mk_temp(), + I3 = hipe_llvm:mk_call(T1, false, [], [], ReturnType, "@" ++ Name, + [{WordTy, Src1}, {WordTy, Src2}], []), + %% T1{0}: result of the operation + I4 = hipe_llvm:mk_extractvalue(TmpDst, ReturnType, T1 , "0", []), + I5 = store_stack_dst(TmpDst, RtlDst), + T2 = mk_temp(), + %% T1{1}: Boolean variable indicating overflow + I6 = hipe_llvm:mk_extractvalue(T2, ReturnType, T1, "1", []), + case hipe_rtl:alub_cond(I) of + Op when Op =:= overflow orelse Op =:= ltu -> + True_label = mk_jump_label(hipe_rtl:alub_true_label(I)), + False_label = mk_jump_label(hipe_rtl:alub_false_label(I)), + MetaData = branch_metadata(hipe_rtl:alub_pred(I)); + not_overflow -> + True_label = mk_jump_label(hipe_rtl:alub_false_label(I)), + False_label = mk_jump_label(hipe_rtl:alub_true_label(I)), + MetaData = branch_metadata(1 - hipe_rtl:alub_pred(I)) + end, + I7 = hipe_llvm:mk_br_cond(T2, True_label, False_label, MetaData), + {[I7, I6, I5, I4, I3, I2, I1], NewRelocs}. + +trans_alub_op(I, Sign) -> + Name = + case Sign of + signed -> + case hipe_rtl:alub_op(I) of + add -> "llvm.sadd.with.overflow."; + mul -> "llvm.smul.with.overflow."; + sub -> "llvm.ssub.with.overflow."; + Op -> exit({?MODULE, trans_alub_op, {"Unknown alub operator", Op}}) + end; + unsigned -> + case hipe_rtl:alub_op(I) of + add -> "llvm.uadd.with.overflow."; + mul -> "llvm.umul.with.overflow."; + sub -> "llvm.usub.with.overflow."; + Op -> exit({?MODULE, trans_alub_op, {"Unknown alub operator", Op}}) + end + end, + Type = + case hipe_rtl_arch:word_size() of + 4 -> "i32"; + 8 -> "i64" + %% Other -> exit({?MODULE, trans_alub_op, {"Unknown type", Other}}) + end, + Name ++ Type. + +trans_alub_no_overflow(I, Relocs) -> + %% alu + T = hipe_rtl:mk_alu(hipe_rtl:alub_dst(I), hipe_rtl:alub_src1(I), + hipe_rtl:alub_op(I), hipe_rtl:alub_src2(I)), + %% A trans_alu instruction cannot change relocations + {I1, _} = trans_alu(T, Relocs), + %% icmp + %% Translate destination as src, to match with the semantics of instruction + {Dst, I2} = trans_src(hipe_rtl:alub_dst(I)), + Cond = trans_rel_op(hipe_rtl:alub_cond(I)), + T3 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I5 = hipe_llvm:mk_icmp(T3, Cond, WordTy, Dst, "0"), + %% br + Metadata = branch_metadata(hipe_rtl:alub_pred(I)), + True_label = mk_jump_label(hipe_rtl:alub_true_label(I)), + False_label = mk_jump_label(hipe_rtl:alub_false_label(I)), + I6 = hipe_llvm:mk_br_cond(T3, True_label, False_label, Metadata), + {[I6, I5, I2, I1], Relocs}. + +%% +%% branch +%% +trans_branch(I, Relocs) -> + {Src1, I1} = trans_src(hipe_rtl:branch_src1(I)), + {Src2, I2} = trans_src(hipe_rtl:branch_src2(I)), + Cond = trans_rel_op(hipe_rtl:branch_cond(I)), + %% icmp + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I3 = hipe_llvm:mk_icmp(T1, Cond, WordTy, Src1, Src2), + %% br + True_label = mk_jump_label(hipe_rtl:branch_true_label(I)), + False_label = mk_jump_label(hipe_rtl:branch_false_label(I)), + Metadata = branch_metadata(hipe_rtl:branch_pred(I)), + I4 = hipe_llvm:mk_br_cond(T1, True_label, False_label, Metadata), + {[I4, I3, I2, I1], Relocs}. + +branch_metadata(X) when X =:= 0.5 -> []; +branch_metadata(X) when X > 0.5 -> ?BRANCH_META_TAKEN; +branch_metadata(X) when X < 0.5 -> ?BRANCH_META_NOT_TAKEN. + +%% +%% call +%% +trans_call(I, Relocs) -> + RtlCallArgList= hipe_rtl:call_arglist(I), + RtlCallName = hipe_rtl:call_fun(I), + {I0, Relocs1} = expose_closure(RtlCallName, RtlCallArgList, Relocs), + TmpDst = mk_temp(), + {CallArgs, I1} = trans_call_args(RtlCallArgList), + FixedRegs = fixed_registers(), + {LoadedFixedRegs, I2} = load_fixed_regs(FixedRegs), + FinalArgs = fix_reg_args(LoadedFixedRegs) ++ CallArgs, + {Name, I3, Relocs2} = + trans_call_name(RtlCallName, Relocs1, CallArgs, FinalArgs), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + I4 = + case hipe_rtl:call_fail(I) of + %% Normal Call + [] -> + hipe_llvm:mk_call(T1, false, "cc 11", [], FunRetTy, Name, FinalArgs, + []); + %% Call With Exception + FailLabelNum -> + TrueLabel = "L" ++ integer_to_list(hipe_rtl:call_normal(I)), + FailLabel = "%FL" ++ integer_to_list(FailLabelNum), + II1 = + hipe_llvm:mk_invoke(T1, "cc 11", [], FunRetTy, Name, FinalArgs, [], + "%" ++ TrueLabel, FailLabel), + II2 = hipe_llvm:mk_label(TrueLabel), + [II2, II1] + end, + I5 = store_fixed_regs(FixedRegs, T1), + I6 = + case hipe_rtl:call_dstlist(I) of + [] -> []; %% No return value + [Destination] -> + II3 = + hipe_llvm:mk_extractvalue(TmpDst, FunRetTy, T1, + integer_to_list(?NR_PINNED_REGS), []), + II4 = store_stack_dst(TmpDst, Destination), + [II4, II3] + end, + I7 = + case hipe_rtl:call_continuation(I) of + [] -> []; %% No continuation + CC -> + {II5, _} = trans_goto(hipe_rtl:mk_goto(CC), Relocs2), + II5 + end, + {[I7, I6, I5, I4, I3, I2, I1, I0], Relocs2}. + +%% In case of call to a register (closure call) with more than ?NR_ARG_REGS +%% arguments we must track the offset this call in the code, in order to +%% to correct the stack descriptor. So, we insert a new Label and add this label +%% to the "table_closures" +%% --------------------------------|-------------------------------------------- +%% Old Code | New Code +%% --------------------------------|-------------------------------------------- +%% | br %ClosureLabel +%% call %reg(Args) | ClosureLabel: +%% | call %reg(Args) +expose_closure(CallName, CallArgs, Relocs) -> + CallArgsNr = length(CallArgs), + case hipe_rtl:is_reg(CallName) andalso CallArgsNr > ?NR_ARG_REGS of + true -> + LabelNum = hipe_gensym:new_label(llvm), + ClosureLabel = hipe_llvm:mk_label(mk_label(LabelNum)), + JumpIns = hipe_llvm:mk_br(mk_jump_label(LabelNum)), + Relocs1 = + relocs_store({CallName, LabelNum}, + {closure_label, LabelNum, CallArgsNr - ?NR_ARG_REGS}, + Relocs), + {[ClosureLabel, JumpIns], Relocs1}; + false -> + {[], Relocs} + end. + +trans_call_name(RtlCallName, Relocs, CallArgs, FinalArgs) -> + case RtlCallName of + PrimOp when is_atom(PrimOp) -> + LlvmName = trans_prim_op(PrimOp), + Relocs1 = relocs_store(LlvmName, {call, {bif, PrimOp, length(CallArgs)}}, + Relocs), + {"@" ++ LlvmName, [], Relocs1}; + {M, F, A} when is_atom(M), is_atom(F), is_integer(A) -> + LlvmName = trans_mfa_name({M,F,A}), + Relocs1 = relocs_store(LlvmName, {call, {M,F,A}}, Relocs), + {"@" ++ LlvmName, [], Relocs1}; + Reg -> + case hipe_rtl:is_reg(Reg) of + true -> + %% In case of a closure call, the register holding the address + %% of the closure must be converted to function type in + %% order to make the call + TT1 = mk_temp(), + {RegName, II1} = trans_src(Reg), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + II2 = + hipe_llvm:mk_conversion(TT1, inttoptr, WordTy, RegName, WordTyPtr), + TT2 = mk_temp(), + ArgsTypeList = lists:duplicate(length(FinalArgs), WordTy), + FunRetTy = + hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + FunType = hipe_llvm:mk_fun(FunRetTy, ArgsTypeList), + FunTypeP = hipe_llvm:mk_pointer(FunType), + II3 = hipe_llvm:mk_conversion(TT2, bitcast, WordTyPtr, TT1, FunTypeP), + {TT2, [II3, II2, II1], Relocs}; + false -> + exit({?MODULE, trans_call, {"Unimplemented call to", RtlCallName}}) + end + end. + +%% +trans_call_args(ArgList) -> + {Args, I} = lists:unzip(trans_args(ArgList)), + %% Reverse arguments that are passed to stack to match with the Erlang + %% calling convention. (Propably not needed in prim calls.) + ReversedArgs = + case erlang:length(Args) > ?NR_ARG_REGS of + false -> + Args; + true -> + {ArgsInRegs, ArgsInStack} = lists:split(?NR_ARG_REGS, Args), + ArgsInRegs ++ lists:reverse(ArgsInStack) + end, + %% Reverse I, because some of the arguments may go out of scope and + %% should be killed(store -5). When two or more arguments are they + %% same, then order matters! + {ReversedArgs, lists:reverse(I)}. + +%% +%% trans_comment +%% +trans_comment(I, Relocs) -> + I1 = hipe_llvm:mk_comment(hipe_rtl:comment_text(I)), + {I1, Relocs}. + +%% +%% enter +%% +trans_enter(I, Relocs) -> + {CallArgs, I0} = trans_call_args(hipe_rtl:enter_arglist(I)), + FixedRegs = fixed_registers(), + {LoadedFixedRegs, I1} = load_fixed_regs(FixedRegs), + FinalArgs = fix_reg_args(LoadedFixedRegs) ++ CallArgs, + {Name, I2, NewRelocs} = + trans_call_name(hipe_rtl:enter_fun(I), Relocs, CallArgs, FinalArgs), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + I3 = hipe_llvm:mk_call(T1, true, "cc 11", [], FunRetTy, Name, FinalArgs, []), + I4 = hipe_llvm:mk_ret([{FunRetTy, T1}]), + {[I4, I3, I2, I1, I0], NewRelocs}. + +%% +%% fconv +%% +trans_fconv(I, Relocs) -> + %% XXX: Can a fconv destination be a precoloured reg? + RtlDst = hipe_rtl:fconv_dst(I), + TmpDst = mk_temp(), + {Src, I1} = trans_float_src(hipe_rtl:fconv_src(I)), + FloatTy = hipe_llvm:mk_double(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I2 = hipe_llvm:mk_conversion(TmpDst, sitofp, WordTy, Src, FloatTy), + I3 = store_float_stack(TmpDst, RtlDst), + {[I3, I2, I1], Relocs}. + + +%% TODO: fload, fstore, fmove, and fp are almost the same with load, store, move +%% and alu. Maybe we should join them. + +%% +%% fload +%% +trans_fload(I, Relocs) -> + RtlDst = hipe_rtl:fload_dst(I), + RtlSrc = hipe_rtl:fload_src(I), + _Offset = hipe_rtl:fload_offset(I), + TmpDst = mk_temp(), + {Src, I1} = trans_float_src(RtlSrc), + {Offset, I2} = trans_float_src(_Offset), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FloatTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_double()), + I3 = hipe_llvm:mk_operation(T1, add, WordTy, Src, Offset, []), + T2 = mk_temp(), + I4 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, FloatTyPtr), + I5 = hipe_llvm:mk_load(TmpDst, FloatTyPtr, T2, [], [], false), + I6 = store_float_stack(TmpDst, RtlDst), + {[I6, I5, I4, I3, I2, I1], Relocs}. + +%% +%% fmove +%% +trans_fmove(I, Relocs) -> + RtlDst = hipe_rtl:fmove_dst(I), + RtlSrc = hipe_rtl:fmove_src(I), + {Src, I1} = trans_float_src(RtlSrc), + I2 = store_float_stack(Src, RtlDst), + {[I2, I1], Relocs}. + +%% +%% fp +%% +trans_fp(I, Relocs) -> + %% XXX: Just copied trans_alu...think again.. + RtlDst = hipe_rtl:fp_dst(I), + RtlSrc1 = hipe_rtl:fp_src1(I), + RtlSrc2 = hipe_rtl:fp_src2(I), + %% Destination cannot be a precoloured register + FloatTy = hipe_llvm:mk_double(), + FloatTyPtr = hipe_llvm:mk_pointer(FloatTy), + TmpDst = mk_temp(), + {Src1, I1} = trans_float_src(RtlSrc1), + {Src2, I2} = trans_float_src(RtlSrc2), + Op = trans_fp_op(hipe_rtl:fp_op(I)), + I3 = hipe_llvm:mk_operation(TmpDst, Op, FloatTy, Src1, Src2, []), + I4 = store_float_stack(TmpDst, RtlDst), + %% Synchronization for floating point exceptions + I5 = hipe_llvm:mk_store(FloatTy, TmpDst, FloatTyPtr, "%exception_sync", [], + [], true), + T1 = mk_temp(), + I6 = hipe_llvm:mk_load(T1, FloatTyPtr, "%exception_sync", [], [], true), + {[I6, I5, I4, I3, I2, I1], Relocs}. + +%% +%% fp_unop +%% +trans_fp_unop(I, Relocs) -> + RtlDst = hipe_rtl:fp_unop_dst(I), + RtlSrc = hipe_rtl:fp_unop_src(I), + %% Destination cannot be a precoloured register + TmpDst = mk_temp(), + {Src, I1} = trans_float_src(RtlSrc), + Op = trans_fp_op(hipe_rtl:fp_unop_op(I)), + FloatTy = hipe_llvm:mk_double(), + I2 = hipe_llvm:mk_operation(TmpDst, Op, FloatTy, "0.0", Src, []), + I3 = store_float_stack(TmpDst, RtlDst), + {[I3, I2, I1], Relocs}. +%% TODO: Fix fp_unop in a way like the following. You must change trans_dest, +%% in order to call float_to_list in a case of float constant. Maybe the type +%% check is expensive... +%% Dst = hipe_rtl:fp_unop_dst(I), +%% Src = hipe_rtl:fp_unop_src(I), +%% Op = hipe_rtl:fp_unop_op(I), +%% Zero = hipe_rtl:mk_imm(0.0), +%% I1 = hipe_rtl:mk_fp(Dst, Zero, Op, Src), +%% trans_fp(I, Relocs1). + +%% +%% fstore +%% +trans_fstore(I, Relocs) -> + Base = hipe_rtl:fstore_base(I), + case isPrecoloured(Base) of + true -> + trans_fstore_reg(I, Relocs); + false -> + exit({?MODULE, trans_fstore ,{"Not implemented yet", false}}) + end. + +trans_fstore_reg(I, Relocs) -> + {Base, I0} = trans_reg(hipe_rtl:fstore_base(I), dst), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + FloatTy = hipe_llvm:mk_double(), + FloatTyPtr = hipe_llvm:mk_pointer(FloatTy), + T1 = mk_temp(), + I1 = hipe_llvm:mk_load(T1, WordTyPtr, Base, [], [], false), + {Offset, I2} = trans_src(hipe_rtl:fstore_offset(I)), + T2 = mk_temp(), + I3 = hipe_llvm:mk_operation(T2, add, WordTy, T1, Offset, []), + T3 = mk_temp(), + I4 = hipe_llvm:mk_conversion(T3, inttoptr, WordTy, T2, FloatTyPtr), + {Value, I5} = trans_src(hipe_rtl:fstore_src(I)), + I6 = hipe_llvm:mk_store(FloatTy, Value, FloatTyPtr, T3, [], [], false), + {[I6, I5, I4, I3, I2, I1, I0], Relocs}. + +%% +%% goto +%% +trans_goto(I, Relocs) -> + I1 = hipe_llvm:mk_br(mk_jump_label(hipe_rtl:goto_label(I))), + {I1, Relocs}. + +%% +%% label +%% +trans_label(I, Relocs) -> + Label = mk_label(hipe_rtl:label_name(I)), + I1 = hipe_llvm:mk_label(Label), + {I1, Relocs}. + +%% +%% load +%% +trans_load(I, Relocs) -> + RtlDst = hipe_rtl:load_dst(I), + TmpDst = mk_temp(), + %% XXX: Why translate them independently? ------------------------ + {Src, I1} = trans_src(hipe_rtl:load_src(I)), + {Offset, I2} = trans_src(hipe_rtl:load_offset(I)), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + I3 = hipe_llvm:mk_operation(T1, add, WordTy, Src, Offset, []), + %%---------------------------------------------------------------- + I4 = case hipe_rtl:load_size(I) of + word -> + T2 = mk_temp(), + II1 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, WordTyPtr), + II2 = hipe_llvm:mk_load(TmpDst, WordTyPtr, T2, [], [], false), + [II2, II1]; + Size -> + LoadType = llvm_type_from_size(Size), + LoadTypeP = hipe_llvm:mk_pointer(LoadType), + T2 = mk_temp(), + II1 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, LoadTypeP), + T3 = mk_temp(), + LoadTypePointer = hipe_llvm:mk_pointer(LoadType), + II2 = hipe_llvm:mk_load(T3, LoadTypePointer, T2, [], [], false), + Conversion = + case hipe_rtl:load_sign(I) of + signed -> sext; + unsigned -> zext + end, + II3 = + hipe_llvm:mk_conversion(TmpDst, Conversion, LoadType, T3, WordTy), + [II3, II2, II1] + end, + I5 = store_stack_dst(TmpDst, RtlDst), + {[I5, I4, I3, I2, I1], Relocs}. + +%% +%% load_address +%% +trans_load_address(I, Relocs) -> + RtlDst = hipe_rtl:load_address_dst(I), + RtlAddr = hipe_rtl:load_address_addr(I), + {Addr, NewRelocs} = + case hipe_rtl:load_address_type(I) of + constant -> + {"%DL" ++ integer_to_list(RtlAddr) ++ "_var", Relocs}; + closure -> + {{_, ClosureName, _}, _, _} = RtlAddr, + FixedClosureName = fix_closure_name(ClosureName), + Relocs1 = relocs_store(FixedClosureName, {closure, RtlAddr}, Relocs), + {"%" ++ FixedClosureName ++ "_var", Relocs1}; + type -> + exit({?MODULE, trans_load_address, + {"Type not implemented in load_address", RtlAddr}}) + end, + I1 = store_stack_dst(Addr, RtlDst), + {[I1], NewRelocs}. + +%% +%% load_atom +%% +trans_load_atom(I, Relocs) -> + RtlDst = hipe_rtl:load_atom_dst(I), + RtlAtom = hipe_rtl:load_atom_atom(I), + AtomName = "atom_" ++ make_llvm_id(atom_to_list(RtlAtom)), + AtomVar = "%" ++ AtomName ++ "_var", + NewRelocs = relocs_store(AtomName, {atom, RtlAtom}, Relocs), + I1 = store_stack_dst(AtomVar, RtlDst), + {[I1], NewRelocs}. + +%% +%% move +%% +trans_move(I, Relocs) -> + RtlDst = hipe_rtl:move_dst(I), + RtlSrc = hipe_rtl:move_src(I), + {Src, I1} = trans_src(RtlSrc), + I2 = store_stack_dst(Src, RtlDst), + {[I2, I1], Relocs}. + +%% +%% return +%% +trans_return(I, Relocs) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + {VarRet, I1} = + case hipe_rtl:return_varlist(I) of + [] -> + {[], []}; + [A] -> + {Name, II1} = trans_src(A), + {[{WordTy, Name}], II1} + end, + FixedRegs = fixed_registers(), + {LoadedFixedRegs, I2} = load_fixed_regs(FixedRegs), + FixedRet = [{WordTy, X} || X <- LoadedFixedRegs], + Ret = FixedRet ++ VarRet, + {RetTypes, _RetNames} = lists:unzip(Ret), + Type = hipe_llvm:mk_struct(RetTypes), + {RetStruct, I3} = mk_return_struct(Ret, Type), + I4 = hipe_llvm:mk_ret([{Type, RetStruct}]), + {[I4, I3, I2, I1], Relocs}. + +%% @doc Create a structure to hold the return value and the precoloured +%% registers. +mk_return_struct(RetValues, Type) -> + mk_return_struct(RetValues, Type, [], "undef", 0). + +mk_return_struct([], _, Acc, StructName, _) -> + {StructName, Acc}; +mk_return_struct([{ElemType, ElemName}|Rest], Type, Acc, StructName, Index) -> + T1 = mk_temp(), + I1 = hipe_llvm:mk_insertvalue(T1, Type, StructName, ElemType, ElemName, + integer_to_list(Index), []), + mk_return_struct(Rest, Type, [I1 | Acc], T1, Index+1). + +%% +%% store +%% +trans_store(I, Relocs) -> + {Base, I1} = trans_src(hipe_rtl:store_base(I)), + {Offset, I2} = trans_src(hipe_rtl:store_offset(I)), + {Value, I3} = trans_src(hipe_rtl:store_src(I)), + T1 = mk_temp(), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + I4 = hipe_llvm:mk_operation(T1, add, WordTy, Base, Offset, []), + I5 = + case hipe_rtl:store_size(I) of + word -> + T2 = mk_temp(), + II1 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, WordTyPtr), + II2 = hipe_llvm:mk_store(WordTy, Value, WordTyPtr, T2, [], [], + false), + [II2, II1]; + Size -> + %% XXX: Is always trunc correct ? + LoadType = llvm_type_from_size(Size), + LoadTypePointer = hipe_llvm:mk_pointer(LoadType), + T2 = mk_temp(), + II1 = hipe_llvm:mk_conversion(T2, inttoptr, WordTy, T1, LoadTypePointer), + T3 = mk_temp(), + II2 = hipe_llvm:mk_conversion(T3, 'trunc', WordTy, Value, LoadType), + II3 = hipe_llvm:mk_store(LoadType, T3, LoadTypePointer, T2, [], [], false), + [II3, II2, II1] + end, + {[I5, I4, I3, I2, I1], Relocs}. + +%% +%% switch +%% +trans_switch(I, Relocs, Data) -> + RtlSrc = hipe_rtl:switch_src(I), + {Src, I1} = trans_src(RtlSrc), + Labels = hipe_rtl:switch_labels(I), + JumpLabels = [mk_jump_label(L) || L <- Labels], + SortOrder = hipe_rtl:switch_sort_order(I), + NrLabels = length(Labels), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + TableType = hipe_llvm:mk_array(NrLabels, ByteTyPtr), + TableTypeP = hipe_llvm:mk_pointer(TableType), + TypedJumpLabels = [{hipe_llvm:mk_label_type(), X} || X <- JumpLabels], + T1 = mk_temp(), + {Src2, []} = trans_dst(RtlSrc), + TableName = "table_" ++ tl(Src2), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I2 = hipe_llvm:mk_getelementptr(T1, TableTypeP, "@"++TableName, + [{WordTy, "0"}, {WordTy, Src}], false), + T2 = mk_temp(), + BYTE_TYPE_PP = hipe_llvm:mk_pointer(ByteTyPtr), + I3 = hipe_llvm:mk_load(T2, BYTE_TYPE_PP, T1, [], [], false), + I4 = hipe_llvm:mk_indirectbr(ByteTyPtr, T2, TypedJumpLabels), + LMap = [{label, L} || L <- Labels], + %% Update data with the info for the jump table + {NewData, JTabLab} = + case hipe_rtl:switch_sort_order(I) of + [] -> + hipe_consttab:insert_block(Data, word, LMap); + SortOrder -> + hipe_consttab:insert_sorted_block(Data, word, LMap, SortOrder) + end, + Relocs2 = relocs_store(TableName, {switch, {TableType, Labels, NrLabels, + SortOrder}, JTabLab}, Relocs), + {[I4, I3, I2, I1], Relocs2, NewData}. + +%% @doc Pass on RTL code in order to fix invoke and closure calls. +fix_code(Code) -> + fix_calls(Code). + +%% @doc Fix invoke calls and closure calls with more than ?NR_ARG_REGS +%% arguments. +fix_calls(Code) -> + fix_calls(Code, [], []). + +fix_calls([], Acc, FailLabels) -> + {lists:reverse(Acc), FailLabels}; +fix_calls([I | Is], Acc, FailLabels) -> + case hipe_rtl:is_call(I) of + true -> + {NewCall, NewFailLabels} = + case hipe_rtl:call_fail(I) of + [] -> + {I, FailLabels}; + FailLabel -> + fix_invoke_call(I, FailLabel, FailLabels) + end, + fix_calls(Is, [NewCall|Acc], NewFailLabels); + false -> + fix_calls(Is, [I|Acc], FailLabels) + end. + +%% @doc When a call has a fail continuation label it must be extended with a +%% normal continuation label to go with the LLVM's invoke instruction. +%% FailLabels is the list of labels of all fail blocks, which are needed to +%% be declared as landing pads. Furtermore, we must add to fail labels a +%% call to hipe_bifs:llvm_fix_pinned_regs/0 in order to avoid reloading old +%% values of pinned registers. This may happen because the result of an +%% invoke instruction is not available at fail-labels, and, thus, we cannot +%% get the correct values of pinned registers. Finally, the stack needs to +%% be re-adjusted when there are stack arguments. +fix_invoke_call(I, FailLabel, FailLabels) -> + NewLabel = hipe_gensym:new_label(llvm), + NewCall1 = hipe_rtl:call_normal_update(I, NewLabel), + SpAdj = find_sp_adj(hipe_rtl:call_arglist(I)), + case lists:keyfind(FailLabel, 1, FailLabels) of + %% Same fail label with same Stack Pointer adjustment + {FailLabel, NewFailLabel, SpAdj} -> + NewCall2 = hipe_rtl:call_fail_update(NewCall1, NewFailLabel), + {NewCall2, FailLabels}; + %% Same fail label but with different Stack Pointer adjustment + {_, _, _} -> + NewFailLabel = hipe_gensym:new_label(llvm), + NewCall2 = hipe_rtl:call_fail_update(NewCall1, NewFailLabel), + {NewCall2, [{FailLabel, NewFailLabel, SpAdj} | FailLabels]}; + %% New Fail label + false -> + NewFailLabel = hipe_gensym:new_label(llvm), + NewCall2 = hipe_rtl:call_fail_update(NewCall1, NewFailLabel), + {NewCall2, [{FailLabel, NewFailLabel, SpAdj} | FailLabels]} + end. + +find_sp_adj(ArgList) -> + NrArgs = length(ArgList), + case NrArgs > ?NR_ARG_REGS of + true -> + (NrArgs - ?NR_ARG_REGS) * hipe_rtl_arch:word_size(); + false -> + 0 + end. + +%% @doc Add landingpad instruction in Fail Blocks. +add_landingpads(LLVM_Code, FailLabels) -> + FailLabels2 = [convert_label(T) || T <- FailLabels], + add_landingpads(LLVM_Code, FailLabels2, []). + +add_landingpads([], _, Acc) -> + lists:reverse(Acc); +add_landingpads([I | Is], FailLabels, Acc) -> + case hipe_llvm:is_label(I) of + true -> + Label = hipe_llvm:label_label(I), + Ins = create_fail_blocks(Label, FailLabels), + add_landingpads(Is, FailLabels, [I | Ins] ++ Acc); + false -> + add_landingpads(Is, FailLabels, [I | Acc]) + end. + +convert_label({X,Y,Z}) -> + {"L" ++ integer_to_list(X), "FL" ++ integer_to_list(Y), Z}. + +%% @doc Create a fail block wich. +create_fail_blocks(_, []) -> []; +create_fail_blocks(Label, FailLabels) -> + create_fail_blocks(Label, FailLabels, []). + +create_fail_blocks(Label, FailLabels, Acc) -> + case lists:keytake(Label, 1, FailLabels) of + false -> + Acc; + {value, {Label, FailLabel, SpAdj}, RestFailLabels} -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + I1 = hipe_llvm:mk_label(FailLabel), + LP = hipe_llvm:mk_landingpad(), + I2 = + case SpAdj > 0 of + true -> + StackPointer = ?ARCH_REGISTERS:reg_name(?ARCH_REGISTERS:sp()), + hipe_llvm:mk_adj_stack(integer_to_list(SpAdj), StackPointer, + WordTy); + false -> [] + end, + T1 = mk_temp(), + FixedRegs = fixed_registers(), + FunRetTy = + hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + I3 = hipe_llvm:mk_call(T1, false, "cc 11", [], FunRetTy, + "@hipe_bifs.llvm_fix_pinned_regs.0", [], []), + I4 = store_fixed_regs(FixedRegs, T1), + I5 = hipe_llvm:mk_br("%" ++ Label), + Ins = lists:flatten([I5, I4, I3, I2, LP,I1]), + create_fail_blocks(Label, RestFailLabels, Ins ++ Acc) + end. + +%%------------------------------------------------------------------------------ +%% Miscellaneous Functions +%%------------------------------------------------------------------------------ + +%% @doc Convert RTL argument list to LLVM argument list. +trans_args(ArgList) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + MakeArg = + fun(A) -> + {Name, I1} = trans_src(A), + {{WordTy, Name}, I1} + end, + [MakeArg(A) || A <- ArgList]. + +%% @doc Convert a list of Precoloured registers to LLVM argument list. +fix_reg_args(ArgList) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + [{WordTy, A} || A <- ArgList]. + +%% @doc Load Precoloured registers. +load_fixed_regs(RegList) -> + Names = [mk_temp_reg(R) || R <- RegList], + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + Fun1 = + fun (X, Y) -> + hipe_llvm:mk_load(X, WordTyPtr, "%" ++ Y ++ "_reg_var", [], [], false) + end, + Ins = lists:zipwith(Fun1, Names, RegList), + {Names, Ins}. + +%% @doc Store Precoloured registers. +store_fixed_regs(RegList, Name) -> + Names = [mk_temp_reg(R) || R <- RegList], + Indexes = lists:seq(0, erlang:length(RegList) - 1), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + Fun1 = + fun(X,Y) -> + hipe_llvm:mk_extractvalue(X, FunRetTy, Name, integer_to_list(Y), []) + end, + I1 = lists:zipwith(Fun1, Names, Indexes), + Fun2 = + fun (X, Y) -> + hipe_llvm:mk_store(WordTy, X, WordTyPtr, "%" ++ Y ++ "_reg_var", [], [], + false) + end, + I2 = lists:zipwith(Fun2, Names, RegList), + [I2, I1]. + +%%------------------------------------------------------------------------------ +%% Translation of Names +%%------------------------------------------------------------------------------ + +%% @doc Fix F in MFA tuple to acceptable LLVM identifier (case of closure). +-spec fix_mfa_name(mfa()) -> mfa(). +fix_mfa_name({Mod_Name, Closure_Name, Arity}) -> + Fun_Name = list_to_atom(fix_closure_name(Closure_Name)), + {Mod_Name, Fun_Name, Arity}. + +%% @doc Make an acceptable LLVM identifier for a closure name. +fix_closure_name(ClosureName) -> + make_llvm_id(atom_to_list(ClosureName)). + +%% @doc Create an acceptable LLVM identifier. +make_llvm_id(Name) -> + case Name of + "" -> "Empty"; + Other -> lists:flatten([llvm_id(C) || C <- Other]) + end. + +llvm_id(C) when C=:=46; C>47 andalso C<58; C>64 andalso C<91; C=:=95; + C>96 andalso C<123 -> + C; +llvm_id(C) -> + io_lib:format("_~2.16.0B_",[C]). + +%% @doc Create an acceptable LLVM identifier for an MFA. +trans_mfa_name({M,F,A}) -> + N = atom_to_list(M) ++ "." ++ atom_to_list(F) ++ "." ++ integer_to_list(A), + make_llvm_id(N). + +%%------------------------------------------------------------------------------ +%% Creation of Labels and Temporaries +%%------------------------------------------------------------------------------ +mk_label(N) -> + "L" ++ integer_to_list(N). + +mk_jump_label(N) -> + "%L" ++ integer_to_list(N). + +mk_temp() -> + "%t" ++ integer_to_list(hipe_gensym:new_var(llvm)). + +mk_temp_reg(Name) -> + "%" ++ Name ++ integer_to_list(hipe_gensym:new_var(llvm)). + +%%---------------------------------------------------------------------------- +%% Translation of Operands +%%---------------------------------------------------------------------------- + +store_stack_dst(TempDst, Dst) -> + {Dst2, II1} = trans_dst(Dst), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + II2 = hipe_llvm:mk_store(WordTy, TempDst, WordTyPtr, Dst2, [], [], false), + [II2, II1]. + +store_float_stack(TempDst, Dst) -> + {Dst2, II1} = trans_dst(Dst), + FloatTy = hipe_llvm:mk_double(), + FloatTyPtr = hipe_llvm:mk_pointer(FloatTy), + II2 = hipe_llvm:mk_store(FloatTy, TempDst, FloatTyPtr, Dst2, [], [], false), + [II2, II1]. + +trans_float_src(Src) -> + case hipe_rtl:is_const_label(Src) of + true -> + Name = "@DL" ++ integer_to_list(hipe_rtl:const_label_label(Src)), + T1 = mk_temp(), + %% XXX: Hardcoded offset + ByteTy = hipe_llvm:mk_int(8), + ByteTyPtr = hipe_llvm:mk_pointer(ByteTy), + I1 = hipe_llvm:mk_getelementptr(T1, ByteTyPtr, Name, + [{ByteTy, integer_to_list(?FLOAT_OFFSET)}], true), + T2 = mk_temp(), + FloatTy = hipe_llvm:mk_double(), + FloatTyPtr = hipe_llvm:mk_pointer(FloatTy), + I2 = hipe_llvm:mk_conversion(T2, bitcast, ByteTyPtr, T1, FloatTyPtr), + T3 = mk_temp(), + I3 = hipe_llvm:mk_load(T3, FloatTyPtr, T2, [], [], false), + {T3, [I3, I2, I1]}; + false -> + trans_src(Src) + end. + +trans_src(A) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + case hipe_rtl:is_imm(A) of + true -> + Value = integer_to_list(hipe_rtl:imm_value(A)), + {Value, []}; + false -> + case hipe_rtl:is_reg(A) of + true -> + case isPrecoloured(A) of + true -> trans_reg(A, src); + false -> + {Name, []} = trans_reg(A, src), + T1 = mk_temp(), + I1 = hipe_llvm:mk_load(T1, WordTyPtr, Name, [], [], false), + {T1, [I1]} + end; + false -> + case hipe_rtl:is_var(A) of + true -> + RootName = "%vr" ++ integer_to_list(hipe_rtl:var_index(A)), + T1 = mk_temp(), + I1 = hipe_llvm:mk_load(T1, WordTyPtr, RootName, [], [], false), + I2 = + case hipe_rtl:var_liveness(A) of + live -> + []; + dead -> + NilValue = hipe_tagscheme:mk_nil(), + hipe_llvm:mk_store(WordTy, integer_to_list(NilValue), WordTyPtr, RootName, + [], [], false) + end, + {T1, [I2, I1]}; + false -> + case hipe_rtl:is_fpreg(A) of + true -> + {Name, []} = trans_dst(A), + T1 = mk_temp(), + FloatTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_double()), + I1 = hipe_llvm:mk_load(T1, FloatTyPtr, Name, [], [], false), + {T1, [I1]}; + false -> trans_dst(A) + end + end + end + end. + +trans_dst(A) -> + case hipe_rtl:is_reg(A) of + true -> + trans_reg(A, dst); + false -> + Name = case hipe_rtl:is_var(A) of + true -> + "%vr" ++ integer_to_list(hipe_rtl:var_index(A)); + false -> + case hipe_rtl:is_fpreg(A) of + true -> "%fr" ++ integer_to_list(hipe_rtl:fpreg_index(A)); + false -> + case hipe_rtl:is_const_label(A) of + true -> + "%DL" ++ integer_to_list(hipe_rtl:const_label_label(A)) ++ "_var"; + false -> + exit({?MODULE, trans_dst, {"Bad RTL argument",A}}) + end + end + end, + {Name, []} + end. + +%% @doc Translate a register. If it is precoloured it must be mapped to the +%% correct stack slot that holds the precoloured register value. +trans_reg(Arg, Position) -> + Index = hipe_rtl:reg_index(Arg), + case isPrecoloured(Arg) of + true -> + Name = map_precoloured_reg(Index), + case Position of + src -> fix_reg_src(Name); + dst -> fix_reg_dst(Name) + end; + false -> + {hipe_rtl_arch:reg_name(Index), []} + end. + +map_precoloured_reg(Index) -> + case hipe_rtl_arch:reg_name(Index) of + "%r15" -> "%hp_reg_var"; + "%rbp" -> "%p_reg_var"; + "%esi" -> "%hp_reg_var"; + "%ebp" -> "%p_reg_var"; + "%fcalls" -> + {"%p_reg_var", ?ARCH_REGISTERS:proc_offset(?ARCH_REGISTERS:fcalls())}; + "%hplim" -> + {"%p_reg_var", ?ARCH_REGISTERS:proc_offset(?ARCH_REGISTERS:heap_limit())}; + _ -> + exit({?MODULE, map_precoloured_reg, {"Register not mapped yet", Index}}) + end. + +%% @doc Load precoloured dst register. +fix_reg_dst(Register) -> + case Register of + {Name, Offset} -> %% Case of %fcalls, %hplim + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + pointer_from_reg(Name, WordTy, Offset); + Name -> %% Case of %p and %hp + {Name, []} + end. + +%% @doc Load precoloured src register. +fix_reg_src(Register) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + case Register of + {Name, Offset} -> %% Case of %fcalls, %hplim + {T1, I1} = pointer_from_reg(Name, WordTy, Offset), + T2 = mk_temp(), + I2 = hipe_llvm:mk_load(T2, WordTyPtr, T1, [], [] , false), + {T2, [I2, I1]}; + Name -> %% Case of %p and %hp + T1 = mk_temp(), + {T1, hipe_llvm:mk_load(T1, WordTyPtr, Name, [], [], false)} + end. + +%% @doc Load %fcalls and %hplim. +pointer_from_reg(RegName, Type, Offset) -> + PointerType = hipe_llvm:mk_pointer(Type), + T1 = mk_temp(), + I1 = hipe_llvm:mk_load(T1, PointerType, RegName, [], [] ,false), + T2 = mk_temp(), + I2 = hipe_llvm:mk_conversion(T2, inttoptr, Type, T1, PointerType), + T3 = mk_temp(), + %% XXX: Offsets should be a power of 2. + I3 = hipe_llvm:mk_getelementptr(T3, PointerType, T2, + [{Type, integer_to_list(Offset div hipe_rtl_arch:word_size())}], true), + {T3, [I3, I2, I1]}. + +isPrecoloured(X) -> + hipe_rtl_arch:is_precoloured(X). + +%%------------------------------------------------------------------------------ +%% Translation of operators +%%------------------------------------------------------------------------------ + +trans_op(Op) -> + case Op of + add -> add; + sub -> sub; + 'or' -> 'or'; + 'and' -> 'and'; + 'xor' -> 'xor'; + sll -> shl; + srl -> lshr; + sra -> ashr; + mul -> mul; + 'fdiv' -> fdiv; + 'sdiv' -> sdiv; + 'srem' -> srem; + Other -> exit({?MODULE, trans_op, {"Unknown RTL operator", Other}}) + end. + +trans_rel_op(Op) -> + case Op of + eq -> eq; + ne -> ne; + gtu -> ugt; + geu -> uge; + ltu -> ult; + leu -> ule; + gt -> sgt; + ge -> sge; + lt -> slt; + le -> sle + end. + +trans_prim_op(Op) -> + case Op of + '+' -> "bif_add"; + '-' -> "bif_sub"; + '*' -> "bif_mul"; + 'div' -> "bif_div"; + '/' -> "bif_div"; + Other -> atom_to_list(Other) + end. + +trans_fp_op(Op) -> + case Op of + fadd -> fadd; + fsub -> fsub; + fdiv -> fdiv; + fmul -> fmul; + fchs -> fsub; + Other -> exit({?MODULE, trans_fp_op, {"Unknown RTL float operator",Other}}) + end. + +%% Misc. +insn_dst(I) -> + case I of + #alu{} -> + [hipe_rtl:alu_dst(I)]; + #alub{} -> + [hipe_rtl:alub_dst(I)]; + #call{} -> + case hipe_rtl:call_dstlist(I) of + [] -> []; + [Dst] -> [Dst] + end; + #load{} -> + [hipe_rtl:load_dst(I)]; + #load_address{} -> + [hipe_rtl:load_address_dst(I)]; + #load_atom{} -> + [hipe_rtl:load_atom_dst(I)]; + #move{} -> + [hipe_rtl:move_dst(I)]; + #phi{} -> + [hipe_rtl:phi_dst(I)]; + #fconv{} -> + [hipe_rtl:fconv_dst(I)]; + #fload{} -> + [hipe_rtl:fload_dst(I)]; + #fmove{} -> + [hipe_rtl:fmove_dst(I)]; + #fp{} -> + [hipe_rtl:fp_dst(I)]; + #fp_unop{} -> + [hipe_rtl:fp_unop_dst(I)]; + _ -> + [] + end. + +llvm_type_from_size(Size) -> + case Size of + byte -> hipe_llvm:mk_int(8); + int16 -> hipe_llvm:mk_int(16); + int32 -> hipe_llvm:mk_int(32); + word -> hipe_llvm:mk_int(64) + end. + +%% @doc Create definition for the compiled function. The parameters that are +%% passed to the stack must be reversed to match with the CC. Also +%% precoloured registers that are passed as arguments must be stored to +%% the corresonding stack slots. +create_function_definition(Fun, Params, Code, LocalVars) -> + FunctionName = trans_mfa_name(Fun), + FixedRegs = fixed_registers(), + %% Reverse parameters to match with the Erlang calling convention + ReversedParams = + case erlang:length(Params) > ?NR_ARG_REGS of + false -> + Params; + true -> + {ParamsInRegs, ParamsInStack} = lists:split(?NR_ARG_REGS, Params), + ParamsInRegs ++ lists:reverse(ParamsInStack) + end, + Args = header_regs(FixedRegs) ++ header_params(ReversedParams), + EntryLabel = hipe_llvm:mk_label("Entry"), + FloatTy = hipe_llvm:mk_double(), + ExceptionSync = hipe_llvm:mk_alloca("%exception_sync", FloatTy, [], []), + I2 = load_regs(FixedRegs), + I3 = hipe_llvm:mk_br(mk_jump_label(get(first_label))), + StoredParams = store_params(Params), + EntryBlock = + lists:flatten([EntryLabel, ExceptionSync, I2, LocalVars, StoredParams, I3]), + Final_Code = EntryBlock ++ Code, + FunctionOptions = [nounwind, noredzone, list_to_atom("gc \"erlang\"")], + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + hipe_llvm:mk_fun_def([], [], "cc 11", [], FunRetTy, FunctionName, Args, + FunctionOptions, [], Final_Code). + +header_params(Params) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + [{WordTy, "%v" ++ integer_to_list(hipe_rtl:var_index(P))} || P <- Params]. + +store_params(Params) -> + Fun1 = + fun(X) -> + Index = hipe_rtl:var_index(X), + {Name, _} = trans_dst(X), + ParamName = "%v" ++ integer_to_list(Index), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + hipe_llvm:mk_store(WordTy, ParamName, WordTyPtr, Name, [], [], false) + end, + lists:map(Fun1, Params). + +fixed_registers() -> + case get(hipe_target_arch) of + x86 -> + ["hp", "p"]; + amd64 -> + ["hp", "p"]; + Other -> + exit({?MODULE, map_registers, {"Unknown architecture", Other}}) + end. + +header_regs(Registers) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + [{WordTy, "%" ++ X ++ "_in"} || X <- Registers]. + +load_regs(Registers) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + Fun1 = + fun(X) -> + I1 = hipe_llvm:mk_alloca("%" ++ X ++ "_reg_var", WordTy, [], []), + I2 = hipe_llvm:mk_store(WordTy, "%" ++ X ++ "_in", WordTyPtr, + "%" ++ X ++ "_reg_var", [], [], false), + [I1, I2] + end, + lists:map(Fun1, Registers). + +%%------------------------------------------------------------------------------ +%% Relocation-specific Stuff +%%------------------------------------------------------------------------------ + +relocs_store(Key, Value, Relocs) -> + dict:store(Key, Value, Relocs). + +relocs_to_list(Relocs) -> + dict:to_list(Relocs). + +%% @doc This function is responsible for the actions needed to handle +%% relocations: +%% 1) Updates relocations with constants and switch jump tables. +%% 2) Creates LLVM code to declare relocations as external +%% functions/constants. +%% 3) Creates LLVM code in order to create local variables for the external +%% constants/labels. +handle_relocations(Relocs, Data, Fun) -> + RelocsList = relocs_to_list(Relocs), + %% Seperate Relocations according to their type + {CallList, AtomList, ClosureList, ClosureLabels, SwitchList} = + seperate_relocs(RelocsList), + %% Create code to declare atoms + AtomDecl = [declare_atom(A) || A <- AtomList], + %% Create code to create local name for atoms + AtomLoad = [load_atom(A) || A <- AtomList], + %% Create code to declare closures + ClosureDecl = [declare_closure(C) || C <- ClosureList], + %% Create code to create local name for closures + ClosureLoad = [load_closure(C) || C <- ClosureList], + %% Find function calls + IsExternalCall = fun (X) -> is_external_call(X, Fun) end, + ExternalCallList = lists:filter(IsExternalCall, CallList), + %% Create code to declare external function + FunDecl = fixed_fun_decl() ++ [call_to_decl(C) || C <- ExternalCallList], + %% Extract constant labels from Constant Map (remove duplicates) + ConstLabels = hipe_consttab:labels(Data), + %% Create code to declare constants + ConstDecl = [declare_constant(C) || C <- ConstLabels], + %% Create code to create local name for constants + ConstLoad = [load_constant(C) || C <- ConstLabels], + %% Create code to create jump tables + SwitchDecl = declare_switches(SwitchList, Fun), + %% Create code to create a table with the labels of all closure calls + {ClosureLabelDecl, Relocs1} = + declare_closure_labels(ClosureLabels, Relocs, Fun), + %% Enter constants to relocations + Relocs2 = lists:foldl(fun const_to_dict/2, Relocs1, ConstLabels), + %% Temporary Store inc_stack and llvm_fix_pinned_regs to Dictionary + %% TODO: Remove this + Relocs3 = dict:store("inc_stack_0", {call, {bif, inc_stack_0, 0}}, Relocs2), + Relocs4 = dict:store("hipe_bifs.llvm_fix_pinned_regs.0", + {call, {hipe_bifs, llvm_fix_pinned_regs, 0}}, Relocs3), + BranchMetaData = [ + hipe_llvm:mk_branch_meta(?BRANCH_META_TAKEN, "99", "1") + , hipe_llvm:mk_branch_meta(?BRANCH_META_NOT_TAKEN, "1", "99") + ], + ExternalDeclarations = AtomDecl ++ ClosureDecl ++ ConstDecl ++ FunDecl ++ + ClosureLabelDecl ++ SwitchDecl ++ BranchMetaData, + LocalVariables = AtomLoad ++ ClosureLoad ++ ConstLoad, + {Relocs4, ExternalDeclarations, LocalVariables}. + +%% @doc Seperate relocations according to their type. +seperate_relocs(Relocs) -> + seperate_relocs(Relocs, [], [], [], [], []). + +seperate_relocs([], CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc) -> + {CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc}; +seperate_relocs([R|Rs], CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc) -> + case R of + {_, {call, _}} -> + seperate_relocs(Rs, [R | CallAcc], AtomAcc, ClosureAcc, LabelAcc, + JmpTableAcc); + {_, {atom, _}} -> + seperate_relocs(Rs, CallAcc, [R | AtomAcc], ClosureAcc, LabelAcc, + JmpTableAcc); + {_, {closure, _}} -> + seperate_relocs(Rs, CallAcc, AtomAcc, [R | ClosureAcc], LabelAcc, + JmpTableAcc); + {_, {switch, _, _}} -> + seperate_relocs(Rs, CallAcc, AtomAcc, ClosureAcc, LabelAcc, + [R | JmpTableAcc]); + {_, {closure_label, _, _}} -> + seperate_relocs(Rs, CallAcc, AtomAcc, ClosureAcc, [R | LabelAcc], + JmpTableAcc) + end. + +%% @doc External declaration of an atom. +declare_atom({AtomName, _}) -> + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + hipe_llvm:mk_const_decl("@" ++ AtomName, "external constant", WordTy, ""). + +%% @doc Creation of local variable for an atom. +load_atom({AtomName, _}) -> + Dst = "%" ++ AtomName ++ "_var", + Name = "@" ++ AtomName, + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + WordTyPtr = hipe_llvm:mk_pointer(WordTy), + hipe_llvm:mk_conversion(Dst, ptrtoint, WordTyPtr, Name, WordTy). + +%% @doc External declaration of a closure. +declare_closure({ClosureName, _})-> + ByteTy = hipe_llvm:mk_int(8), + hipe_llvm:mk_const_decl("@" ++ ClosureName, "external constant", ByteTy, ""). + +%% @doc Creation of local variable for a closure. +load_closure({ClosureName, _})-> + Dst = "%" ++ ClosureName ++ "_var", + Name = "@" ++ ClosureName, + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + hipe_llvm:mk_conversion(Dst, ptrtoint, ByteTyPtr, Name, WordTy). + +%% @doc Declaration of a local variable for a switch jump table. +declare_switches(JumpTableList, Fun) -> + FunName = trans_mfa_name(Fun), + [declare_switch_table(X, FunName) || X <- JumpTableList]. + +declare_switch_table({Name, {switch, {TableType, Labels, _, _}, _}}, FunName) -> + LabelList = [mk_jump_label(L) || L <- Labels], + Fun1 = fun(X) -> "i8* blockaddress(@" ++ FunName ++ ", " ++ X ++ ")" end, + List2 = lists:map(Fun1, LabelList), + List3 = string:join(List2, ",\n"), + List4 = "[\n" ++ List3 ++ "\n]\n", + hipe_llvm:mk_const_decl("@" ++ Name, "constant", TableType, List4). + +%% @doc Declaration of a variable for a table with the labels of all closure +%% calls in the code. +declare_closure_labels([], Relocs, _Fun) -> + {[], Relocs}; +declare_closure_labels(ClosureLabels, Relocs, Fun) -> + FunName = trans_mfa_name(Fun), + {LabelList, ArityList} = + lists:unzip([{mk_jump_label(Label), A} || + {_, {closure_label, Label, A}} <- ClosureLabels]), + Relocs1 = relocs_store("table_closures", {table_closures, ArityList}, Relocs), + List2 = + ["i8* blockaddress(@" ++ FunName ++ ", " ++ L ++ ")" || L <- LabelList], + List3 = string:join(List2, ",\n"), + List4 = "[\n" ++ List3 ++ "\n]\n", + NrLabels = length(LabelList), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + TableType = hipe_llvm:mk_array(NrLabels, ByteTyPtr), + ConstDecl = + hipe_llvm:mk_const_decl("@table_closures", "constant", TableType, List4), + {[ConstDecl], Relocs1}. + +%% @doc A call is treated as non external only in a case of a recursive +%% function. +is_external_call({_, {call, Fun}}, Fun) -> false; +is_external_call(_, _) -> true. + +%% @doc External declaration of a function. +call_to_decl({Name, {call, MFA}}) -> + {M, _F, A} = MFA, + CConv = "cc 11", + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + {Type, Args} = + case M of + llvm -> + {hipe_llvm:mk_struct([WordTy, hipe_llvm:mk_int(1)]), [1, 2]}; + %% +precoloured regs + _ -> + {FunRetTy, lists:seq(1, A + ?NR_PINNED_REGS)} + end, + ArgsTypes = lists:duplicate(length(Args), WordTy), + hipe_llvm:mk_fun_decl([], [], CConv, [], Type, "@" ++ Name, ArgsTypes, []). + +%% @doc These functions are always declared, even if not used. +fixed_fun_decl() -> + ByteTy = hipe_llvm:mk_int(8), + ByteTyPtr = hipe_llvm:mk_pointer(ByteTy), + LandPad = hipe_llvm:mk_fun_decl([], [], [], [], hipe_llvm:mk_int(32), + "@__gcc_personality_v0", [hipe_llvm:mk_int(32), hipe_llvm:mk_int(64), + ByteTyPtr, ByteTyPtr], []), + GCROOTDecl = hipe_llvm:mk_fun_decl([], [], [], [], hipe_llvm:mk_void(), + "@llvm.gcroot", [hipe_llvm:mk_pointer(ByteTyPtr), ByteTyPtr], []), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)), + FixPinnedRegs = hipe_llvm:mk_fun_decl([], [], [], [], FunRetTy, + "@hipe_bifs.llvm_fix_pinned_regs.0", [], []), + GcMetadata = hipe_llvm:mk_const_decl("@gc_metadata", "external constant", + ByteTy, ""), + [LandPad, GCROOTDecl, FixPinnedRegs, GcMetadata]. + +%% @doc Declare an External Consant. We declare all constants as i8 in order to +%% be able to calcucate pointers of the form DL+6, with the getelementptr +%% instruction. Otherwise we have to convert constants form pointers to +%% values, add the offset and convert them again to pointers. +declare_constant(Label) -> + Name = "@DL" ++ integer_to_list(Label), + ByteTy = hipe_llvm:mk_int(8), + hipe_llvm:mk_const_decl(Name, "external constant", ByteTy, ""). + +%% @doc Load a constant is achieved by converting a pointer to an integer of +%% the correct width. +load_constant(Label) -> + Dst = "%DL" ++ integer_to_list(Label) ++ "_var", + Name = "@DL" ++ integer_to_list(Label), + WordTy = hipe_llvm:mk_int(?WORD_WIDTH), + ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(8)), + hipe_llvm:mk_conversion(Dst, ptrtoint, ByteTyPtr, Name, WordTy). + +%% @doc Store external constants and calls to dictionary. +const_to_dict(Elem, Dict) -> + Name = "DL" ++ integer_to_list(Elem), + dict:store(Name, {'constant', Elem}, Dict). diff --git a/lib/hipe/main/hipe.app.src b/lib/hipe/main/hipe.app.src index bcdfcb0e03..e81212d4dc 100644 --- a/lib/hipe/main/hipe.app.src +++ b/lib/hipe/main/hipe.app.src @@ -30,6 +30,7 @@ cerl_prettypr, cerl_to_icode, cerl_typean, + elf_format, erl_bif_types, erl_types, hipe, @@ -108,6 +109,10 @@ hipe_ig, hipe_ig_moves, hipe_jit, + hipe_llvm, + hipe_llvm_liveness, + hipe_llvm_main, + hipe_llvm_merge, hipe_ls_regalloc, hipe_main, hipe_moves, @@ -159,6 +164,7 @@ hipe_rtl_symbolic, hipe_rtl_to_amd64, hipe_rtl_to_arm, + hipe_rtl_to_llvm, hipe_rtl_to_ppc, hipe_rtl_to_sparc, hipe_rtl_to_x86, @@ -216,4 +222,6 @@ hipe_x86_x87]}, {registered,[]}, {applications, [kernel,stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["syntax_tools-1.6.14","stdlib-2.0","kernel-3.0", + "erts-6.0","compiler-5.0"]}]}. diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index 434d5c3061..d47eced6d8 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -821,7 +821,9 @@ finalize_fun_sequential({MFA, Icode}, Opts, Servers) -> ?debug_msg("Compiled ~w in ~.2f s\n", [MFA,(T2-T1)/1000])), {MFA, Code}; {rtl, LinearRtl} -> - {MFA, LinearRtl} + {MFA, LinearRtl}; + {llvm_binary, Binary} -> + {MFA, Binary} catch error:Error -> ?when_option(verbose, Opts, ?debug_untagged_msg("\n", [])), @@ -890,21 +892,27 @@ do_load(Mod, Bin, BeamBinOrPath) when is_binary(BeamBinOrPath); end. assemble(CompiledCode, Closures, Exports, Options) -> - case get(hipe_target_arch) of - ultrasparc -> - hipe_sparc_assemble:assemble(CompiledCode, Closures, Exports, Options); - powerpc -> - hipe_ppc_assemble:assemble(CompiledCode, Closures, Exports, Options); - ppc64 -> - hipe_ppc_assemble:assemble(CompiledCode, Closures, Exports, Options); - arm -> - hipe_arm_assemble:assemble(CompiledCode, Closures, Exports, Options); - x86 -> - hipe_x86_assemble:assemble(CompiledCode, Closures, Exports, Options); - amd64 -> - hipe_amd64_assemble:assemble(CompiledCode, Closures, Exports, Options); - Arch -> - ?EXIT({executing_on_an_unsupported_architecture, Arch}) + case proplists:get_bool(to_llvm, Options) of + false -> + case get(hipe_target_arch) of + ultrasparc -> + hipe_sparc_assemble:assemble(CompiledCode, Closures, Exports, Options); + powerpc -> + hipe_ppc_assemble:assemble(CompiledCode, Closures, Exports, Options); + ppc64 -> + hipe_ppc_assemble:assemble(CompiledCode, Closures, Exports, Options); + arm -> + hipe_arm_assemble:assemble(CompiledCode, Closures, Exports, Options); + x86 -> + hipe_x86_assemble:assemble(CompiledCode, Closures, Exports, Options); + amd64 -> + hipe_amd64_assemble:assemble(CompiledCode, Closures, Exports, Options); + Arch -> + ?EXIT({executing_on_an_unsupported_architecture, Arch}) + end; + true -> + %% Merge already compiled code (per MFA) to a single binary. + hipe_llvm_merge:finalize(CompiledCode, Closures, Exports) end. %% -------------------------------------------------------------------- @@ -1330,6 +1338,11 @@ opt_keys() -> timeregalloc, timers, to_rtl, + to_llvm, % Use the LLVM backend for compilation. + llvm_save_temps, % Save the LLVM intermediate files in the current + % directory; useful for debugging. + llvm_llc, % Specify llc optimization-level: o1, o2, o3, undefined. + llvm_opt, % Specify opt optimization-level: o1, o2, o3, undefined. use_indexing, use_inline_atom_search, use_callgraph, @@ -1468,11 +1481,19 @@ opt_expansions() -> [{o1, o1_opts()}, {o2, o2_opts()}, {o3, o3_opts()}, + {to_llvm, llvm_opts(o3)}, + {{to_llvm, o0}, llvm_opts(o0)}, + {{to_llvm, o1}, llvm_opts(o1)}, + {{to_llvm, o2}, llvm_opts(o2)}, + {{to_llvm, o3}, llvm_opts(o3)}, {x87, [x87, inline_fp]}, {inline_fp, case get(hipe_target_arch) of %% XXX: Temporary until x86 x86 -> [x87, inline_fp]; %% has sse2 _ -> [inline_fp] end}]. +llvm_opts(O) -> + [to_llvm, {llvm_opt, O}, {llvm_llc, O}]. + %% This expands "basic" options, which may be tested early and cannot be %% in conflict with options found in the source code. diff --git a/lib/hipe/main/hipe_main.erl b/lib/hipe/main/hipe_main.erl index 99028cc3c1..89b79998be 100644 --- a/lib/hipe/main/hipe_main.erl +++ b/lib/hipe/main/hipe_main.erl @@ -49,7 +49,7 @@ %%===================================================================== -type comp_icode_ret() :: {'native',hipe_architecture(),{'unprofiled',_}} - | {'rtl',tuple()}. + | {'rtl',tuple()} | {'llvm_binary',term()}. %%===================================================================== @@ -115,11 +115,18 @@ compile_icode(MFA, LinearIcode0, Options, Servers, DebugState) -> pp(IcodeCfg7, MFA, icode_liveness, pp_icode_liveness, Options, Servers), FinalIcode = hipe_icode_cfg:cfg_to_linear(IcodeCfg7), ?opt_stop_timer("Icode"), - LinearRTL = ?option_time(icode_to_rtl(MFA,FinalIcode,Options, Servers), - "RTL", Options), + {LinearRTL, Roots} = ?option_time(icode_to_rtl(MFA, FinalIcode, Options, Servers), + "RTL", Options), case proplists:get_bool(to_rtl, Options) of false -> - rtl_to_native(MFA, LinearRTL, Options, DebugState); + case proplists:get_bool(to_llvm, Options) of + false -> + rtl_to_native(MFA, LinearRTL, Options, DebugState); + true -> + %% The LLVM backend returns binary code, unlike the rest of the HiPE + %% backends which return native assembly. + rtl_to_llvm_to_binary(MFA, LinearRTL, Roots, Options, DebugState) + end; true -> put(hipe_debug, DebugState), {rtl, LinearRTL} @@ -385,11 +392,21 @@ icode_to_rtl(MFA, Icode, Options, Servers) -> %% hipe_rtl_cfg:pp(RtlCfg3), pp(RtlCfg3, MFA, rtl_liveness, pp_rtl_liveness, Options, Servers), RtlCfg4 = rtl_lcm(RtlCfg3, Options), - pp(RtlCfg4, MFA, rtl, pp_rtl, Options, Servers), - LinearRTL1 = hipe_rtl_cfg:linearize(RtlCfg4), + %% LLVM: A liveness analysis on RTL must be performed in order to find the GC + %% roots and explicitly mark them (in RTL) when they go out of scope (only + %% when the LLVM backend is used). + {RtlCfg5, Roots} = + case proplists:get_bool(to_llvm, Options) of + false -> + {RtlCfg4, []}; + true -> + hipe_llvm_liveness:analyze(RtlCfg4) + end, + pp(RtlCfg5, MFA, rtl, pp_rtl, Options, Servers), + LinearRTL1 = hipe_rtl_cfg:linearize(RtlCfg5), LinearRTL2 = hipe_rtl_cleanup_const:cleanup(LinearRTL1), %% hipe_rtl:pp(standard_io, LinearRTL2), - LinearRTL2. + {LinearRTL2, Roots}. translate_to_rtl(Icode, Options) -> %% GC tests should have been added in the conversion to Icode. @@ -540,6 +557,17 @@ rtl_to_native(MFA, LinearRTL, Options, DebugState) -> put(hipe_debug, DebugState), LinearNativeCode. +%% Translate Linear RTL to binary code using LLVM. +rtl_to_llvm_to_binary(MFA, LinearRTL, Roots, Options, DebugState) -> + ?opt_start_timer("LLVM native code"), + %% BinaryCode is a tuple, as defined in llvm/hipe_llvm_main module, which + %% contains the binary code together with info needed by the loader, e.g. + %% ConstTab, Refs, LabelMap, etc. + BinaryCode = hipe_llvm_main:rtl_to_native(MFA, LinearRTL, Roots, Options), + ?opt_stop_timer("LLVM native code"), + put(hipe_debug, DebugState), + {llvm_binary, BinaryCode}. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Debugging stuff ... %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/hipe/misc/hipe_gensym.erl b/lib/hipe/misc/hipe_gensym.erl index 84fc8fa7e8..4d2a237188 100644 --- a/lib/hipe/misc/hipe_gensym.erl +++ b/lib/hipe/misc/hipe_gensym.erl @@ -44,7 +44,7 @@ %% Types of allowable entities to set global variables for %%----------------------------------------------------------------------- --type gvarname() :: 'icode' | 'rtl' | 'arm' | 'ppc' | 'sparc' | 'x86'. +-type gvarname() :: 'icode' | 'rtl' | 'arm' | 'ppc' | 'sparc' | 'x86' | 'llvm'. %%----------------------------------------------------------------------- diff --git a/lib/hipe/misc/hipe_pack_constants.erl b/lib/hipe/misc/hipe_pack_constants.erl index ca8a9e6bf7..300f9ae43a 100644 --- a/lib/hipe/misc/hipe_pack_constants.erl +++ b/lib/hipe/misc/hipe_pack_constants.erl @@ -20,30 +20,48 @@ %% -module(hipe_pack_constants). --export([pack_constants/2, slim_refs/1, slim_constmap/1]). +-export([pack_constants/2, slim_refs/1, slim_constmap/1, + find_const/2, mk_data_relocs/2, slim_sorted_exportmap/3]). -include("hipe_consttab.hrl"). -include("../../kernel/src/hipe_ext_format.hrl"). +-include("../main/hipe.hrl"). % Needed for the EXIT macro in find_const/2. %%----------------------------------------------------------------------------- --type raw_data() :: binary() | number() | list() | tuple(). --type tbl_ref() :: {hipe_constlbl(), non_neg_integer()}. +-type const_num() :: non_neg_integer(). +-type raw_data() :: binary() | number() | list() | tuple(). + +-type addr() :: non_neg_integer(). +-type ref_p() :: {DataPos :: hipe_constlbl(), CodeOffset :: addr()}. +-type ref() :: ref_p() | {'sorted', Base :: addr(), [ref_p()]}. + +-type mfa_refs() :: {mfa(), [ref()]}. + +%% XXX: these types may not belong here: FIX! +-type fa() :: {atom(), arity()}. +-type export_map() :: [{addr(), module(), atom(), arity()}]. -record(pcm_entry, {mfa :: mfa(), label :: hipe_constlbl(), - const_num :: non_neg_integer(), - start :: non_neg_integer(), + const_num :: const_num(), + start :: addr(), type :: 0 | 1 | 2, raw_data :: raw_data()}). +-type pcm_entry() :: #pcm_entry{}. + +-type label_map() :: gb_trees:tree({mfa(), hipe_constlbl()}, addr()). + +%% Some of the following types may possibly need to be exported +-type data_relocs() :: [ref()]. +-type packed_const_map() :: [pcm_entry()]. +-type mfa_refs_map() :: [mfa_refs()]. +-type slim_export_map() :: [addr() | module() | atom() | arity() | boolean()]. %%----------------------------------------------------------------------------- -spec pack_constants([{mfa(),[_],hipe_consttab()}], ct_alignment()) -> - {ct_alignment(), - non_neg_integer(), - [#pcm_entry{}], - [{mfa(),[tbl_ref() | {'sorted',non_neg_integer(),[tbl_ref()]}]}]}. + {ct_alignment(), non_neg_integer(), packed_const_map(), mfa_refs_map()}. pack_constants(Data, Align) -> pack_constants(Data, 0, Align, 0, [], []). @@ -194,13 +212,12 @@ compact_dests([], Dest, AccofDest, Acc) -> %% to the slimmed and flattened format ConstMap which is put in object %% files. %% --spec slim_constmap([#pcm_entry{}]) -> [raw_data()]. +-spec slim_constmap(packed_const_map()) -> [raw_data()]. slim_constmap(Map) -> slim_constmap(Map, gb_sets:new(), []). --spec slim_constmap([#pcm_entry{}], gb_sets:set(), [raw_data()]) -> [raw_data()]. -slim_constmap([#pcm_entry{const_num=ConstNo, start=Offset, - type=Type, raw_data=Term}|Rest], Inserted, Acc) -> +slim_constmap([#pcm_entry{const_num = ConstNo, start = Offset, + type = Type, raw_data = Term}|Rest], Inserted, Acc) -> case gb_sets:is_member(ConstNo, Inserted) of true -> slim_constmap(Rest, Inserted, Acc); @@ -209,3 +226,60 @@ slim_constmap([#pcm_entry{const_num=ConstNo, start=Offset, slim_constmap(Rest, NewInserted, [ConstNo, Offset, Type, Term|Acc]) end; slim_constmap([], _Inserted, Acc) -> Acc. + +%% +%% Lookup a constant in a ConstMap. +%% +-spec find_const({mfa(), hipe_constlbl()}, packed_const_map()) -> const_num(). + +find_const({MFA, Label}, [E = #pcm_entry{mfa = MFA, label = Label}|_]) -> + E#pcm_entry.const_num; +find_const(N, [_|R]) -> + find_const(N, R); +find_const(C, []) -> + ?EXIT({constant_not_found, C}). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%% +%% Functions to build and handle Refs, ExportMap and LabelMap. +%% Note: Moved here because they are used by all backends in +%% hipe_{arm,sparc,ppc,x86}_assemble.erl +%% XXX: Is this the right place for them? +%% + +-spec mk_data_relocs(mfa_refs_map(), label_map()) -> data_relocs(). + +mk_data_relocs(RefsFromConsts, LabelMap) -> + lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). + +mk_data_relocs([{MFA, Labels} | Rest], LabelMap, Acc) -> + Map = [case Label of + {L,Pos} -> + Offset = find({MFA,L}, LabelMap), + {Pos,Offset}; + {sorted,Base,OrderedLabels} -> + {sorted, Base, [begin + Offset = find({MFA,L}, LabelMap), + {Order, Offset} + end + || {L,Order} <- OrderedLabels]} + end + || Label <- Labels], + %% msg("Map: ~w Map\n", [Map]), + mk_data_relocs(Rest, LabelMap, [Map,Acc]); +mk_data_relocs([], _, Acc) -> Acc. + +find({MFA,L}, LabelMap) -> + gb_trees:get({MFA,L}, LabelMap). + +-spec slim_sorted_exportmap(export_map(), [mfa()], [fa()]) -> slim_export_map(). + +slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> + IsClosure = lists:member({M,F,A}, Closures), + IsExported = is_exported(F, A, Exports), + [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; +slim_sorted_exportmap([], _, _) -> []. + +is_exported(F, A, Exports) -> + lists:member({F,A}, Exports). diff --git a/lib/hipe/ppc/hipe_ppc_assemble.erl b/lib/hipe/ppc/hipe_ppc_assemble.erl index b2fd50517b..3ad91f4051 100644 --- a/lib/hipe/ppc/hipe_ppc_assemble.erl +++ b/lib/hipe/ppc/hipe_ppc_assemble.erl @@ -46,8 +46,8 @@ assemble(CompiledCode, Closures, Exports, Options) -> print("Total num bytes=~w\n", [CodeSize], Options), %% SC = hipe_pack_constants:slim_constmap(ConstMap), - DataRelocs = mk_data_relocs(RefsFromConsts, LabelMap), - SSE = slim_sorted_exportmap(ExportMap,Closures,Exports), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap,Closures,Exports), SlimRefs = hipe_pack_constants:slim_refs(AccRefs), Bin = term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, ConstAlign, ConstSize, @@ -288,7 +288,7 @@ do_pseudo_li(I, MFA, ConstMap) -> %%% end, %%% {load_address, {Tag,untag_mfa_or_prim(MFAorPrim)}}; {Label,constant} -> - ConstNo = find_const({MFA,Label}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,Label}, ConstMap), {load_address, {constant,ConstNo}}; {Label,closure} -> {load_address, {closure,Label}}; @@ -574,37 +574,6 @@ mk_y(Pred, BD) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -mk_data_relocs(RefsFromConsts, LabelMap) -> - lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). - -mk_data_relocs([{MFA,Labels} | Rest], LabelMap, Acc) -> - Map = [case Label of - {L,Pos} -> - Offset = find({MFA,L}, LabelMap), - {Pos,Offset}; - {sorted,Base,OrderedLabels} -> - {sorted, Base, [begin - Offset = find({MFA,L}, LabelMap), - {Order, Offset} - end - || {L,Order} <- OrderedLabels]} - end - || Label <- Labels], - %% msg("Map: ~w Map\n",[Map]), - mk_data_relocs(Rest, LabelMap, [Map,Acc]); -mk_data_relocs([],_,Acc) -> Acc. - -find({_MFA,_L} = MFAL,LabelMap) -> - gb_trees:get(MFAL, LabelMap). - -slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> - IsClosure = lists:member({M,F,A}, Closures), - IsExported = is_exported(F, A, Exports), - [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; -slim_sorted_exportmap([],_,_) -> []. - -is_exported(F, A, Exports) -> lists:member({F,A}, Exports). - %%% %%% Assembly listing support (pp_asm option). %%% @@ -642,14 +611,3 @@ fill_spaces(N) when N > 0 -> fill_spaces(N-1); fill_spaces(0) -> []. - -%%% -%%% Lookup a constant in a ConstMap. -%%% - -find_const({MFA,Label}, [{pcm_entry,MFA,Label,ConstNo,_,_,_}|_]) -> - ConstNo; -find_const(N, [_|R]) -> - find_const(N, R); -find_const(C, []) -> - ?EXIT({constant_not_found,C}). diff --git a/lib/hipe/rtl/hipe_rtl.erl b/lib/hipe/rtl/hipe_rtl.erl index 4bf4eb6bd7..bc61bec0bd 100644 --- a/lib/hipe/rtl/hipe_rtl.erl +++ b/lib/hipe/rtl/hipe_rtl.erl @@ -29,7 +29,7 @@ %% <li> {alu, Dst, Src1, Op, Src2} </li> %% <li> {alub, Dst, Src1, Op, Src2, RelOp, TrueLabel, FalseLabel, P} </li> %% <li> {branch, Src1, Src2, RelOp, TrueLabel, FalseLabel, P} </li> -%% <li> {call, DsListt, Fun, ArgList, Type, Continuation, FailContinuation} +%% <li> {call, DsListt, Fun, ArgList, Type, Continuation, FailContinuation, NormalContinuation} %% Type is one of {local, remote, primop, closure} </li> %% <li> {comment, Text} </li> %% <li> {enter, Fun, ArgList, Type} @@ -106,7 +106,7 @@ %% rtl_data_update/2, %% rtl_var_range/1, %% rtl_var_range_update/2, - %% rtl_label_range/1, + rtl_label_range/1, %% rtl_label_range_update/2, rtl_info/1, rtl_info_update/2]). @@ -226,6 +226,7 @@ %% goto_label_update/2, mk_call/6, + mk_call/7, call_fun/1, call_dstlist/1, call_dstlist_update/2, @@ -233,8 +234,10 @@ call_continuation/1, call_fail/1, call_type/1, + call_normal/1, + call_normal_update/2, %% call_continuation_update/2, - %% call_fail_update/2, + call_fail_update/2, is_call/1, mk_enter/3, @@ -290,10 +293,13 @@ %% fconv_src_update/2, %% is_fconv/1, - %% mk_var/1, + mk_var/1, + mk_var/2, mk_new_var/0, is_var/1, var_index/1, + var_liveness/1, + var_liveness_update/2, %% change_vars_to_regs/1, @@ -350,10 +356,15 @@ %% move_dst_update/2, fixnumop_dst_update/2, pp_instr/2, - %% pp_arg/2, + %% Uber hack! + pp_var/2, + pp_reg/2, + pp_arg/2, phi_arglist_update/2, phi_redirect_pred/3]). +-export([subst_uses_llvm/2]). + -export_type([alub_cond/0]). %% @@ -387,7 +398,7 @@ rtl_data(#rtl{data=Data}) -> Data. %% rtl_data_update(Rtl, Data) -> Rtl#rtl{data=Data}. %% rtl_var_range(#rtl{var_range=VarRange}) -> VarRange. %% rtl_var_range_update(Rtl, VarRange) -> Rtl#rtl{var_range=VarRange}. -%% rtl_label_range(#rtl{label_range=LabelRange}) -> LabelRange. +rtl_label_range(#rtl{label_range=LabelRange}) -> LabelRange. %% rtl_label_range_update(Rtl, LabelRange) -> Rtl#rtl{label_range=LabelRange}. rtl_info(#rtl{info=Info}) -> Info. rtl_info_update(Rtl, Info) -> Rtl#rtl{info=Info}. @@ -643,6 +654,17 @@ is_goto(_) -> false. %% call %% +%% LLVM: Call with normal continuation +mk_call(DstList, Fun, ArgList, Continuation, FailContinuation, + NormalContinuation, Type) -> + case Type of + remote -> ok; + not_remote -> ok + end, + #call{dstlist=DstList, 'fun'=Fun, arglist=ArgList, type=Type, + continuation=Continuation, failcontinuation=FailContinuation, + normalcontinuation=NormalContinuation}. + mk_call(DstList, Fun, ArgList, Continuation, FailContinuation, Type) -> case Type of remote -> ok; @@ -651,6 +673,10 @@ mk_call(DstList, Fun, ArgList, Continuation, FailContinuation, Type) -> #call{dstlist=DstList, 'fun'=Fun, arglist=ArgList, type=Type, continuation=Continuation, failcontinuation=FailContinuation}. + +call_normal(#call{normalcontinuation=NormalContinuation}) -> NormalContinuation. +call_normal_update(C, NewNormalContinuation) -> + C#call{normalcontinuation=NewNormalContinuation}. call_dstlist(#call{dstlist=DstList}) -> DstList. call_dstlist_update(C, NewDstList) -> C#call{dstlist=NewDstList}. call_fun(#call{'fun'=Fun}) -> Fun. @@ -853,11 +879,14 @@ reg_is_gcsafe(#rtl_reg{is_gc_safe=IsGcSafe}) -> IsGcSafe. is_reg(#rtl_reg{}) -> true; is_reg(_) -> false. --record(rtl_var, {index :: non_neg_integer()}). +-record(rtl_var, {index :: non_neg_integer(), liveness=live :: dead | live}). mk_var(Num) when is_integer(Num), Num >= 0 -> #rtl_var{index=Num}. +mk_var(Num, Liveness) when is_integer(Num), Num>=0 -> #rtl_var{index=Num, liveness=Liveness}. mk_new_var() -> mk_var(hipe_gensym:get_next_var(rtl)). var_index(#rtl_var{index=Index}) -> Index. +var_liveness(#rtl_var{liveness=Liveness}) -> Liveness. +var_liveness_update(RtlVar, Liveness) -> RtlVar#rtl_var{liveness=Liveness}. is_var(#rtl_var{}) -> true; is_var(_) -> false. @@ -1077,6 +1106,131 @@ subst_uses(Subst, I) -> switch_src_update(I, subst1(Subst, switch_src(I))) end. +subst_uses_llvm(Subst, I) -> + case I of + #alu{} -> + {NewSrc2, Subst1} = subst1_llvm(Subst, alu_src2(I)), + {NewSrc1, _ } = subst1_llvm(Subst1, alu_src1(I)), + I0 = alu_src1_update(I, NewSrc1), + alu_src2_update(I0, NewSrc2); + #alub{} -> + {NewSrc2, Subst1} = subst1_llvm(Subst, alub_src2(I)), + {NewSrc1, _ } = subst1_llvm(Subst1, alub_src1(I)), + I0 = alub_src1_update(I, NewSrc1), + alub_src2_update(I0, NewSrc2); + #branch{} -> + {NewSrc2, Subst1} = subst1_llvm(Subst, branch_src2(I)), + {NewSrc1, _ } = subst1_llvm(Subst1, branch_src1(I)), + I0 = branch_src1_update(I, NewSrc1), + branch_src2_update(I0, NewSrc2); + #call{} -> + case call_is_known(I) of + false -> + {NewFun, Subst1} = subst1_llvm(Subst, call_fun(I)), + {NewArgList, _} = subst_list_llvm(Subst1, call_arglist(I)), + I0 = call_fun_update(I, NewFun), + call_arglist_update(I0, NewArgList); + true -> + {NewArgList, _} = subst_list_llvm(Subst, call_arglist(I)), + call_arglist_update(I, NewArgList) + end; + #comment{} -> + I; + #enter{} -> + case enter_is_known(I) of + false -> + {NewFun, Subst1} = subst1_llvm(Subst, enter_fun(I)), + {NewArgList, _} = subst_list_llvm(Subst1, enter_arglist(I)), + I0 = enter_fun_update(I, NewFun), + enter_arglist_update(I0, NewArgList); + true -> + {NewArgList, _} = subst_list_llvm(Subst, enter_arglist(I)), + enter_arglist_update(I, NewArgList) + end; + #fconv{} -> + {NewSrc, _ } = subst1_llvm(Subst, fconv_src(I)), + fconv_src_update(I, NewSrc); + #fixnumop{} -> + {NewSrc, _ } = subst1_llvm(Subst, fixnumop_src(I)), + fixnumop_src_update(I, NewSrc); + #fload{} -> + {NewSrc, Subst1} = subst1_llvm(Subst, fload_src(I)), + {NewOffset, _ } = subst1_llvm(Subst1, fload_offset(I)), + I0 = fload_src_update(I, NewSrc), + fload_offset_update(I0, NewOffset); + #fmove{} -> + {NewSrc, _ } = subst1_llvm(Subst, fmove_src(I)), + fmove_src_update(I, NewSrc); + #fp{} -> + {NewSrc2, Subst1} = subst1_llvm(Subst, fp_src2(I)), + {NewSrc1, _ } = subst1_llvm(Subst1, fp_src1(I)), + I0 = fp_src1_update(I, NewSrc1), + fp_src2_update(I0, NewSrc2); + #fp_unop{} -> + {NewSrc, _ } = subst1_llvm(Subst, fp_unop_src(I)), + fp_unop_src_update(I, NewSrc); + #fstore{} -> + {NewSrc, Subst1} = subst1_llvm(Subst, fstore_src(I)), + {NewBase, Subst2} = subst1_llvm(Subst1, fstore_base(I)), + {NewOffset, _ } = subst1_llvm(Subst2, fstore_offset(I)), + I0 = fstore_src_update(I, NewSrc), + I1 = fstore_base_update(I0, NewBase), + fstore_offset_update(I1, NewOffset); + #goto{} -> + I; + #goto_index{} -> + I; + #gctest{} -> + {NewWords, _ } = subst1_llvm(Subst, gctest_words(I)), + gctest_words_update(I, NewWords); + #label{} -> + I; + #load{} -> + {NewSrc, Subst1} = subst1_llvm(Subst, load_src(I)), + {NewOffset, _ } = subst1_llvm(Subst1, load_offset(I)), + I0 = load_src_update(I, NewSrc), + load_offset_update(I0, NewOffset); + #load_address{} -> + I; + #load_atom{} -> + I; + #load_word_index{} -> + I; + #move{} -> + {NewSrc, _ } = subst1_llvm(Subst, move_src(I)), + move_src_update(I, NewSrc); + #multimove{} -> + {NewSrcList, _} = subst_list_llvm(Subst, multimove_srclist(I)), + multimove_srclist_update(I, NewSrcList); + #phi{} -> + phi_argvar_subst(I, Subst); + #return{} -> + {NewVarList, _} = subst_list_llvm(Subst, return_varlist(I)), + return_varlist_update(I, NewVarList); + #store{} -> + {NewSrc, Subst1} = subst1_llvm(Subst, store_src(I)), + {NewBase, Subst2} = subst1_llvm(Subst1, store_base(I)), + {NewOffset, _ } = subst1_llvm(Subst2, store_offset(I)), + I0 = store_src_update(I, NewSrc), + I1 = store_base_update(I0, NewBase), + store_offset_update(I1, NewOffset); + #switch{} -> + {NewSrc, _ } = subst1_llvm(Subst, switch_src(I)), + switch_src_update(I, NewSrc) + end. + +subst_list_llvm(S,X) -> subst_list_llvm(S, lists:reverse(X), []). +subst_list_llvm(S, [], Acc) -> {Acc, S}; +subst_list_llvm(S, [X|Xs], Acc) -> + {NewX, RestS} = subst1_llvm(S, X), + subst_list_llvm(RestS, Xs, [NewX|Acc]). + +subst1_llvm(A,B) -> subst1_llvm(A,B,[]). + +subst1_llvm([], X, Acc) -> {X, Acc}; +subst1_llvm([{X,Y}|Rs], X, Acc) -> {Y, Acc++Rs}; +subst1_llvm([R|Xs], X, Acc) -> subst1_llvm(Xs,X,[R|Acc]). + subst_defines(Subst, I)-> case I of #alu{} -> @@ -1614,7 +1768,11 @@ pp_var(Dev, Arg) -> true -> pp_hard_reg(Dev, var_index(Arg)); false -> - io:format(Dev, "v~w", [var_index(Arg)]) + io:format(Dev, "v~w", [var_index(Arg)]), + case var_liveness(Arg) of + dead -> io:format(Dev, "(dead)", []); + _ -> ok + end end. pp_arg(Dev, A) -> diff --git a/lib/hipe/rtl/hipe_rtl.hrl b/lib/hipe/rtl/hipe_rtl.hrl index 974e40f830..fbdf9ac524 100644 --- a/lib/hipe/rtl/hipe_rtl.hrl +++ b/lib/hipe/rtl/hipe_rtl.hrl @@ -28,7 +28,8 @@ -record(alu, {dst, src1, op, src2}). -record(alub, {dst, src1, op, src2, 'cond', true_label, false_label, p}). -record(branch, {src1, src2, 'cond', true_label, false_label, p}). --record(call, {dstlist, 'fun', arglist, type, continuation, failcontinuation}). +-record(call, {dstlist, 'fun', arglist, type, continuation, + failcontinuation, normalcontinuation = []}). -record(comment, {text}). -record(enter, {'fun', arglist, type}). -record(fconv, {dst, src}). diff --git a/lib/hipe/rtl/hipe_rtl_liveness.erl b/lib/hipe/rtl/hipe_rtl_liveness.erl index 3cfada9d6c..0c4b6b2e11 100644 --- a/lib/hipe/rtl/hipe_rtl_liveness.erl +++ b/lib/hipe/rtl/hipe_rtl_liveness.erl @@ -34,7 +34,8 @@ -module(hipe_rtl_liveness). -%% -define(LIVEOUT_NEEDED,true). % needed for liveness.inc below. +%% -define(DEBUG_LIVENESS,true). +-define(LIVEOUT_NEEDED,true). % needed for liveness.inc below. -define(PRETTY_PRINT,false). -include("hipe_rtl.hrl"). diff --git a/lib/hipe/sparc/hipe_sparc_assemble.erl b/lib/hipe/sparc/hipe_sparc_assemble.erl index b534fe20ec..68a4e1b349 100644 --- a/lib/hipe/sparc/hipe_sparc_assemble.erl +++ b/lib/hipe/sparc/hipe_sparc_assemble.erl @@ -45,8 +45,8 @@ assemble(CompiledCode, Closures, Exports, Options) -> print("Total num bytes=~w\n", [CodeSize], Options), %% SC = hipe_pack_constants:slim_constmap(ConstMap), - DataRelocs = mk_data_relocs(RefsFromConsts, LabelMap), - SSE = slim_sorted_exportmap(ExportMap,Closures,Exports), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap,Closures,Exports), SlimRefs = hipe_pack_constants:slim_refs(AccRefs), Bin = term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, ConstAlign, ConstSize, @@ -222,7 +222,7 @@ do_pseudo_set(I, MFA, ConstMap) -> %%% end, %%% {load_address, {Tag,untag_mfa_or_prim(MFAorPrim)}}; {Label,constant} -> - ConstNo = find_const({MFA,Label}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,Label}, ConstMap), {load_address, {constant,ConstNo}}; {Label,closure} -> {load_address, {closure,Label}}; @@ -507,37 +507,6 @@ px({pred,Pred}) -> % XXX: use pt/pn throughout entire backend %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -mk_data_relocs(RefsFromConsts, LabelMap) -> - lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). - -mk_data_relocs([{MFA,Labels} | Rest], LabelMap, Acc) -> - Map = [case Label of - {L,Pos} -> - Offset = find({MFA,L}, LabelMap), - {Pos,Offset}; - {sorted,Base,OrderedLabels} -> - {sorted, Base, [begin - Offset = find({MFA,L}, LabelMap), - {Order, Offset} - end - || {L,Order} <- OrderedLabels]} - end - || Label <- Labels], - %% msg("Map: ~w Map\n",[Map]), - mk_data_relocs(Rest, LabelMap, [Map,Acc]); -mk_data_relocs([],_,Acc) -> Acc. - -find({_MFA,_L} = MFAL, LabelMap) -> - gb_trees:get(MFAL, LabelMap). - -slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> - IsClosure = lists:member({M,F,A}, Closures), - IsExported = is_exported(F, A, Exports), - [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; -slim_sorted_exportmap([],_,_) -> []. - -is_exported(F, A, Exports) -> lists:member({F,A}, Exports). - %%% %%% Assembly listing support (pp_asm option). %%% @@ -575,14 +544,3 @@ fill_spaces(N) when N > 0 -> fill_spaces(N-1); fill_spaces(0) -> []. - -%%% -%%% Lookup a constant in a ConstMap. -%%% - -find_const({MFA,Label},[{pcm_entry,MFA,Label,ConstNo,_,_,_}|_]) -> - ConstNo; -find_const(N,[_|R]) -> - find_const(N,R); -find_const(C,[]) -> - ?EXIT({constant_not_found,C}). diff --git a/lib/hipe/vsn.mk b/lib/hipe/vsn.mk index ed4b4dc8d2..fb7e4b91a0 100644 --- a/lib/hipe/vsn.mk +++ b/lib/hipe/vsn.mk @@ -1 +1 @@ -HIPE_VSN = 3.10.2.2 +HIPE_VSN = 3.10.3 diff --git a/lib/hipe/x86/hipe_x86_assemble.erl b/lib/hipe/x86/hipe_x86_assemble.erl index 7878c7219d..3f756769c4 100644 --- a/lib/hipe/x86/hipe_x86_assemble.erl +++ b/lib/hipe/x86/hipe_x86_assemble.erl @@ -21,7 +21,6 @@ %%% %%% TODO: %%% - Simplify combine_label_maps and mk_data_relocs. -%%% - Move find_const to hipe_pack_constants? -ifdef(HIPE_AMD64). -define(HIPE_X86_ASSEMBLE, hipe_amd64_assemble). @@ -80,8 +79,8 @@ assemble(CompiledCode, Closures, Exports, Options) -> %% ?debug_msg("Constants are ~w bytes\n",[ConstSize])), %% SC = hipe_pack_constants:slim_constmap(ConstMap), - DataRelocs = mk_data_relocs(RefsFromConsts, LabelMap), - SSE = slim_sorted_exportmap(ExportMap,Closures,Exports), + DataRelocs = hipe_pack_constants:mk_data_relocs(RefsFromConsts, LabelMap), + SSE = hipe_pack_constants:slim_sorted_exportmap(ExportMap,Closures,Exports), SlimRefs = hipe_pack_constants:slim_refs(AccRefs), Bin = term_to_binary([{?VERSION_STRING(),?HIPE_SYSTEM_CRC}, ConstAlign, ConstSize, @@ -442,7 +441,7 @@ translate_imm(#x86_imm{value=Imm}, Context, MayTrunc8) -> case Imm of {Label,constant} -> {MFA,ConstMap} = Context, - ConstNo = find_const({MFA,Label}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,Label}, ConstMap), {constant,ConstNo}; {Label,closure} -> {closure,Label}; @@ -712,7 +711,7 @@ resolve_jmp_switch_arg(I, _Context) -> {rm64,hipe_amd64_encode:rm_mem(EA)}. -else. resolve_jmp_switch_arg(I, {MFA,ConstMap}) -> - ConstNo = find_const({MFA,hipe_x86:jmp_switch_jtab(I)}, ConstMap), + ConstNo = hipe_pack_constants:find_const({MFA,hipe_x86:jmp_switch_jtab(I)}, ConstMap), Disp32 = {?LOAD_ADDRESS,{constant,ConstNo}}, SINDEX = ?HIPE_X86_ENCODE:sindex(2, hipe_x86:temp_reg(hipe_x86:jmp_switch_temp(I))), EA = ?HIPE_X86_ENCODE:ea_disp32_sindex(Disp32, SINDEX), % this creates a SIB implicitly @@ -932,37 +931,6 @@ resolve_x87_binop_args(Src=#x86_fpreg{}, Dst=#x86_fpreg{})-> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -mk_data_relocs(RefsFromConsts, LabelMap) -> - lists:flatten(mk_data_relocs(RefsFromConsts, LabelMap, [])). - -mk_data_relocs([{MFA,Labels} | Rest], LabelMap, Acc) -> - Map = [case Label of - {L,Pos} -> - Offset = find({MFA,L}, LabelMap), - {Pos,Offset}; - {sorted,Base,OrderedLabels} -> - {sorted, Base, [begin - Offset = find({MFA,L}, LabelMap), - {Order, Offset} - end - || {L,Order} <- OrderedLabels]} - end - || Label <- Labels], - %% msg("Map: ~w Map\n",[Map]), - mk_data_relocs(Rest, LabelMap, [Map,Acc]); -mk_data_relocs([],_,Acc) -> Acc. - -find({MFA,L},LabelMap) -> - gb_trees:get({MFA,L}, LabelMap). - -slim_sorted_exportmap([{Addr,M,F,A}|Rest], Closures, Exports) -> - IsClosure = lists:member({M,F,A}, Closures), - IsExported = is_exported(F, A, Exports), - [Addr,M,F,A,IsClosure,IsExported | slim_sorted_exportmap(Rest, Closures, Exports)]; -slim_sorted_exportmap([],_,_) -> []. - -is_exported(F, A, Exports) -> lists:member({F,A}, Exports). - %%% %%% Assembly listing support (pp_asm option). %%% @@ -1001,14 +969,3 @@ fill_spaces(N) when N > 0 -> fill_spaces(N-1); fill_spaces(0) -> []. - -%%% -%%% Lookup a constant in a ConstMap. -%%% - -find_const({MFA,Label},[{pcm_entry,MFA,Label,ConstNo,_,_,_}|_]) -> - ConstNo; -find_const(N,[_|R]) -> - find_const(N,R); -find_const(C,[]) -> - ?EXIT({constant_not_found,C}). diff --git a/lib/ic/src/ic.app.src b/lib/ic/src/ic.app.src index 29aa6def00..7dd47ac9c6 100644 --- a/lib/ic/src/ic.app.src +++ b/lib/ic/src/ic.app.src @@ -46,7 +46,8 @@ {registered, []}, {applications, [stdlib, kernel]}, {env, []}, - {mod, {ic, []}} + {mod, {ic, []}}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/ic/vsn.mk b/lib/ic/vsn.mk index fe27d095d3..2ffbbad444 100644 --- a/lib/ic/vsn.mk +++ b/lib/ic/vsn.mk @@ -1 +1 @@ -IC_VSN = 4.3.4 +IC_VSN = 4.3.5 diff --git a/lib/inets/src/http_client/httpc_cookie.erl b/lib/inets/src/http_client/httpc_cookie.erl index 69900bae65..134115bdfa 100644 --- a/lib/inets/src/http_client/httpc_cookie.erl +++ b/lib/inets/src/http_client/httpc_cookie.erl @@ -335,7 +335,8 @@ add_domain(Str, #http_cookie{domain = Domain}) -> Str ++ "; $Domain=" ++ Domain. parse_set_cookies(CookieHeaders, DefaultPathDomain) -> - SetCookieHeaders = [Value || {"set-cookie", Value} <- CookieHeaders], + %% empty Set-Cookie header is invalid according to RFC but some sites violate it + SetCookieHeaders = [Value || {"set-cookie", Value} <- CookieHeaders, Value /= ""], Cookies = [parse_set_cookie(SetCookieHeader, DefaultPathDomain) || SetCookieHeader <- SetCookieHeaders], %% print_cookies("Parsed Cookies", Cookies), diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 5598185ad3..88e08be789 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1119,15 +1119,8 @@ handle_http_body(Body, #state{headers = Headers, handle_response(State#state{headers = NewHeaders, body = NewBody}) end; - Encoding when is_list(Encoding) -> - ?hcrt("handle_http_body - encoding", [{encoding, Encoding}]), - NewState = answer_request(Request, - httpc_response:error(Request, - unknown_encoding), - State), - {stop, normal, NewState}; - _ -> - ?hcrt("handle_http_body - other", []), + Enc when Enc =:= "identity"; Enc =:= undefined -> + ?hcrt("handle_http_body - identity", []), Length = list_to_integer(Headers#http_response_h.'content-length'), case ((Length =< MaxBodySize) orelse (MaxBodySize =:= nolimit)) of @@ -1149,12 +1142,19 @@ handle_http_body(Body, #state{headers = Headers, body_too_big), State), {stop, normal, NewState} - end + end; + Encoding when is_list(Encoding) -> + ?hcrt("handle_http_body - other", [{encoding, Encoding}]), + NewState = answer_request(Request, + httpc_response:error(Request, + unknown_encoding), + State), + {stop, normal, NewState} end. handle_response(#state{status = new} = State) -> ?hcrd("handle response - status = new", []), - handle_response(check_persistent(State)); + handle_response(try_to_enable_pipeline_or_keep_alive(State)); handle_response(#state{request = Request, status = Status, @@ -1429,22 +1429,39 @@ is_keep_alive_enabled_server(_,_) -> is_keep_alive_connection(Headers, #session{client_close = ClientClose}) -> (not ((ClientClose) orelse httpc_response:is_server_closing(Headers))). -check_persistent( - #state{session = #session{type = Type} = Session, +try_to_enable_pipeline_or_keep_alive( + #state{session = Session, + request = #request{method = Method}, status_line = {Version, _, _}, headers = Headers, - profile_name = ProfileName} = State) -> + profile_name = ProfileName} = State) -> + ?hcrd("try to enable pipeline or keep-alive", + [{version, Version}, + {headers, Headers}, + {session, Session}]), case is_keep_alive_enabled_server(Version, Headers) andalso - is_keep_alive_connection(Headers, Session) of + is_keep_alive_connection(Headers, Session) of true -> - mark_persistent(ProfileName, Session), - State#state{status = Type}; + case (is_pipeline_enabled_client(Session) andalso + httpc_request:is_idempotent(Method)) of + true -> + insert_session(Session, ProfileName), + State#state{status = pipeline}; + false -> + insert_session(Session, ProfileName), + %% Make sure type is keep_alive in session + %% as it in this case might be pipeline + NewSession = Session#session{type = keep_alive}, + State#state{status = keep_alive, + session = NewSession} + end; false -> State#state{status = close} end. answer_request(#request{id = RequestId, from = From} = Request, Msg, - #state{timers = Timers, + #state{session = Session, + timers = Timers, profile_name = ProfileName} = State) -> ?hcrt("answer request", [{request, Request}, {msg, Msg}]), httpc_response:send(From, Msg), @@ -1454,14 +1471,19 @@ answer_request(#request{id = RequestId, from = From} = Request, Msg, Timer = {RequestId, TimerRef}, cancel_timer(TimerRef, {timeout, Request#request.id}), httpc_manager:request_done(RequestId, ProfileName), + NewSession = maybe_make_session_available(ProfileName, Session), Timers2 = Timers#timers{request_timers = lists:delete(Timer, RequestTimers)}, State#state{request = Request#request{from = answer_sent}, + session = NewSession, timers = Timers2}. -mark_persistent(ProfileName, Session) -> - update_session(ProfileName, Session, #session.persistent, true), - Session#session{persistent = true}. +maybe_make_session_available(ProfileName, + #session{available = false} = Session) -> + update_session(ProfileName, Session, #session.available, true), + Session#session{available = true}; +maybe_make_session_available(_ProfileName, Session) -> + Session. cancel_timers(#timers{request_timers = ReqTmrs, queue_timer = QTmr}) -> cancel_timer(QTmr, timeout_queue), diff --git a/lib/inets/src/http_client/httpc_internal.hrl b/lib/inets/src/http_client/httpc_internal.hrl index d5b3dd2a2a..add5d11dfa 100644 --- a/lib/inets/src/http_client/httpc_internal.hrl +++ b/lib/inets/src/http_client/httpc_internal.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -143,8 +143,8 @@ %% true | false %% This will be true, when a response has been received for - %% the first request and the server has not closed the connection - persistent = false + %% the first request. See type above. + available = false }). diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index a3ed371e61..48a9c32454 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2013. All Rights Reserved. +%% Copyright Ericsson AB 2002-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -451,7 +451,7 @@ do_init(ProfileName, CookiesDir) -> %%-------------------------------------------------------------------- handle_call({request, Request}, _, State) -> ?hcri("request", [{request, Request}]), - case (catch handle_request(Request, State, false)) of + case (catch handle_request(Request, State)) of {reply, Msg, NewState} -> {reply, Msg, NewState}; Error -> @@ -511,7 +511,7 @@ handle_cast({retry_or_redirect_request, {Time, Request}}, {noreply, State}; handle_cast({retry_or_redirect_request, Request}, State) -> - case (catch handle_request(Request, State, true)) of + case (catch handle_request(Request, State)) of {reply, {ok, _}, NewState} -> {noreply, NewState}; Error -> @@ -724,7 +724,7 @@ get_handler_info(Tab) -> handle_request(#request{settings = #http_options{version = "HTTP/0.9"}} = Request, - State, _) -> + State) -> %% Act as an HTTP/0.9 client that does not know anything %% about persistent connections @@ -737,7 +737,7 @@ handle_request(#request{settings = handle_request(#request{settings = #http_options{version = "HTTP/1.0"}} = Request, - State, _) -> + State) -> %% Act as an HTTP/1.0 client that does not %% use persistent connections @@ -748,13 +748,13 @@ handle_request(#request{settings = start_handler(NewRequest#request{headers = NewHeaders}, State), {reply, {ok, NewRequest#request.id}, State}; -handle_request(Request, State = #state{options = Options}, Retry) -> +handle_request(Request, State = #state{options = Options}) -> NewRequest = handle_cookies(generate_request_id(Request), State), SessionType = session_type(Options), case select_session(Request#request.method, Request#request.address, - Request#request.scheme, SessionType, State, Retry) of + Request#request.scheme, SessionType, State) of {ok, HandlerPid} -> pipeline_or_keep_alive(NewRequest, HandlerPid, State); no_connection -> @@ -778,7 +778,6 @@ start_handler(#request{id = Id, #state{profile_name = ProfileName, handler_db = HandlerDb, options = Options}) -> - ClientClose = httpc_request:is_client_closing(Request#request.headers), {ok, Pid} = case is_inets_manager() of true -> @@ -789,18 +788,13 @@ start_handler(#request{id = Id, end, HandlerInfo = {Id, Pid, From}, ets:insert(HandlerDb, HandlerInfo), - insert_session(#session{id = {Request#request.address, Pid}, - scheme = Request#request.scheme, - client_close = ClientClose, - type = session_type(Options) - }, ProfileName), erlang:monitor(process, Pid). select_session(Method, HostPort, Scheme, SessionType, #state{options = #options{max_pipeline_length = MaxPipe, max_keep_alive_length = MaxKeepAlive}, - session_db = SessionDb}, Retry) -> + session_db = SessionDb}) -> ?hcrd("select session", [{session_type, SessionType}, {max_pipeline_length, MaxPipe}, {max_keep_alive_length, MaxKeepAlive}]), @@ -813,23 +807,13 @@ select_session(Method, HostPort, Scheme, SessionType, %% client_close, scheme and type specified. %% The fields id (part of: HandlerPid) and queue_length %% specified. - Pattern = case (Retry andalso SessionType == pipeline) of - true -> - #session{id = {HostPort, '$1'}, - client_close = false, - scheme = Scheme, - queue_length = '$2', - type = SessionType, - persistent = true, - _ = '_'}; - false -> - #session{id = {HostPort, '$1'}, - client_close = false, - scheme = Scheme, - queue_length = '$2', - type = SessionType, - _ = '_'} - end, + Pattern = #session{id = {HostPort, '$1'}, + client_close = false, + scheme = Scheme, + queue_length = '$2', + type = SessionType, + available = true, + _ = '_'}, %% {'_', {HostPort, '$1'}, false, Scheme, '_', '$2', SessionTyp}, Candidates = ets:match(SessionDb, Pattern), ?hcrd("select session", [{host_port, HostPort}, diff --git a/lib/inets/src/http_server/httpd_example.erl b/lib/inets/src/http_server/httpd_example.erl index 16a080f8e2..6fc07f033c 100644 --- a/lib/inets/src/http_server/httpd_example.erl +++ b/lib/inets/src/http_server/httpd_example.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -66,7 +66,7 @@ get_bin(_Env,_Input) -> <INPUT TYPE=\"text\" NAME=\"input2\"> <INPUT TYPE=\"submit\"><BR> </FORM>" ++ "\n"), - footer()]. + list_to_binary(footer())]. post(_Env,[]) -> [header(), diff --git a/lib/inets/src/inets_app/inets.app.src b/lib/inets/src/inets_app/inets.app.src index a6dd364c2d..9eae962d03 100644 --- a/lib/inets/src/inets_app/inets.app.src +++ b/lib/inets/src/inets_app/inets.app.src @@ -110,4 +110,6 @@ {registered,[inets_sup, httpc_manager]}, %% If the "new" ssl is used then 'crypto' must be started before inets. {applications,[kernel,stdlib]}, - {mod,{inets_app,[]}}]}. + {mod,{inets_app,[]}}, + {runtime_dependencies, ["stdlib-2.0","ssl-5.3.4","runtime_tools-1.8.14", + "mnesia-4.12","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/inets/test/Makefile b/lib/inets/test/Makefile index c156b34406..609396273d 100644 --- a/lib/inets/test/Makefile +++ b/lib/inets/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2013. All Rights Reserved. +# Copyright Ericsson AB 1997-2014. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index fe6edd504e..b1b799c953 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2013. All Rights Reserved. +%% Copyright Ericsson AB 2004-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -91,6 +91,7 @@ only_simulated() -> [ cookie, cookie_profile, + empty_set_cookie, trace, stream_once, no_content_204, @@ -104,6 +105,7 @@ only_simulated() -> remote_socket_close, remote_socket_close_async, transfer_encoding, + transfer_encoding_identity, redirect_loop, redirect_moved_permanently, redirect_multiple_choises, @@ -296,6 +298,9 @@ trace(Config) when is_list(Config) -> pipeline(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, {ok, _} = httpc:request(get, Request, [], [], pipeline), + + %% Make sure pipeline session is registerd + test_server:sleep(4000), keep_alive_requests(Request, pipeline). %%-------------------------------------------------------------------- @@ -303,6 +308,9 @@ pipeline(Config) when is_list(Config) -> persistent_connection(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, {ok, _} = httpc:request(get, Request, [], [], persistent), + + %% Make sure pipeline session is registerd + test_server:sleep(4000), keep_alive_requests(Request, persistent). %%------------------------------------------------------------------------- @@ -530,6 +538,19 @@ cookie_profile(Config) when is_list(Config) -> inets:stop(httpc, cookie_test). %%------------------------------------------------------------------------- +empty_set_cookie() -> + [{doc, "Test empty Set-Cookie header."}]. +empty_set_cookie(Config) when is_list(Config) -> + ok = httpc:set_options([{cookies, enabled}]), + + Request0 = {url(group_name(Config), "/empty_set_cookie.html", Config), []}, + + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, Request0, [], []), + + ok = httpc:set_options([{cookies, disabled}]). + +%%------------------------------------------------------------------------- headers_as_is(doc) -> ["Test the option headers_as_is"]; headers_as_is(Config) when is_list(Config) -> @@ -624,6 +645,12 @@ transfer_encoding(Config) when is_list(Config) -> %%------------------------------------------------------------------------- +transfer_encoding_identity(Config) when is_list(Config) -> + URL = url(group_name(Config), "/identity_transfer_encoding.html", Config), + {ok, {{_,200,_}, [_|_], "IDENTITY"}} = httpc:request(URL). + +%%------------------------------------------------------------------------- + empty_response_header() -> [{doc, "Test the case that the HTTP server does not send any headers. Solves OTP-6830"}]. empty_response_header(Config) when is_list(Config) -> @@ -1609,6 +1636,13 @@ handle_uri(_,"/capital_transfer_encoding.html",_,_,Socket,_) -> send(Socket, http_chunk:encode("obar</BODY></HTML>")), http_chunk:encode_last(); +handle_uri(_,"/identity_transfer_encoding.html",_,_,_,_) -> + "HTTP/1.0 200 OK\r\n" + "Transfer-Encoding:identity\r\n" + "Content-Length:8\r\n" + "\r\n" + "IDENTITY"; + handle_uri(_,"/cookie.html",_,_,_,_) -> "HTTP/1.1 200 ok\r\n" ++ "set-cookie:" ++ "test_cookie=true; path=/;" ++ @@ -1616,6 +1650,12 @@ handle_uri(_,"/cookie.html",_,_,_,_) -> "Content-Length:32\r\n\r\n"++ "<HTML><BODY>foobar</BODY></HTML>"; +handle_uri(_,"/empty_set_cookie.html",_,_,_,_) -> + "HTTP/1.1 200 ok\r\n" ++ + "set-cookie: \r\n" ++ + "Content-Length:32\r\n\r\n"++ + "<HTML><BODY>foobar</BODY></HTML>"; + handle_uri(_,"/missing_crlf.html",_,_,_,_) -> "HTTP/1.1 200 ok" ++ "Content-Length:32\r\n" ++ diff --git a/lib/inets/test/httpd_1_0.erl b/lib/inets/test/httpd_1_0.erl index 53f23b12e0..0836c9e881 100644 --- a/lib/inets/test/httpd_1_0.erl +++ b/lib/inets/test/httpd_1_0.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2013. All Rights Reserved. +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,7 +20,7 @@ -module(httpd_1_0). --export([host/4]). +-export([host/4, trace/4]). %%------------------------------------------------------------------------- %% Test cases @@ -31,3 +31,8 @@ host(Type, Port, Host, Node) -> "GET / HTTP/1.0\r\n\r\n", [{statuscode, 200}, {version, "HTTP/1.0"}]). +trace(Type, Port, Host, Node)-> + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "TRACE / HTTP/1.0\r\n\r\n", + [{statuscode, 501}, + {version, "HTTP/1.0"}]). diff --git a/lib/inets/test/httpd_1_1.erl b/lib/inets/test/httpd_1_1.erl index 4b2a5f619d..6a5fc4a18f 100644 --- a/lib/inets/test/httpd_1_1.erl +++ b/lib/inets/test/httpd_1_1.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,7 +22,7 @@ -include_lib("kernel/include/file.hrl"). --export([host/4, chunked/4, expect/4, range/4, if_test/5, http_trace/4, +-export([host/4, chunked/4, expect/4, range/4, if_test/5, trace/4, head/4, mod_cgi_chunked_encoding_test/5]). %% -define(all_keys_lower_case,true). @@ -152,13 +152,13 @@ if_test(Type, Port, Host, Node, DocRoot)-> calendar:datetime_to_gregorian_seconds(FileInfo#file_info.mtime), Mod = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec-1)), + CreatedSec-1)), %% Test that we get the data when the file is modified ok = httpd_test_lib:verify_request(Type, Host, Port, Node, "GET / HTTP/1.1\r\nHost:" ++ Host ++ - "\r\nIf-Modified-Since:" ++ - Mod ++ "\r\n\r\n", + "\r\nIf-Modified-Since:" ++ + Mod ++ "\r\n\r\n", [{statuscode, 200}]), Mod1 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( CreatedSec+100)), @@ -168,74 +168,69 @@ if_test(Type, Port, Host, Node, DocRoot)-> ++ Mod1 ++"\r\n\r\n", [{statuscode, 304}]), - + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, "GET / HTTP/1.1\r\nHost:" ++ Host ++ - "\r\nIf-Modified-Since:" ++ - "AAA[...]AAAA" ++ "\r\n\r\n", + "\r\nIf-Modified-Since:" ++ + "AAA[...]AAAA" ++ "\r\n\r\n", [{statuscode, 400}]), - - - Mod2 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( + + Mod2 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( CreatedSec+1)), - %% Control that the If-Unmodified-Header lmits the response - ok = httpd_test_lib:verify_request(Type,Host,Port,Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ - "\r\nIf-Unmodified-Since:" ++ Mod2 - ++ "\r\n\r\n", - [{statuscode, 200}]), - Mod3 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( + %% Control that the If-Unmodified-Header lmits the response + ok = httpd_test_lib:verify_request(Type,Host,Port,Node, + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ + "\r\nIf-Unmodified-Since:" ++ Mod2 + ++ "\r\n\r\n", + [{statuscode, 200}]), + Mod3 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( CreatedSec-1)), ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ - "\r\nIf-Unmodified-Since:"++ Mod3 + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ + "\r\nIf-Unmodified-Since:"++ Mod3 ++"\r\n\r\n", - [{statuscode, 412}]), + [{statuscode, 412}]), - %% Control that we get the body when the etag match + %% Control that we get the body when the etag match ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ Host - ++"\r\n"++ - "If-Match:"++ - httpd_util:create_etag(FileInfo)++ - "\r\n\r\n", - [{statuscode, 200}]), - ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ - Host ++ "\r\n"++ - "If-Match:NotEtag\r\n\r\n", - [{statuscode, 412}]), + "GET / HTTP/1.1\r\nHost:" ++ Host + ++"\r\n"++ + "If-Match:"++ + httpd_util:create_etag(FileInfo)++ + "\r\n\r\n", + [{statuscode, 200}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "GET / HTTP/1.1\r\nHost:" ++ + Host ++ "\r\n"++ + "If-Match:NotEtag\r\n\r\n", + [{statuscode, 412}]), - %% Control the response when the if-none-match header is there - ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++"\r\n"++ - "If-None-Match:NoTaag," ++ - httpd_util:create_etag(FileInfo) ++ - "\r\n\r\n", - [{statuscode, 304}]), + %% Control the response when the if-none-match header is there + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "GET / HTTP/1.1\r\nHost:" + ++ Host ++"\r\n"++ + "If-None-Match:NoTaag," ++ + httpd_util:create_etag(FileInfo) ++ + "\r\n\r\n", + [{statuscode, 304}]), ok = httpd_test_lib:verify_request(Type, Host, Port, Node, "GET / HTTP/1.1\r\nHost:" - ++ Host ++ "\r\n"++ - "If-None-Match:NotEtag," - "NeihterEtag\r\n\r\n", + ++ Host ++ "\r\n"++ + "If-None-Match:NotEtag," + "NeihterEtag\r\n\r\n", [{statuscode,200}]), ok. - -http_trace(Type, Port, Host, Node)-> + +trace(Type, Port, Host, Node)-> ok = httpd_test_lib:verify_request(Type, Host, Port, Node, "TRACE / HTTP/1.1\r\n" ++ "Host:" ++ Host ++ "\r\n" ++ "Max-Forwards:2\r\n\r\n", - [{statuscode, 200}]), - ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "TRACE / HTTP/1.0\r\n\r\n", - [{statuscode, 501}, - {version, "HTTP/1.0"}]). + [{statuscode, 200}]). head(Type, Port, Host, Node)-> %% mod_include ok = httpd_test_lib:verify_request(Type, Host, Port, Node, @@ -283,7 +278,7 @@ mod_cgi_chunked_encoding_test(Type, Port, Host, Node, [Request| Rest])-> %%-------------------------------------------------------------------- validateRangeRequest(Socket,Response,ValidBody,C,O,DE)-> receive - {tcp,Socket,Data} -> + {_,Socket,Data} -> case string:str(Data,"\r\n") of 0-> validateRangeRequest(Socket, @@ -312,7 +307,7 @@ validateRangeRequest1(Socket, Response, ValidBody) -> case end_of_header(Response) of false -> receive - {tcp,Socket,Data} -> + {_,Socket,Data} -> validateRangeRequest1(Socket, Response ++ Data, ValidBody); _-> @@ -331,10 +326,10 @@ validateRangeRequest2(Socket, Head, Body, ValidBody, {multiPart,Boundary})-> validateMultiPartRangeRequest(Body, ValidBody, Boundary); false-> receive - {tcp, Socket, Data} -> + {_, Socket, Data} -> validateRangeRequest2(Socket, Head, Body ++ Data, ValidBody, {multiPart, Boundary}); - {tcp_closed, Socket} -> + {_, Socket} -> error; _ -> error @@ -353,7 +348,7 @@ validateRangeRequest2(Socket, Head, Body, ValidBody, BodySize) end; Size when Size < BodySize -> receive - {tcp, Socket, Data} -> + {_, Socket, Data} -> validateRangeRequest2(Socket, Head, Body ++ Data, ValidBody, BodySize); _ -> diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index c0d73663d3..3eb8a0818f 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -26,6 +26,7 @@ -include_lib("kernel/include/file.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("public_key/include/public_key.hrl"). -include("inets_test_lib.hrl"). %% Note: This directive should only be used in test suites. @@ -33,6 +34,11 @@ -record(httpd_user, {user_name, password, user_data}). -record(httpd_group, {group_name, userlist}). +-define(MAX_HEADER_SIZE, 256). +%% Minutes before failed auths timeout. +-define(FAIL_EXPIRE_TIME,1). +%% Seconds before successful auths timeout. +-define(AUTH_TIMEOUT,5). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- @@ -42,22 +48,59 @@ suite() -> all() -> [ - {group, http}, - {group, http_limit} - %%{group, https} + {group, http_basic}, + {group, https_basic}, + {group, http_limit}, + {group, https_limit}, + {group, http_basic_auth}, + {group, https_basic_auth}, + {group, http_auth_api}, + {group, https_auth_api}, + {group, http_auth_api_dets}, + {group, https_auth_api_dets}, + {group, http_auth_api_mnesia}, + {group, https_auth_api_mnesia}, + {group, http_htaccess}, + {group, https_htaccess}, + {group, http_security}, + {group, https_security} ]. groups() -> [ - {http, [], all_groups()}, - %%{https, [], all_groups()}, - {http_limit, [], [max_clients_1_1, max_clients_1_0, max_clients_0_9]}, - {http_1_1, [], [host, chunked, expect, cgi] ++ http_head() ++ http_get()}, - {http_1_0, [], [host, cgi] ++ http_head() ++ http_get()}, - {http_0_9, [], http_head() ++ http_get()} + {http_basic, [], basic_groups()}, + {https_basic, [], basic_groups()}, + {http_limit, [], [{group, limit}]}, + {https_limit, [], [{group, limit}]}, + {http_basic_auth, [], [{group, basic_auth}]}, + {https_basic_auth, [], [{group, basic_auth}]}, + {http_auth_api, [], [{group, auth_api}]}, + {https_auth_api, [], [{group, auth_api}]}, + {http_auth_api_dets, [], [{group, auth_api_dets}]}, + {https_auth_api_dets, [], [{group, auth_api_dets}]}, + {http_auth_api_mnesia, [], [{group, auth_api_mnesia}]}, + {https_auth_api_mnesia, [], [{group, auth_api_mnesia}]}, + {http_htaccess, [], [{group, htaccess}]}, + {https_htaccess, [], [{group, htaccess}]}, + {http_security, [], [{group, security}]}, + {https_security, [], [{group, security}]}, + {limit, [], [max_clients_1_1, max_clients_1_0, max_clients_0_9]}, + {basic_auth, [], [basic_auth_1_1, basic_auth_1_0, basic_auth_0_9]}, + {auth_api, [], [auth_api_1_1, auth_api_1_0, auth_api_0_9 + ]}, + {auth_api_dets, [], [auth_api_1_1, auth_api_1_0, auth_api_0_9 + ]}, + {auth_api_mnesia, [], [auth_api_1_1, auth_api_1_0, auth_api_0_9 + ]}, + {htaccess, [], [htaccess_1_1, htaccess_1_0, htaccess_0_9]}, + {security, [], [security_1_1, security_1_0]}, %% Skip 0.9 as causes timing issus in test code + {http_1_1, [], [host, chunked, expect, cgi, cgi_chunked_encoding_test, + trace, range, if_modified_since] ++ http_head() ++ http_get() ++ load()}, + {http_1_0, [], [host, cgi, trace] ++ http_head() ++ http_get() ++ load()}, + {http_0_9, [], http_head() ++ http_get() ++ load()} ]. -all_groups ()-> +basic_groups ()-> [{group, http_1_1}, {group, http_1_0}, {group, http_0_9} @@ -66,15 +109,27 @@ all_groups ()-> http_head() -> [head]. http_get() -> - [alias, get, - basic_auth, - esi, ssi]. + [alias, + get, + %%actions, Add configuration so that this test mod_action + esi, + ssi, + content_length, + bad_hex, + missing_CR, + max_header, + ipv6 + ]. +load() -> + [light, medium + %%,heavy + ]. + init_per_suite(Config) -> PrivDir = ?config(priv_dir, Config), DataDir = ?config(data_dir, Config), inets_test_lib:stop_apps([inets]), - inets_test_lib:start_apps([inets]), ServerRoot = filename:join(PrivDir, "server_root"), inets_test_lib:del_dirs(ServerRoot), DocRoot = filename:join(ServerRoot, "htdocs"), @@ -82,21 +137,31 @@ init_per_suite(Config) -> [{server_root, ServerRoot}, {doc_root, DocRoot}, {node, node()}, - {host, inets_test_lib:hostname()} | Config]. + {host, inets_test_lib:hostname()}, + {address, getaddr()} | Config]. end_per_suite(_Config) -> ok. %%-------------------------------------------------------------------- -init_per_group(https = Group, Config0) -> - case start_apps(Group) of - ok -> - init_httpd(Group, [{type, ssl} | Config0]); - _ -> - {skip, "Could not start https apps"} - end; - -init_per_group(Group, Config0) when Group == http; Group == http_limit -> +init_per_group(Group, Config0) when Group == https_basic; + Group == https_limit; + Group == https_basic_auth; + Group == https_auth_api; + Group == https_auth_api_dets; + Group == https_auth_api_mnesia; + Group == https_security + -> + init_ssl(Group, Config0); +init_per_group(Group, Config0) when Group == http_basic; + Group == http_limit; + Group == http_basic_auth; + Group == http_auth_api; + Group == http_auth_api_dets; + Group == http_auth_api_mnesia; + Group == http_security + -> + ok = start_apps(Group), init_httpd(Group, [{type, ip_comm} | Config0]); init_per_group(http_1_1, Config) -> [{http_version, "HTTP/1.1"} | Config]; @@ -104,22 +169,57 @@ init_per_group(http_1_0, Config) -> [{http_version, "HTTP/1.0"} | Config]; init_per_group(http_0_9, Config) -> [{http_version, "HTTP/0.9"} | Config]; +init_per_group(http_htaccess = Group, Config) -> + Path = ?config(doc_root, Config), + catch remove_htaccess(Path), + create_htaccess_data(Path, ?config(address, Config)), + ok = start_apps(Group), + init_httpd(Group, [{type, ip_comm} | Config]); +init_per_group(https_htaccess = Group, Config) -> + Path = ?config(doc_root, Config), + catch remove_htaccess(Path), + create_htaccess_data(Path, ?config(address, Config)), + init_ssl(Group, Config); +init_per_group(auth_api, Config) -> + [{auth_prefix, ""} | Config]; +init_per_group(auth_api_dets, Config) -> + [{auth_prefix, "dets_"} | Config]; +init_per_group(auth_api_mnesia, Config) -> + start_mnesia(?config(node, Config)), + [{auth_prefix, "mnesia_"} | Config]; init_per_group(_, Config) -> Config. -end_per_group(http, _Config) -> - ok; -end_per_group(https, _Config) -> - ssl:stop(); + +end_per_group(Group, _Config) when Group == http_basic; + Group == http_limit; + Group == http_basic_auth; + Group == http_auth_api; + Group == http_auth_api_dets; + Group == http_auth_api_mnesia; + Group == http_htaccess; + Group == http_security + -> + inets:stop(); +end_per_group(Group, _Config) when Group == https_basic; + Group == https_limit; + Group == https_basic_auth; + Group == https_auth_api; + Group == http_auth_api_dets; + Group == http_auth_api_mnesia; + Group == https_htaccess; + Group == http_security + -> + ssl:stop(), + inets:stop(); + +end_per_group(auth_api_mnesia, _) -> + cleanup_mnesia(); + end_per_group(_, _) -> ok. -init_httpd(Group, Config0) -> - Config1 = proplists:delete(port, Config0), - Config = proplists:delete(server_pid, Config1), - {Pid, Port} = server_start(Group, server_config(Group, Config)), - [{server_pid, Pid}, {port, Port} | Config]. %%-------------------------------------------------------------------- -init_per_testcase(host, Config) -> +init_per_testcase(Case, Config) when Case == host; Case == trace -> Prop = ?config(tc_group_properties, Config), Name = proplists:get_value(name, Prop), Cb = case Name of @@ -129,15 +229,15 @@ init_per_testcase(host, Config) -> httpd_1_1 end, [{version_cb, Cb} | proplists:delete(version_cb, Config)]; + +init_per_testcase(range, Config) -> + DocRoot = ?config(doc_root, Config), + create_range_data(DocRoot), + Config; + init_per_testcase(_, Config) -> Config. -%% init_per_testcase(basic_auth = Case, Config) -> -%% start_mnesia(?config(node, Config)), -%% common_init_per_test_case(Case, Config); - -%% end_per_testcase(basic_auth, Config) -> -%% cleanup_mnesia(); end_per_testcase(_Case, _Config) -> ok. @@ -163,8 +263,11 @@ get() -> get(Config) when is_list(Config) -> Version = ?config(http_version, Config), Host = ?config(host, Config), + Type = ?config(type, Config), ok = httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), http_request("GET /index.html ", Version, Host), [{statuscode, 200}, {header, "Content-Type", "text/html"}, @@ -172,6 +275,15 @@ get(Config) when is_list(Config) -> {header, "Server"}, {version, Version}]). +basic_auth_1_1(Config) when is_list(Config) -> + basic_auth([{http_version, "HTTP/1.1"} | Config]). + +basic_auth_1_0(Config) when is_list(Config) -> + basic_auth([{http_version, "HTTP/1.0"} | Config]). + +basic_auth_0_9(Config) when is_list(Config) -> + basic_auth([{http_version, "HTTP/0.9"} | Config]). + basic_auth() -> [{doc, "Test Basic authentication with WWW-Authenticate header"}]. @@ -203,13 +315,211 @@ basic_auth(Config) -> Config, [{statuscode, 200}]), %% Authentication still required! basic_auth_requiered(Config). - + +auth_api_1_1(Config) when is_list(Config) -> + auth_api([{http_version, "HTTP/1.1"} | Config]). + +auth_api_1_0(Config) when is_list(Config) -> + auth_api([{http_version, "HTTP/1.0"} | Config]). + +auth_api_0_9(Config) when is_list(Config) -> + auth_api([{http_version, "HTTP/0.9"} | Config]). + +auth_api() -> + [{doc, "Test mod_auth API"}]. + +auth_api(Config) when is_list(Config) -> + Prefix = ?config(auth_prefix, Config), + do_auth_api(Prefix, Config). + +do_auth_api(AuthPrefix, Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + Port = ?config(port, Config), + Node = ?config(node, Config), + ServerRoot = ?config(server_root, Config), + ok = http_status("GET / ", Config, + [{statuscode, 200}]), + ok = auth_status(auth_request("/", "one", "WrongPassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Make sure Authenticate header is received even the second time + %% we try a incorrect password! Otherwise a browser client will hang! + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "dummy", "WrongPassword", Version, Host), Config, + [{statuscode, 401}, + {header, "WWW-Authenticate"}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", "dummy", "WrongPassword", + Version, Host), Config, [{statuscode, 401}, + {header, "WWW-Authenticate"}]), + + %% Change the password to DummyPassword then try to add a user + %% Get an error and set it to NoPassword + ok = update_password(Node, ServerRoot, Host, Port, AuthPrefix, + "open", "NoPassword", "DummyPassword"), + {error,bad_password} = + add_user(Node, ServerRoot, Port, AuthPrefix, "open", "one", + "onePassword", []), + ok = update_password(Node, ServerRoot, Host, Port, AuthPrefix, "open", + "DummyPassword", "NoPassword"), + + %% Test /*open, require user one Aladdin + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "open"), + + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "one", "onePassword", Version, Host), Config, + [{statuscode, 401}]), + + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "two", "twoPassword", Version, Host), Config, + [{statuscode, 401}]), + + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "Aladdin", "onePassword", Version, Host), + Config, [{statuscode, 401}]), + + true = add_user(Node, ServerRoot, Port, AuthPrefix, "open", "one", + "onePassword", []), + true = add_user(Node, ServerRoot, Port, AuthPrefix, "open", "two", + "twoPassword", []), + true = add_user(Node, ServerRoot, Port, AuthPrefix, "open", "Aladdin", + "AladdinPassword", []), + {ok, [_|_]} = list_users(Node, ServerRoot, Host, Port, + AuthPrefix, "open"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "one", "WrongPassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "one", "onePassword", Version, Host), + Config, [{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "two", "twoPassword", Version, Host), + Config,[{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "Aladdin", "WrongPassword", Version, Host), + Config,[{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "open/", + "Aladdin", "AladdinPassword", Version, Host), + Config, [{statuscode, 200}]), + + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "open"), + {ok, []} = list_users(Node, ServerRoot, Host, Port, + AuthPrefix, "open"), + + %% Phase 2 + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "secret"), + {ok, []} = list_users(Node, ServerRoot, Host, Port, AuthPrefix, + "secret"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "one", "onePassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "two", "twoPassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "three", "threePassword", Version, Host), + Config, [{statuscode, 401}]), + add_user(Node, ServerRoot, Port, AuthPrefix, "secret", "one", + "onePassword", + []), + add_user(Node, ServerRoot, Port, AuthPrefix, "secret", + "two", "twoPassword", []), + add_user(Node, ServerRoot, Port, AuthPrefix, "secret", "Aladdin", + "AladdinPassword",[]), + add_group_member(Node, ServerRoot, Port, AuthPrefix, "secret", + "one", "group1"), + add_group_member(Node, ServerRoot, Port, AuthPrefix, "secret", + "two", "group1"), + add_group_member(Node, ServerRoot, Port, AuthPrefix, + "secret", "Aladdin", "group2"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "one", "onePassword", Version, Host), + Config, [{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "two", "twoPassword", Version, Host), + Config,[{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "Aladdin", "AladdinPassword", Version, Host), + Config, [{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ "secret/", + "three", "threePassword", Version, Host), + Config, [{statuscode, 401}]), + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "secret"), + {ok, []} = list_users(Node, ServerRoot, Host, Port, + AuthPrefix, "secret"), + remove_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret"), + + {ok, []} = list_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret"), + + %% Phase 3 + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + remove_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", + "three", "threePassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", "two", "twoPassword", Version, Host), + Config, [{statuscode, 401}]), + add_user(Node, ServerRoot, Port, AuthPrefix, + "secret/top_secret","three", + "threePassword",[]), + add_user(Node, ServerRoot, Port, AuthPrefix, "secret/top_secret", + "two","twoPassword", []), + add_group_member(Node, ServerRoot, Port, AuthPrefix, "secret/top_secret", "three", "group3"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", "three", "threePassword", + Version, Host), + Config, [{statuscode, 200}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", "two", "twoPassword", Version, Host), + Config, [{statuscode, 401}]), + add_group_member(Node, ServerRoot, Port, AuthPrefix, "secret/top_secret", "two", "group3"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", + "two", "twoPassword", Version, Host), + Config, [{statuscode, 200}]), + remove_users(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + {ok, []} = list_users(Node, ServerRoot, Host, Port, + AuthPrefix, "secret/top_secret"), + remove_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + {ok, []} = list_groups(Node, ServerRoot, Host, Port, AuthPrefix, "secret/top_secret"), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/", "two", "twoPassword", Version, Host), + Config, [{statuscode, 401}]), + ok = auth_status(auth_request("/" ++ AuthPrefix ++ + "secret/top_secret/","three", "threePassword", Version, Host), + Config, [{statuscde, 401}]). +%%------------------------------------------------------------------------- +ipv6() -> + [{require, ipv6_hosts}, + {doc,"Test ipv6."}]. +ipv6(Config) when is_list(Config) -> + {ok, Hostname0} = inet:gethostname(), + case lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) of + true -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + URI = http_request("GET /", Version, Host), + httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), [inet6], + ?config(code, Config), + URI, + [{statuscode, 200}, {version, Version}]); + false -> + {skip, "Host does not support IPv6"} + end. + +%%------------------------------------------------------------------------- ssi() -> [{doc, "HTTP GET server side include test"}]. ssi(Config) when is_list(Config) -> Version = ?config(http_version, Config), Host = ?config(host, Config), + Type = ?config(type, Config), ok = httpd_test_lib:verify_request(?config(type, Config), Host, ?config(port, Config), + transport_opts(Type, Config), ?config(node, Config), http_request("GET /fsize.shtml ", Version, Host), [{statuscode, 200}, @@ -217,6 +527,131 @@ ssi(Config) when is_list(Config) -> {header, "Date"}, {header, "Server"}, {version, Version}]). +%%------------------------------------------------------------------------- +htaccess_1_1(Config) when is_list(Config) -> + htaccess([{http_version, "HTTP/1.1"} | Config]). + +htaccess_1_0(Config) when is_list(Config) -> + htaccess([{http_version, "HTTP/1.0"} | Config]). + +htaccess_0_9(Config) when is_list(Config) -> + htaccess([{http_version, "HTTP/0.9"} | Config]). + +htaccess() -> + [{doc, "Test mod_auth API"}]. + +htaccess(Config) when is_list(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + Type = ?config(type, Config), + Port = ?config(port, Config), + Node = ?config(node, Config), + %% Control that authentication required! + %% Control that the pages that shall be + %% authenticated really need authenticatin + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/open/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/secret/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/secret/top_secret/ ", + Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + + %% Make sure Authenticate header is received even the second time + %% we try a incorrect password! Otherwise a browser client will hang! + ok = auth_status(auth_request("/ht/open/", + "dummy", "WrongPassword", Version, Host), Config, + [{statuscode, 401}, + {header, "WWW-Authenticate"}]), + ok = auth_status(auth_request("/ht/open/", + "dummy", "WrongPassword", Version, Host), Config, + [{statuscode, 401}, + {header, "WWW-Authenticate"}]), + + %% Control that not just the first user in the list is valid + %% Control the first user + %% Authennticating ["one:OnePassword" user first in user list] + ok = auth_status(auth_request("/ht/open/dummy.html", "one", "OnePassword", + Version, Host), Config, + [{statuscode, 200}]), + + %% Control the second user + %% Authentication OK and a directory listing is supplied! + %% ["Aladdin:open sesame" user second in user list] + ok = auth_status(auth_request("/ht/open/","Aladdin", + "AladdinPassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Contro that bad passwords and userids get a good denial + %% User correct but wrong password! ["one:one" user first in user list] + ok = auth_status(auth_request("/ht/open/", "one", "one", Version, Host), Config, + [{statuscode, 401}]), + %% Neither user or password correct! ["dummy:dummy"] + ok = auth_status(auth_request("/ht/open/", "dummy", "dummy", Version, Host), Config, + [{statuscode, 401}]), + + %% Control that authetication still works, even if its a member in a group + %% Authentication OK! ["two:TwoPassword" user in first group] + ok = auth_status(auth_request("/ht/secret/dummy.html", "two", + "TwoPassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Authentication OK and a directory listing is supplied! + %% ["three:ThreePassword" user in second group] + ok = auth_status(auth_request("/ht/secret/", "three", + "ThreePassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Deny users with bad passwords even if the user is a group member + %% User correct but wrong password! ["two:two" user in first group] + ok = auth_status(auth_request("/ht/secret/", "two", "two", Version, Host), Config, + [{statuscode, 401}]), + %% Neither user or password correct! ["dummy:dummy"] + ok = auth_status(auth_request("/ht/secret/", "dummy", "dummy", Version, Host), Config, + [{statuscode, 401}]), + + %% control that we deny the users that are in subnet above the allowed + ok = auth_status(auth_request("/ht/blocknet/dummy.html", "four", + "FourPassword", Version, Host), Config, + [{statuscode, 403}]), + %% Control that we only applies the rules to the right methods + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("HEAD /ht/blocknet/dummy.html ", Version, Host), + [{statuscode, head_status(Version)}, + {version, Version}]), + + %% Control that the rerquire directive can be overrideen + ok = auth_status(auth_request("/ht/secret/top_secret/ ", "Aladdin", "AladdinPassword", + Version, Host), Config, + [{statuscode, 401}]), + + %% Authentication still required! + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/open/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/secret/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + http_request("GET /ht/secret/top_secret/ ", Version, Host), + [{statuscode, 401}, + {version, Version}, + {header, "WWW-Authenticate"}]). + +%%------------------------------------------------------------------------- host() -> [{doc, "Test host header"}]. @@ -224,21 +659,21 @@ host(Config) when is_list(Config) -> Cb = ?config(version_cb, Config), Cb:host(?config(type, Config), ?config(port, Config), ?config(host, Config), ?config(node, Config)). - +%%------------------------------------------------------------------------- chunked() -> [{doc, "Check that the server accepts chunked requests."}]. chunked(Config) when is_list(Config) -> httpd_1_1:chunked(?config(type, Config), ?config(port, Config), ?config(host, Config), ?config(node, Config)). - +%%------------------------------------------------------------------------- expect() -> ["Check that the server handles request with the expect header " "field appropiate"]. expect(Config) when is_list(Config) -> httpd_1_1:expect(?config(type, Config), ?config(port, Config), ?config(host, Config), ?config(node, Config)). - +%%------------------------------------------------------------------------- max_clients_1_1() -> [{doc, "Test max clients limit"}]. @@ -256,7 +691,7 @@ max_clients_0_9() -> max_clients_0_9(Config) when is_list(Config) -> do_max_clients([{http_version, "HTTP/0.9"} | Config]). - +%%------------------------------------------------------------------------- esi() -> [{doc, "Test mod_esi"}]. @@ -286,7 +721,7 @@ esi(Config) when is_list(Config) -> ok = http_status("GET /cgi-bin/erl/httpd_example:get ", Config, [{statuscode, 200}, {no_header, "cache-control"}]). - +%%------------------------------------------------------------------------- cgi() -> [{doc, "Test mod_cgi"}]. @@ -361,7 +796,27 @@ cgi(Config) when is_list(Config) -> ok = http_status("GET /cgi-bin/" ++ Script ++ " ", Config, [{statuscode, 200}, {no_header, "cache-control"}]). - +%%------------------------------------------------------------------------- +cgi_chunked_encoding_test() -> + [{doc, "Test chunked encoding together with mod_cgi "}]. +cgi_chunked_encoding_test(Config) when is_list(Config) -> + Host = ?config(host, Config), + Script = + case test_server:os_type() of + {win32, _} -> + "/cgi-bin/printenv.bat"; + _ -> + "/cgi-bin/printenv.sh" + end, + Requests = + ["GET " ++ Script ++ " HTTP/1.1\r\nHost:"++ Host ++"\r\n\r\n", + "GET /cgi-bin/erl/httpd_example/newformat HTTP/1.1\r\nHost:" + ++ Host ++"\r\n\r\n"], + httpd_1_1:mod_cgi_chunked_encoding_test(?config(type, Config), ?config(port, Config), + Host, + ?config(node, Config), + Requests). +%%------------------------------------------------------------------------- alias() -> [{doc, "Test mod_alias"}]. @@ -389,160 +844,246 @@ alias(Config) when is_list(Config) -> [{statuscode, 301}, {header, "Location"}, {header, "Content-Type","text/html"}]). +%%------------------------------------------------------------------------- +actions() -> + [{doc, "Test mod_actions"}]. +actions(Config) when is_list(Config) -> + ok = http_status("GET /", Config, [{statuscode, 200}]). -%% auth_api() -> -%% [{doc, "Test mod_auth API"}]. - -%% auth_api(Config) when is_list(Config) -> -%% Version = ?config(http_version, Config), -%% Host = ?config(host, Config), -%% ok = http_status("GET / ", Config, -%% [{statuscode, 200}]), -%% ok = auth_status(auth_request("/", "one", "WrongPassword", Version, Host), Config, -%% [{statuscode, 200}]), - -%% %% Make sure Authenticate header is received even the second time -%% %% we try a incorrect password! Otherwise a browser client will hang! -%% ok = auth_status(auth_request("/" ++ AuthStoreType ++ "open/", -%% "dummy", "WrongPassword", Host), Config, -%% [{statuscode, 401}, -%% {header, "WWW-Authenticate"}]), -%% ok = auth_status(auth_request("/" ++ AuthStoreType ++ "open/", "dummy", "WrongPassword", -%% Host), Config, [{statuscode, 401}, -%% {header, "WWW-Authenticate"}]), - -%% %% Change the password to DummyPassword then try to add a user -%% %% Get an error and set it to NoPassword -%% ok = update_password(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "open", "NoPassword", "DummyPassword"), -%% {error,bad_password} = -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "open", "one", -%% "onePassword", []), -%% ok = update_password(Node, ServerRoot, Host, Port, AuthStoreType ++"open", -%% "DummyPassword", "NoPassword"), - -%% %% Test /*open, require user one Aladdin -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType ++ "open"), +%%------------------------------------------------------------------------- +range() -> + [{doc, "Test Range header"}]. + +range(Config) when is_list(Config) -> + httpd_1_1:range(?config(type, Config), ?config(port, Config), + ?config(host, Config), ?config(node, Config)). + +%%------------------------------------------------------------------------- +if_modified_since() -> + [{doc, "Test If-Modified-Since header"}]. + +if_modified_since(Config) when is_list(Config) -> + httpd_1_1:if_test(?config(type, Config), ?config(port, Config), + ?config(host, Config), ?config(node, Config), + ?config(doc_root, Config)). +%%------------------------------------------------------------------------- +trace() -> + [{doc, "Test TRACE method"}]. + +trace(Config) when is_list(Config) -> + Cb = ?config(version_cb, Config), + Cb:trace(?config(type, Config), ?config(port, Config), + ?config(host, Config), ?config(node, Config)). + +%%------------------------------------------------------------------------- +light() -> + ["Test light load"]. +light(Config) when is_list(Config) -> + httpd_load:load_test(?config(type, Config), ?config(port, Config), ?config(host, Config), + ?config(node, Config), 10). +%%------------------------------------------------------------------------- +medium() -> + ["Test medium load"]. +medium(Config) when is_list(Config) -> + httpd_load:load_test(?config(type, Config), ?config(port, Config), ?config(host, Config), + ?config(node, Config), 100). +%%------------------------------------------------------------------------- +heavy() -> + ["Test heavy load"]. +heavy(Config) when is_list(Config) -> + httpd_load:load_test(?config(type, Config), ?config(port, Config), ?config(host, Config), + ?config(node, Config), + 1000). +%%------------------------------------------------------------------------- +content_length() -> + ["Tests that content-length is correct OTP-5775"]. +content_length(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + ok = httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), ?config(node, Config), + http_request("GET /cgi-bin/erl/httpd_example:get_bin ", + Version, Host), + [{statuscode, 200}, + {content_length, 274}, + {version, Version}]). +%%------------------------------------------------------------------------- +bad_hex() -> + ["Tests that a URI with a bad hexadecimal code is handled OTP-6003"]. +bad_hex(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + ok = httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), ?config(node, Config), + http_request("GET http://www.erlang.org/%skalle ", + Version, Host), + [{statuscode, 400}, + {version, Version}]). +%%------------------------------------------------------------------------- +missing_CR() -> + ["Tests missing CR in delimiter OTP-7304"]. +missing_CR(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + ok = httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), ?config(node, Config), + http_request_missing_CR("GET /index.html ", Version, Host), + [{statuscode, 200}, + {version, Version}]). + +%%------------------------------------------------------------------------- +max_header() -> + ["Denial Of Service (DOS) attack, prevented by max_header"]. +max_header(Config) when is_list(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + case Version of + "HTTP/0.9" -> + {skip, no_implemented}; + _ -> + dos_hostname(?config(type, Config), ?config(port, Config), Host, + ?config(node, Config), Version, ?MAX_HEADER_SIZE) + end. + +%%------------------------------------------------------------------------- +security_1_1(Config) when is_list(Config) -> + security([{http_version, "HTTP/1.1"} | Config]). + +security_1_0(Config) when is_list(Config) -> + security([{http_version, "HTTP/1.0"} | Config]). + +security() -> + ["Test mod_security"]. +security(Config) -> + Version = ?config(http_version, Config), + Host = ?config(host, Config), + Port = ?config(port, Config), + Node = ?config(node, Config), + ServerRoot = ?config(server_root, Config), + + global:register_name(mod_security_test, self()), % Receive events + + test_server:sleep(5000), + + OpenDir = filename:join([ServerRoot, "htdocs", "open"]), + + %% Test blocking / unblocking of users. -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "one", "onePassword", [{statuscode, 401}]), + %% /open, require user one Aladdin + remove_users(Node, ServerRoot, Host, Port, "", "open"), + + ok = auth_status(auth_request("/open/", + "one", "onePassword", Version, Host), Config, + [{statuscode, 401}]), + + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "one"}, {password, "onePassword"}]}, + Node, Port), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "two", "twoPassword", [{statuscode, 401}]), + ok = auth_status(auth_request("/open/", + "two", "twoPassword", Version, Host), Config, + [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "Aladdin", "onePassword", [{statuscode, 401}]), - -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "open", "one", -%% "onePassword", []), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "open", "two", -%% "twoPassword", []), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "open", "Aladdin", -%% "AladdinPassword", []), + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "two"}, {password, "twoPassword"}]}, + Node, Port), + + ok = auth_status(auth_request("/open/", + "Aladdin", "AladdinPassword", Version, Host), + Config, [{statuscode, 401}]), + + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "Aladdin"}, + {password, "AladdinPassword"}]}, + Node, Port), + + add_user(Node, ServerRoot, Port, "", "open", "one", "onePassword", []), + add_user(Node, ServerRoot, Port, "", "open", "two", "twoPassword", []), + + ok = auth_status(auth_request("/open/", "one", "WrongPassword", Version, Host), Config, + [{statuscode, 401}]), + + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "one"}, {password, "WrongPassword"}]}, + Node, Port), + + ok = auth_status(auth_request("/open/", "one", "WrongPassword", Version, Host), Config, + [{statuscode, 401}]), + + receive_security_event({event, auth_fail, Port, OpenDir, + [{user, "one"}, {password, "WrongPassword"}]}, + Node, Port), + receive_security_event({event, user_block, Port, OpenDir, + [{user, "one"}]}, Node, Port), + + global:unregister_name(mod_security_test), % No more events. + + ok = auth_status(auth_request("/open/", "one", "WrongPassword", Version, Host), Config, + [{statuscode, 401}]), + + %% User "one" should be blocked now.. + case list_blocked_users(Node, Port) of + [{"one",_, Port, OpenDir,_}] -> + ok; + Blocked -> + ct:fail({unexpected_blocked, Blocked}) + end, + + [{"one",_, Port, OpenDir,_}] = list_blocked_users(Node, Port, OpenDir), + + true = unblock_user(Node, "one", Port, OpenDir), + %% User "one" should not be blocked any more. + + [] = list_blocked_users(Node, Port), + + ok = auth_status(auth_request("/open/", "one", "onePassword", Version, Host), Config, + [{statuscode, 200}]), + + %% Test list_auth_users & auth_timeout + + ["one"] = list_auth_users(Node, Port), + + ok = auth_status(auth_request("/open/", "two", "onePassword", Version, Host), Config, + [{statuscode, 401}]), + + ["one"] = list_auth_users(Node, Port), + -%% {ok, [_|_]} = list_users(Node, ServerRoot, Host, Port, -%% AuthStoreType++"open"), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ "open/", -%% "one", "WrongPassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ "open/", -%% "one", "onePassword", [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "two", "twoPassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ "open/", -%% "Aladdin", "WrongPassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "open/", -%% "Aladdin", "AladdinPassword", [{statuscode, 200}]), + ["one"] = list_auth_users(Node, Port, OpenDir), + -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType++"open"), -%% {ok, []} = list_users(Node, ServerRoot, Host, Port, -%% AuthStoreType++"open"), - -%% %% Phase 2 -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType++"secret"), -%% {ok, []} = list_users(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret"), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "one", "onePassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "two", "twoPassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ "secret/", -%% "three", "threePassword", [{statuscode, 401}]), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "secret", "one", -%% "onePassword", -%% []), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "secret", -%% "two", "twoPassword", []), -%% add_user(Node, ServerRoot, Port, AuthStoreType++"secret", "Aladdin", -%% "AladdinPassword",[]), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ "secret", -%% "one", "group1"), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ "secret", -%% "two", "group1"), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ -%% "secret", "Aladdin", "group2"), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "one", "onePassword", [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "two", "twoPassword", [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "Aladdin", "AladdinPassword", [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ "secret/", -%% "three", "threePassword", [{statuscode, 401}]), -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType ++ "secret"), -%% {ok, []} = list_users(Node, ServerRoot, Host, Port, -%% AuthStoreType ++ "secret"), -%% remove_groups(Node, ServerRoot, Host, Port, AuthStoreType ++ "secret"), -%% Directory = filename:join([ServerRoot, "htdocs", AuthStoreType ++ -%% "secret"]), -%% {ok, []} = list_groups(Node, ServerRoot, Host, Port, Directory), - -%% %% Phase 3 -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret/top_secret"), -%% remove_groups(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret/top_secret"), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", -%% "three", "threePassword", [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", "two", "twoPassword", -%% [{statuscode, 401}]), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ -%% "secret/top_secret","three", -%% "threePassword",[]), -%% add_user(Node, ServerRoot, Port, AuthStoreType ++ "secret/top_secret", -%% "two","twoPassword", []), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ -%% "secret/top_secret", -%% "three", "group3"), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", "three", "threePassword", -%% [{statuscode, 200}]), -%% auth_request(Type, Host, Port, Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", "two", "twoPassword", -%% [{statuscode, 401}]), -%% add_group_member(Node, ServerRoot, Port, AuthStoreType ++ -%% "secret/top_secret", -%% "two", "group3"), -%% auth_request(Type,Host,Port,Node,"/" ++ AuthStoreType ++ -%% "secret/top_secret/", -%% "two", "twoPassword", [{statuscode, 200}]), -%% remove_users(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret/top_secret"), -%% {ok, []} = list_users(Node, ServerRoot, Host, Port, -%% AuthStoreType ++ "secret/top_secret"), -%% remove_groups(Node, ServerRoot, Host, Port, AuthStoreType ++ -%% "secret/top_secret"), -%% Directory2 = filename:join([ServerRoot, "htdocs", -%% AuthStoreType ++ "secret/top_secret"]), -%% {ok, []} = list_groups(Node, ServerRoot, Host, Port, Directory2), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ -%% "secret/top_secret/", "two", "twoPassword", -%% [{statuscode, 401}]), -%% auth_request(Type, Host, Port, Node, "/" ++ AuthStoreType ++ -%% "secret/top_secret/","three", "threePassword", -%% [{statuscode, 401}]). + ok = auth_status(auth_request("/open/", "two", "twoPassword", Version, Host), Config, + [{statuscode, 401}]), + + ["one"] = list_auth_users(Node, Port), + + + ["one"] = list_auth_users(Node, Port, OpenDir), + + %% Wait for successful auth to timeout. + test_server:sleep(?AUTH_TIMEOUT*1001), + + [] = list_auth_users(Node, Port), + + [] = list_auth_users(Node, Port, OpenDir), + + %% "two" is blocked. + + true = unblock_user(Node, "two", Port, OpenDir), + + + %% Test explicit blocking. Block user 'two'. + + [] = list_blocked_users(Node,Port,OpenDir), + + true = block_user(Node, "two", Port, OpenDir, 10), + + ok = auth_status(auth_request("/open/", "two", "twoPassword", Version, Host), Config, + [{statuscode, 401}]), + + true = unblock_user(Node, "two", Port, OpenDir). + %%-------------------------------------------------------------------- @@ -550,21 +1091,34 @@ alias(Config) when is_list(Config) -> %%-------------------------------------------------------------------- do_max_clients(Config) -> Version = ?config(http_version, Config), - Host = ?config(host, Config), - start_blocker(Config), - ok = httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), - http_request("GET /index.html ", Version, Host), + Host = ?config(host, Config), + Port = ?config(port, Config), + Type = ?config(type, Config), + + Request = http_request("GET /index.html ", Version, Host), + BlockRequest = http_request("GET /eval?httpd_example:delay(2000) ", Version, Host), + {ok, Socket} = inets_test_lib:connect_bin(Type, Host, Port, transport_opts(Type, Config)), + inets_test_lib:send(Type, Socket, BlockRequest), + ct:sleep(100), + ok = httpd_test_lib:verify_request(Type, Host, + Port, + transport_opts(Type, Config), + ?config(node, Config), + Request, [{statuscode, 503}, {version, Version}]), receive - after 2000 -> - ok = httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), - http_request("GET /index.html ", Version, Host), - [{statuscode, 200}, - {version, Version}]) - end. + {_, Socket, _Msg} -> + ok + end, + inets_test_lib:close(Type, Socket), + ok = httpd_test_lib:verify_request(Type, Host, + Port, + transport_opts(Type, Config), + ?config(node, Config), + Request, + [{statuscode, 200}, + {version, Version}]). setup_server_dirs(ServerRoot, DocRoot, DataDir) -> CgiDir = filename:join(ServerRoot, "cgi-bin"), @@ -604,10 +1158,24 @@ setup_server_dirs(ServerRoot, DocRoot, DataDir) -> ok = file:write_file_info(EnvCGI, FileInfo1#file_info{mode = 8#00755}). -start_apps(https) -> - inets_test_lib:start_apps([crypto, public_key, ssl]); -start_apps(_) -> - ok. +start_apps(Group) when Group == https_basic; + Group == https_limit; + Group == https_basic_auth; + Group == https_auth_api; + Group == https_auth_api_dets; + Group == https_auth_api_mnesia; + Group == http_htaccess; + Group == http_security -> + inets_test_lib:start_apps([inets, asn1, crypto, public_key, ssl]); +start_apps(Group) when Group == http_basic; + Group == http_limit; + Group == http_basic_auth; + Group == http_auth_api; + Group == http_auth_api_dets; + Group == http_auth_api_mnesia; + Group == https_htaccess; + Group == https_security -> + inets_test_lib:start_apps([inets]). server_start(_, HttpdConfig) -> {ok, Pid} = inets:start(httpd, HttpdConfig), @@ -615,6 +1183,80 @@ server_start(_, HttpdConfig) -> {value, {_, _, Info}} = lists:keysearch(Pid, 2, Serv), {Pid, proplists:get_value(port, Info)}. +init_ssl(Group, Config) -> + PrivDir = ?config(priv_dir, Config), + CaKey = {_Trusted,_} = + erl_make_certs:make_cert([{key, dsa}, + {subject, + [{name, "Public Key"}, + {?'id-at-name', + {printableString, "public_key"}}, + {?'id-at-pseudonym', + {printableString, "pubkey"}}, + {city, "Stockholm"}, + {country, "SE"}, + {org, "erlang"}, + {org_unit, "testing dep"} + ]} + ]), + ok = erl_make_certs:write_pem(PrivDir, "public_key_cacert", CaKey), + + CertK1 = {_Cert1, _} = erl_make_certs:make_cert([{issuer, CaKey}]), + CertK2 = {_Cert2,_} = erl_make_certs:make_cert([{issuer, CertK1}, + {digest, md5}, + {extensions, false}]), + ok = erl_make_certs:write_pem(PrivDir, "public_key_cert", CertK2), + + case start_apps(Group) of + ok -> + init_httpd(Group, [{type, ssl} | Config]); + _ -> + {skip, "Could not start https apps"} + end. + +server_config(http_basic, Config) -> + basic_conf() ++ server_config(http, Config); +server_config(https_basic, Config) -> + basic_conf() ++ server_config(https, Config); +server_config(http_limit, Config) -> + [{max_clients, 1}] ++ server_config(http, Config); +server_config(https_limit, Config) -> + [{max_clients, 1}] ++ server_config(https, Config); +server_config(http_basic_auth, Config) -> + ServerRoot = ?config(server_root, Config), + auth_conf(ServerRoot) ++ server_config(http, Config); +server_config(https_basic_auth, Config) -> + ServerRoot = ?config(server_root, Config), + auth_conf(ServerRoot) ++ server_config(https, Config); +server_config(http_auth_api, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, plain) ++ server_config(http, Config); +server_config(https_auth_api, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, plain) ++ server_config(https, Config); +server_config(http_auth_api_dets, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, dets) ++ server_config(http, Config); +server_config(https_auth_api_dets, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, dets) ++ server_config(https, Config); +server_config(http_auth_api_mnesia, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, mnesia) ++ server_config(http, Config); +server_config(https_auth_api_mnesia, Config) -> + ServerRoot = ?config(server_root, Config), + auth_api_conf(ServerRoot, mnesia) ++ server_config(https, Config); +server_config(http_htaccess, Config) -> + auth_access_conf() ++ server_config(http, Config); +server_config(https_htaccess, Config) -> + auth_access_conf() ++ server_config(https, Config); +server_config(http_security, Config) -> + ServerRoot = ?config(server_root, Config), + tl(auth_conf(ServerRoot)) ++ security_conf(ServerRoot) ++ server_config(http, Config); +server_config(https_security, Config) -> + ServerRoot = ?config(server_root, Config), + tl(auth_conf(ServerRoot)) ++ security_conf(ServerRoot) ++ server_config(https, Config); + server_config(http, Config) -> ServerRoot = ?config(server_root, Config), [{port, 0}, @@ -625,6 +1267,7 @@ server_config(http, Config) -> {ipfamily, inet}, {max_header_size, 256}, {max_header_action, close}, + {directory_index, ["index.html", "welcome.html"]}, {mime_types, [{"html","text/html"},{"htm","text/html"}, {"shtml","text/html"}, {"gif", "image/gif"}]}, {alias, {"/icons/", filename:join(ServerRoot,"icons") ++ "/"}}, @@ -633,13 +1276,24 @@ server_config(http, Config) -> {script_alias, {"/htbin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}}, {erl_script_alias, {"/cgi-bin/erl", [httpd_example, io]}}, {eval_script_alias, {"/eval", [httpd_example, io]}} - ] ++ auth_conf(ServerRoot); + ]; -server_config(http_limit, Config) -> - [{max_clients, 1}] ++ server_config(http, Config); - -server_config(_, _) -> - []. +server_config(https, Config) -> + PrivDir = ?config(priv_dir, Config), + [{socket_type, {essl, + [{cacertfile, + filename:join(PrivDir, "public_key_cacert.pem")}, + {certfile, + filename:join(PrivDir, "public_key_cert.pem")}, + {keyfile, + filename:join(PrivDir, "public_key_cert_key.pem")} + ]}}] ++ server_config(http, Config). + +init_httpd(Group, Config0) -> + Config1 = proplists:delete(port, Config0), + Config = proplists:delete(server_pid, Config1), + {Pid, Port} = server_start(Group, server_config(Group, Config)), + [{server_pid, Pid}, {port, Port} | Config]. http_request(Request, "HTTP/1.1" = Version, Host, {Headers, Body}) -> Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n" ++ Headers ++ "\r\n" ++ Body; @@ -662,19 +1316,33 @@ auth_request(Path, User, Passwd, Version, _Host) -> base64:encode_to_string(User++":"++Passwd) ++ "\r\n\r\n". +http_request_missing_CR(Request, "HTTP/1.1" = Version, Host) -> + Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n\r\n\n"; +http_request_missing_CR(Request, Version, _) -> + Request ++ Version ++ "\r\n\n". + head_status("HTTP/0.9") -> 501; %% Not implemented in HTTP/0.9 head_status(_) -> 200. +basic_conf() -> + [{modules, [mod_alias, mod_range, mod_responsecontrol, + mod_trace, mod_esi, mod_cgi, mod_dir, mod_get, mod_head]}]. + +auth_access_conf() -> + [{modules, [mod_alias, mod_htaccess, mod_dir, mod_get, mod_head]}, + {access_files, [".htaccess"]}]. + auth_conf(Root) -> - [{directory, {filename:join(Root, "htdocs/open"), + [{modules, [mod_alias, mod_auth, mod_dir, mod_get, mod_head]}, + {directory, {filename:join(Root, "htdocs/open"), [{auth_type, plain}, {auth_name, "Open Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, {auth_group_file, filename:join(Root, "auth/group")}, {require_user, ["one", "Aladdin"]}]}}, - {directory, {filename:join(Root, "htdocs/secret"), + {directory, {filename:join(Root, "htdocs/secret"), [{auth_type, plain}, {auth_name, "Secret Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, @@ -685,43 +1353,134 @@ auth_conf(Root) -> {auth_name, "Top Secret Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, {auth_group_file, filename:join(Root, "auth/group")}, - {require_group, ["group3"]}]}}, + {require_group, ["group3"]}]}}]. + +auth_api_conf(Root, plain) -> + [{modules, [mod_alias, mod_auth, mod_dir, mod_get, mod_head]}, {directory, {filename:join(Root, "htdocs/open"), - [{auth_type, mnesia}, + [{auth_type, plain}, {auth_name, "Open Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, {auth_group_file, filename:join(Root, "auth/group")}, {require_user, ["one", "Aladdin"]}]}}, {directory, {filename:join(Root, "htdocs/secret"), - [{auth_type, mnesia}, + [{auth_type, plain}, {auth_name, "Secret Area"}, {auth_user_file, filename:join(Root, "auth/passwd")}, {auth_group_file, filename:join(Root, "auth/group")}, - {require_group, ["group1", "group2"]}]}} - ]. + {require_group, ["group1", "group2"]}]}}, + {directory, {filename:join(Root, "htdocs/secret/top_secret"), + [{auth_type, plain}, + {auth_name, "Top Secret Area"}, + {auth_user_file, filename:join(Root, "auth/passwd")}, + {auth_group_file, filename:join(Root, "auth/group")}, + {require_group, ["group3"]}]}}]; +auth_api_conf(Root, dets) -> + [ + {modules, [mod_alias, mod_auth, mod_dir, mod_get, mod_head]}, + {directory, {filename:join(Root, "htdocs/dets_open"), + [{auth_type, dets}, + {auth_name, "Dets Open Area"}, + {auth_user_file, filename:join(Root, "passwd")}, + {auth_group_file, filename:join(Root, "group")}, + {require_user, ["one", "Aladdin"]}]}}, + {directory, {filename:join(Root, "htdocs/dets_secret"), + [{auth_type, dets}, + {auth_name, "Dests Secret Area"}, + {auth_user_file, filename:join(Root, "passwd")}, + {auth_group_file, filename:join(Root, "group")}, + {require_group, ["group1", "group2"]}]}}, + {directory, {filename:join(Root, "htdocs/dets_secret/top_secret"), + [{auth_type, dets}, + {auth_name, "Dets Top Secret Area"}, + {auth_user_file, filename:join(Root, "passwd")}, + {auth_group_file, filename:join(Root, "group")}, + {require_group, ["group3"]}]}} + ]; + +auth_api_conf(Root, mnesia) -> + [{modules, [mod_alias, mod_auth, mod_dir, mod_get, mod_head]}, + {directory, {filename:join(Root, "htdocs/mnesia_open"), + [{auth_type, mnesia}, + {auth_name, "Mnesia Open Area"}, + {require_user, ["one", "Aladdin"]}]}}, + {directory, {filename:join(Root, "htdocs/mnesia_secret"), + [{auth_type, mnesia}, + {auth_name, "Mnesia Secret Area"}, + {require_group, ["group1", "group2"]}]}}, + {directory, {filename:join(Root, "htdocs/mnesia_secret/top_secret"), + [{auth_type, mnesia}, + {auth_name, "Mnesia Top Secret Area"}, + {require_group, ["group3"]}]}}]. + +security_conf(Root) -> + SecFile = filename:join(Root, "security_data"), + Open = filename:join(Root, "htdocs/open"), + Secret = filename:join(Root, "htdocs/secret"), + TopSecret = filename:join(Root, "htdocs/secret/top_secret"), + + [{modules, [mod_alias, mod_auth, mod_security, mod_dir, mod_get, mod_head]}, + {security_directory, {Open, + [{auth_name, "Open Area"}, + {auth_user_file, filename:join(Root, "auth/passwd")}, + {auth_group_file, filename:join(Root, "auth/group")}, + {require_user, ["one", "Aladdin"]} | + mod_security_conf(SecFile, Open)]}}, + {security_directory, {Secret, + [{auth_name, "Secret Area"}, + {auth_user_file, filename:join(Root, "auth/passwd")}, + {auth_group_file, filename:join(Root, "auth/group")}, + {require_group, ["group1", "group2"]} | + mod_security_conf(SecFile, Secret)]}}, + {security_directory, {TopSecret, + [{auth_name, "Top Secret Area"}, + {auth_user_file, filename:join(Root, "auth/passwd")}, + {auth_group_file, filename:join(Root, "auth/group")}, + {require_group, ["group3"]} | + mod_security_conf(SecFile, TopSecret)]}}]. + +mod_security_conf(SecFile, Dir) -> + [{data_file, SecFile}, + {max_retries, 3}, + {fail_expire_time, ?FAIL_EXPIRE_TIME}, + {block_time, 1}, + {auth_timeout, ?AUTH_TIMEOUT}, + {callback_module, ?MODULE}, + {path, Dir} %% This is should not be needed, but is atm, awful design! + ]. + http_status(Request, Config, Expected) -> Version = ?config(http_version, Config), Host = ?config(host, Config), + Type = ?config(type, Config), httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), http_request(Request, Version, Host), Expected ++ [{version, Version}]). http_status(Request, HeadersAndBody, Config, Expected) -> Version = ?config(http_version, Config), - Host = ?config(host, Config), + Host = ?config(host, Config), + Type = ?config(type, Config), httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), http_request(Request, Version, Host, HeadersAndBody), Expected ++ [{version, Version}]). auth_status(AuthRequest, Config, Expected) -> Version = ?config(http_version, Config), Host = ?config(host, Config), + Type = ?config(type, Config), httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), AuthRequest, Expected ++ [{version, Version}]). @@ -772,23 +1531,258 @@ cleanup_mnesia() -> mnesia:delete_schema([node()]), ok. -start_blocker(Config) -> - spawn(httpd_SUITE, init_blocker, [self(), Config]), - receive - blocker_start -> +transport_opts(ssl, Config) -> + PrivDir = ?config(priv_dir, Config), + [{cacertfile, filename:join(PrivDir, "public_key_cacert.pem")}]; +transport_opts(_, _) -> + []. + + +%%% mod_range +create_range_data(Path) -> + PathAndFileName=filename:join([Path,"range.txt"]), + case file:read_file(PathAndFileName) of + {error, enoent} -> + file:write_file(PathAndFileName,list_to_binary(["12345678901234567890", + "12345678901234567890", + "12345678901234567890", + "12345678901234567890", + "12345678901234567890"])); + _ -> ok end. + +%%% mod_htaccess +create_htaccess_data(Path, IpAddress)-> + create_htaccess_dirs(Path), + + create_html_file(filename:join([Path,"ht/open/dummy.html"])), + create_html_file(filename:join([Path,"ht/blocknet/dummy.html"])), + create_html_file(filename:join([Path,"ht/secret/dummy.html"])), + create_html_file(filename:join([Path,"ht/secret/top_secret/dummy.html"])), -init_blocker(From, Config) -> - From ! blocker_start, - block(Config). + create_htaccess_file(filename:join([Path,"ht/open/.htaccess"]), + Path, "user one Aladdin"), + create_htaccess_file(filename:join([Path,"ht/secret/.htaccess"]), + Path, "group group1 group2"), + create_htaccess_file(filename:join([Path, + "ht/secret/top_secret/.htaccess"]), + Path, "user four"), + create_htaccess_file(filename:join([Path,"ht/blocknet/.htaccess"]), + Path, nouser, IpAddress), + + create_user_group_file(filename:join([Path,"ht","users.file"]), + "one:OnePassword\ntwo:TwoPassword\nthree:" + "ThreePassword\nfour:FourPassword\nAladdin:" + "AladdinPassword"), + create_user_group_file(filename:join([Path,"ht","groups.file"]), + "group1: two one\ngroup2: two three"). + +create_html_file(PathAndFileName)-> + file:write_file(PathAndFileName,list_to_binary( + "<html><head><title>test</title></head> + <body>testar</body></html>")). + +create_htaccess_file(PathAndFileName, BaseDir, RequireData)-> + file:write_file(PathAndFileName, + list_to_binary( + "AuthUserFile "++ BaseDir ++ + "/ht/users.file\nAuthGroupFile "++ BaseDir + ++ "/ht/groups.file\nAuthName Test\nAuthType" + " Basic\n<Limit>\nrequire " ++ RequireData ++ + "\n</Limit>")). + +create_htaccess_file(PathAndFileName, BaseDir, nouser, IpAddress)-> + file:write_file(PathAndFileName,list_to_binary( + "AuthUserFile "++ BaseDir ++ + "/ht/users.file\nAuthGroupFile " ++ + BaseDir ++ "/ht/groups.file\nAuthName" + " Test\nAuthType" + " Basic\n<Limit GET>\n\tallow from " ++ + format_ip(IpAddress, + string:rchr(IpAddress,$.)) ++ + "\n</Limit>")). + +create_user_group_file(PathAndFileName, Data)-> + file:write_file(PathAndFileName, list_to_binary(Data)). + +create_htaccess_dirs(Path)-> + ok = file:make_dir(filename:join([Path,"ht"])), + ok = file:make_dir(filename:join([Path,"ht/open"])), + ok = file:make_dir(filename:join([Path,"ht/blocknet"])), + ok = file:make_dir(filename:join([Path,"ht/secret"])), + ok = file:make_dir(filename:join([Path,"ht/secret/top_secret"])). + +remove_htaccess_dirs(Path)-> + file:del_dir(filename:join([Path,"ht/secret/top_secret"])), + file:del_dir(filename:join([Path,"ht/secret"])), + file:del_dir(filename:join([Path,"ht/blocknet"])), + file:del_dir(filename:join([Path,"ht/open"])), + file:del_dir(filename:join([Path,"ht"])). + +format_ip(IpAddress,Pos)when Pos > 0-> + case lists:nth(Pos,IpAddress) of + $.-> + case lists:nth(Pos-2,IpAddress) of + $.-> + format_ip(IpAddress,Pos-3); + _-> + lists:sublist(IpAddress,Pos-2) ++ "." + end; + _ -> + format_ip(IpAddress,Pos-1) + end; + +format_ip(IpAddress, _Pos)-> + "1" ++ IpAddress. + +remove_htaccess(Path)-> + file:delete(filename:join([Path,"ht/open/dummy.html"])), + file:delete(filename:join([Path,"ht/secret/dummy.html"])), + file:delete(filename:join([Path,"ht/secret/top_secret/dummy.html"])), + file:delete(filename:join([Path,"ht/blocknet/dummy.html"])), + file:delete(filename:join([Path,"ht/blocknet/.htaccess"])), + file:delete(filename:join([Path,"ht/open/.htaccess"])), + file:delete(filename:join([Path,"ht/secret/.htaccess"])), + file:delete(filename:join([Path,"ht/secret/top_secret/.htaccess"])), + file:delete(filename:join([Path,"ht","users.file"])), + file:delete(filename:join([Path,"ht","groups.file"])), + remove_htaccess_dirs(Path). + +dos_hostname(Type, Port, Host, Node, Version, Max) -> + TooLongHeader = lists:append(lists:duplicate(Max + 1, "a")), + + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + dos_hostname_request("", Version), + [{statuscode, 200}, + {version, Version}]), + + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + dos_hostname_request("dummy-host.ericsson.se", Version), + [{statuscode, 200}, + {version, Version}]), + + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + dos_hostname_request(TooLongHeader, Version), + [{statuscode, dos_code(Version)}, + {version, Version}]). +dos_hostname_request(Host, Version) -> + dos_http_request("GET / ", Version, Host). + +dos_http_request(Request, "HTTP/1.1" = Version, Host) -> + http_request(Request, Version, Host); +dos_http_request(Request, Version, Host) -> + Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n\r\n". + +dos_code("HTTP/1.0") -> + 403; %% 413 not defined in HTTP/1.0 +dos_code(_) -> + 413. + +update_password(Node, ServerRoot, _Address, Port, AuthPrefix, Dir, Old, New)-> + Directory = filename:join([ServerRoot, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, update_password, + [undefined, Port, Directory, Old, New, New]). + +add_user(Node, Root, Port, AuthPrefix, Dir, User, Password, UserData) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, add_user, + [User, Password, UserData, Addr, Port, Directory]). + + +delete_user(Node, Root, _Host, Port, AuthPrefix, Dir, User) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, delete_user, [User, Addr, Port, Directory]). +remove_users(Node, ServerRoot, Host, Port, AuthPrefix, Dir) -> + %% List users, delete them, and make sure they are gone. + case list_users(Node, ServerRoot, Host, Port, AuthPrefix, Dir) of + {ok, Users} -> + lists:foreach(fun(User) -> + delete_user(Node, ServerRoot, Host, + Port, AuthPrefix, Dir, User) + end, + Users), + {ok, []} = list_users(Node, ServerRoot, Host, Port, AuthPrefix, Dir); + _ -> + ok + end. + +list_users(Node, Root, _Host, Port, AuthPrefix, Dir) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, list_users, [Addr, Port, Directory]). + +remove_groups(Node, ServerRoot, Host, Port, AuthPrefix, Dir) -> + {ok, Groups} = list_groups(Node, ServerRoot, Host, Port, AuthPrefix, Dir), + lists:foreach(fun(Group) -> + delete_group(Node, Group, Port, ServerRoot, AuthPrefix, Dir) + end, + Groups), + {ok, []} = list_groups(Node, ServerRoot, Host, Port, AuthPrefix, Dir). + +delete_group(Node, Group, Port, Root, AuthPrefix, Dir) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, delete_group, [Group, Addr, Port, Directory]). + +list_groups(Node, Root, _, Port, AuthPrefix, Dir) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, list_groups, [Addr, Port, Directory]). + +add_group_member(Node, Root, Port, AuthPrefix, Dir, User, Group) -> + Addr = undefined, + Directory = filename:join([Root, "htdocs", AuthPrefix ++ Dir]), + rpc:call(Node, mod_auth, add_group_member, [Group, User, Addr, Port, + Directory]). +getaddr() -> + {ok,HostName} = inet:gethostname(), + {ok,{A1,A2,A3,A4}} = inet:getaddr(HostName,inet), + lists:flatten(io_lib:format("~p.~p.~p.~p",[A1,A2,A3,A4])). + +receive_security_event(Event, Node, Port) -> + receive + Event -> + ok; + {'EXIT', _, _} -> + receive_security_event(Event, Node, Port) + after 5000 -> + %% Flush the message queue, to see if we got something... + inets_test_lib:flush() + end. + +list_blocked_users(Node,Port) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, list_blocked_users, [Addr,Port]). + +list_blocked_users(Node,Port,Dir) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, list_blocked_users, [Addr,Port,Dir]). + +block_user(Node,User,Port,Dir,Sec) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, block_user, [User, Addr, Port, Dir, Sec]). + +unblock_user(Node,User,Port,Dir) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, unblock_user, [User, Addr, Port, Dir]). + +list_auth_users(Node,Port) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, list_auth_users, [Addr,Port]). + +list_auth_users(Node,Port,Dir) -> + Addr = undefined, % Assumed to be on the same host + rpc:call(Node, mod_security, list_auth_users, [Addr,Port,Dir]). + +event(What, Port, Dir, Data) -> + Msg = {event, What, Port, Dir, Data}, + case global:whereis_name(mod_security_test) of + undefined -> + ok; + _Pid -> + global:send(mod_security_test, Msg) + end. -block(Config) -> - Version = ?config(http_version, Config), - Host = ?config(host, Config), - httpd_test_lib:verify_request(?config(type, Config), Host, - ?config(port, Config), ?config(node, Config), - http_request("GET /eval?httpd_example:delay(1000) ", - Version, Host), - [{statuscode, 200}, - {version, Version}]). diff --git a/lib/inets/test/httpd_basic_SUITE.erl b/lib/inets/test/httpd_basic_SUITE.erl index 2d06f3e70c..fbe65145dc 100644 --- a/lib/inets/test/httpd_basic_SUITE.erl +++ b/lib/inets/test/httpd_basic_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2012. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -65,7 +65,8 @@ end_per_group(_GroupName, Config) -> init_per_suite(Config) -> tsp("init_per_suite -> entry with" "~n Config: ~p", [Config]), - ok = inets:start(), + inets_test_lib:stop_apps([inets]), + inets_test_lib:start_apps([inets]), PrivDir = ?config(priv_dir, Config), DataDir = ?config(data_dir, Config), diff --git a/lib/inets/test/httpd_test_lib.erl b/lib/inets/test/httpd_test_lib.erl index 6406eeae79..ed466fd727 100644 --- a/lib/inets/test/httpd_test_lib.erl +++ b/lib/inets/test/httpd_test_lib.erl @@ -92,16 +92,6 @@ verify_request(SocketType, Host, Port, Node, RequestStr, Options, TimeOut) verify_request(SocketType, Host, Port, [], Node, RequestStr, Options, TimeOut). verify_request(SocketType, Host, Port, TranspOpts0, Node, RequestStr, Options, TimeOut) -> - tsp("verify_request -> entry with" - "~n SocketType: ~p" - "~n Host: ~p" - "~n Port: ~p" - "~n TranspOpts: ~p" - "~n Node: ~p" - "~n Options: ~p" - "~n TimeOut: ~p", - [SocketType, Host, Port, TranspOpts0, Node, Options, TimeOut]), - %% For now, until we modernize the httpd tests TranspOpts = case lists:member(inet6, TranspOpts0) of @@ -113,10 +103,7 @@ verify_request(SocketType, Host, Port, TranspOpts0, Node, RequestStr, Options, T try inets_test_lib:connect_bin(SocketType, Host, Port, TranspOpts) of {ok, Socket} -> - tsp("verify_request -> connected - now send message"), SendRes = inets_test_lib:send(SocketType, Socket, RequestStr), - tsp("verify_request -> send result: " - "~n ~p", [SendRes]), State = case inets_regexp:match(RequestStr, "printenv") of nomatch -> #state{}; @@ -127,37 +114,24 @@ verify_request(SocketType, Host, Port, TranspOpts0, Node, RequestStr, Options, T case request(State#state{request = RequestStr, socket = Socket}, TimeOut) of {error, Reason} -> - tsp("verify_request -> request failed: " - "~n Reason: ~p", [Reason]), {error, Reason}; NewState -> - tsp("verify_request -> validate reply: " - "~n NewState: ~p", [NewState]), ValidateResult = validate(RequestStr, NewState, Options, Node, Port), - tsp("verify_request -> validation result: " - "~n ~p", [ValidateResult]), inets_test_lib:close(SocketType, Socket), ValidateResult end; ConnectError -> - tsp("verify_request -> connect error: " - "~n ~p" - "~n", [ConnectError]), - tsf({connect_error, ConnectError, - [SocketType, Host, Port, TranspOpts]}) + ct:fail({connect_error, ConnectError, + [SocketType, Host, Port, TranspOpts]}) catch T:E -> - tsp("verify_request -> connect failed: " - "~n E: ~p" - "~n T: ~p" - "~n", [E, T]), - tsf({connect_failure, - [{type, T}, - {error, E}, - {stacktrace, erlang:get_stacktrace()}, - {args, [SocketType, Host, Port, TranspOpts]}]}) + ct:fail({connect_failure, + [{type, T}, + {error, E}, + {stacktrace, erlang:get_stacktrace()}, + {args, [SocketType, Host, Port, TranspOpts]}]}) end. request(#state{mfa = {Module, Function, Args}, @@ -166,10 +140,6 @@ request(#state{mfa = {Module, Function, Args}, HeadRequest = lists:sublist(RequestStr, 1, 4), receive {tcp, Socket, Data} -> - io:format("~p ~w[~w]request -> received (tcp) data" - "~n Data: ~p" - "~n", [self(), ?MODULE, ?LINE, Data]), - print(tcp, Data, State), case Module:Function([Data | Args]) of {ok, Parsed} -> handle_http_msg(Parsed, State); @@ -179,22 +149,12 @@ request(#state{mfa = {Module, Function, Args}, request(State#state{mfa = NewMFA}, TimeOut) end; {tcp_closed, Socket} when Function =:= whole_body -> - io:format("~p ~w[~w]request -> " - "received (tcp) closed when whole_body" - "~n", [self(), ?MODULE, ?LINE]), - print(tcp, "closed", State), State#state{body = hd(Args)}; {tcp_closed, Socket} -> - io:format("~p ~w[~w]request -> received (tcp) closed" - "~n", [self(), ?MODULE, ?LINE]), exit({test_failed, connection_closed}); {tcp_error, Socket, Reason} -> - io:format("~p ~w[~w]request -> received (tcp) error" - "~n Reason: ~p" - "~n", [self(), ?MODULE, ?LINE, Reason]), ct:fail({tcp_error, Reason}); {ssl, Socket, Data} -> - print(ssl, Data, State), case Module:Function([Data | Args]) of {ok, Parsed} -> handle_http_msg(Parsed, State); @@ -204,28 +164,19 @@ request(#state{mfa = {Module, Function, Args}, request(State#state{mfa = NewMFA}, TimeOut) end; {ssl_closed, Socket} when Function =:= whole_body -> - print(ssl, "closed", State), State#state{body = hd(Args)}; {ssl_closed, Socket} -> exit({test_failed, connection_closed}); {ssl_error, Socket, Reason} -> ct:fail({ssl_error, Reason}) after TimeOut -> - io:format("~p ~w[~w]request -> timeout" - "~n", [self(), ?MODULE, ?LINE]), + ct:pal("~p ~w[~w]request -> timeout" + "~n", [self(), ?MODULE, ?LINE]), ct:fail(connection_timed_out) end. handle_http_msg({Version, StatusCode, ReasonPharse, Headers, Body}, State = #state{request = RequestStr}) -> - io:format("~p ~w[~w]handle_http_msg -> entry with" - "~n Version: ~p" - "~n StatusCode: ~p" - "~n ReasonPharse: ~p" - "~n Headers: ~p" - "~n Body: ~p" - "~n", [self(), ?MODULE, ?LINE, - Version, StatusCode, ReasonPharse, Headers, Body]), case is_expect(RequestStr) of true -> State#state{status_line = {Version, @@ -285,11 +236,6 @@ validate(RequestStr, #state{status_line = {Version, StatusCode, _}, headers = Headers, body = Body}, Options, N, P) -> - tsp("validate -> entry with" - "~n StatusCode: ~p" - "~n Headers: ~p" - "~n Body: ~p", [StatusCode, Headers, Body]), - check_version(Version, Options), case lists:keysearch(statuscode, 1, Options) of {value, _} -> @@ -311,20 +257,20 @@ check_version(Version, Options) -> {value, {version, Version}} -> ok; {value, {version, Ver}} -> - tsf({wrong_version, [{got, Version}, - {expected, Ver}]}); + ct:fail({wrong_version, [{got, Version}, + {expected, Ver}]}); _ -> - case Version of - "HTTP/1.1" -> - ok; + case Version of + "HTTP/1.1" -> + ok; _ -> - tsf({wrong_version, [{got, Version}, - {expected, "HTTP/1.1"}]}) - end + ct:fail({wrong_version, [{got, Version}, + {expected, "HTTP/1.1"}]}) + end end. check_status_code(StatusCode, [], Options) -> - tsf({wrong_status_code, [{got, StatusCode}, {expected, Options}]}); + ct:fail({wrong_status_code, [{got, StatusCode}, {expected, Options}]}); check_status_code(StatusCode, Current = [_ | Rest], Options) -> case lists:keysearch(statuscode, 1, Current) of {value, {statuscode, StatusCode}} -> @@ -332,7 +278,7 @@ check_status_code(StatusCode, Current = [_ | Rest], Options) -> {value, {statuscode, _OtherStatus}} -> check_status_code(StatusCode, Rest, Options); false -> - tsf({wrong_status_code, [{got, StatusCode}, {expected, Options}]}) + ct:fail({wrong_status_code, [{got, StatusCode}, {expected, Options}]}) end. do_validate(_, [], _, _) -> @@ -345,9 +291,9 @@ do_validate(Header, [{header, HeaderField}|Rest], N, P) -> {value, {LowerHeaderField, _Value}} -> ok; false -> - tsf({missing_header_field, LowerHeaderField, Header}); + ct:fail({missing_header_field, LowerHeaderField, Header}); _ -> - tsf({missing_header_field, LowerHeaderField, Header}) + ct:fail({missing_header_field, LowerHeaderField, Header}) end, do_validate(Header, Rest, N, P); do_validate(Header, [{header, HeaderField, Value}|Rest],N,P) -> @@ -356,15 +302,15 @@ do_validate(Header, [{header, HeaderField, Value}|Rest],N,P) -> {value, {LowerHeaderField, Value}} -> ok; false -> - tsf({wrong_header_field_value, LowerHeaderField, Header}); + ct:fail({wrong_header_field_value, LowerHeaderField, Header}); _ -> - tsf({wrong_header_field_value, LowerHeaderField, Header}) + ct:fail({wrong_header_field_value, LowerHeaderField, Header}) end, do_validate(Header, Rest, N, P); do_validate(Header,[{no_header, HeaderField}|Rest],N,P) -> case lists:keysearch(HeaderField,1,Header) of {value,_} -> - tsf({wrong_header_field_value, HeaderField, Header}); + ct:fail({wrong_header_field_value, HeaderField, Header}); _ -> ok end, @@ -382,14 +328,14 @@ is_expect(RequestStr) -> %% OTP-5775, content-length check_body("GET /cgi-bin/erl/httpd_example:get_bin HTTP/1.0\r\n\r\n", 200, "text/html", Length, _Body) when (Length =/= 274) -> - tsf(content_length_error); + ct:fail(content_length_error); check_body("GET /cgi-bin/cgi_echo HTTP/1.0\r\n\r\n", 200, "text/plain", _, Body) -> case size(Body) of 100 -> ok; _ -> - tsf(content_length_error) + ct:fail(content_length_error) end; check_body(RequestStr, 200, "text/html", _, Body) -> @@ -404,16 +350,3 @@ check_body(RequestStr, 200, "text/html", _, Body) -> check_body(_, _, _, _,_) -> ok. -print(Proto, Data, #state{print = true}) -> - ct:pal("Received ~p: ~p~n", [Proto, Data]); -print(_, _, #state{print = false}) -> - ok. - - -tsp(F) -> - inets_test_lib:tsp(F). -tsp(F, A) -> - inets_test_lib:tsp(F, A). - -tsf(Reason) -> - inets_test_lib:tsf(Reason). diff --git a/lib/inets/test/old_httpd_SUITE.erl b/lib/inets/test/old_httpd_SUITE.erl index de9aa4562e..3e1a1a3845 100644 --- a/lib/inets/test/old_httpd_SUITE.erl +++ b/lib/inets/test/old_httpd_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -155,65 +155,103 @@ all() -> [ {group, ip}, {group, ssl}, - {group, http_1_1_ip}, - {group, http_1_0_ip}, - {group, http_0_9_ip}, - {group, ipv6}, + %%{group, http_1_1_ip}, + %%{group, http_1_0_ip}, + %%{group, http_0_9_ip}, + %%{group, ipv6}, {group, tickets} ]. groups() -> [ {ip, [], - [ip_mod_alias, ip_mod_actions, ip_mod_security, - ip_mod_auth, ip_mod_auth_api, ip_mod_auth_mnesia_api, - ip_mod_htaccess, ip_mod_cgi, ip_mod_esi, ip_mod_get, - ip_mod_head, ip_mod_all, ip_load_light, ip_load_medium, - ip_load_heavy, ip_dos_hostname, ip_time_test, - ip_restart_no_block, ip_restart_disturbing_block, - ip_restart_non_disturbing_block, - ip_block_disturbing_idle, ip_block_non_disturbing_idle, - ip_block_503, ip_block_disturbing_active, - ip_block_non_disturbing_active, - ip_block_disturbing_active_timeout_not_released, - ip_block_disturbing_active_timeout_released, - ip_block_non_disturbing_active_timeout_not_released, - ip_block_non_disturbing_active_timeout_released, - ip_block_disturbing_blocker_dies, - ip_block_non_disturbing_blocker_dies]}, + [ + %%ip_mod_alias, + ip_mod_actions, + %%ip_mod_security, + %% ip_mod_auth, + %% ip_mod_auth_api, + ip_mod_auth_mnesia_api, + %%ip_mod_htaccess, + %%ip_mod_cgi, + %%ip_mod_esi, + %%ip_mod_get, + %%ip_mod_head, + %%ip_mod_all, + %% ip_load_light, + %% ip_load_medium, + %% ip_load_heavy, + %%ip_dos_hostname, + ip_time_test + %% Replaced by load_config + %% ip_restart_no_block, + %% ip_restart_disturbing_block, + %% ip_restart_non_disturbing_block, + %% ip_block_disturbing_idle, + %% ip_block_non_disturbing_idle, + %% ip_block_503, + %% ip_block_disturbing_active, + %% ip_block_non_disturbing_active, + %% ip_block_disturbing_active_timeout_not_released, + %% ip_block_disturbing_active_timeout_released, + %% ip_block_non_disturbing_active_timeout_not_released, + %% ip_block_non_disturbing_active_timeout_released, + %% ip_block_disturbing_blocker_dies, + %% ip_block_non_disturbing_blocker_dies + ]}, {ssl, [], [{group, essl}]}, {essl, [], - [essl_mod_alias, essl_mod_actions, essl_mod_security, - essl_mod_auth, essl_mod_auth_api, - essl_mod_auth_mnesia_api, essl_mod_htaccess, - essl_mod_cgi, essl_mod_esi, essl_mod_get, essl_mod_head, - essl_mod_all, essl_load_light, essl_load_medium, - essl_load_heavy, essl_dos_hostname, essl_time_test, - essl_restart_no_block, essl_restart_disturbing_block, - essl_restart_non_disturbing_block, - essl_block_disturbing_idle, - essl_block_non_disturbing_idle, essl_block_503, - essl_block_disturbing_active, - essl_block_non_disturbing_active, - essl_block_disturbing_active_timeout_not_released, - essl_block_disturbing_active_timeout_released, - essl_block_non_disturbing_active_timeout_not_released, - essl_block_non_disturbing_active_timeout_released, - essl_block_disturbing_blocker_dies, - essl_block_non_disturbing_blocker_dies]}, - {http_1_1_ip, [], - [ip_host, ip_chunked, ip_expect, ip_range, ip_if_test, - ip_http_trace, ip_http1_1_head, - ip_mod_cgi_chunked_encoding_test]}, - {http_1_0_ip, [], - [ip_head_1_0, ip_get_1_0, ip_post_1_0]}, - {http_0_9_ip, [], [ip_get_0_9]}, - {ipv6, [], [ipv6_hostname_ipcomm, ipv6_address_ipcomm, - ipv6_hostname_essl, ipv6_address_essl]}, + [ + %%essl_mod_alias, + essl_mod_actions, + %% essl_mod_security, + %% essl_mod_auth, + %% essl_mod_auth_api, + essl_mod_auth_mnesia_api, + %%essl_mod_htaccess, + %%essl_mod_cgi, + %%essl_mod_esi, + %%essl_mod_get, + %%essl_mod_head, + %% essl_mod_all, + %% essl_load_light, + %% essl_load_medium, + %% essl_load_heavy, + %%essl_dos_hostname, + essl_time_test + %% Replaced by load_config + %% essl_restart_no_block, + %% essl_restart_disturbing_block, + %% essl_restart_non_disturbing_block, + %% essl_block_disturbing_idle, + %% essl_block_non_disturbing_idle, essl_block_503, + %% essl_block_disturbing_active, + %% essl_block_non_disturbing_active, + %% essl_block_disturbing_active_timeout_not_released, + %% essl_block_disturbing_active_timeout_released, + %% essl_block_non_disturbing_active_timeout_not_released, + %% essl_block_non_disturbing_active_timeout_released, + %% essl_block_disturbing_blocker_dies, + %% essl_block_non_disturbing_blocker_dies + ]}, + %% {http_1_1_ip, [], + %% [ + %% %%ip_host, ip_chunked, ip_expect, + %% %%ip_range, + %% %%ip_if_test + %% %%ip_http_trace, ip_http1_1_head, + %% %%ip_mod_cgi_chunked_encoding_test + %% ]}, + %%{http_1_0_ip, [], + %%[ip_head_1_0, ip_get_1_0, ip_post_1_0]}, + %%{http_0_9_ip, [], [ip_get_0_9]}, + %% {ipv6, [], [ipv6_hostname_ipcomm, ipv6_address_ipcomm, + %% ipv6_hostname_essl, ipv6_address_essl]}, {tickets, [], - [ticket_5775, ticket_5865, ticket_5913, ticket_6003, - ticket_7304]}]. - + [%%ticket_5775, ticket_5865, + ticket_5913%%, ticket_6003, + %%ticket_7304 + ]}]. init_per_group(ipv6 = _GroupName, Config) -> case inets_test_lib:has_ipv6_support() of diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 6b1f149cc8..cbcf0362c9 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 5.9.8 +INETS_VSN = 5.10 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/jinterface/vsn.mk b/lib/jinterface/vsn.mk index 1954040c3d..c50200fab6 100644 --- a/lib/jinterface/vsn.mk +++ b/lib/jinterface/vsn.mk @@ -1 +1 @@ -JINTERFACE_VSN = 1.5.8 +JINTERFACE_VSN = 1.5.9 diff --git a/lib/kernel/doc/src/app.xml b/lib/kernel/doc/src/app.xml index 7c9d6eecec..8575d94048 100644 --- a/lib/kernel/doc/src/app.xml +++ b/lib/kernel/doc/src/app.xml @@ -61,7 +61,8 @@ {applications, Apps}, {env, Env}, {mod, Start}, - {start_phases, Phases}]}. + {start_phases, Phases}, + {runtime_dependencies, RTDeps}]}. Value Default ----- ------- @@ -77,8 +78,10 @@ Apps [App] [] Env [{Par,Val}] [] Start {Module,StartArgs} [] Phases [{Phase,PhaseArgs}] undefined +RTDeps [ApplicationVersion] [] Module = Name = App = Par = Phase = atom() - Val = StartArgs = PhaseArgs = term()</code> + Val = StartArgs = PhaseArgs = term() + ApplicationVersion = string()</code> <p><c>Application</c> is the name of the application.</p> <p>For the application controller, all keys are optional. The respective default values are used for any omitted keys.</p> @@ -87,6 +90,8 @@ Phases [{Phase,PhaseArgs}] undefined <c>description</c>, <c>vsn</c>, <c>modules</c>, <c>registered</c> and <c>applications</c>. The other keys are ignored by <c>systools</c>.</p> + <warning><p>The <c>RTDeps</c> type was introduced in OTP 17.0 and + might be subject to changes during the OTP 17 release.</p></warning> <taglist> <tag><c>description</c></tag> <item> @@ -185,6 +190,33 @@ Phases [{Phase,PhaseArgs}] undefined start phases must be a subset of the set of phases defined for the primary application. Refer to <em>OTP Design Principles</em> for more information.</p> </item> + <tag><marker id="runtime_dependencies"><c>runtime_dependencies</c></marker></tag> + <item><p>A list of application versions that the application + depends on. An example of such an application version is + <c>"kernel-3.0"</c>. Application versions specified as runtime + dependencies are minimum requirements. That is, a larger + application version than the one specified in the + dependency satisfies the requirement. For information on + how to compare application versions see + <seealso marker="doc/system_principles:versions">the + documentation of versions in the system principles + guide</seealso>. Note that that the application version + specifies a source code version. An additional indirect + requirement is that installed binary application of + the specified version has been built so that it is + compatible with the rest of the system.</p> + <p>Some dependencies might only be required in specific runtime + scenarios. In the case such optional dependencies exist, these are + specified and documented in the corresponding "App" documentation + of the specific application.</p> + <warning><p>The <c>runtime_dependencies</c> key was introduced in + OTP 17.0. The type of its value might be subject to changes during + the OTP 17 release.</p></warning> + <warning><p>All runtime dependencies specified in OTP applications + during the OTP 17 release may not be completely correct. This + is actively being worked on. Declared runtime dependencies in OTP + applications are expected to be correct in OTP 18.</p></warning> + </item> </taglist> </section> diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 6d4b1cb2db..8dae34431b 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -642,6 +642,11 @@ <item> <p>Symbolic links are not supported on this platform.</p> </item> + <tag><c>eperm</c></tag> + <item> + <p>User does not have privileges to create symbolic links + (<c>SeCreateSymbolicLinkPrivilege</c> on Windows).</p> + </item> </taglist> </desc> </func> diff --git a/lib/kernel/src/disk_log_server.erl b/lib/kernel/src/disk_log_server.erl index 684ea5b5db..45334912eb 100644 --- a/lib/kernel/src/disk_log_server.erl +++ b/lib/kernel/src/disk_log_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -199,7 +199,7 @@ do_open({open, W, #arg{name = Name}=A}=Req, From, State) -> false when W =:= local -> case A#arg.distributed of {true, Nodes} -> - Fun = fun() -> open_distr_rpc(Nodes, A, From) end, + Fun = open_distr_rpc_fun(Nodes, A, From), _Pid = spawn(Fun), %% No pending reply is expected, but don't reply yet. {pending, State}; @@ -225,11 +225,15 @@ do_open({open, W, #arg{name = Name}=A}=Req, From, State) -> end end. +-spec open_distr_rpc_fun([node()], _, _) -> % XXX: underspecified + fun(() -> no_return()). + +open_distr_rpc_fun(Nodes, A, From) -> + fun() -> open_distr_rpc(Nodes, A, From) end. + %% Spawning a process is a means to avoid deadlock when %% disk_log_servers mutually open disk_logs. --spec open_distr_rpc([node()], _, _) -> no_return(). % XXX: underspecified - open_distr_rpc(Nodes, A, From) -> {AllReplies, BadNodes} = rpc:multicall(Nodes, ?MODULE, dist_open, [A]), {Ok, Bad} = cr(AllReplies, [], []), diff --git a/lib/kernel/src/global.erl b/lib/kernel/src/global.erl index ef878b8d0c..0a4edea452 100644 --- a/lib/kernel/src/global.erl +++ b/lib/kernel/src/global.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1513,14 +1513,18 @@ delete_global_name(_Name, _Pid) -> -record(him, {node, locker, vsn, my_tag}). start_the_locker(DoTrace) -> - spawn_link(fun() -> init_the_locker(DoTrace) end). - -init_the_locker(DoTrace) -> - process_flag(trap_exit, true), % needed? - S0 = #multi{do_trace = DoTrace}, - S1 = update_locker_known({add, get_known()}, S0), - loop_the_locker(S1), - erlang:error(locker_exited). + spawn_link(init_the_locker_fun(DoTrace)). + +-spec init_the_locker_fun(boolean()) -> fun(() -> no_return()). + +init_the_locker_fun(DoTrace) -> + fun() -> + process_flag(trap_exit, true), % needed? + S0 = #multi{do_trace = DoTrace}, + S1 = update_locker_known({add, get_known()}, S0), + loop_the_locker(S1), + erlang:error(locker_exited) + end. loop_the_locker(S) -> ?trace({loop_the_locker,S}), diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index 976d5e35cb..e5928c7b63 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -194,6 +194,13 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> CodeSize, CodeBinary, Refs, 0,[] % ColdSize, CRrefs ] = binary_to_term(Bin), + ?debug_msg("***** ErLLVM *****~nVersion: ~s~nCheckSum: ~w~nConstAlign: ~w~n" ++ + "ConstSize: ~w~nConstMap: ~w~nLabelMap: ~w~nExportMap ~w~nRefs ~w~n", + [Version, CheckSum, ConstAlign, ConstSize, ConstMap, LabelMap, ExportMap, + Refs]), + %% Write HiPE binary code to a file in the current directory in order to + %% debug by disassembling. + %% file:write_file("erl.o", CodeBinary, [binary]), %% Check that we are loading up-to-date code. version_check(Version, Mod), case hipe_bifs:check_crc(CheckSum) of @@ -221,6 +228,7 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> {MFAs,Addresses} = exports(ExportMap, CodeAddress), %% Remove references to old versions of the module. ReferencesToPatch = get_refs_from(MFAs, []), + %% io:format("References to patch: ~w~n", [ReferencesToPatch]), ok = remove_refs_from(MFAs), %% Patch all dynamic references in the code. %% Function calls, Atoms, Constants, System calls @@ -246,8 +254,7 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> AddressesOfClosuresToPatch = calculate_addresses(ClosurePatches, CodeAddress, Addresses), export_funs(Addresses), - export_funs(Mod, BeamBinary, Addresses, AddressesOfClosuresToPatch), - ok + export_funs(Mod, BeamBinary, Addresses, AddressesOfClosuresToPatch) end, %% Redirect references to the old module to the new module's BEAM stub. patch_to_emu_step2(OldReferencesToPatch), diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index cb8c98ab06..5658c6b6cf 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -114,6 +114,7 @@ pg2]}, {applications, []}, {env, [{error_logger, tty}]}, - {mod, {kernel, []}} + {mod, {kernel, []}}, + {runtime_dependencies, ["erts-6.0", "stdlib-2.0", "sasl-2.4"]} ] }. diff --git a/lib/kernel/src/os.erl b/lib/kernel/src/os.erl index 9ffa9adeab..3bda391b8e 100644 --- a/lib/kernel/src/os.erl +++ b/lib/kernel/src/os.erl @@ -230,7 +230,9 @@ unix_cmd(Cmd) -> %% and the commands are read from standard input. We set the %% $1 parameter for easy identification of the resident shell. %% --define(SHELL, "/bin/sh -s unix:cmd 2>&1"). +-define(ROOT, "/"). +-define(ROOT_ANDROID, "/system"). +-define(SHELL, "bin/sh -s unix:cmd 2>&1"). -define(PORT_CREATOR_NAME, os_cmd_port_creator). %% @@ -280,7 +282,12 @@ start_port_srv(Request) -> end. start_port_srv_handle({Ref,Client}) -> - Reply = try open_port({spawn, ?SHELL},[stream]) of + Path = case lists:reverse(erlang:system_info(system_architecture)) of + % androideabi + "ibaediordna" ++ _ -> filename:join([?ROOT_ANDROID, ?SHELL]); + _ -> filename:join([?ROOT, ?SHELL]) + end, + Reply = try open_port({spawn, Path},[stream]) of Port when is_port(Port) -> (catch port_connect(Port, Client)), unlink(Port), diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 6b52493f46..f6d6cd94ab 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -428,7 +428,13 @@ make_del_dir(Config) when is_list(Config) -> % because there are processes having that directory as current. ?line ok = ?FILE_MODULE:make_dir(NewDir), ?line {ok,CurrentDir} = file:get_cwd(), - ?line ok = ?FILE_MODULE:set_cwd(NewDir), + case {os:type(), length(NewDir) >= 260 } of + {{win32,_}, true} -> + io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH)\n", []), + io:format("\nNewDir = ~p\n", [NewDir]); + _ -> + ?line ok = ?FILE_MODULE:set_cwd(NewDir) + end, try %% Check that we get an error when trying to create... %% a deep directory @@ -485,32 +491,39 @@ cur_dir_0(Config) when is_list(Config) -> atom_to_list(?MODULE) ++"_curdir"), ?line ok = ?FILE_MODULE:make_dir(NewDir), - ?line io:format("cd to ~s",[NewDir]), - ?line ok = ?FILE_MODULE:set_cwd(NewDir), - - %% Create a file in the new current directory, and check that it - %% really is created there - ?line UncommonName = "uncommon.fil", - ?line {ok,Fd} = ?FILE_MODULE:open(UncommonName,read_write), - ?line ok = ?FILE_MODULE:close(Fd), - ?line {ok,NewDirFiles} = ?FILE_MODULE:list_dir("."), - ?line true = lists:member(UncommonName,NewDirFiles), - - %% Delete the directory and return to the old current directory - %% and check that the created file isn't there (too!) - ?line expect({error, einval}, {error, eacces}, - ?FILE_MODULE:del_dir(NewDir)), - ?line ?FILE_MODULE:delete(UncommonName), - ?line {ok,[]} = ?FILE_MODULE:list_dir("."), - ?line ok = ?FILE_MODULE:set_cwd(Dir1), - ?line io:format("cd back to ~s",[Dir1]), - ?line ok = ?FILE_MODULE:del_dir(NewDir), - ?line {error, enoent} = ?FILE_MODULE:set_cwd(NewDir), - ?line ok = ?FILE_MODULE:set_cwd(Dir1), - ?line io:format("cd back to ~s",[Dir1]), - ?line {ok,OldDirFiles} = ?FILE_MODULE:list_dir("."), - ?line false = lists:member(UncommonName,OldDirFiles), - + case {os:type(), length(NewDir) >= 260} of + {{win32,_}, true} -> + io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH):\n"), + io:format("\nNewDir = ~p\n", [NewDir]); + _ -> + io:format("cd to ~s",[NewDir]), + ok = ?FILE_MODULE:set_cwd(NewDir), + + %% Create a file in the new current directory, and check that it + %% really is created there + UncommonName = "uncommon.fil", + {ok,Fd} = ?FILE_MODULE:open(UncommonName,read_write), + ok = ?FILE_MODULE:close(Fd), + {ok,NewDirFiles} = ?FILE_MODULE:list_dir("."), + true = lists:member(UncommonName,NewDirFiles), + + %% Delete the directory and return to the old current directory + %% and check that the created file isn't there (too!) + expect({error, einval}, {error, eacces}, + ?FILE_MODULE:del_dir(NewDir)), + ?FILE_MODULE:delete(UncommonName), + {ok,[]} = ?FILE_MODULE:list_dir("."), + ok = ?FILE_MODULE:set_cwd(Dir1), + io:format("cd back to ~s",[Dir1]), + + ok = ?FILE_MODULE:del_dir(NewDir), + {error, enoent} = ?FILE_MODULE:set_cwd(NewDir), + ok = ?FILE_MODULE:set_cwd(Dir1), + io:format("cd back to ~s",[Dir1]), + {ok,OldDirFiles} = ?FILE_MODULE:list_dir("."), + false = lists:member(UncommonName,OldDirFiles) + end, + %% Try doing some bad things ?line {error, badarg} = ?FILE_MODULE:set_cwd({foo,bar}), ?line {error, enoent} = ?FILE_MODULE:set_cwd(""), @@ -1982,7 +1995,6 @@ names(Config) when is_list(Config) -> ?line Name1 = filename:join(RootDir, FileName), ?line Name2 = [RootDir,"/","foo1",".","fil"], ?line Name3 = [RootDir,"/",foo,$1,[[[],[],'.']],"f",il], - ?line Name4 = list_to_atom(Name1), ?line {ok,Fd0} = ?FILE_MODULE:open(Name1,write), ?line ok = ?FILE_MODULE:close(Fd0), @@ -1995,23 +2007,33 @@ names(Config) when is_list(Config) -> ?line ok = ?FILE_MODULE:close(Fd2), ?line {ok,Fd3} = ?FILE_MODULE:open(Name3,read), ?line ok = ?FILE_MODULE:close(Fd3), - ?line {ok,Fd4} = ?FILE_MODULE:open(Name4,read), - ?line ok = ?FILE_MODULE:close(Fd4), + case length(Name1) > 255 of + true -> + io:format("Path too long for an atom:\n\n~p\n", [Name1]); + false -> + Name4 = list_to_atom(Name1), + {ok,Fd4} = ?FILE_MODULE:open(Name4,read), + ok = ?FILE_MODULE:close(Fd4) + end, %% Try some path names ?line Path1 = RootDir, ?line Path2 = [RootDir], ?line Path3 = ['',[],[RootDir,[[]]]], - ?line Path4 = list_to_atom(Path1), ?line {ok,Fd11,_} = ?FILE_MODULE:path_open([Path1],FileName,read), ?line ok = ?FILE_MODULE:close(Fd11), ?line {ok,Fd12,_} = ?FILE_MODULE:path_open([Path2],FileName,read), ?line ok = ?FILE_MODULE:close(Fd12), ?line {ok,Fd13,_} = ?FILE_MODULE:path_open([Path3],FileName,read), ?line ok = ?FILE_MODULE:close(Fd13), - ?line {ok,Fd14,_} = ?FILE_MODULE:path_open([Path4],FileName,read), - ?line ok = ?FILE_MODULE:close(Fd14), - + case length(Path1) > 255 of + true-> + io:format("Path too long for an atom:\n\n~p\n", [Path1]); + false -> + Path4 = list_to_atom(Path1), + {ok,Fd14,_} = ?FILE_MODULE:path_open([Path4],FileName,read), + ok = ?FILE_MODULE:close(Fd14) + end, ?line [] = flush(), ?line test_server:timetrap_cancel(Dog), ok. @@ -2673,6 +2695,9 @@ symlinks(Config) when is_list(Config) -> case ?FILE_MODULE:make_symlink(Name, Alias) of {error, enotsup} -> {skipped, "Links not supported on this platform"}; + {error, eperm} -> + {win32,_} = os:type(), + {skipped, "Windows user not privileged to create symlinks"}; ok -> ?line {ok, Info1} = ?FILE_MODULE:read_file_info(Name), ?line {ok, Info1} = ?FILE_MODULE:read_file_info(Alias), @@ -3599,7 +3624,11 @@ otp_10852(Config) when is_list(Config) -> ok = rpc_call(Node, list_dir_all, [B]), ok = rpc_call(Node, read_file, [B]), ok = rpc_call(Node, make_link, [B,B]), - ok = rpc_call(Node, make_symlink, [B,B]), + case rpc_call(Node, make_symlink, [B,B]) of + ok -> ok; + {error, E} when (E =:= enotsup) or (E =:= eperm) -> + {win32,_} = os:type() + end, ok = rpc_call(Node, delete, [B]), ok = rpc_call(Node, make_dir, [B]), ok = rpc_call(Node, del_dir, [B]), diff --git a/lib/kernel/test/prim_file_SUITE.erl b/lib/kernel/test/prim_file_SUITE.erl index 3e6a85eadd..05bd5b3a3d 100644 --- a/lib/kernel/test/prim_file_SUITE.erl +++ b/lib/kernel/test/prim_file_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2013. All Rights Reserved. +%% Copyright Ericsson AB 2000-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -183,7 +183,6 @@ time_dist({_D1, _T1} = DT1, {_D2, _T2} = DT2) -> read_write_file(suite) -> []; read_write_file(doc) -> []; read_write_file(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line Name = filename:join(RootDir, atom_to_list(?MODULE) @@ -232,7 +231,6 @@ read_write_file(Config) when is_list(Config) -> ?line {ok,Bin5} = ?PRIM_FILE:read_file(Name), ?line {Bin1,Bin2} = split_binary(Bin5,byte_size(Bin1)), - ?line test_server:timetrap_cancel(Dog), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -254,7 +252,6 @@ make_del_dir_b(Config) when is_list(Config) -> Result. make_del_dir(Config, Handle, Suffix) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line NewDir = filename:join(RootDir, atom_to_list(?MODULE) @@ -269,7 +266,13 @@ make_del_dir(Config, Handle, Suffix) -> % because there are processes having that directory as current. ?line ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), ?line {ok, CurrentDir} = ?PRIM_FILE_call(get_cwd, Handle, []), - ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), + case {os:type(), length(NewDir) >= 260 } of + {{win32,_}, true} -> + io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH)\n", []), + io:format("\nNewDir = ~p\n", [NewDir]); + _ -> + ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]) + end, try %% Check that we get an error when trying to create... %% a deep directory @@ -302,9 +305,7 @@ make_del_dir(Config, Handle, Suffix) -> {error, einval} -> ok %FreeBSD end, ?line {error, enoent} = ?PRIM_FILE_call(del_dir, Handle, [""]), - ?line {error, badarg} = ?PRIM_FILE_call(del_dir, Handle, [[3,2,1,{}]]), - - ?line test_server:timetrap_cancel(Dog) + ?line {error, badarg} = ?PRIM_FILE_call(del_dir, Handle, [[3,2,1,{}]]) after ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [CurrentDir]) end, @@ -324,7 +325,6 @@ cur_dir_0b(Config) when is_list(Config) -> Result. cur_dir_0(Config, Handle) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), %% Find out the current dir, and cd to it ;-) ?line {ok,BaseDir} = ?PRIM_FILE_call(get_cwd, Handle, []), ?line Dir1 = BaseDir ++ "", %% Check that it's a string @@ -341,31 +341,37 @@ cur_dir_0(Config, Handle) -> ?line RootDir = ?config(priv_dir,Config), ?line NewDir = filename:join(RootDir, DirName), ?line ok = ?PRIM_FILE_call(make_dir, Handle, [NewDir]), - ?line io:format("cd to ~s",[NewDir]), - ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), - - %% Create a file in the new current directory, and check that it - %% really is created there - ?line UncommonName = "uncommon.fil", - ?line {ok,Fd} = ?PRIM_FILE:open(UncommonName, [read, write]), - ?line ok = ?PRIM_FILE:close(Fd), - ?line {ok,NewDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), - ?line true = lists:member(UncommonName,NewDirFiles), - - %% Delete the directory and return to the old current directory - %% and check that the created file isn't there (too!) - ?line expect({error, einval}, {error, eacces}, {error, eexist}, + case {os:type(), length(NewDir) >= 260} of + {{win32,_}, true} -> + io:format("Skip set_cwd for windows path longer than 260 (MAX_PATH):\n"), + io:format("\nNewDir = ~p\n", [NewDir]); + _ -> + io:format("cd to ~s",[NewDir]), + ok = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), + + %% Create a file in the new current directory, and check that it + %% really is created there + UncommonName = "uncommon.fil", + {ok,Fd} = ?PRIM_FILE:open(UncommonName, [read, write]), + ok = ?PRIM_FILE:close(Fd), + {ok,NewDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), + true = lists:member(UncommonName,NewDirFiles), + + %% Delete the directory and return to the old current directory + %% and check that the created file isn't there (too!) + expect({error, einval}, {error, eacces}, {error, eexist}, ?PRIM_FILE_call(del_dir, Handle, [NewDir])), - ?line ?PRIM_FILE_call(delete, Handle, [UncommonName]), - ?line {ok,[]} = ?PRIM_FILE_call(list_dir, Handle, ["."]), - ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), - ?line io:format("cd back to ~s",[Dir1]), - ?line ok = ?PRIM_FILE_call(del_dir, Handle, [NewDir]), - ?line {error, enoent} = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), - ?line ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), - ?line io:format("cd back to ~s",[Dir1]), - ?line {ok,OldDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), - ?line false = lists:member(UncommonName,OldDirFiles), + ?PRIM_FILE_call(delete, Handle, [UncommonName]), + {ok,[]} = ?PRIM_FILE_call(list_dir, Handle, ["."]), + ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), + io:format("cd back to ~s",[Dir1]), + ok = ?PRIM_FILE_call(del_dir, Handle, [NewDir]), + {error, enoent} = ?PRIM_FILE_call(set_cwd, Handle, [NewDir]), + ok = ?PRIM_FILE_call(set_cwd, Handle, [Dir1]), + io:format("cd back to ~s",[Dir1]), + {ok,OldDirFiles} = ?PRIM_FILE_call(list_dir, Handle, ["."]), + false = lists:member(UncommonName,OldDirFiles) + end, %% Try doing some bad things ?line {error, badarg} = @@ -385,7 +391,6 @@ cur_dir_0(Config, Handle) -> ?line {ok, BaseDir} = ?PRIM_FILE_call(get_cwd, Handle, []), ?line false = lists:member($\\, BaseDir), - ?line test_server:timetrap_cancel(Dog), ok. %% Tests ?PRIM_FILE:get_cwd/1. @@ -404,8 +409,6 @@ cur_dir_1b(Config) when is_list(Config) -> Result. cur_dir_1(Config, Handle) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), - ?line case os:type() of {win32, _} -> win_cur_dir_1(Config, Handle); @@ -413,7 +416,6 @@ cur_dir_1(Config, Handle) -> ?line {error, enotsup} = ?PRIM_FILE_call(get_cwd, Handle, ["d:"]) end, - ?line test_server:timetrap_cancel(Dog), ok. win_cur_dir_1(_Config, Handle) -> @@ -439,7 +441,6 @@ win_cur_dir_1(_Config, Handle) -> open1(suite) -> []; open1(doc) -> []; open1(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line NewDir = filename:join(RootDir, atom_to_list(?MODULE) @@ -465,7 +466,6 @@ open1(Config) when is_list(Config) -> ?line {ok,Fd3} = ?PRIM_FILE:open(Name, [read]), ?line eof = ?PRIM_FILE:read(Fd3,Length), ?line ok = ?PRIM_FILE:close(Fd3), - ?line test_server:timetrap_cancel(Dog), ok. %% Tests all open modes. @@ -517,7 +517,6 @@ modes(Config) when is_list(Config) -> close(suite) -> []; close(doc) -> []; close(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line Name = filename:join(RootDir, atom_to_list(?MODULE) @@ -534,13 +533,11 @@ close(Config) when is_list(Config) -> ?line Val = ?PRIM_FILE:close(Fd1), ?line io:format("Second close gave: ~p", [Val]), - ?line test_server:timetrap_cancel(Dog), ok. access(suite) -> []; access(doc) -> []; access(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line Name = filename:join(RootDir, atom_to_list(?MODULE) @@ -562,7 +559,6 @@ access(Config) when is_list(Config) -> ?line {ok, Str} = ?PRIM_FILE:read(Fd3,length(Str)), ?line ok = ?PRIM_FILE:close(Fd3), - ?line test_server:timetrap_cancel(Dog), ok. %% Tests ?PRIM_FILE:read/2 and ?PRIM_FILE:write/2. @@ -570,7 +566,6 @@ access(Config) when is_list(Config) -> read_write(suite) -> []; read_write(doc) -> []; read_write(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir, Config), ?line NewDir = filename:join(RootDir, atom_to_list(?MODULE) @@ -582,7 +577,6 @@ read_write(Config) when is_list(Config) -> ?line {ok, Fd} = ?PRIM_FILE:open(Name, [read, write]), ?line read_write_test(Fd), - ?line test_server:timetrap_cancel(Dog), ok. read_write_test(File) -> @@ -600,7 +594,6 @@ read_write_test(File) -> pread_write(suite) -> []; pread_write(doc) -> []; pread_write(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir, Config), ?line NewDir = filename:join(RootDir, atom_to_list(?MODULE) @@ -612,7 +605,6 @@ pread_write(Config) when is_list(Config) -> ?line {ok, Fd} = ?PRIM_FILE:open(Name, [read, write]), ?line pread_write_test(Fd), - ?line test_server:timetrap_cancel(Dog), ok. pread_write_test(File) -> @@ -632,7 +624,6 @@ pread_write_test(File) -> append(doc) -> "Test appending to a file."; append(suite) -> []; append(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir, Config), ?line NewDir = filename:join(RootDir, atom_to_list(?MODULE) @@ -659,13 +650,11 @@ append(Config) when is_list(Config) -> ?line Expected = list_to_binary([First, Second, Third]), ?line {ok, Expected} = ?PRIM_FILE:read_file(Name1), - ?line test_server:timetrap_cancel(Dog), ok. exclusive(suite) -> []; exclusive(doc) -> "Test exclusive access to a file."; exclusive(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line NewDir = filename:join(RootDir, atom_to_list(?MODULE) @@ -675,7 +664,6 @@ exclusive(Config) when is_list(Config) -> ?line {ok,Fd} = ?PRIM_FILE:open(Name, [write, exclusive]), ?line {error, eexist} = ?PRIM_FILE:open(Name, [write, exclusive]), ?line ok = ?PRIM_FILE:close(Fd), - ?line test_server:timetrap_cancel(Dog), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -684,7 +672,6 @@ exclusive(Config) when is_list(Config) -> pos1(suite) -> []; pos1(doc) -> []; pos1(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line Name = filename:join(RootDir, atom_to_list(?MODULE) @@ -741,13 +728,11 @@ pos1(Config) when is_list(Config) -> ?line {ok, 0} = ?PRIM_FILE:position(Fd2,{eof,-8}), ?line {ok, "A"} = ?PRIM_FILE:read(Fd2,1), ?line {error, einval} = ?PRIM_FILE:position(Fd2,{eof,-9}), - ?line test_server:timetrap_cancel(Dog), ok. pos2(suite) -> []; pos2(doc) -> []; pos2(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line Name = filename:join(RootDir, atom_to_list(?MODULE) @@ -764,7 +749,6 @@ pos2(Config) when is_list(Config) -> ?line {ok, "D"} = ?PRIM_FILE:read(Fd2,1), ?line io:format("DONE"), - ?line test_server:timetrap_cancel(Dog), ok. @@ -782,7 +766,6 @@ file_info_basic_file_b(Config) when is_list(Config) -> Result. file_info_basic_file(Config, Handle, Suffix) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir, Config), %% Create a short file. @@ -811,7 +794,6 @@ file_info_basic_file(Config, Handle, Suffix) -> ?line {MD, MT} = ModifyTime, ?line all_integers(tuple_to_list(MD) ++ tuple_to_list(MT)), - ?line test_server:timetrap_cancel(Dog), ok. file_info_basic_directory_a(suite) -> []; @@ -828,8 +810,6 @@ file_info_basic_directory_b(Config) when is_list(Config) -> Result. file_info_basic_directory(Config, Handle) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), - %% Note: filename:join/1 removes any trailing slash, %% which is essential for ?PRIM_FILE:read_file_info/1 to work on %% platforms such as Windows95. @@ -849,7 +829,7 @@ file_info_basic_directory(Config, Handle) -> _ -> ?line test_directory("/", read, Handle) end, - ?line test_server:timetrap_cancel(Dog). + ok. test_directory(Name, ExpectedAccess, Handle) -> ?line {ok, FileInfo} = ?PRIM_FILE_call(read_file_info, Handle, [Name]), @@ -890,14 +870,12 @@ file_info_bad_b(Config) when is_list(Config) -> Result. file_info_bad(Config, Handle) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = filename:join([?config(priv_dir, Config)]), ?line {error, enoent} = ?PRIM_FILE_call( read_file_info, Handle, [filename:join(RootDir, atom_to_list(?MODULE)++"_nonexistent")]), - ?line test_server:timetrap_cancel(Dog), ok. %% Test that the file times behave as they should. @@ -1192,7 +1170,6 @@ get_good_directory(Config) -> truncate(suite) -> []; truncate(doc) -> []; truncate(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line Name = filename:join(RootDir, atom_to_list(?MODULE) @@ -1218,14 +1195,12 @@ truncate(Config) when is_list(Config) -> ?line {ok, 5} = ?PRIM_FILE:position(Fd2, 5), ?line {error, _} = ?PRIM_FILE:truncate(Fd2), - ?line test_server:timetrap_cancel(Dog), ok. datasync(suite) -> []; datasync(doc) -> "Tests that ?PRIM_FILE:datasync/1 at least doesn't crash."; datasync(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line PrivDir = ?config(priv_dir, Config), ?line Sync = filename:join(PrivDir, atom_to_list(?MODULE) @@ -1236,14 +1211,12 @@ datasync(Config) when is_list(Config) -> ?line ok = ?PRIM_FILE:datasync(Fd), ?line ok = ?PRIM_FILE:close(Fd), - ?line test_server:timetrap_cancel(Dog), ok. sync(suite) -> []; sync(doc) -> "Tests that ?PRIM_FILE:sync/1 at least doesn't crash."; sync(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line PrivDir = ?config(priv_dir, Config), ?line Sync = filename:join(PrivDir, atom_to_list(?MODULE) @@ -1254,14 +1227,12 @@ sync(Config) when is_list(Config) -> ?line ok = ?PRIM_FILE:sync(Fd), ?line ok = ?PRIM_FILE:close(Fd), - ?line test_server:timetrap_cancel(Dog), ok. advise(suite) -> []; advise(doc) -> "Tests that ?PRIM_FILE:advise/4 at least doesn't crash."; advise(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line PrivDir = ?config(priv_dir, Config), ?line Advise = filename:join(PrivDir, atom_to_list(?MODULE) @@ -1325,7 +1296,6 @@ advise(Config) when is_list(Config) -> ?line eof = ?PRIM_FILE:read_line(Fd9), ?line ok = ?PRIM_FILE:close(Fd9), - ?line test_server:timetrap_cancel(Dog), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1369,7 +1339,6 @@ check_large_write(Dog, Fd, _, _, []) -> allocate(suite) -> []; allocate(doc) -> "Tests that ?PRIM_FILE:allocate/3 at least doesn't crash."; allocate(Config) when is_list(Config) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line PrivDir = ?config(priv_dir, Config), ?line Allocate = filename:join(PrivDir, atom_to_list(?MODULE) @@ -1402,7 +1371,6 @@ allocate(Config) when is_list(Config) -> ?line ok = ?PRIM_FILE:write(Fd4, Line2), ?line ok = ?PRIM_FILE:close(Fd4), - ?line test_server:timetrap_cancel(Dog), ok. allocate_and_assert(Fd, Offset, Length) -> @@ -1450,7 +1418,6 @@ delete_b(Config) when is_list(Config) -> Result. delete(Config, Handle, Suffix) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line Name = filename:join(RootDir, atom_to_list(?MODULE) @@ -1466,7 +1433,6 @@ delete(Config, Handle, Suffix) -> ?line {error, _} = ?PRIM_FILE:open(Name, [read]), %% Try deleting a nonexistent file ?line {error, enoent} = ?PRIM_FILE_call(delete, Handle, [Name]), - ?line test_server:timetrap_cancel(Dog), ok. rename_a(suite) ->[]; @@ -1483,7 +1449,6 @@ rename_b(Config) when is_list(Config) -> Result. rename(Config, Handle, Suffix) -> - ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = ?config(priv_dir,Config), ?line FileName1 = atom_to_list(?MODULE)++"_rename"++Suffix++".fil", ?line FileName2 = atom_to_list(?MODULE)++"_rename"++Suffix++".ful", @@ -1536,7 +1501,6 @@ rename(Config, Handle, Suffix) -> ?PRIM_FILE_call(rename, Handle, [DirName2, Name2foo]), ?line io:format("Errmsg2: ~p",[Msg2]), - ?line test_server:timetrap_cancel(Dog), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2029,6 +1993,9 @@ symlinks(Config, Handle, Suffix) -> case ?PRIM_FILE_call(make_symlink, Handle, [Name, Alias]) of {error, enotsup} -> {skipped, "Links not supported on this platform"}; + {error, eperm} -> + {win32,_} = os:type(), + {skipped, "Windows user not privileged to create links"}; ok -> ?line {ok, Info1} = ?PRIM_FILE_call(read_file_info, Handle, [Name]), diff --git a/lib/megaco/src/app/megaco.app.src b/lib/megaco/src/app/megaco.app.src index 40265166ae..6ab85a1bbc 100644 --- a/lib/megaco/src/app/megaco.app.src +++ b/lib/megaco/src/app/megaco.app.src @@ -112,7 +112,10 @@ megaco_trans_sup, megaco_misc_sup, megaco_sup]}, {applications, [stdlib, kernel]}, {env, []}, - {mod, {megaco_sup, []}} + {mod, {megaco_sup, []}}, + {runtime_dependencies, ["stdlib-2.0","runtime_tools-1.8.14","kernel-3.0", + "et-1.5","erts-6.0","debugger-4.0", + "asn1-3.0"]} ]}. diff --git a/lib/megaco/vsn.mk b/lib/megaco/vsn.mk index 01d429d0ae..373f5199bf 100644 --- a/lib/megaco/vsn.mk +++ b/lib/megaco/vsn.mk @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = megaco -MEGACO_VSN = 3.17.0.3 +MEGACO_VSN = 3.17.1 PRE_VSN = APP_VSN = "$(APPLICATION)-$(MEGACO_VSN)$(PRE_VSN)" diff --git a/lib/mnesia/src/mnesia.app.src b/lib/mnesia/src/mnesia.app.src index 3715488ec2..e755864792 100644 --- a/lib/mnesia/src/mnesia.app.src +++ b/lib/mnesia/src/mnesia.app.src @@ -47,6 +47,7 @@ mnesia_tm ]}, {applications, [kernel, stdlib]}, - {mod, {mnesia_sup, []}}]}. + {mod, {mnesia_sup, []}}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index 064ba43791..c596f98c81 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.11 +MNESIA_VSN = 4.12 diff --git a/lib/observer/src/observer.app.src b/lib/observer/src/observer.app.src index f14f0ee849..97a54cd6f9 100644 --- a/lib/observer/src/observer.app.src +++ b/lib/observer/src/observer.app.src @@ -60,6 +60,9 @@ ttb_et]}, {registered, []}, {applications, [kernel, stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["wx-1.2","stdlib-2.0","runtime_tools-1.8.14", + "kernel-3.0","inets-5.10","et-1.5", + "erts-6.0"]}]}. diff --git a/lib/observer/vsn.mk b/lib/observer/vsn.mk index f48809a839..a6300eeb18 100644 --- a/lib/observer/vsn.mk +++ b/lib/observer/vsn.mk @@ -1 +1 @@ -OBSERVER_VSN = 1.3.1.2 +OBSERVER_VSN = 2.0 diff --git a/lib/odbc/src/odbc.app.src b/lib/odbc/src/odbc.app.src index 5229b28c08..b2c5775de2 100644 --- a/lib/odbc/src/odbc.app.src +++ b/lib/odbc/src/odbc.app.src @@ -11,5 +11,6 @@ ]}, {applications, [kernel, stdlib]}, {env,[]}, - {mod, {odbc_app, []}}]}. + {mod, {odbc_app, []}}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/odbc/vsn.mk b/lib/odbc/vsn.mk index d9e2ab26a9..1af4751248 100644 --- a/lib/odbc/vsn.mk +++ b/lib/odbc/vsn.mk @@ -1 +1 @@ -ODBC_VSN = 2.10.19 +ODBC_VSN = 2.10.20 diff --git a/lib/orber/src/orber.app.src b/lib/orber/src/orber.app.src index 88df4162b6..30bd90347d 100644 --- a/lib/orber/src/orber.app.src +++ b/lib/orber/src/orber.app.src @@ -103,7 +103,9 @@ orber_iiop_pm, orber_env]}, {applications, [stdlib, kernel, mnesia]}, {env, []}, - {mod, {orber, []}} + {mod, {orber, []}}, + {runtime_dependencies, ["stdlib-2.0","ssl-5.3.4","mnesia-4.12","kernel-3.0", + "inets-5.10","erts-6.0"]} ]}. diff --git a/lib/orber/vsn.mk b/lib/orber/vsn.mk index 7bbebc65dc..3ea64b1ff6 100644 --- a/lib/orber/vsn.mk +++ b/lib/orber/vsn.mk @@ -1,2 +1,2 @@ -ORBER_VSN = 3.6.26.1 +ORBER_VSN = 3.6.27 diff --git a/lib/os_mon/c_src/memsup.c b/lib/os_mon/c_src/memsup.c index b5114d10ed..409db84aa7 100644 --- a/lib/os_mon/c_src/memsup.c +++ b/lib/os_mon/c_src/memsup.c @@ -324,7 +324,7 @@ get_mem_procfs(memory_ext *me){ /* arch specific functions */ -#if defined(__linux__) /* ifdef SYSINFO */ +#if defined(__linux__) && !defined(__ANDROID__)/* ifdef SYSINFO */ /* sysinfo does not include cached memory which is a problem. */ static int get_extended_mem_sysinfo(memory_ext *me) { @@ -395,8 +395,12 @@ get_extended_mem_sgi(memory_ext *me) { static void get_extended_mem(memory_ext *me) { +/* android */ +#if defined(__ANDROID__) + if (get_mem_procfs(me)) return; + /* linux */ -#if defined(__linux__) +#elif defined(__linux__) if (get_mem_procfs(me)) return; if (get_extended_mem_sysinfo(me)) return; diff --git a/lib/os_mon/src/os_mon.app.src b/lib/os_mon/src/os_mon.app.src index 15bbd2663c..cc08cebe3d 100644 --- a/lib/os_mon/src/os_mon.app.src +++ b/lib/os_mon/src/os_mon.app.src @@ -29,4 +29,7 @@ {start_disksup, true}, {start_memsup, true}, {start_os_sup, false}]}, - {mod, {os_mon, []}}]}. + {mod, {os_mon, []}}, + {runtime_dependencies, ["stdlib-2.0","snmp-4.25.1","sasl-2.4", + "otp_mibs-1.0.9","mnesia-4.12","kernel-3.0", + "erts-6.0"]}]}. diff --git a/lib/os_mon/vsn.mk b/lib/os_mon/vsn.mk index e9e90729f2..74397c2bc6 100644 --- a/lib/os_mon/vsn.mk +++ b/lib/os_mon/vsn.mk @@ -1 +1 @@ -OS_MON_VSN = 2.2.14 +OS_MON_VSN = 2.2.15 diff --git a/lib/ose/src/ose.app.src b/lib/ose/src/ose.app.src index c39d3f2d05..60699c369b 100644 --- a/lib/ose/src/ose.app.src +++ b/lib/ose/src/ose.app.src @@ -23,4 +23,5 @@ {modules, [ose]}, {registered,[]}, {applications, [stdlib,kernel]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["stdlib-2.0","erts-6.0"]}]}. diff --git a/lib/otp_mibs/src/otp_mibs.app.src b/lib/otp_mibs/src/otp_mibs.app.src index b177af0709..ebc656b0b2 100644 --- a/lib/otp_mibs/src/otp_mibs.app.src +++ b/lib/otp_mibs/src/otp_mibs.app.src @@ -23,5 +23,7 @@ {modules, [otp_mib]}, {registered, []}, {applications, [kernel, stdlib, snmp]}, - {env,[]}]}. + {env,[]}, + {runtime_dependencies, ["stdlib-2.0","snmp-4.25.1","mnesia-4.12", + "kernel-3.0","erts-6.0"]}]}. diff --git a/lib/otp_mibs/vsn.mk b/lib/otp_mibs/vsn.mk index 96d3088224..98db21c132 100644 --- a/lib/otp_mibs/vsn.mk +++ b/lib/otp_mibs/vsn.mk @@ -1,4 +1,4 @@ -OTP_MIBS_VSN = 1.0.8 +OTP_MIBS_VSN = 1.0.9 # Note: The branch 'otp_mibs' is defunct as of otp_mibs-1.0.4 and # should NOT be used again. diff --git a/lib/parsetools/src/parsetools.app.src b/lib/parsetools/src/parsetools.app.src index af62fc4f6b..9eeb8fcc05 100644 --- a/lib/parsetools/src/parsetools.app.src +++ b/lib/parsetools/src/parsetools.app.src @@ -11,7 +11,8 @@ {applications, [kernel,stdlib]}, {env, [{file_util_search_methods,[{"", ""}, {"ebin", "esrc"}, {"ebin", "src"}]} ] - } + }, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} ] }. diff --git a/lib/parsetools/vsn.mk b/lib/parsetools/vsn.mk index d62962c54a..8fd7422c1c 100644 --- a/lib/parsetools/vsn.mk +++ b/lib/parsetools/vsn.mk @@ -1 +1 @@ -PARSETOOLS_VSN = 2.0.10 +PARSETOOLS_VSN = 2.0.11 diff --git a/lib/percept/src/percept.app.src b/lib/percept/src/percept.app.src index cf4a9fc438..f8991ee577 100644 --- a/lib/percept/src/percept.app.src +++ b/lib/percept/src/percept.app.src @@ -35,7 +35,9 @@ ]}, {registered, [percept_db,percept_port]}, {applications, [kernel,stdlib]}, - {env,[]} + {env,[]}, + {runtime_dependencies, ["stdlib-2.0","runtime_tools-1.8.14","kernel-3.0", + "inets-5.10","erts-6.0"]} ]}. diff --git a/lib/percept/vsn.mk b/lib/percept/vsn.mk index 99729c11e2..935a9d1336 100644 --- a/lib/percept/vsn.mk +++ b/lib/percept/vsn.mk @@ -1 +1 @@ -PERCEPT_VSN = 0.8.8.2 +PERCEPT_VSN = 0.8.9 diff --git a/lib/public_key/src/public_key.app.src b/lib/public_key/src/public_key.app.src index 736a778a4b..88ef07c5a6 100644 --- a/lib/public_key/src/public_key.app.src +++ b/lib/public_key/src/public_key.app.src @@ -13,7 +13,9 @@ ]}, {applications, [asn1, crypto, kernel, stdlib]}, {registered, []}, - {env, []} + {env, []}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0","crypto-3.3", + "asn1-3.0"]} ] }. diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index 3473757c5f..f0450918aa 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 0.21 +PUBLIC_KEY_VSN = 0.22 diff --git a/lib/reltool/src/reltool.app.src b/lib/reltool/src/reltool.app.src index 4188f341f1..65fcf4aae5 100644 --- a/lib/reltool/src/reltool.app.src +++ b/lib/reltool/src/reltool.app.src @@ -34,5 +34,7 @@ ]}, {registered, []}, {applications, [stdlib, kernel]}, - {env, []} + {env, []}, + {runtime_dependencies, ["wx-1.2","tools-2.6.14","stdlib-2.0","sasl-2.4", + "kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/reltool/src/reltool.hrl b/lib/reltool/src/reltool.hrl index f0d8b38519..56161a152a 100644 --- a/lib/reltool/src/reltool.hrl +++ b/lib/reltool/src/reltool.hrl @@ -164,7 +164,8 @@ applications = [] :: [app_name()], env = [] :: [{atom(), term()}], mod = undefined :: {mod_name(), [term()]} | undefined, - start_phases = undefined :: [{atom(), term()}] | undefined + start_phases = undefined :: [{atom(), term()}] | undefined, + runtime_dependencies = [] :: [string()] }). -record(regexp, {source, compiled}). diff --git a/lib/reltool/src/reltool_server.erl b/lib/reltool/src/reltool_server.erl index 97785ca7f8..98eeed5c27 100644 --- a/lib/reltool/src/reltool_server.erl +++ b/lib/reltool/src/reltool_server.erl @@ -1125,6 +1125,9 @@ parse_app_info(File, [{Key, Val} | KeyVals], AI, Status) -> start_phases -> parse_app_info(File, KeyVals, AI#app_info{start_phases = Val}, Status); + runtime_dependencies -> + parse_app_info(File, KeyVals, AI#app_info{runtime_dependencies = Val}, + Status); _ -> Status2 = reltool_utils:add_warning("Unexpected item ~p in app file ~tp.", diff --git a/lib/reltool/vsn.mk b/lib/reltool/vsn.mk index 16ec570d22..163b77dfa0 100644 --- a/lib/reltool/vsn.mk +++ b/lib/reltool/vsn.mk @@ -1 +1 @@ -RELTOOL_VSN = 0.6.4.1 +RELTOOL_VSN = 0.6.5 diff --git a/lib/runtime_tools/doc/specs/.gitignore b/lib/runtime_tools/doc/specs/.gitignore new file mode 100644 index 0000000000..322eebcb06 --- /dev/null +++ b/lib/runtime_tools/doc/specs/.gitignore @@ -0,0 +1 @@ +specs_*.xml diff --git a/lib/runtime_tools/doc/src/Makefile b/lib/runtime_tools/doc/src/Makefile index 51d93df418..07c63197e9 100644 --- a/lib/runtime_tools/doc/src/Makefile +++ b/lib/runtime_tools/doc/src/Makefile @@ -40,7 +40,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # Target Specs # ---------------------------------------------------- XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = dbg.xml dyntrace.xml erts_alloc_config.xml +XML_REF3_FILES = dbg.xml dyntrace.xml erts_alloc_config.xml system_information.xml XML_REF6_FILES = runtime_tools_app.xml XML_PART_FILES = part_notes.xml part_notes_history.xml part.xml @@ -71,12 +71,20 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf +SPECS_FILES = $(XML_REF3_FILES:%.xml=$(SPECDIR)/specs_%.xml) + +TOP_SPECS_FILE = specs.xml + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- XML_FLAGS += DVIPS_FLAGS += +SPECS_ESRC = ../../src + +SPECS_FLAGS = -I../../include -I../../../kernel/src + # ---------------------------------------------------- # Targets # ---------------------------------------------------- diff --git a/lib/runtime_tools/doc/src/ref_man.xml b/lib/runtime_tools/doc/src/ref_man.xml index 6017f3cdaa..25fa97896b 100644 --- a/lib/runtime_tools/doc/src/ref_man.xml +++ b/lib/runtime_tools/doc/src/ref_man.xml @@ -35,5 +35,6 @@ <xi:include href="dbg.xml"/> <xi:include href="dyntrace.xml"/> <xi:include href="erts_alloc_config.xml"/> + <xi:include href="system_information.xml"/> </application> diff --git a/lib/runtime_tools/doc/src/specs.xml b/lib/runtime_tools/doc/src/specs.xml new file mode 100644 index 0000000000..d4c3c9dfe6 --- /dev/null +++ b/lib/runtime_tools/doc/src/specs.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="utf-8" ?> +<specs xmlns:xi="http://www.w3.org/2001/XInclude"> + <xi:include href="../specs/specs_system_information.xml"/> +</specs> diff --git a/lib/runtime_tools/doc/src/system_information.xml b/lib/runtime_tools/doc/src/system_information.xml new file mode 100644 index 0000000000..b586334ae7 --- /dev/null +++ b/lib/runtime_tools/doc/src/system_information.xml @@ -0,0 +1,98 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2014</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title></title> + <prepared></prepared> + <responsible></responsible> + <docno>1</docno> + <approved></approved> + <checked></checked> + <date></date> + <rev></rev> + <file>system_information.xml</file> + </header> + <module>system_information</module> + <modulesummary>System Information</modulesummary> + <description> + <p></p> + </description> + <funcs> + <func> + <name name="sanity_check" arity="0"/> + <fsummary>Perform a sanity check</fsummary> + <desc> + <p>Performs a sanity check on the system. If no issues + were found, <c>ok</c> is returned. If issues were + found, <c>{failed, <anno>Failures</anno>}</c> is + returned. All failures found will be part of the + <c><anno>Failures</anno></c> list. Currently defined + <c><anno>Failure</anno></c> elements in the + <c><anno>Failures</anno></c> list:</p> + <taglist> + <tag><c><anno>InvalidAppFile</anno></c></tag> + <item><p>An application has an invalid <c>.app</c> file. The + second element identifies the application which has the + invalid <c>.app</c> file.</p></item> + <tag><c><anno>InvalidApplicationVersion</anno></c></tag> + <item><p>An application has an invalid application version. + The second element identifies the application version that + is invalid.</p></item> + <tag><c><anno>MissingRuntimeDependencies</anno></c></tag> + <item><p>An application is missing + <seealso marker="kernel:app#runtime_dependencies">runtime + dependencies</seealso>. The second element identifies the + application (with version) that has missing dependencies. + The third element contains the missing dependencies.</p> + <p>Note that this check use application versions that + are loaded, or will be loaded when used. You might have + application versions that satisfies all dependencies + installed in the system, but if those are not loaded this + check will fail. The system will of course also fail when + used like this. This may happen when you have multiple + <seealso marker="doc/system_principles:versions">branched + versions</seealso> of the same application installed in the + system, but you do not use a + <seealso marker="doc/system_principles:system_principles#BOOTSCRIPT">boot + script</seealso> identifing the correct application version.</p> + </item> + </taglist> + <p>Currently the sanity check is limited to verifying + runtime dependencies found in the <c>.app</c> files of + all applications. More checks will be introduced in the + future. This implies that the return type <em>will</em> + change in the future.</p> + <note><p>An <c>ok</c> return value only means that + <c>sanity_check/0</c> did not find any issues, <em>not</em> + that no issues exist.</p></note> + </desc> + </func> + <func> + <name name="to_file" arity="1"/> + <fsummary>Write miscellaneous system information to file</fsummary> + <desc><p>Writes miscellaneous system information to file. This + information will typically be requested by the Erlang/OTP team + at Ericsson AB when reporting an issue.</p></desc> + </func> + </funcs> + </erlref> + diff --git a/lib/runtime_tools/src/runtime_tools.app.src b/lib/runtime_tools/src/runtime_tools.app.src index d46cfe1f32..0a70802c08 100644 --- a/lib/runtime_tools/src/runtime_tools.app.src +++ b/lib/runtime_tools/src/runtime_tools.app.src @@ -25,6 +25,8 @@ {registered, [runtime_tools_sup]}, {applications, [kernel, stdlib]}, {env, []}, - {mod, {runtime_tools, []}}]}. + {mod, {runtime_tools, []}}, + {runtime_dependencies, ["stdlib-2.0","mnesia-4.12","kernel-3.0", + "erts-6.0"]}]}. diff --git a/lib/runtime_tools/src/system_information.erl b/lib/runtime_tools/src/system_information.erl index 603b698d5e..f541d6e449 100644 --- a/lib/runtime_tools/src/system_information.erl +++ b/lib/runtime_tools/src/system_information.erl @@ -39,7 +39,8 @@ application/1, application/2, environment/0, environment/1, module/1, module/2, - modules/1 + modules/1, + sanity_check/0 ]). %% gen_server callbacks @@ -85,9 +86,14 @@ report() -> [ {erts_compile_info, erlang:system_info(compile_info)}, {beam_dynamic_libraries, get_dynamic_libraries()}, {environment_erts, os_getenv_erts_specific()}, - {environment, [split_env(Env) || Env <- os:getenv()]} + {environment, [split_env(Env) || Env <- os:getenv()]}, + {sanity_check, sanity_check()} ]. +-spec to_file(FileName) -> ok | {error, Reason} when + FileName :: file:name_all(), + Reason :: file:posix() | badarg | terminated | system_limit. + to_file(File) -> file:write_file(File, iolist_to_binary([ io_lib:format("{system_information_version, ~p}.~n", [ @@ -130,6 +136,27 @@ module(M, Opts) when is_atom(M), is_list(Opts) -> modules(Opt) when is_atom(Opt) -> gen_server:call(?SERVER, {modules, Opt}). + +-spec sanity_check() -> ok | {failed, Failures} when + Application :: atom(), + ApplicationVersion :: string(), + MissingRuntimeDependencies :: {missing_runtime_dependencies, + ApplicationVersion, + [ApplicationVersion]}, + InvalidApplicationVersion :: {invalid_application_version, + ApplicationVersion}, + InvalidAppFile :: {invalid_app_file, Application}, + Failure :: MissingRuntimeDependencies + | InvalidApplicationVersion + | InvalidAppFile, + Failures :: [Failure]. + +sanity_check() -> + case check_runtime_dependencies() of + [] -> ok; + Issues -> {failed, Issues} + end. + %%=================================================================== %% gen_server callbacks %%=================================================================== @@ -457,6 +484,8 @@ get_application_from_path(Path) -> {description, proplists:get_value(description, Info, [])}, {vsn, proplists:get_value(vsn, Info, [])}, {path, Path}, + {runtime_dependencies, + proplists:get_value(runtime_dependencies, Info, [])}, {modules, get_modules_from_path(Path)} ]} end. @@ -552,3 +581,252 @@ get_beam_name() -> Value -> Value end, Beam ++ Type ++ Flavor. + +%% Check runtime dependencies... + +vsnstr2vsn(VsnStr) -> + list_to_tuple(lists:map(fun (Part) -> + list_to_integer(Part) + end, + string:tokens(VsnStr, "."))). + +rtdepstrs2rtdeps([]) -> + []; +rtdepstrs2rtdeps([RTDep | RTDeps]) -> + [AppStr, VsnStr] = string:tokens(RTDep, "-"), + [{list_to_atom(AppStr), vsnstr2vsn(VsnStr)} | rtdepstrs2rtdeps(RTDeps)]. + +build_app_table([], AppTab) -> + AppTab; +build_app_table([App | Apps], AppTab0) -> + AppTab1 = try + %% We may have multiple application versions installed + %% of the same application! It is therefore important + %% to look up the application version that actually will + %% be used via code server. + AppFile = code:where_is_file(atom_to_list(App) ++ ".app"), + {ok, [{application, App, Info}]} = file:consult(AppFile), + VsnStr = proplists:get_value(vsn, Info), + Vsn = vsnstr2vsn(VsnStr), + RTDepStrs = proplists:get_value(runtime_dependencies, + Info, []), + RTDeps = rtdepstrs2rtdeps(RTDepStrs), + gb_trees:insert(App, {Vsn, RTDeps}, AppTab0) + catch + _ : _ -> + AppTab0 + end, + build_app_table(Apps, AppTab1). + +meets_min_req(Vsn, Vsn) -> + true; +meets_min_req({X}, VsnReq) -> + meets_min_req({X, 0, 0}, VsnReq); +meets_min_req({X, Y}, VsnReq) -> + meets_min_req({X, Y, 0}, VsnReq); +meets_min_req(Vsn, {X}) -> + meets_min_req(Vsn, {X, 0, 0}); +meets_min_req(Vsn, {X, Y}) -> + meets_min_req(Vsn, {X, Y, 0}); +meets_min_req({X, _Y, _Z}, {XReq, _YReq, _ZReq}) when X > XReq -> + true; +meets_min_req({X, Y, _Z}, {X, YReq, _ZReq}) when Y > YReq -> + true; +meets_min_req({X, Y, Z}, {X, Y, ZReq}) when Z > ZReq -> + true; +meets_min_req({_X, _Y, _Z}, {_XReq, _YReq, _ZReq}) -> + false; +meets_min_req(Vsn, VsnReq) -> + gp_meets_min_req(mk_gp_vsn_list(Vsn), mk_gp_vsn_list(VsnReq)). + +gp_meets_min_req([X, Y, Z | _Vs], [X, Y, Z]) -> + true; +gp_meets_min_req([X, Y, Z | _Vs], [XReq, YReq, ZReq]) -> + meets_min_req({X, Y, Z}, {XReq, YReq, ZReq}); +gp_meets_min_req([X, Y, Z | Vs], [X, Y, Z | VReqs]) -> + gp_meets_min_req_tail(Vs, VReqs); +gp_meets_min_req(_Vsn, _VReq) -> + %% Versions on different version branches, i.e., the minimum + %% required functionality is not included in Vsn. + false. + +gp_meets_min_req_tail([V | Vs], [V | VReqs]) -> + gp_meets_min_req_tail(Vs, VReqs); +gp_meets_min_req_tail([], []) -> + true; +gp_meets_min_req_tail([_V | _Vs], []) -> + true; +gp_meets_min_req_tail([V | _Vs], [VReq]) when V > VReq -> + true; +gp_meets_min_req_tail(_Vs, _VReqs) -> + %% Versions on different version branches, i.e., the minimum + %% required functionality is not included in Vsn. + false. + +mk_gp_vsn_list(Vsn) -> + [X, Y, Z | Tail] = tuple_to_list(Vsn), + [X, Y, Z | remove_trailing_zeroes(Tail)]. + +remove_trailing_zeroes([]) -> + []; +remove_trailing_zeroes([0 | Vs]) -> + case remove_trailing_zeroes(Vs) of + [] -> []; + NewVs -> [0 | NewVs] + end; +remove_trailing_zeroes([V | Vs]) -> + [V | remove_trailing_zeroes(Vs)]. + +mk_app_vsn_str({App, Vsn}) -> + mk_app_vsn_str(App, Vsn). + +mk_app_vsn_str(App, Vsn) -> + VsnList = tuple_to_list(Vsn), + lists:flatten([atom_to_list(App), + $-, + integer_to_list(hd(VsnList)), + lists:map(fun (Part) -> + [$., integer_to_list(Part)] + end, tl(VsnList))]). + +otp_17_0_vsns_orddict() -> + [{asn1,{3,0}}, + {common_test,{1,8}}, + {compiler,{5,0}}, + {cosEvent,{2,1,15}}, + {cosEventDomain,{1,1,14}}, + {cosFileTransfer,{1,1,16}}, + {cosNotification,{1,1,21}}, + {cosProperty,{1,1,17}}, + {cosTime,{1,1,14}}, + {cosTransactions,{1,2,14}}, + {crypto,{3,3}}, + {debugger,{4,0}}, + {dialyzer,{2,7}}, + {diameter,{1,6}}, + {edoc,{0,7,13}}, + {eldap,{1,0,3}}, + {erl_docgen,{0,3,5}}, + {erl_interface,{3,7,16}}, + {erts,{6,0}}, + {et,{1,5}}, + {eunit,{2,2,7}}, + {gs,{1,5,16}}, + {hipe,{3,10,3}}, + {ic,{4,3,5}}, + {inets,{5,10}}, + {jinterface,{1,5,9}}, + {kernel,{3,0}}, + {megaco,{3,17,1}}, + {mnesia,{4,12}}, + {observer,{2,0}}, + {odbc,{2,10,20}}, + {orber,{3,6,27}}, + {os_mon,{2,2,15}}, + {ose,{1,0}}, + {otp_mibs,{1,0,9}}, + {parsetools,{2,0,11}}, + {percept,{0,8,9}}, + {public_key,{0,22}}, + {reltool,{0,6,5}}, + {runtime_tools,{1,8,14}}, + {sasl,{2,4}}, + {snmp,{4,25,1}}, + {ssh,{3,0,1}}, + {ssl,{5,3,4}}, + {stdlib,{2,0}}, + {syntax_tools,{1,6,14}}, + {test_server,{3,7}}, + {tools,{2,6,14}}, + {typer,{0,9,6}}, + {webtool,{0,8,10}}, + {wx,{1,2}}, + {xmerl,{1,3,7}}]. + +otp_17_0_vsns_tab() -> + gb_trees:from_orddict(otp_17_0_vsns_orddict()). + +check_runtime_dependency({App, DepVsn}, AppTab) -> + case gb_trees:lookup(App, AppTab) of + none -> + false; + {value, {Vsn, _}} -> + meets_min_req(Vsn, DepVsn) + end. + +check_runtime_dependencies(App, AppTab, OtpMinVsnTab) -> + case gb_trees:lookup(App, AppTab) of + none -> + [{invalid_app_file, App}]; + {value, {Vsn, RTDeps}} -> + RTD = case lists:foldl( + fun (RTDep, Acc) -> + case check_runtime_dependency(RTDep, AppTab) of + true -> + Acc; + false -> + [mk_app_vsn_str(RTDep) | Acc] + end + end, + [], + RTDeps) of + [] -> + []; + MissingDeps -> + [{missing_runtime_dependencies, + mk_app_vsn_str(App, Vsn), + MissingDeps}] + end, + case gb_trees:lookup(App, OtpMinVsnTab) of + none -> + RTD; + {value, MinVsn} -> + case meets_min_req(Vsn, MinVsn) of + true -> + RTD; + false -> + [{invalid_application_version, + mk_app_vsn_str(App, Vsn)} | RTD] + end + end + end. + +app_file_to_app(AF) -> + list_to_atom(filename:basename(AF, ".app")). + +get_apps() -> + get_apps(code:get_path(), []). + +get_apps([], Apps) -> + lists:usort(Apps); +get_apps([Path|Paths], Apps) -> + case filelib:wildcard(filename:join(Path, "*.app")) of + [] -> + %% Not app or invalid app + get_apps(Paths, Apps); + [AppFile] -> + get_apps(Paths, [app_file_to_app(AppFile) | Apps]); + [_AppFile| _] = AppFiles -> + %% Strange with multple .app files... Lets put them + %% all in the list and see what we get... + lists:map(fun (AF) -> + app_file_to_app(AF) + end, AppFiles) ++ Apps + end. + +check_runtime_dependencies() -> + OtpMinVsnTab = otp_17_0_vsns_tab(), + Apps = get_apps(), + AppTab = build_app_table(Apps, gb_trees:empty()), + lists:foldl(fun (App, Acc) -> + case check_runtime_dependencies(App, + AppTab, + OtpMinVsnTab) of + [] -> Acc; + Issues -> Issues ++ Acc + end + end, + [], + Apps). + +%% End of runtime dependency checks diff --git a/lib/runtime_tools/test/system_information_SUITE.erl b/lib/runtime_tools/test/system_information_SUITE.erl index fb9455a30f..53d20060e7 100644 --- a/lib/runtime_tools/test/system_information_SUITE.erl +++ b/lib/runtime_tools/test/system_information_SUITE.erl @@ -33,6 +33,7 @@ api_report/1, api_to_file/1, api_from_file/1, + sanity_check/1, %% server api_start_stop/1, validate_server_interface/1 @@ -84,7 +85,8 @@ all() -> [ api_to_file, api_from_file, api_start_stop, - validate_server_interface + validate_server_interface, + sanity_check ]. @@ -262,6 +264,9 @@ validate_server_interface(Config) -> ok = system_information:stop(), ok. +sanity_check(Config) when is_list(Config) -> + ok = system_information:sanity_check(). + %% aux @@ -288,7 +293,8 @@ validate_report(Report) -> erts_compile_info, beam_dynamic_libraries, environment_erts, - environment + environment, + sanity_check ], Report). ensure_report_keys([], _) -> ok; diff --git a/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat b/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat index 0900eadd4a..18938372a3 100644 --- a/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat +++ b/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat @@ -9870,4 +9870,5 @@ {"MANPATH", "/usr/local/man:/usr/share/man:/usr/X11R6/man:/opt/gnome/share/man"}, {"LESSKEY","/etc/lesskey.bin"}, - {"LC_PAPER","sv_SE.UTF-8"}]}]}. + {"LC_PAPER","sv_SE.UTF-8"}]}, + {sanity_check,ok}]}. diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index c282661a61..32953dfc5a 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.8.13 +RUNTIME_TOOLS_VSN = 1.8.14 diff --git a/lib/sasl/src/sasl.app.src b/lib/sasl/src/sasl.app.src index 8c814cfaf5..8e95197a2a 100644 --- a/lib/sasl/src/sasl.app.src +++ b/lib/sasl/src/sasl.app.src @@ -44,5 +44,7 @@ {applications, [kernel, stdlib]}, {env, [{sasl_error_logger, tty}, {errlog_type, all}]}, - {mod, {sasl, []}}]}. + {mod, {sasl, []}}, + {runtime_dependencies, ["tools-2.6.14","stdlib-2.0","kernel-3.0", + "erts-6.0"]}]}. diff --git a/lib/snmp/doc/src/snmpa_mib_data.xml b/lib/snmp/doc/src/snmpa_mib_data.xml index c1ea0a91f9..95a33e603e 100644 --- a/lib/snmp/doc/src/snmpa_mib_data.xml +++ b/lib/snmp/doc/src/snmpa_mib_data.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="iso-8859-1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> <header> <copyright> - <year>2013</year><year>2013</year> + <year>2013</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/snmp/doc/src/snmpa_mib_storage.xml b/lib/snmp/doc/src/snmpa_mib_storage.xml index a857ce79e8..791fbc80fe 100644 --- a/lib/snmp/doc/src/snmpa_mib_storage.xml +++ b/lib/snmp/doc/src/snmpa_mib_storage.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="iso-8859-1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> <header> <copyright> - <year>2013</year><year>2013</year> + <year>2013</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/snmp/src/app/snmp.app.src b/lib/snmp/src/app/snmp.app.src index 904d17954b..cbd292e4c3 100644 --- a/lib/snmp/src/app/snmp.app.src +++ b/lib/snmp/src/app/snmp.app.src @@ -136,4 +136,6 @@ %% configuration and use), and in that case mnesia must also be started, %% before snmp. {applications, [kernel, stdlib]}, - {mod, {snmp_app, []}}]}. + {mod, {snmp_app, []}}, + {runtime_dependencies, ["stdlib-2.0","runtime_tools-1.8.14","mnesia-4.12", + "kernel-3.0","erts-6.0","crypto-3.3"]}]}. diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index 533e313bdb..04c3cc9392 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = snmp -SNMP_VSN = 4.25.0.1 +SNMP_VSN = 4.25.1 PRE_VSN = APP_VSN = "$(APPLICATION)-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 74d7293be0..e0a51b3574 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -38,6 +38,8 @@ {registered, []}, {applications, [kernel, stdlib, crypto, public_key]}, {env, []}, - {mod, {ssh_app, []}}]}. + {mod, {ssh_app, []}}, + {runtime_dependencies, ["stdlib-2.0","public_key-0.22","kernel-3.0", + "erts-6.0","crypto-3.3"]}]}. diff --git a/lib/ssh/test/ssh_unicode_SUITE.erl b/lib/ssh/test/ssh_unicode_SUITE.erl index a896a425b9..cc916673b3 100644 --- a/lib/ssh/test/ssh_unicode_SUITE.erl +++ b/lib/ssh/test/ssh_unicode_SUITE.erl @@ -1,10 +1,7 @@ -%% Next line needed to enable utf8-strings in Erlang: -%% -*- coding: utf-8 -*- - %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2013. All Rights Reserved. +%% Copyright Ericsson AB 2005-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index 68ebc49e4a..99839f6149 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -47,6 +47,8 @@ {registered, [ssl_sup, ssl_manager]}, {applications, [crypto, public_key, kernel, stdlib]}, {env, []}, - {mod, {ssl_app, []}}]}. + {mod, {ssl_app, []}}, + {runtime_dependencies, ["stdlib-2.0","public_key-0.22","kernel-3.0", + "erts-6.0","crypto-3.3"]}]}. diff --git a/lib/stdlib/doc/src/epp.xml b/lib/stdlib/doc/src/epp.xml index 50baad04a9..452341f7d2 100644 --- a/lib/stdlib/doc/src/epp.xml +++ b/lib/stdlib/doc/src/epp.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2013</year> + <year>1996</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -46,8 +46,10 @@ valid encodings are <c>Latin-1</c> and <c>UTF-8</c> where the case of the characters can be chosen freely. Examples:</p> <pre> -%% coding: utf-8 -%% For this file we have chosen encoding = Latin-1 +%% coding: utf-8</pre> + <pre> +%% For this file we have chosen encoding = Latin-1</pre> + <pre> %% -*- coding: latin-1 -*-</pre> </description> <datatypes> diff --git a/lib/stdlib/doc/src/erl_parse.xml b/lib/stdlib/doc/src/erl_parse.xml index 2d5aff3c6c..cf0bff48cd 100644 --- a/lib/stdlib/doc/src/erl_parse.xml +++ b/lib/stdlib/doc/src/erl_parse.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2013</year> + <year>1996</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -173,6 +173,7 @@ </func> <func> <name name="abstract" arity="2"/> + <type name="encoding_func"/> <fsummary>Convert an Erlang term into an abstract form</fsummary> <desc> <p>Converts the Erlang data structure <c><anno>Data</anno></c> into an @@ -183,7 +184,12 @@ selecting which integer lists will be considered as strings. The default is to use the encoding returned by <seealso marker="epp#default_encoding/0"> - <c>epp:default_encoding/0</c></seealso></p> + <c>epp:default_encoding/0</c></seealso>. + The value <c>none</c> means that no integer lists will be + considered as strings. The <c>encoding_func()</c> will be + called with one integer of a list at a time, and if it + returns <c>true</c> for every integer the list will be + considered a string.</p> </desc> </func> </funcs> diff --git a/lib/stdlib/doc/src/sys.xml b/lib/stdlib/doc/src/sys.xml index ab8b380f49..a46fa1289f 100644 --- a/lib/stdlib/doc/src/sys.xml +++ b/lib/stdlib/doc/src/sys.xml @@ -246,6 +246,22 @@ <c>{Module, Id, HandlerState}</c>, where <c>Module</c> is the event handler's module name, <c>Id</c> is the handler's ID (which is the value <c>false</c> if it was registered without an ID), and <c>HandlerState</c> is the handler's state.</p> + <p>If the callback module exports a <c>system_get_state/1</c> function, it will be called in the + target process to get its state. Its argument is the same as the <c>Misc</c> value returned by + <seealso marker="#get_status-1">get_status/1,2</seealso>, and the <c>system_get_state/1</c> + function is expected to extract the callback module's state from it. The <c>system_get_state/1</c> + function must return <c>{ok, State}</c> where <c>State</c> is the callback module's state.</p> + <p>If the callback module does not export a <c>system_get_state/1</c> function, <c>get_state/1,2</c> + assumes the <c>Misc</c> value is the callback module's state and returns it directly instead.</p> + <p>If the callback module's <c>system_get_state/1</c> function crashes or throws an exception, the + caller exits with error <c>{callback_failed, {Module, system_get_state}, {Class, Reason}}</c> where + <c>Module</c> is the name of the callback module and <c>Class</c> and <c>Reason</c> indicate + details of the exception.</p> + <p>The <c>system_get_state/1</c> function is primarily useful for user-defined + behaviours and modules that implement OTP <seealso marker="#special_process">special + processes</seealso>. The <c>gen_server</c>, <c>gen_fsm</c>, and <c>gen_event</c> OTP + behaviour modules export this function, and so callback modules for those behaviours + need not supply their own.</p> <p>To obtain more information about a process, including its state, see <seealso marker="#get_status-1">get_status/1</seealso> and <seealso marker="#get_status-2">get_status/2</seealso>.</p> @@ -289,6 +305,28 @@ function means that only the state of the particular event handler it was working on when it failed or crashed is unchanged; it can still succeed in changing the states of other event handlers registered in the same <c>gen_event</c> process.</p> + <p>If the callback module exports a <c>system_replace_state/2</c> function, it will be called in the + target process to replace its state using <c>StateFun</c>. Its two arguments are <c>StateFun</c> + and <c>Misc</c>, where <c>Misc</c> is the same as the <c>Misc</c> value returned by + <seealso marker="#get_status-1">get_status/1,2</seealso>. A <c>system_replace_state/2</c> function + is expected to return <c>{ok, NewState, NewMisc}</c> where <c>NewState</c> is the callback module's + new state obtained by calling <c>StateFun</c>, and <c>NewMisc</c> is a possibly new value used to + replace the original <c>Misc</c> (required since <c>Misc</c> often contains the callback + module's state within it).</p> + <p>If the callback module does not export a <c>system_replace_state/2</c> function, + <c>replace_state/2,3</c> assumes the <c>Misc</c> value is the callback module's state, passes it + to <c>StateFun</c> and uses the return value as both the new state and as the new value of + <c>Misc</c>.</p> + <p>If the callback module's <c>system_replace_state/2</c> function crashes or throws an exception, + the caller exits with error <c>{callback_failed, {Module, system_replace_state}, {Class, Reason}}</c> + where <c>Module</c> is the name of the callback module and <c>Class</c> and <c>Reason</c> indicate details + of the exception. If the callback module does not provide a <c>system_replace_state/2</c> function and + <c>StateFun</c> crashes or throws an exception, the caller exits with error + <c>{callback_failed, StateFun, {Class, Reason}}</c>.</p> + <p>The <c>system_replace_state/2</c> function is primarily useful for user-defined behaviours and + modules that implement OTP <seealso marker="#special_process">special processes</seealso>. The + <c>gen_server</c>, <c>gen_fsm</c>, and <c>gen_event</c> OTP behaviour modules export this function, + and so callback modules for those behaviours need not supply their own.</p> </desc> </func> <func> @@ -322,7 +360,7 @@ <section> <title>Process Implementation Functions</title> - <p>The following functions are used when implementing a + <p><marker id="special_process"/>The following functions are used when implementing a special process. This is an ordinary process which does not use a standard behaviour, but a process which understands the standard system messages.</p> </section> @@ -375,8 +413,9 @@ process continues the execution, or <c><anno>Module</anno>:system_terminate(Reason, <anno>Parent</anno>, <anno>Debug</anno>, <anno>Misc</anno>)</c> if the process should terminate. The <c><anno>Module</anno></c> must export - <c>system_continue/3</c>, <c>system_terminate/4</c>, and - <c>system_code_change/4</c> (see below). + <c>system_continue/3</c>, <c>system_terminate/4</c>, + <c>system_code_change/4</c>, <c>system_get_state/1</c> and + <c>system_replace_state/2</c> (see below). </p> <p>The <c><anno>Misc</anno></c> argument can be used to save internal data in a process, for example its state. It is sent to @@ -444,6 +483,34 @@ defined, the atom <c>undefined</c> is sent.</p> </desc> </func> + <func> + <name>Mod:system_get_state(Misc) -> {ok, State}</name> + <fsummary>Called when the process should return its current state</fsummary> + <type> + <v>Misc = term()</v> + <v>State = term()</v> + </type> + <desc> + <p>This function is called from <c>sys:handle_system_msg/6</c> when the process + should return a term that reflects its current state. <c>State</c> is the + value returned by <c>sys:get_state/2</c>.</p> + </desc> + </func> + <func> + <name>Mod:system_replace_state(StateFun, Misc) -> {ok, NState, NMisc}</name> + <fsummary>Called when the process should replace its current state</fsummary> + <type> + <v>StateFun = fun((State :: term()) -> NState)</v> + <v>Misc = term()</v> + <v>NState = term()</v> + <v>NMisc = term()</v> + </type> + <desc> + <p>This function is called from <c>sys:handle_system_msg/6</c> when the process + should replace its current state. <c>NState</c> is the value returned by + <c>sys:replace_state/3</c>.</p> + </desc> + </func> </funcs> </erlref> diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index c843ef7736..bebfbd4514 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -848,6 +848,7 @@ Eshell V5.10.1 (abort with ^G) </section> <section> <title>Unicode in Environment and Parameters</title> + <marker id="unicode_in_environment_and_parameters"/> <p>Environment variables and their interpretation is handled much in the same way as file names. If Unicode file names are enabled, environment variables as well as parameters to the Erlang VM are diff --git a/lib/stdlib/src/dict.erl b/lib/stdlib/src/dict.erl index 6088e1a2dd..cf8fb3114a 100644 --- a/lib/stdlib/src/dict.erl +++ b/lib/stdlib/src/dict.erl @@ -55,8 +55,7 @@ -define(exp_size, (?seg_size * ?expand_load)). -define(con_size, (?seg_size * ?contract_load)). --type segs(K, V) :: tuple() - | {K, V}. % dummy +-type segs(_Key, _Value) :: tuple(). %% Define a hashtable. The default values are the standard ones. -record(dict, diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index d212a55b47..9b506b0a44 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -33,23 +33,33 @@ -export_type([source_encoding/0]). --type macros() :: [{atom(), term()}]. +-type macros() :: [atom() | {atom(), term()}]. -type epp_handle() :: pid(). -type source_encoding() :: latin1 | utf8. +-type ifdef() :: 'ifdef' | 'ifndef' | 'else'. + +-type name() :: {'atom', atom()}. +-type argspec() :: 'none' %No arguments + | non_neg_integer(). %Number of arguments +-type tokens() :: [erl_scan:token()]. +-type used() :: {name(), argspec()}. + -define(DEFAULT_ENCODING, utf8). %% Epp state record. --record(epp, {file, %Current file +-record(epp, {file :: file:io_device(), %Current file location=1, %Current location - delta, %Offset from Location (-file) - name="", %Current file name - name2="", %-"-, modified by -file - istk=[], %Ifdef stack - sstk=[], %State stack - path=[], %Include-path - macs = dict:new() :: dict:dict(),%Macros (don't care locations) - uses = dict:new() :: dict:dict(),%Macro use structure + delta=0 :: non_neg_integer(), %Offset from Location (-file) + name="" :: file:name(), %Current file name + name2="" :: file:name(), %-"-, modified by -file + istk=[] :: [ifdef()], %Ifdef stack + sstk=[] :: [#epp{}], %State stack + path=[] :: [file:name()], %Include-path + macs = dict:new() %Macros (don't care locations) + :: dict:dict(name(), {argspec(), tokens()}), + uses = dict:new() %Macro use structure + :: dict:dict(name(), [{argspec(), [used()]}]), default_encoding = ?DEFAULT_ENCODING :: source_encoding(), pre_opened = false :: boolean() }). diff --git a/lib/stdlib/src/erl_compile.erl b/lib/stdlib/src/erl_compile.erl index ed8fea5d78..caed4d41d6 100644 --- a/lib/stdlib/src/erl_compile.erl +++ b/lib/stdlib/src/erl_compile.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -66,7 +66,7 @@ my_halt(Reason) -> compile(List) -> process_flag(trap_exit, true), - Pid = spawn_link(fun() -> compiler_runner(List) end), + Pid = spawn_link(compiler_runner(List)), receive {'EXIT', Pid, {compiler_result, Result}} -> Result; @@ -79,14 +79,16 @@ compile(List) -> error end. --spec compiler_runner([cmd_line_arg()]) -> no_return(). +-spec compiler_runner([cmd_line_arg()]) -> fun(() -> no_return()). compiler_runner(List) -> - %% We don't want the current directory in the code path. - %% Remove it. - Path = [D || D <- code:get_path(), D =/= "."], - true = code:set_path(Path), - exit({compiler_result, compile1(List)}). + fun() -> + %% We don't want the current directory in the code path. + %% Remove it. + Path = [D || D <- code:get_path(), D =/= "."], + true = code:set_path(Path), + exit({compiler_result, compile1(List)}) + end. %% Parses the first part of the option list. diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 4c0261a1ad..c4c94fbee4 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -80,13 +80,17 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> -type fa() :: {atom(), arity()}. % function+arity -type ta() :: {atom(), arity()}. % type+arity +-record(typeinfo, {attr, line}). + %% Usage of records, functions, and imports. The variable table, which %% is passed on as an argument, holds the usage of variables. -record(usage, { calls = dict:new(), %Who calls who imported = [], %Actually imported functions - used_records=sets:new() :: sets:set(),%Used record definitions - used_types = dict:new() :: dict:dict()%Used type definitions + used_records = sets:new() %Used record definitions + :: sets:set(atom()), + used_types = dict:new() %Used type definitions + :: dict:dict(ta(), line()) }). %% Define the lint state record. @@ -95,13 +99,17 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> -record(lint, {state=start :: 'start' | 'attribute' | 'function', module=[], %Module behaviour=[], %Behaviour - exports=gb_sets:empty() :: gb_sets:set(),%Exports - imports=[], %Imports + exports=gb_sets:empty() :: gb_sets:set(fa()),%Exports + imports=[] :: [fa()], %Imports, an orddict() compile=[], %Compile flags - records=dict:new() :: dict:dict(), %Record definitions - locals=gb_sets:empty() :: gb_sets:set(),%All defined functions (prescanned) - no_auto=gb_sets:empty() :: gb_sets:set() | 'all',%Functions explicitly not autoimported - defined=gb_sets:empty() :: gb_sets:set(),%Defined fuctions + records=dict:new() %Record definitions + :: dict:dict(atom(), {line(),Fields :: term()}), + locals=gb_sets:empty() %All defined functions (prescanned) + :: gb_sets:set(fa()), + no_auto=gb_sets:empty() %Functions explicitly not autoimported + :: gb_sets:set(fa()) | 'all', + defined=gb_sets:empty() %Defined fuctions + :: gb_sets:set(fa()), on_load=[] :: [fa()], %On-load function on_load_line=0 :: line(), %Line for on_load clashes=[], %Exported functions named as BIFs @@ -116,12 +124,16 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> %outside any fun or lc xqlc= false :: boolean(), %true if qlc.hrl included new = false :: boolean(), %Has user-defined 'new/N' - called= [] :: [{fa(),line()}], %Called functions + called= [] :: [{fa(),line()}], %Called functions usage = #usage{} :: #usage{}, - specs = dict:new() :: dict:dict(), %Type specifications - callbacks = dict:new() :: dict:dict(), %Callback types - types = dict:new() :: dict:dict(), %Type definitions - exp_types=gb_sets:empty():: gb_sets:set()%Exported types + specs = dict:new() %Type specifications + :: dict:dict(mfa(), line()), + callbacks = dict:new() %Callback types + :: dict:dict(mfa(), line()), + types = dict:new() %Type definitions + :: dict:dict(ta(), #typeinfo{}), + exp_types=gb_sets:empty() %Exported types + :: gb_sets:set(ta()) }). -type lint_state() :: #lint{}. @@ -319,10 +331,14 @@ format_error({undefined_type, {TypeName, Arity}}) -> io_lib:format("type ~w~s undefined", [TypeName, gen_type_paren(Arity)]); format_error({unused_type, {TypeName, Arity}}) -> io_lib:format("type ~w~s is unused", [TypeName, gen_type_paren(Arity)]); -format_error({new_builtin_type, {TypeName, Arity}}) -> - io_lib:format("type ~w~s is a new builtin type; " +%% format_error({new_builtin_type, {TypeName, Arity}}) -> +%% io_lib:format("type ~w~s is a new builtin type; " +%% "its (re)definition is allowed only until the next release", +%% [TypeName, gen_type_paren(Arity)]); +format_error({new_var_arity_type, TypeName}) -> + io_lib:format("type ~w is a new builtin type; " "its (re)definition is allowed only until the next release", - [TypeName, gen_type_paren(Arity)]); + [TypeName]); format_error({builtin_type, {TypeName, Arity}}) -> io_lib:format("type ~w~s is a builtin type; it cannot be redefined", [TypeName, gen_type_paren(Arity)]); @@ -1170,7 +1186,7 @@ export_type(Line, ETs, #lint{usage = Usage, exp_types = ETs0} = St0) -> add_error(Line, {bad_export_type, ETs}, St0) end. --spec exports(lint_state()) -> gb_sets:set(). +-spec exports(lint_state()) -> gb_sets:set(fa()). exports(#lint{compile = Opts, defined = Defs, exports = Es}) -> case lists:member(export_all, Opts) of @@ -2574,8 +2590,6 @@ find_field(_F, []) -> error. %% Attr :: 'type' | 'opaque' %% Checks that a type definition is valid. --record(typeinfo, {attr, line}). - type_def(_Attr, _Line, {record, _RecName}, Fields, [], St0) -> %% The record field names and such are checked in the record format. %% We only need to check the types. @@ -2596,23 +2610,30 @@ type_def(Attr, Line, TypeName, ProtoType, Args, St0) -> true -> case is_obsolete_builtin_type(TypePair) of true -> StoreType(St0); - false -> - case is_newly_introduced_builtin_type(TypePair) of - %% allow some types just for bootstrapping - true -> - Warn = {new_builtin_type, TypePair}, - St1 = add_warning(Line, Warn, St0), - StoreType(St1); - false -> - add_error(Line, {builtin_type, TypePair}, St0) - end + false -> add_error(Line, {builtin_type, TypePair}, St0) +%% case is_newly_introduced_builtin_type(TypePair) of +%% %% allow some types just for bootstrapping +%% true -> +%% Warn = {new_builtin_type, TypePair}, +%% St1 = add_warning(Line, Warn, St0), +%% StoreType(St1); +%% false -> +%% add_error(Line, {builtin_type, TypePair}, St0) +%% end end; false -> case - dict:is_key(TypePair, TypeDefs) - orelse is_var_arity_type(TypeName) + dict:is_key(TypePair, TypeDefs) orelse + is_var_arity_type(TypeName) of - true -> add_error(Line, {redefine_type, TypePair}, St0); + true -> + case is_newly_introduced_var_arity_type(TypeName) of + true -> + Warn = {new_var_arity_type, TypeName}, + add_warning(Line, Warn, St0); + false -> + add_error(Line, {redefine_type, TypePair}, St0) + end; false -> St1 = case Attr =:= opaque andalso @@ -2847,8 +2868,10 @@ is_default_type({timeout, 0}) -> true; is_default_type({var, 1}) -> true; is_default_type(_) -> false. -%% OTP 17.0 -is_newly_introduced_builtin_type({Name, _}) when is_atom(Name) -> false. +is_newly_introduced_var_arity_type(map) -> true; +is_newly_introduced_var_arity_type(_) -> false. + +%% is_newly_introduced_builtin_type({Name, _}) when is_atom(Name) -> false. is_obsolete_builtin_type(TypePair) -> obsolete_builtin_type(TypePair) =/= no. diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 6316db7054..1dc5fc52a7 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -919,59 +919,63 @@ normalise_list([]) -> Data :: term(), AbsTerm :: abstract_expr(). abstract(T) -> - abstract(T, 0, epp:default_encoding()). + abstract(T, 0, enc_func(epp:default_encoding())). + +-type encoding_func() :: fun((non_neg_integer()) -> boolean()). %%% abstract/2 takes line and encoding options -spec abstract(Data, Options) -> AbsTerm when Data :: term(), Options :: Line | [Option], Option :: {line, Line} | {encoding, Encoding}, - Encoding :: latin1 | unicode | utf8, + Encoding :: 'latin1' | 'unicode' | 'utf8' | 'none' | encoding_func(), Line :: erl_scan:line(), AbsTerm :: abstract_expr(). abstract(T, Line) when is_integer(Line) -> - abstract(T, Line, epp:default_encoding()); + abstract(T, Line, enc_func(epp:default_encoding())); abstract(T, Options) when is_list(Options) -> Line = proplists:get_value(line, Options, 0), Encoding = proplists:get_value(encoding, Options,epp:default_encoding()), - abstract(T, Line, Encoding). + EncFunc = enc_func(Encoding), + abstract(T, Line, EncFunc). -define(UNICODE(C), - is_integer(C) andalso - (C >= 0 andalso C < 16#D800 orelse + (C < 16#D800 orelse C > 16#DFFF andalso C < 16#FFFE orelse C > 16#FFFF andalso C =< 16#10FFFF)). +enc_func(latin1) -> fun(C) -> C < 256 end; +enc_func(unicode) -> fun(C) -> ?UNICODE(C) end; +enc_func(utf8) -> fun(C) -> ?UNICODE(C) end; +enc_func(none) -> none; +enc_func(Fun) when is_function(Fun, 1) -> Fun; +enc_func(Term) -> erlang:error({badarg, Term}). + abstract(T, L, _E) when is_integer(T) -> {integer,L,T}; abstract(T, L, _E) when is_float(T) -> {float,L,T}; abstract(T, L, _E) when is_atom(T) -> {atom,L,T}; abstract([], L, _E) -> {nil,L}; abstract(B, L, _E) when is_bitstring(B) -> {bin, L, [abstract_byte(Byte, L) || Byte <- bitstring_to_list(B)]}; -abstract([C|T], L, unicode=E) when ?UNICODE(C) -> - abstract_unicode_string(T, [C], L, E); -abstract([C|T], L, utf8=E) when ?UNICODE(C) -> - abstract_unicode_string(T, [C], L, E); -abstract([C|T], L, latin1=E) when is_integer(C), 0 =< C, C < 256 -> - abstract_string(T, [C], L, E); -abstract([H|T], L, E) -> +abstract([H|T], L, none=E) -> {cons,L,abstract(H, L, E),abstract(T, L, E)}; +abstract(List, L, E) when is_list(List) -> + abstract_list(List, [], L, E); abstract(Tuple, L, E) when is_tuple(Tuple) -> - {tuple,L,abstract_list(tuple_to_list(Tuple), L, E)}. - -abstract_string([C|T], String, L, E) when is_integer(C), 0 =< C, C < 256 -> - abstract_string(T, [C|String], L, E); -abstract_string([], String, L, _E) -> - {string, L, lists:reverse(String)}; -abstract_string(T, String, L, E) -> - not_string(String, abstract(T, L, E), L, E). - -abstract_unicode_string([C|T], String, L, E) when ?UNICODE(C) -> - abstract_unicode_string(T, [C|String], L, E); -abstract_unicode_string([], String, L, _E) -> + {tuple,L,abstract_tuple_list(tuple_to_list(Tuple), L, E)}. + +abstract_list([H|T], String, L, E) -> + case is_integer(H) andalso H >= 0 andalso E(H) of + true -> + abstract_list(T, [H|String], L, E); + false -> + AbstrList = {cons,L,abstract(H, L, E),abstract(T, L, E)}, + not_string(String, AbstrList, L, E) + end; +abstract_list([], String, L, _E) -> {string, L, lists:reverse(String)}; -abstract_unicode_string(T, String, L, E) -> +abstract_list(T, String, L, E) -> not_string(String, abstract(T, L, E), L, E). not_string([C|T], Result, L, E) -> @@ -979,9 +983,9 @@ not_string([C|T], Result, L, E) -> not_string([], Result, _L, _E) -> Result. -abstract_list([H|T], L, E) -> - [abstract(H, L, E)|abstract_list(T, L, E)]; -abstract_list([], _L, _E) -> +abstract_tuple_list([H|T], L, E) -> + [abstract(H, L, E)|abstract_tuple_list(T, L, E)]; +abstract_tuple_list([], _L, _E) -> []. abstract_byte(Byte, L) when is_integer(Byte) -> diff --git a/lib/stdlib/src/escript.erl b/lib/stdlib/src/escript.erl index 35f6dff57e..a8a82272d6 100644 --- a/lib/stdlib/src/escript.erl +++ b/lib/stdlib/src/escript.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2013. All Rights Reserved. +%% Copyright Ericsson AB 2007-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -289,6 +289,8 @@ start(EscriptOptions) -> my_halt(127) end. +-spec parse_and_run(_, _, _) -> no_return(). + parse_and_run(File, Args, Options) -> CheckOnly = lists:member("s", Options), {Source, Module, FormsOrBin, HasRecs, Mode} = @@ -727,6 +729,8 @@ epp_parse_file2(Epp, S, Forms, Parsed) -> %% Evaluate script %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-spec debug(_, _, _) -> no_return(). + debug(Module, AbsMod, Args) -> case hidden_apply(debugger, debugger, start, []) of {ok, _} -> @@ -742,6 +746,8 @@ debug(Module, AbsMod, Args) -> fatal("Cannot start the debugger") end. +-spec run(_, _) -> no_return(). + run(Module, Args) -> try Module:main(Args), @@ -751,6 +757,8 @@ run(Module, Args) -> fatal(format_exception(Class, Reason)) end. +-spec interpret(_, _, _, _) -> no_return(). + interpret(Forms, HasRecs, File, Args) -> %% Basic validation before execution case erl_lint:module(Forms) of diff --git a/lib/stdlib/src/gen_event.erl b/lib/stdlib/src/gen_event.erl index 7629e88fbf..d39dd89d3a 100644 --- a/lib/stdlib/src/gen_event.erl +++ b/lib/stdlib/src/gen_event.erl @@ -40,6 +40,8 @@ system_continue/3, system_terminate/4, system_code_change/4, + system_get_state/1, + system_replace_state/2, format_status/2]). -export_type([handler/0, handler_args/0, add_handler_ret/0, @@ -229,24 +231,6 @@ wake_hib(Parent, ServerName, MSL, Debug) -> fetch_msg(Parent, ServerName, MSL, Debug, Hib) -> receive - {system, From, get_state} -> - States = [{Mod,Id,State} || #handler{module=Mod, id=Id, state=State} <- MSL], - sys:handle_system_msg(get_state, From, Parent, ?MODULE, Debug, - {States, [ServerName, MSL, Hib]}, Hib); - {system, From, {replace_state, StateFun}} -> - {NMSL, NStates} = - lists:unzip([begin - Cur = {Mod,Id,State}, - try - NState = {Mod,Id,NS} = StateFun(Cur), - {HS#handler{state=NS}, NState} - catch - _:_ -> - {HS, Cur} - end - end || #handler{module=Mod, id=Id, state=State}=HS <- MSL]), - sys:handle_system_msg(replace_state, From, Parent, ?MODULE, Debug, - {NStates, [ServerName, NMSL, Hib]}, Hib); {system, From, Req} -> sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, [ServerName, MSL, Hib],Hib); @@ -383,6 +367,23 @@ system_code_change([ServerName, MSL, Hib], Module, OldVsn, Extra) -> MSL), {ok, [ServerName, MSL1, Hib]}. +system_get_state([_ServerName, MSL, _Hib]) -> + {ok, [{Mod,Id,State} || #handler{module=Mod, id=Id, state=State} <- MSL]}. + +system_replace_state(StateFun, [ServerName, MSL, Hib]) -> + {NMSL, NStates} = + lists:unzip([begin + Cur = {Mod,Id,State}, + try + NState = {Mod,Id,NS} = StateFun(Cur), + {HS#handler{state=NS}, NState} + catch + _:_ -> + {HS, Cur} + end + end || #handler{module=Mod, id=Id, state=State}=HS <- MSL]), + {ok, NStates, [ServerName, NMSL, Hib]}. + %%----------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees %% them, not as the real erlang messages. Use trace for that. diff --git a/lib/stdlib/src/gen_fsm.erl b/lib/stdlib/src/gen_fsm.erl index e9654322f1..e914f7d0b2 100644 --- a/lib/stdlib/src/gen_fsm.erl +++ b/lib/stdlib/src/gen_fsm.erl @@ -118,6 +118,8 @@ system_continue/3, system_terminate/4, system_code_change/4, + system_get_state/1, + system_replace_state/2, format_status/2]). -import(error_logger, [format/2]). @@ -422,17 +424,6 @@ wake_hib(Parent, Name, StateName, StateData, Mod, Debug) -> decode_msg(Msg,Parent, Name, StateName, StateData, Mod, Time, Debug, Hib) -> case Msg of - {system, From, get_state} -> - Misc = [Name, StateName, StateData, Mod, Time], - sys:handle_system_msg(get_state, From, Parent, ?MODULE, Debug, - {{StateName, StateData}, Misc}, Hib); - {system, From, {replace_state, StateFun}} -> - State = {StateName, StateData}, - NState = {NStateName, NStateData} = try StateFun(State) - catch _:_ -> State end, - NMisc = [Name, NStateName, NStateData, Mod, Time], - sys:handle_system_msg(replace_state, From, Parent, ?MODULE, Debug, - {NState, NMisc}, Hib); {system, From, Req} -> sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, [Name, StateName, StateData, Mod, Time], Hib); @@ -467,6 +458,13 @@ system_code_change([Name, StateName, StateData, Mod, Time], Else -> Else end. +system_get_state([_Name, StateName, StateData, _Mod, _Time]) -> + {ok, {StateName, StateData}}. + +system_replace_state(StateFun, [Name, StateName, StateData, Mod, Time]) -> + Result = {NStateName, NStateData} = StateFun({StateName, StateData}), + {ok, Result, [Name, NStateName, NStateData, Mod, Time]}. + %%----------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees %% them, not as the real erlang messages. Use trace for that. diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index 5f14e48b0a..202a931fae 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -98,6 +98,8 @@ -export([system_continue/3, system_terminate/4, system_code_change/4, + system_get_state/1, + system_replace_state/2, format_status/2]). %% Internal exports @@ -372,13 +374,6 @@ wake_hib(Parent, Name, State, Mod, Debug) -> decode_msg(Msg, Parent, Name, State, Mod, Time, Debug, Hib) -> case Msg of - {system, From, get_state} -> - sys:handle_system_msg(get_state, From, Parent, ?MODULE, Debug, - {State, [Name, State, Mod, Time]}, Hib); - {system, From, {replace_state, StateFun}} -> - NState = try StateFun(State) catch _:_ -> State end, - sys:handle_system_msg(replace_state, From, Parent, ?MODULE, Debug, - {NState, [Name, NState, Mod, Time]}, Hib); {system, From, Req} -> sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, [Name, State, Mod, Time], Hib); @@ -687,6 +682,13 @@ system_code_change([Name, State, Mod, Time], _Module, OldVsn, Extra) -> Else -> Else end. +system_get_state([_Name, State, _Mod, _Time]) -> + {ok, State}. + +system_replace_state(StateFun, [Name, State, Mod, Time]) -> + NState = StateFun(State), + {ok, NState, [Name, NState, Mod, Time]}. + %%----------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees %% them, not as the real erlang messages. Use trace for that. diff --git a/lib/stdlib/src/io.erl b/lib/stdlib/src/io.erl index b11d41e2eb..27e2a82b41 100644 --- a/lib/stdlib/src/io.erl +++ b/lib/stdlib/src/io.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -177,13 +177,15 @@ get_password(Io) -> | {'expand_fun', expand_fun()} | {'encoding', encoding()}. --spec getopts() -> [opt_pair()]. +-spec getopts() -> [opt_pair()] | {'error', Reason} when + Reason :: term(). getopts() -> getopts(default_input()). --spec getopts(IoDevice) -> [opt_pair()] when - IoDevice :: device(). +-spec getopts(IoDevice) -> [opt_pair()] | {'error', Reason} when + IoDevice :: device(), + Reason :: term(). getopts(Io) -> request(Io, getopts). diff --git a/lib/stdlib/src/sets.erl b/lib/stdlib/src/sets.erl index be4b600f25..167a676281 100644 --- a/lib/stdlib/src/sets.erl +++ b/lib/stdlib/src/sets.erl @@ -55,9 +55,8 @@ %%------------------------------------------------------------------------------ --type seg() :: tuple(). --type segs(E) :: tuple() - | E. % dummy +-type seg() :: tuple(). +-type segs(_Element) :: tuple(). %% Define a hash set. The default values are the standard ones. -record(set, diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index a64b8e13c0..d388410de0 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -102,5 +102,8 @@ {registered,[timer_server,rsh_starter,take_over_monitor,pool_master, dets]}, {applications, [kernel]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["sasl-2.4","kernel-3.0","erts-6.0","crypto-3.3", + "compiler-5.0"]} +]}. diff --git a/lib/stdlib/src/sys.erl b/lib/stdlib/src/sys.erl index 04f8dfb61b..e25cc25f57 100644 --- a/lib/stdlib/src/sys.erl +++ b/lib/stdlib/src/sys.erl @@ -102,20 +102,31 @@ get_status(Name, Timeout) -> send_system_msg(Name, get_status, Timeout). -spec get_state(Name) -> State when Name :: name(), State :: term(). -get_state(Name) -> send_system_msg(Name, get_state). +get_state(Name) -> + case send_system_msg(Name, get_state) of + {error, Reason} -> error(Reason); + State -> State + end. -spec get_state(Name, Timeout) -> State when Name :: name(), Timeout :: timeout(), State :: term(). -get_state(Name, Timeout) -> send_system_msg(Name, get_state, Timeout). +get_state(Name, Timeout) -> + case send_system_msg(Name, get_state, Timeout) of + {error, Reason} -> error(Reason); + State -> State + end. -spec replace_state(Name, StateFun) -> NewState when Name :: name(), StateFun :: fun((State :: term()) -> NewState :: term()), NewState :: term(). replace_state(Name, StateFun) -> - send_system_msg(Name, {replace_state, StateFun}). + case send_system_msg(Name, {replace_state, StateFun}) of + {error, Reason} -> error(Reason); + State -> State + end. -spec replace_state(Name, StateFun, Timeout) -> NewState when Name :: name(), @@ -123,7 +134,10 @@ replace_state(Name, StateFun) -> Timeout :: timeout(), NewState :: term(). replace_state(Name, StateFun, Timeout) -> - send_system_msg(Name, {replace_state, StateFun}, Timeout). + case send_system_msg(Name, {replace_state, StateFun}, Timeout) of + {error, Reason} -> error(Reason); + State -> State + end. -spec change_code(Name, Module, OldVsn, Extra) -> 'ok' | {error, Reason} when Name :: name(), @@ -390,10 +404,11 @@ do_cmd(_, suspend, _Parent, _Mod, Debug, Misc) -> {suspended, ok, Debug, Misc}; do_cmd(_, resume, _Parent, _Mod, Debug, Misc) -> {running, ok, Debug, Misc}; -do_cmd(SysState, get_state, _Parent, _Mod, Debug, {State, Misc}) -> - {SysState, State, Debug, Misc}; -do_cmd(SysState, replace_state, _Parent, _Mod, Debug, {State, Misc}) -> - {SysState, State, Debug, Misc}; +do_cmd(SysState, get_state, _Parent, Mod, Debug, Misc) -> + {SysState, do_get_state(Mod, Misc), Debug, Misc}; +do_cmd(SysState, {replace_state, StateFun}, _Parent, Mod, Debug, Misc) -> + {Res, NMisc} = do_replace_state(StateFun, Mod, Misc), + {SysState, Res, Debug, NMisc}; do_cmd(SysState, get_status, Parent, Mod, Debug, Misc) -> Res = get_status(SysState, Parent, Mod, Debug, Misc), {SysState, Res, Debug, Misc}; @@ -407,6 +422,40 @@ do_cmd(suspended, {change_code, Module, Vsn, Extra}, _Parent, do_cmd(SysState, Other, _Parent, _Mod, Debug, Misc) -> {SysState, {error, {unknown_system_msg, Other}}, Debug, Misc}. +do_get_state(Mod, Misc) -> + case erlang:function_exported(Mod, system_get_state, 1) of + true -> + try + {ok, State} = Mod:system_get_state(Misc), + State + catch + Cl:Exc -> + {error, {callback_failed,{Mod,system_get_state},{Cl,Exc}}} + end; + false -> + Misc + end. + +do_replace_state(StateFun, Mod, Misc) -> + case erlang:function_exported(Mod, system_replace_state, 2) of + true -> + try + {ok, State, NMisc} = Mod:system_replace_state(StateFun, Misc), + {State, NMisc} + catch + Cl:Exc -> + {{error, {callback_failed,{Mod,system_replace_state},{Cl,Exc}}}, Misc} + end; + false -> + try + NMisc = StateFun(Misc), + {NMisc, NMisc} + catch + Cl:Exc -> + {{error, {callback_failed,StateFun,{Cl,Exc}}}, Misc} + end + end. + get_status(SysState, Parent, Mod, Debug, Misc) -> PDict = get(), FmtMisc = diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile index af82f22b21..39f6ce423a 100644 --- a/lib/stdlib/test/Makefile +++ b/lib/stdlib/test/Makefile @@ -73,6 +73,8 @@ MODULES= \ supervisor_SUITE \ supervisor_bridge_SUITE \ sys_SUITE \ + sys_sp1 \ + sys_sp2 \ tar_SUITE \ timer_SUITE \ timer_simple_SUITE \ diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index 673a3cf159..f822986981 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -92,7 +92,7 @@ all() -> bif_clash, behaviour_basic, behaviour_multiple, otp_7550, otp_8051, format_warn, {group, on_load}, too_many_arguments, basic_errors, bin_syntax_errors, predef, - maps,maps_type]. + maps, maps_type]. groups() -> [{unused_vars_warn, [], @@ -3439,11 +3439,10 @@ maps_type(Config) when is_list(Config) -> t(M) -> M. ">>, [], - {errors,[{3,erl_lint,{redefine_type,{map,0}}}],[]}}], + {warnings,[{3,erl_lint,{new_var_arity_type,map}}]}}], [] = run(Config, Ts), ok. - run(Config, Tests) -> F = fun({N,P,Ws,E}, BadL) -> case catch run_test(Config, P, Ws) of diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index 447e159cd4..35067e8116 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,7 +21,7 @@ init_per_group/2,end_per_group/2]). -export([ error_1/1, error_2/1, iso88591/1, otp_7810/1, otp_10302/1, - otp_10990/1, otp_10992/1]). + otp_10990/1, otp_10992/1, otp_11807/1]). -import(lists, [nth/2,flatten/1]). -import(io_lib, [print/1]). @@ -60,7 +60,8 @@ end_per_testcase(_Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992]. + [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992, + otp_11807]. groups() -> [{error, [], [error_1, error_2]}]. @@ -1144,6 +1145,25 @@ otp_10992(Config) when is_list(Config) -> erl_parse:abstract([$A,42.0], [{encoding,utf8}]), ok. +otp_11807(doc) -> + "OTP-11807. Generalize erl_parse:abstract/2."; +otp_11807(suite) -> + []; +otp_11807(Config) when is_list(Config) -> + {cons,0,{integer,0,97},{cons,0,{integer,0,98},{nil,0}}} = + erl_parse:abstract("ab", [{encoding,none}]), + {cons,0,{integer,0,-1},{nil,0}} = + erl_parse:abstract([-1], [{encoding,latin1}]), + ASCII = fun(I) -> I >= 0 andalso I < 128 end, + {string,0,"xyz"} = erl_parse:abstract("xyz", [{encoding,ASCII}]), + {cons,0,{integer,0,228},{nil,0}} = + erl_parse:abstract([228], [{encoding,ASCII}]), + {cons,0,{integer,0,97},{atom,0,a}} = + erl_parse:abstract("a"++a, [{encoding,latin1}]), + {'EXIT', {{badarg,bad},_}} = % minor backward incompatibility + (catch erl_parse:abstract("string", [{encoding,bad}])), + ok. + test_string(String, Expected) -> {ok, Expected, _End} = erl_scan:string(String), test(String). diff --git a/lib/stdlib/test/gen_event_SUITE.erl b/lib/stdlib/test/gen_event_SUITE.erl index 5819ef3890..60a1ba8c60 100644 --- a/lib/stdlib/test/gen_event_SUITE.erl +++ b/lib/stdlib/test/gen_event_SUITE.erl @@ -974,6 +974,10 @@ get_state(Config) when is_list(Config) -> [{dummy1_h,false,State1},{dummy1_h,id,State2}] = lists:sort(Result1), Result2 = sys:get_state(Pid, 5000), [{dummy1_h,false,State1},{dummy1_h,id,State2}] = lists:sort(Result2), + ok = sys:suspend(Pid), + Result3 = sys:get_state(Pid), + [{dummy1_h,false,State1},{dummy1_h,id,State2}] = lists:sort(Result3), + ok = sys:resume(Pid), ok = gen_event:stop(Pid), ok. @@ -998,4 +1002,11 @@ replace_state(Config) when is_list(Config) -> Replace3 = fun(_) -> exit(fail) end, [{dummy1_h,false,NState2}] = sys:replace_state(Pid, Replace3), [{dummy1_h,false,NState2}] = sys:get_state(Pid), + %% verify state replaced if process sys suspended + NState3 = "replaced again and again", + Replace4 = fun({dummy1_h,false,_}=S) -> setelement(3,S,NState3) end, + ok = sys:suspend(Pid), + [{dummy1_h,false,NState3}] = sys:replace_state(Pid, Replace4), + ok = sys:resume(Pid), + [{dummy1_h,false,NState3}] = sys:get_state(Pid), ok. diff --git a/lib/stdlib/test/gen_fsm_SUITE.erl b/lib/stdlib/test/gen_fsm_SUITE.erl index fd15838b7d..8aeec07ae8 100644 --- a/lib/stdlib/test/gen_fsm_SUITE.erl +++ b/lib/stdlib/test/gen_fsm_SUITE.erl @@ -426,6 +426,14 @@ get_state(Config) when is_list(Config) -> {idle, State} = sys:get_state(gfsm), {idle, State} = sys:get_state(gfsm, 5000), stop_it(Pid2), + + %% check that get_state works when pid is sys suspended + {ok, Pid3} = gen_fsm:start(gen_fsm_SUITE, {state_data, State}, []), + {idle, State} = sys:get_state(Pid3), + ok = sys:suspend(Pid3), + {idle, State} = sys:get_state(Pid3, 5000), + ok = sys:resume(Pid3), + stop_it(Pid3), ok. replace_state(Config) when is_list(Config) -> @@ -442,8 +450,18 @@ replace_state(Config) when is_list(Config) -> {state0, NState2} = sys:get_state(Pid), %% verify no change in state if replace function crashes Replace3 = fun(_) -> error(fail) end, - {state0, NState2} = sys:replace_state(Pid, Replace3), + {'EXIT',{{callback_failed, + {gen_fsm,system_replace_state},{error,fail}},_}} = + (catch sys:replace_state(Pid, Replace3)), {state0, NState2} = sys:get_state(Pid), + %% verify state replaced if process sys suspended + ok = sys:suspend(Pid), + Suffix2 = " and again", + NState3 = NState2 ++ Suffix2, + Replace4 = fun({StateName, _}) -> {StateName, NState3} end, + {state0, NState3} = sys:replace_state(Pid, Replace4), + ok = sys:resume(Pid), + {state0, NState3} = sys:get_state(Pid, 5000), stop_it(Pid), ok. diff --git a/lib/stdlib/test/gen_server_SUITE.erl b/lib/stdlib/test/gen_server_SUITE.erl index a360a0809b..960e7f60e7 100644 --- a/lib/stdlib/test/gen_server_SUITE.erl +++ b/lib/stdlib/test/gen_server_SUITE.erl @@ -1049,6 +1049,9 @@ get_state(Config) when is_list(Config) -> {ok, Pid} = gen_server:start_link(?MODULE, {state,State}, []), State = sys:get_state(Pid), State = sys:get_state(Pid, 5000), + ok = sys:suspend(Pid), + State = sys:get_state(Pid), + ok = sys:resume(Pid), ok. %% Verify that sys:replace_state correctly replaces gen_server state @@ -1075,8 +1078,18 @@ replace_state(Config) when is_list(Config) -> NState2 = sys:get_state(Pid, 5000), %% verify no change in state if replace function crashes Replace3 = fun(_) -> throw(fail) end, - NState2 = sys:replace_state(Pid, Replace3), + {'EXIT',{{callback_failed, + {gen_server,system_replace_state},{throw,fail}},_}} = + (catch sys:replace_state(Pid, Replace3)), NState2 = sys:get_state(Pid, 5000), + %% verify state replaced if process sys suspended + ok = sys:suspend(Pid), + Suffix2 = " and again", + NState3 = NState2 ++ Suffix2, + Replace4 = fun(S) -> S ++ Suffix2 end, + NState3 = sys:replace_state(Pid, Replace4), + ok = sys:resume(Pid), + NState3 = sys:get_state(Pid, 5000), ok. %% Test that the time for a huge message queue is not diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index 692dfe0faa..e016432f4d 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -54,7 +54,7 @@ config(priv_dir,_) -> -include_lib("test_server/include/test_server.hrl"). -export([init_per_testcase/2, end_per_testcase/2]). % Default timetrap timeout (set in init_per_testcase). --define(default_timeout, ?t:minutes(2)). +-define(default_timeout, ?t:minutes(10)). init_per_testcase(_Case, Config) -> ?line Dog = ?t:timetrap(?default_timeout), ?line OrigPath = code:get_path(), diff --git a/lib/stdlib/test/sys_SUITE.erl b/lib/stdlib/test/sys_SUITE.erl index c06ba545e7..f38bc87ae5 100644 --- a/lib/stdlib/test/sys_SUITE.erl +++ b/lib/stdlib/test/sys_SUITE.erl @@ -19,7 +19,7 @@ -module(sys_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2,log/1,log_to_file/1, - stats/1,trace/1,suspend/1,install/1]). + stats/1,trace/1,suspend/1,install/1,special_process/1]). -export([handle_call/3,terminate/2,init/1]). -include_lib("test_server/include/test_server.hrl"). @@ -27,14 +27,12 @@ %% Doesn't look into change_code at all -%% Doesn't address writing your own process that understands -%% system messages at all. suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [log, log_to_file, stats, trace, suspend, install]. + [log, log_to_file, stats, trace, suspend, install, special_process]. groups() -> []. @@ -157,6 +155,84 @@ install(Config) when is_list(Config) -> ?line stop(), ok. +special_process(suite) -> []; +special_process(Config) when is_list(Config) -> + ok = spec_proc(sys_sp1), + ok = spec_proc(sys_sp2). + +spec_proc(Mod) -> + {ok,_} = Mod:start_link(100), + ok = sys:statistics(Mod,true), + ok = sys:trace(Mod,true), + 1 = Ch = Mod:alloc(), + Free = lists:seq(2,100), + Replace = case sys:get_state(Mod) of + {[Ch],Free} -> + fun({A,F}) -> + Free = F, + {A,[2,3,4]} + end; + {state,[Ch],Free} -> + fun({state,A,F}) -> + Free = F, + {state,A,[2,3,4]} + end + end, + case sys:replace_state(Mod, Replace) of + {[Ch],[2,3,4]} -> ok; + {state,[Ch],[2,3,4]} -> ok + end, + ok = Mod:free(Ch), + case sys:get_state(Mod) of + {[],[1,2,3,4]} -> ok; + {state,[],[1,2,3,4]} -> ok + end, + {ok,[{start_time,_}, + {current_time,_}, + {reductions,_}, + {messages_in,2}, + {messages_out,1}]} = sys:statistics(Mod,get), + ok = sys:statistics(Mod,false), + [] = sys:replace_state(Mod, fun(_) -> [] end), + process_flag(trap_exit,true), + ok = case catch sys:get_state(Mod) of + [] -> + ok; + {'EXIT',{{callback_failed, + {Mod,system_get_state},{throw,fail}},_}} -> + ok + end, + Mod:stop(), + WaitForUnregister = fun W() -> + case whereis(Mod) of + undefined -> ok; + _ -> timer:sleep(10), W() + end + end, + WaitForUnregister(), + {ok,_} = Mod:start_link(4), + ok = case catch sys:replace_state(Mod, fun(_) -> {} end) of + {} -> + ok; + {'EXIT',{{callback_failed, + {Mod,system_replace_state},{throw,fail}},_}} -> + ok + end, + Mod:stop(), + WaitForUnregister(), + {ok,_} = Mod:start_link(4), + StateFun = fun(_) -> error(fail) end, + ok = case catch sys:replace_state(Mod, StateFun) of + {} -> + ok; + {'EXIT',{{callback_failed, + {Mod,system_replace_state},{error,fail}},_}} -> + ok; + {'EXIT',{{callback_failed,StateFun,{error,fail}},_}} -> + ok + end, + Mod:stop(). + %%%%%%%%%%%%%%%%%%%% %% Dummy server diff --git a/lib/stdlib/test/sys_sp1.erl b/lib/stdlib/test/sys_sp1.erl new file mode 100644 index 0000000000..e84ffcfa12 --- /dev/null +++ b/lib/stdlib/test/sys_sp1.erl @@ -0,0 +1,114 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(sys_sp1). +-export([start_link/1, stop/0]). +-export([alloc/0, free/1]). +-export([init/1]). +-export([system_continue/3, system_terminate/4, + write_debug/3, + system_get_state/1, system_replace_state/2]). + +%% Implements the ch4 example from the Design Principles doc. Same as +%% sys_sp2 except this module exports system_get_state/1 and +%% system_replace_state/2 + +start_link(NumCh) -> + proc_lib:start_link(?MODULE, init, [[self(),NumCh]]). + +stop() -> + ?MODULE ! stop, + ok. + +alloc() -> + ?MODULE ! {self(), alloc}, + receive + {?MODULE, Res} -> + Res + end. + +free(Ch) -> + ?MODULE ! {free, Ch}, + ok. + +init([Parent,NumCh]) -> + register(?MODULE, self()), + Chs = channels(NumCh), + Deb = sys:debug_options([]), + proc_lib:init_ack(Parent, {ok, self()}), + loop(Chs, Parent, Deb). + +loop(Chs, Parent, Deb) -> + receive + {From, alloc} -> + Deb2 = sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, alloc, From}), + {Ch, Chs2} = alloc(Chs), + From ! {?MODULE, Ch}, + Deb3 = sys:handle_debug(Deb2, fun write_debug/3, + ?MODULE, {out, {?MODULE, Ch}, From}), + loop(Chs2, Parent, Deb3); + {free, Ch} -> + Deb2 = sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, {free, Ch}}), + Chs2 = free(Ch, Chs), + loop(Chs2, Parent, Deb2); + {system, From, Request} -> + sys:handle_system_msg(Request, From, Parent, + ?MODULE, Deb, Chs); + stop -> + sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, stop}), + ok + end. + +system_continue(Parent, Deb, Chs) -> + loop(Chs, Parent, Deb). + +system_terminate(Reason, _Parent, _Deb, _Chs) -> + exit(Reason). + +system_get_state([]) -> + throw(fail); +system_get_state(Chs) -> + {ok, Chs}. + +system_replace_state(_StateFun, {}) -> + throw(fail); +system_replace_state(StateFun, Chs) -> + NChs = StateFun(Chs), + {ok, NChs, NChs}. + +write_debug(Dev, Event, Name) -> + io:format(Dev, "~p event = ~p~n", [Name, Event]). + +channels(NumCh) -> + {_Allocated=[], _Free=lists:seq(1,NumCh)}. + +alloc({_, []}) -> + {error, "no channels available"}; +alloc({Allocated, [H|T]}) -> + {H, {[H|Allocated], T}}. + +free(Ch, {Alloc, Free}=Channels) -> + case lists:member(Ch, Alloc) of + true -> + {lists:delete(Ch, Alloc), [Ch|Free]}; + false -> + Channels + end. diff --git a/lib/stdlib/test/sys_sp2.erl b/lib/stdlib/test/sys_sp2.erl new file mode 100644 index 0000000000..56a5e4d071 --- /dev/null +++ b/lib/stdlib/test/sys_sp2.erl @@ -0,0 +1,107 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(sys_sp2). +-export([start_link/1, stop/0]). +-export([alloc/0, free/1]). +-export([init/1]). +-export([system_continue/3, system_terminate/4, + write_debug/3]). + +%% Implements the ch4 example from the Design Principles doc. Same as +%% sys_sp1 except this module does not export system_get_state/1 or +%% system_replace_state/2 + +start_link(NumCh) -> + proc_lib:start_link(?MODULE, init, [[self(),NumCh]]). + +stop() -> + ?MODULE ! stop, + ok. + +alloc() -> + ?MODULE ! {self(), alloc}, + receive + {?MODULE, Res} -> + Res + end. + +free(Ch) -> + ?MODULE ! {free, Ch}, + ok. + +%% can't use 2-tuple for state here as we do in sys_sp1, since the 2-tuple +%% is not compatible with the backward compatibility handling for +%% sys:get_state in sys.erl +-record(state, {alloc,free}). + +init([Parent,NumCh]) -> + register(?MODULE, self()), + Chs = channels(NumCh), + Deb = sys:debug_options([]), + proc_lib:init_ack(Parent, {ok, self()}), + loop(Chs, Parent, Deb). + +loop(Chs, Parent, Deb) -> + receive + {From, alloc} -> + Deb2 = sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, alloc, From}), + {Ch, Chs2} = alloc(Chs), + From ! {?MODULE, Ch}, + Deb3 = sys:handle_debug(Deb2, fun write_debug/3, + ?MODULE, {out, {?MODULE, Ch}, From}), + loop(Chs2, Parent, Deb3); + {free, Ch} -> + Deb2 = sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, {free, Ch}}), + Chs2 = free(Ch, Chs), + loop(Chs2, Parent, Deb2); + {system, From, Request} -> + sys:handle_system_msg(Request, From, Parent, + ?MODULE, Deb, Chs); + stop -> + sys:handle_debug(Deb, fun write_debug/3, + ?MODULE, {in, stop}), + ok + end. + +system_continue(Parent, Deb, Chs) -> + loop(Chs, Parent, Deb). + +system_terminate(Reason, _Parent, _Deb, _Chs) -> + exit(Reason). + +write_debug(Dev, Event, Name) -> + io:format(Dev, "~p event = ~p~n", [Name, Event]). + +channels(NumCh) -> + #state{alloc=[], free=lists:seq(1,NumCh)}. + +alloc(#state{free=[]}=Channels) -> + {{error, "no channels available"}, Channels}; +alloc(#state{alloc=Allocated, free=[H|T]}) -> + {H, #state{alloc=[H|Allocated], free=T}}. + +free(Ch, #state{alloc=Alloc, free=Free}=Channels) -> + case lists:member(Ch, Alloc) of + true -> + #state{alloc=lists:delete(Ch, Alloc), free=[Ch|Free]}; + false -> + Channels + end. diff --git a/lib/syntax_tools/src/syntax_tools.app.src b/lib/syntax_tools/src/syntax_tools.app.src index dc0b9edd62..83dcb5fe23 100644 --- a/lib/syntax_tools/src/syntax_tools.app.src +++ b/lib/syntax_tools/src/syntax_tools.app.src @@ -14,4 +14,5 @@ prettypr]}, {registered,[]}, {applications, [stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/syntax_tools/vsn.mk b/lib/syntax_tools/vsn.mk index 26153a55f1..cf396ce636 100644 --- a/lib/syntax_tools/vsn.mk +++ b/lib/syntax_tools/vsn.mk @@ -1 +1 @@ -SYNTAX_TOOLS_VSN = 1.6.13 +SYNTAX_TOOLS_VSN = 1.6.14 diff --git a/lib/test_server/src/test_server.app.src b/lib/test_server/src/test_server.app.src index 42e78ed279..5672baa6ef 100644 --- a/lib/test_server/src/test_server.app.src +++ b/lib/test_server/src/test_server.app.src @@ -31,5 +31,8 @@ test_server, test_server_break_process]}, {applications, [kernel,stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["tools-2.6.14","stdlib-2.0","runtime_tools-1.8.14", + "observer-2.0","kernel-3.0","inets-5.10", + "erts-6.0"]}]}. diff --git a/lib/test_server/src/test_server_sup.erl b/lib/test_server/src/test_server_sup.erl index 3cfa84a52f..96e369a138 100644 --- a/lib/test_server/src/test_server_sup.erl +++ b/lib/test_server/src/test_server_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -469,7 +469,7 @@ check_instruction(_, {add_application, Application, Type}, _, _) -> check_instruction(_, Instr, _, _) -> throw({error, {low_level_or_invalid_instruction, Instr}}). -check_module(Module, Modules) when is_atom(Module) -> +check_module(Module, Modules) -> case {is_atom(Module), lists:member(Module, Modules)} of {true, true} -> ok; {true, false} -> throw({error, {unknown_module, Module}}); diff --git a/lib/test_server/src/ts.unix.config b/lib/test_server/src/ts.unix.config index a34857b9e5..1ba5d9033e 100644 --- a/lib/test_server/src/ts.unix.config +++ b/lib/test_server/src/ts.unix.config @@ -3,4 +3,4 @@ %% Always run a (VNC) X server on host %% {xserver, "xserver.example.com:66"}. -{unix,[{telnet,"belegost"},{username,"bofh"},{password,"root"},{keep_alive,true}]}. +{unix,[{telnet,"belegost"},{username,"telnet-test"},{password,"tset-tenlet"},{keep_alive,true}]}. diff --git a/lib/test_server/vsn.mk b/lib/test_server/vsn.mk index 6871b5bd14..4eb70aa2cd 100644 --- a/lib/test_server/vsn.mk +++ b/lib/test_server/vsn.mk @@ -1 +1 @@ -TEST_SERVER_VSN = 3.6.4 +TEST_SERVER_VSN = 3.7 diff --git a/lib/tools/src/tools.app.src b/lib/tools/src/tools.app.src index 553c5eb96b..ec5b6f3a82 100644 --- a/lib/tools/src/tools.app.src +++ b/lib/tools/src/tools.app.src @@ -39,23 +39,9 @@ {applications, [kernel, stdlib]}, {env, [{file_util_search_methods,[{"", ""}, {"ebin", "esrc"}, {"ebin", "src"}]} ] - } + }, + {runtime_dependencies, ["webtool-0.8.10","stdlib-2.0","runtime_tools-1.8.14", + "kernel-3.0","inets-5.10","erts-6.0", + "compiler-5.0"]} ] }. - - - - - - - - - - - - - - - - - diff --git a/lib/tools/test/cover_SUITE.erl b/lib/tools/test/cover_SUITE.erl index ec61c57cec..80807b1d38 100644 --- a/lib/tools/test/cover_SUITE.erl +++ b/lib/tools/test/cover_SUITE.erl @@ -516,13 +516,11 @@ reconnect(Config) -> cover:flush(N1), rpc:call(N1,f,f1,[]), - %% This will cause a call to f:f2() when nodes()==[] on N1 + %% This will cause first casue the N1 node to initiate a + %% disconnect and then call f:f2() when nodes() =:= [] on N1. rpc:cast(N1,f,call_f2_when_isolated,[]), - - %% Disconnect and check that node is removed from main cover node - net_kernel:disconnect(N1), timer:sleep(500), % allow some to detect disconnect and for f:f2() call - [] = cover:which_nodes(), + cover_which_nodes([]), %% Do some add one module (b) and remove one module (a) code:purge(a), @@ -530,7 +528,7 @@ reconnect(Config) -> {ok,b} = cover:compile(b), cover_compiled = code:which(b), - [] = cover:which_nodes(), + cover_which_nodes([]), check_f_calls(1,0), % only the first call - before the flush %% Reconnect the node and check that b and f are cover compiled but not a @@ -573,7 +571,7 @@ die_and_reconnect(Config) -> %% Kill the node rpc:call(N1,erlang,halt,[]), - [] = cover:which_nodes(), + cover_which_nodes([]), check_f_calls(1,0), % only the first call - before the flush @@ -614,7 +612,7 @@ dont_reconnect_after_stop(Config) -> %% Stop cover on the node, then terminate the node cover:stop(N1), rpc:call(N1,erlang,halt,[]), - [] = cover:which_nodes(), + cover_which_nodes([]), check_f_calls(1,0), @@ -622,7 +620,7 @@ dont_reconnect_after_stop(Config) -> {ok,N1} = ?t:start_node(NodeName,peer, [{args," -pa " ++ DataDir},{start_cover,false}]), timer:sleep(300), - [] = cover:which_nodes(), + cover_which_nodes([]), Beam = rpc:call(N1,code,which,[f]), false = (Beam==cover_compiled), @@ -667,7 +665,7 @@ stop_node_after_disconnect(Config) -> {ok,N1} = ?t:start_node(NodeName,peer, [{args," -pa " ++ DataDir},{start_cover,false}]), timer:sleep(300), - [] = cover:which_nodes(), + cover_which_nodes([]), Beam = rpc:call(N1,code,which,[f]), false = (Beam==cover_compiled), @@ -1575,3 +1573,21 @@ is_unloaded(What) -> check_f_calls(F1,F2) -> {ok,[{{f,f1,0},F1},{{f,f2,0},F2}|_]} = cover:analyse(f,calls,function). + +cover_which_nodes(Expected) -> + case cover:which_nodes() of + Expected -> + ok; + Other -> + {Time,ok} = timer:tc(fun Retry() -> + case cover:which_nodes() of + Expected -> ok; + _ -> + ?t:sleep(100), + Retry() + end + end), + io:format("~p ms before cover:which_nodes() returned ~p", + [Time,Expected]), + Expected = Other + end. diff --git a/lib/tools/test/cover_SUITE_data/f.erl b/lib/tools/test/cover_SUITE_data/f.erl index ce2963014a..a29a67b388 100644 --- a/lib/tools/test/cover_SUITE_data/f.erl +++ b/lib/tools/test/cover_SUITE_data/f.erl @@ -10,10 +10,15 @@ f2() -> f2_line2. call_f2_when_isolated() -> + [Other] = nodes(), + net_kernel:disconnect(Other), + do_call_f2_when_isolated(). + +do_call_f2_when_isolated() -> case nodes() of [] -> f2(); _ -> timer:sleep(100), - call_f2_when_isolated() + do_call_f2_when_isolated() end. diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index 0cead00554..2d2970de3a 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 2.6.13 +TOOLS_VSN = 2.6.14 diff --git a/lib/typer/src/typer.app.src b/lib/typer/src/typer.app.src index 850829e1dc..974091b44c 100644 --- a/lib/typer/src/typer.app.src +++ b/lib/typer/src/typer.app.src @@ -6,4 +6,6 @@ {modules, [typer]}, {registered, []}, {applications, [compiler, dialyzer, hipe, kernel, stdlib]}, - {env, []}]}. + {env, []}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","hipe-3.10.3","erts-6.0", + "dialyzer-2.7","compiler-5.0"]}]}. diff --git a/lib/typer/vsn.mk b/lib/typer/vsn.mk index 5ac145d9ff..49fdda756e 100644 --- a/lib/typer/vsn.mk +++ b/lib/typer/vsn.mk @@ -1 +1 @@ -TYPER_VSN = 0.9.5 +TYPER_VSN = 0.9.6 diff --git a/lib/webtool/src/webtool.app.src b/lib/webtool/src/webtool.app.src index 8c6774c533..3d8d11ea60 100644 --- a/lib/webtool/src/webtool.app.src +++ b/lib/webtool/src/webtool.app.src @@ -21,5 +21,7 @@ {vsn,"%VSN%"}, {modules,[webtool,webtool_sup]}, {registered,[web_tool,websup]}, - {applications,[kernel,stdlib]}]}. + {applications,[kernel,stdlib]}, + {runtime_dependencies, ["stdlib-2.0","observer-2.0","kernel-3.0", + "inets-5.10","erts-6.0"]}]}. diff --git a/lib/webtool/vsn.mk b/lib/webtool/vsn.mk index d356a8954d..a79c273d9f 100644 --- a/lib/webtool/vsn.mk +++ b/lib/webtool/vsn.mk @@ -1 +1 @@ -WEBTOOL_VSN=0.8.9.2 +WEBTOOL_VSN=0.8.10 diff --git a/lib/wx/src/wx.app.src b/lib/wx/src/wx.app.src index e13982b0c1..d5ac478f20 100644 --- a/lib/wx/src/wx.app.src +++ b/lib/wx/src/wx.app.src @@ -33,5 +33,6 @@ ]}, {registered, []}, {applications, [stdlib, kernel]}, - {env, []} + {env, []}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/wx/vsn.mk b/lib/wx/vsn.mk index c018b4fb86..5523c20440 100644 --- a/lib/wx/vsn.mk +++ b/lib/wx/vsn.mk @@ -1 +1 @@ -WX_VSN = 1.1.2 +WX_VSN = 1.2 diff --git a/lib/xmerl/src/xmerl.app.src b/lib/xmerl/src/xmerl.app.src index b471447bbd..45cfe9d250 100644 --- a/lib/xmerl/src/xmerl.app.src +++ b/lib/xmerl/src/xmerl.app.src @@ -39,5 +39,6 @@ {registered, []}, {env, []}, - {applications, [kernel, stdlib]} + {applications, [kernel, stdlib]}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/xmerl/vsn.mk b/lib/xmerl/vsn.mk index 333466c11e..aab2a37d6c 100644 --- a/lib/xmerl/vsn.mk +++ b/lib/xmerl/vsn.mk @@ -1 +1 @@ -XMERL_VSN = 1.3.6 +XMERL_VSN = 1.3.7 diff --git a/make/otp_released_app.mk b/make/otp_released_app.mk new file mode 100644 index 0000000000..fb5205ab23 --- /dev/null +++ b/make/otp_released_app.mk @@ -0,0 +1,43 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2014. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% + +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk +include $(APP_PWD)/vsn.mk +include $(ERL_TOP)/make/otp_default_release_path.mk + +RELEASED_APP_VSN=$(APP)-$($(APP_VSN)) +ifeq ($(TESTROOT),) +REL_DIR=$(OTP_DEFAULT_RELEASE_PATH)/releases/$(SYSTEM_VSN) +else +REL_DIR=$(TESTROOT)/releases/$(SYSTEM_VSN) +endif +INST_APP_VSNS=$(REL_DIR)/installed_application_versions + +.PHONY: update + +update: + test -d "$(REL_DIR)" || mkdir -p "$(REL_DIR)" ; \ + if test ! -f "$(INST_APP_VSNS)" ; then \ + echo "$(RELEASED_APP_VSN)" > "$(INST_APP_VSNS)" || exit 1; \ + else \ + if test x = x`grep $(RELEASED_APP_VSN) "$(INST_APP_VSNS)"` ; then \ + echo $(RELEASED_APP_VSN) >> "$(INST_APP_VSNS)" || exit 1; \ + fi ; \ + fi + diff --git a/make/otp_subdir.mk b/make/otp_subdir.mk index f31ab05c87..e6a75cce17 100644 --- a/make/otp_subdir.mk +++ b/make/otp_subdir.mk @@ -44,5 +44,14 @@ opt debug release docs release_docs tests release_tests clean depend valgrind st fi ; \ done ; \ if test -f vsn.mk; then \ + if test release = $@ && test ! -f SKIP; then \ + app=`basename $$app_pwd` ; \ + app_vsn=`echo $$app | sed "y|abcdefghijklmnopqrstuvwxyz|ABCDEFGHIJKLMNOPQRSTUVWXYZ|"` ; \ + app_vsn=$${app_vsn}_VSN ; \ + ( $(MAKE) -f "$(ERL_TOP)/make/otp_released_app.mk" \ + APP_PWD="$$app_pwd" APP_VSN=$$app_vsn APP=$$app \ + TESTROOT="$(TESTROOT)" update) \ + || exit $$? ; \ + fi ; \ echo "=== Leaving application" `basename $$app_pwd` ; \ fi diff --git a/make/verify_runtime_dependencies b/make/verify_runtime_dependencies new file mode 100755 index 0000000000..b8eea06b6e --- /dev/null +++ b/make/verify_runtime_dependencies @@ -0,0 +1,313 @@ +#!/usr/bin/env escript +%% -*- erlang -*- + +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% @author Rickard Green <[email protected]> +%%% @copyright (C) 2014, Rickard Green +%%% @doc +%%% Verify runtime dependencies when patching OTP applications. +%%% @end +%%% Created : 4 Mar 2014 by Rickard Green <[email protected]> +%%%------------------------------------------------------------------- + +-mode(compile). + +-export([main/1]). + +main(Args) -> + {Force, Release, SourceDir, TargetDir, AppList} = parse_args(Args, + false, + [], + [], + [], + []), + SourceAppInfo = read_source_app_info(AppList, SourceDir), + AppVsnsTab0 = current_target_app_vsns(TargetDir, Release), + AppVsnsTab1 = add_source_app_vsns(SourceAppInfo, AppVsnsTab0), + case verify_runtime_deps(SourceAppInfo, AppVsnsTab1, true) of + true -> + ok; + false -> + case Force of + true -> + warn("Your OTP development system was updated with " + "unfulfilled runtime dependencies. The system " + "may not be working as expected.", []); + false -> + err("Unfulfilled runtime dependencies. " + "See warnings above.~n", []) + end + end, + halt(0). + +parse_args(["-force" | Args], _, Release, SourceDir, TargetDir, Apps) -> + parse_args(Args, true, Release, SourceDir, TargetDir, Apps); +parse_args(["-release", Release | Args], Force, _, SourceDir, TargetDir, Apps) -> + parse_args(Args, Force, Release, SourceDir, TargetDir, Apps); +parse_args(["-source", SourceDir | Args], Force, Release, _, TargetDir, Apps) -> + parse_args(Args, Force, Release, SourceDir, TargetDir, Apps); +parse_args(["-target", TargetDir | Args], Force, Release, SourceDir, _, Apps) -> + parse_args(Args, Force, Release, SourceDir, TargetDir, Apps); +parse_args([App | Args], Force, Release, SourceDir, TargetDir, OldApps) -> + parse_args(Args, Force, Release, SourceDir, TargetDir, [App | OldApps]); +parse_args([], _, [], _, _, _) -> + err("Missing release~n", []); +parse_args([], _, _, [], _, _) -> + err("Missing source directory~n", []); +parse_args([], _, _, _, [], _) -> + err("Missing target directory~n", []); +parse_args([], _, _, _, _, []) -> + err("Missing applications~n"); +parse_args([], Force, Release, SourceDir, TargetDir, Apps) -> + {Force, Release, SourceDir, TargetDir, Apps}. + + +%warn(Format) -> +% warn(Format, []). + +warn(Format, Args) -> + io:format(standard_error, "WARNING: " ++ Format, Args). + +err(Format) -> + err(Format, []). + +err(Format, Args) -> + io:format(standard_error, "ERROR: " ++ Format, Args), + halt(1). + +read_file(FileName) -> + case file:read_file(FileName) of + {ok, Content} -> + binary_to_list(Content); + {error, Error} -> + err("Failed to read ~s: ~p~n", [FileName, Error]) + end. + +consult_file(FileName) -> + case file:consult(FileName) of + {ok, Terms} -> + Terms; + {error, Error} -> + err("Failed to consult ~s: ~p~n", [FileName, Error]) + end. + +current_target_app_vsns(TargetDir, Release) -> + IAV = read_file(filename:join([TargetDir, "releases", Release, + "installed_application_versions"])), + DirList = string:tokens(IAV, "\n\r\t "), + LibDir = filename:join(TargetDir, "lib"), + make_app_vsns_tab(DirList, LibDir, gb_trees:empty()). + +make_app_vsns_tab([], _LibDir, GBT) -> + GBT; +make_app_vsns_tab([AppVer | AppVsns], LibDir, GBT0) -> + GBT1 = try + case file:read_file_info(filename:join(LibDir, AppVer)) of + {ok, _FInfo} -> + [App, Vsn] = string:tokens(AppVer, "-"), + add_app_vsn(App, Vsn, GBT0); + _ -> + GBT0 + end + catch + _:_ -> + warn("Unexpected directory: ~p~n", + [filename:join(LibDir, AppVer)]), + GBT0 + end, + make_app_vsns_tab(AppVsns, LibDir, GBT1). + +add_app_vsn(App, VsnList, GBT) when is_atom(App) -> + Vsn = parse_vsn(VsnList), + case gb_trees:lookup(App, GBT) of + none -> + gb_trees:insert(App, [Vsn], GBT); + {value, Vsns} -> + gb_trees:update(App, [Vsn | Vsns], GBT) + end; +add_app_vsn(AppStr, VsnList, GBT) -> + add_app_vsn(list_to_atom(AppStr), VsnList, GBT). + +add_source_app_vsns([], AppVsnsTab) -> + AppVsnsTab; +add_source_app_vsns([{App, Vsn, _IReqs} | AI], AppVsnsTab) -> + add_source_app_vsns(AI, add_app_vsn(App, Vsn, AppVsnsTab)). + +read_source_app_info([], _SourceDir) -> + []; +read_source_app_info([App | Apps], SourceDir) -> + AppFile = case App of + "erts" -> + filename:join([SourceDir, "erts", "preloaded", "ebin", + "erts.app"]); + _ -> + filename:join([SourceDir, "lib", App, "ebin", + App ++ ".app"]) + end, + AppAtom = list_to_atom(App), + case consult_file(AppFile) of + [{application, AppAtom, InfoList}] -> + Vsn = case lists:keyfind(vsn, 1, InfoList) of + {vsn, V} -> + V; + _ -> + err("Missing vsn in ~p~n", AppFile) + end, + AI = case lists:keyfind(runtime_dependencies, 1, InfoList) of + {runtime_dependencies, IReqs} -> + case parse_inst_reqs(IReqs) of + error -> + err("Failed to parse runtime_dependencies in ~p~n", + [AppFile]); + ParsedIReqs -> + {AppAtom, Vsn, ParsedIReqs} + end; + _ -> + {AppAtom, Vsn, []} + end, + [AI | read_source_app_info(Apps, SourceDir)]; + _ -> + err("Failed to parse ~p~n", [AppFile]) + end. + +parse_vsn(VsnStr) -> + list_to_tuple(lists:map(fun (IL) -> + list_to_integer(IL) + end, string:tokens(VsnStr, "."))). + +parse_inst_reqs(InstReqs) -> + try + parse_inst_reqs_aux(InstReqs) + catch + _ : _ -> + error + end. + +parse_inst_reqs_aux([]) -> + []; +parse_inst_reqs_aux([IR | IRs]) -> + [App, VsnStr] = string:tokens(IR, "-"), + [{list_to_atom(App), parse_vsn(VsnStr)} | parse_inst_reqs_aux(IRs)]. + +make_app_vsn_str({App, VsnTup}) -> + make_app_vsn_str(tuple_to_list(VsnTup), [atom_to_list(App), $-]). + +make_app_vsn_str([I], Acc) -> + lists:flatten([Acc, integer_to_list(I)]); +make_app_vsn_str([I | Is], Acc) -> + make_app_vsn_str(Is, [Acc, integer_to_list(I), $.]). + +missing_min_req(App, AppVsn, IReq) -> + warn("Unfulfilled runtime dependency for application ~p-~s: ~s~n", + [App, AppVsn, make_app_vsn_str(IReq)]). + +verify_runtime_deps([], _AppVsnsTab, Res) -> + Res; +verify_runtime_deps([{App, Vsn, IReqs} | SAIs], AppVsnsTab, Res0) -> + Res = lists:foldl( + fun ({IRApp, IRMinVsn} = InstReq, AccRes) -> + case gb_trees:lookup(IRApp, AppVsnsTab) of + none -> + missing_min_req(App, Vsn, InstReq), + false; + {value, AppVsns} -> + try + lists:foreach( + fun (AppVsn) -> + case meets_min_req(AppVsn, IRMinVsn) of + true -> + throw(true); + false -> + false + end + end, + AppVsns), + missing_min_req(App, Vsn, InstReq), + false + catch + throw : true -> + AccRes + end + end + end, + Res0, + IReqs), + verify_runtime_deps(SAIs, AppVsnsTab, Res). + +meets_min_req(Vsn, Vsn) -> + true; +meets_min_req({X}, VsnReq) -> + meets_min_req({X, 0, 0}, VsnReq); +meets_min_req({X, Y}, VsnReq) -> + meets_min_req({X, Y, 0}, VsnReq); +meets_min_req(Vsn, {X}) -> + meets_min_req(Vsn, {X, 0, 0}); +meets_min_req(Vsn, {X, Y}) -> + meets_min_req(Vsn, {X, Y, 0}); +meets_min_req({X, _Y, _Z}, {XReq, _YReq, _ZReq}) when X > XReq -> + true; +meets_min_req({X, Y, _Z}, {X, YReq, _ZReq}) when Y > YReq -> + true; +meets_min_req({X, Y, Z}, {X, Y, ZReq}) when Z > ZReq -> + true; +meets_min_req({_X, _Y, _Z}, {_XReq, _YReq, _ZReq}) -> + false; +meets_min_req(Vsn, VsnReq) -> + gp_meets_min_req(mk_gp_vsn_list(Vsn), mk_gp_vsn_list(VsnReq)). + +gp_meets_min_req([X, Y, Z | _Vs], [X, Y, Z]) -> + true; +gp_meets_min_req([X, Y, Z | _Vs], [XReq, YReq, ZReq]) -> + meets_min_req({X, Y, Z}, {XReq, YReq, ZReq}); +gp_meets_min_req([X, Y, Z | Vs], [X, Y, Z | VReqs]) -> + gp_meets_min_req_tail(Vs, VReqs); +gp_meets_min_req(_Vsn, _VReq) -> + %% Versions on different version branches, i.e., the minimum + %% required functionality is not included in Vsn. + false. + +gp_meets_min_req_tail([V | Vs], [V | VReqs]) -> + gp_meets_min_req_tail(Vs, VReqs); +gp_meets_min_req_tail([], []) -> + true; +gp_meets_min_req_tail([_V | _Vs], []) -> + true; +gp_meets_min_req_tail([V | _Vs], [VReq]) when V > VReq -> + true; +gp_meets_min_req_tail(_Vs, _VReqs) -> + %% Versions on different version branches, i.e., the minimum + %% required functionality is not included in Vsn. + false. + +mk_gp_vsn_list(Vsn) -> + [X, Y, Z | Tail] = tuple_to_list(Vsn), + [X, Y, Z | remove_trailing_zeroes(Tail)]. + +remove_trailing_zeroes([]) -> + []; +remove_trailing_zeroes([0 | Vs]) -> + case remove_trailing_zeroes(Vs) of + [] -> []; + NewVs -> [0 | NewVs] + end; +remove_trailing_zeroes([V | Vs]) -> + [V | remove_trailing_zeroes(Vs)]. @@ -52,7 +52,7 @@ usage () echo " release [-a] <target_dir> - creates full release to <target_dir>" echo " smp [-a] - build an Erlang system, smp flavor only" echo " tests <dir> - Build testsuites to <dir>" - echo " patch_app <target_dir> <app1>... - build given apps to <target_dir>" + echo " patch_app [-f] <target_dir> <app1>... - build given apps to <target_dir>" echo " If core apps are patched, new start scripts will be created" echo " and 'Install' must be run again." echo "" @@ -1209,25 +1209,37 @@ do_patch_app () if [ X`$MAKE is_cross_configured` = Xyes ]; then TARGET=`$MAKE target_configured` fi + if [ "x$1" = "x-f" ]; then + force="-force" + shift + else + force= + fi target_dir=$1 - if [ ! -d $target_dir/releases/$otp_major_vsn ]; then + if [ ! -d "$target_dir/releases/$otp_major_vsn" ]; then echo "No OTP $otp_major_vsn installation in $target_dir" 1>&2 exit 1 fi shift - otp_version=`cat "$target_dir/OTP_VERSION"` || { echo "Not able to read $target_dir/OTP_VERSION" 1>&2; exit 1; } - { echo "$otp_version" | sed "s|^\([^\*]*\)\**|\1\*\*|g" > $target_dir/OTP_VERSION; } 2>/dev/null || { echo "Not able to update $target_dir/OTP_VERSION" 1>&2; exit 1; } + otp_version=`cat "$target_dir/releases/$otp_major_vsn/OTP_VERSION"` || { echo "Not able to read $target_dir/releases/$otp_major_vsn/OTP_VERSION" 1>&2; exit 1; } + { echo "$otp_version" | sed "s|^\([^\*]*\)\**|\1\*\*|g" > "$target_dir/releases/$otp_major_vsn/OTP_VERSION"; } 2>/dev/null || { echo "Not able to update $target_dir/OTP_VERSION" 1>&2; exit 1; } + + PATH="$ERL_TOP/bootstrap/bin:$PATH" $ERL_TOP/make/verify_runtime_dependencies -release "$otp_major_vsn" -source "$ERL_TOP" -target "$target_dir" $force "$@" + + if [ $? -ne 0 ]; then + exit $? + fi # Build all applications to target for app in "$@"; do if [ "$app" = "erts" ] && [ -d $ERL_TOP/$app ]; then (cd $ERL_TOP/$app && $MAKE MAKE="$MAKE" TARGET=$TARGET \ - TESTROOT=$target_dir release) || exit 1 + TESTROOT="$target_dir" release) || exit 1 elif [ "$app" != "erts" ] && [ -d $ERL_TOP/lib/$app ]; then (cd $ERL_TOP/lib/$app && $MAKE MAKE="$MAKE" TARGET=$TARGET \ - TESTROOT=$target_dir release) || exit 1 + TESTROOT="$target_dir" release) || exit 1 else echo "Invalid application $app" 1>&2 exit 1 @@ -1254,62 +1266,62 @@ do_patch_app () # and find the old versions for those not included if [ "X$update_rel" != "X" ]; then if [ "X$erts_vsn" = "X" ]; then - erts_vsns=`ls -d $target_dir/erts-* | sed "s|$target_dir/erts-\([0-9\.].*\)|\1|g"` + erts_vsns=`ls -d "$target_dir"/erts-* | sed "s|$target_dir/erts-\([0-9\.].*\)|\1|g"` erts_vsn=`echo "$erts_vsns" | sort -t '.' -g | tail -n 1` fi if [ "X$kernel_vsn" = "X" ]; then - kernel_vsns=`ls -d $target_dir/lib/kernel-* | sed "s|$target_dir/lib/kernel-\([0-9\.].*\)|\1|g"` + kernel_vsns=`ls -d "$target_dir"/lib/kernel-* | sed "s|$target_dir/lib/kernel-\([0-9\.].*\)|\1|g"` kernel_vsn=`echo "$kernel_vsns" | sort -t '.' -g | tail -n 1` fi if [ "X$stdlib_vsn" = "X" ]; then - stdlib_vsns=`ls -d $target_dir/lib/stdlib-* | sed "s|$target_dir/lib/stdlib-\([0-9\.].*\)|\1|g"` + stdlib_vsns=`ls -d "$target_dir"/lib/stdlib-* | sed "s|$target_dir/lib/stdlib-\([0-9\.].*\)|\1|g"` stdlib_vsn=`echo "$stdlib_vsns" | sort -t '.' -g | tail -n 1` fi if [ "X$sasl_vsn" = "X" ]; then - sasl_vsns=`ls -d $target_dir/lib/sasl-* | sed "s|$target_dir/lib/sasl-\([0-9\.].*\)|\1|g"` + sasl_vsns=`ls -d "$target_dir"/lib/sasl-* | sed "s|$target_dir/lib/sasl-\([0-9\.].*\)|\1|g"` sasl_vsn=`echo "$sasl_vsns" | sort -t '.' -g | tail -n 1` fi # Generate .rel, .script and .boot - to tmp dir - start_clean="{release, {\"OTP APN 181 01\",\"$otp_major_vsn\"}, {erts, \"$erts_vsn\"},\n [{kernel,\"$kernel_vsn\"},\n {stdlib,\"$stdlib_vsn\"}]}.\n" - start_sasl="{release, {\"OTP APN 181 01\",\"$otp_major_vsn\"}, {erts, \"$erts_vsn\"},\n [{kernel,\"$kernel_vsn\"},\n {stdlib,\"$stdlib_vsn\"},\n {sasl,\"$sasl_vsn\"}]}.\n" + start_clean="{release, {\"Erlang/OTP\",\"$otp_major_vsn\"}, {erts, \"$erts_vsn\"},\n [{kernel,\"$kernel_vsn\"},\n {stdlib,\"$stdlib_vsn\"}]}.\n" + start_sasl="{release, {\"Erlang/OTP\",\"$otp_major_vsn\"}, {erts, \"$erts_vsn\"},\n [{kernel,\"$kernel_vsn\"},\n {stdlib,\"$stdlib_vsn\"},\n {sasl,\"$sasl_vsn\"}]}.\n" - tmp_dir=$target_dir/tmp; - if [ ! -d $tmp_dir ]; then - mkdir $tmp_dir + tmp_dir="$target_dir/tmp"; + if [ ! -d "$tmp_dir" ]; then + mkdir "$tmp_dir" fi - echo $start_sasl > $tmp_dir/start_sasl.rel - echo $start_clean > $tmp_dir/start_clean.rel - echo $start_clean > $tmp_dir/no_dot_erlang.rel + echo "$start_sasl" > "$tmp_dir/start_sasl.rel" + echo "$start_clean" > "$tmp_dir/start_clean.rel" + echo "$start_clean" > "$tmp_dir/no_dot_erlang.rel" - erlc=$ERL_TOP/bootstrap/bin/erlc - if [ ! -x $erlc ]; then + erlc="$ERL_TOP/bootstrap/bin/erlc" + if [ ! -x "$erlc" ]; then echo "erlc not found, can not create .script and .boot files" 1>&2 exit 1 fi - $erlc -I$target_dir/lib/*/ebin -o$tmp_dir $tmp_dir/start_sasl.rel || exit 1 - $erlc -I$target_dir/lib/*/ebin -o$tmp_dir +no_warn_sasl $tmp_dir/start_clean.rel || exit 1 - $erlc -I$target_dir/lib/*/ebin -o$tmp_dir +no_warn_sasl +no_dot_erlang $tmp_dir/no_dot_erlang.rel || exit 1 + $erlc -I"$target_dir"/lib/*/ebin -o$tmp_dir $tmp_dir/start_sasl.rel || exit 1 + $erlc -I"$target_dir"/lib/*/ebin -o$tmp_dir +no_warn_sasl $tmp_dir/start_clean.rel || exit 1 + $erlc -I"$target_dir"/lib/*/ebin -o$tmp_dir +no_warn_sasl +no_dot_erlang $tmp_dir/no_dot_erlang.rel || exit 1 # Generate RELEASES file - erl=$ERL_TOP/bootstrap/bin/erl - if [ ! -x $erl ]; then + erl="$ERL_TOP/bootstrap/bin/erl" + if [ ! -x "$erl" ]; then echo "erl not found, can not create RELEASES file" 1>&2 exit 1 fi - $erl -noinput +B -eval "release_handler:create_RELEASES(\"%ERL_ROOT%\", \"$tmp_dir\", \"$tmp_dir/start_sasl.rel\", []), halt()" || exit 1 + "$erl" -noinput +B -eval "release_handler:create_RELEASES(\"%ERL_ROOT%\", \"$tmp_dir\", \"$tmp_dir/start_sasl.rel\", []), halt()" || exit 1 # If all good so far, move generated files into target area - mv $tmp_dir/RELEASES $target_dir/releases/RELEASES.src - mv $tmp_dir/* $target_dir/releases/$otp_major_vsn - rmdir $tmp_dir + mv "$tmp_dir/RELEASES" "$target_dir/releases/RELEASES.src" + mv "$tmp_dir"/* "$target_dir/releases/$otp_major_vsn" + rmdir "$tmp_dir" # Remove old start scripts (forces a new run of Install) - rm -f $target_dir/releases/RELEASES - rm -f $target_dir/bin/*.script - rm -f $target_dir/bin/*.boot - rm -f $target_dir/bin/erl + rm -f "$target_dir"/releases/RELEASES + rm -f "$target_dir"/bin/*.script + rm -f "$target_dir"/bin/*.boot + rm -f "$target_dir"/bin/erl fi } diff --git a/otp_versions.table b/otp_versions.table new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/otp_versions.table diff --git a/system/doc/design_principles/spec_proc.xml b/system/doc/design_principles/spec_proc.xml index 8de7a5fe03..69bf2e0448 100644 --- a/system/doc/design_principles/spec_proc.xml +++ b/system/doc/design_principles/spec_proc.xml @@ -130,7 +130,8 @@ ok -export([alloc/0, free/1]). -export([init/1]). -export([system_continue/3, system_terminate/4, - write_debug/3]). + write_debug/3, + system_get_state/1, system_replace_state/2]). start_link() -> proc_lib:start_link(ch4, init, [self()]). @@ -156,15 +157,15 @@ init(Parent) -> loop(Chs, Parent, Deb) -> receive {From, alloc} -> - Deb2 = sys:handle_debug(Deb, {ch4, write_debug}, + Deb2 = sys:handle_debug(Deb, fun ch4:write_debug/3, ch4, {in, alloc, From}), {Ch, Chs2} = alloc(Chs), From ! {ch4, Ch}, - Deb3 = sys:handle_debug(Deb2, {ch4, write_debug}, + Deb3 = sys:handle_debug(Deb2, fun ch4:write_debug/3, ch4, {out, {ch4, Ch}, From}), loop(Chs2, Parent, Deb3); {free, Ch} -> - Deb2 = sys:handle_debug(Deb, {ch4, write_debug}, + Deb2 = sys:handle_debug(Deb, fun ch4:write_debug/3, ch4, {in, {free, Ch}}), Chs2 = free(Ch, Chs), loop(Chs2, Parent, Deb2); @@ -177,9 +178,16 @@ loop(Chs, Parent, Deb) -> system_continue(Parent, Deb, Chs) -> loop(Chs, Parent, Deb). -system_terminate(Reason, Parent, Deb, Chs) -> +system_terminate(Reason, _Parent, _Deb, _Chs) -> exit(Reason). +system_get_state(Chs) -> + {ok, Chs}. + +system_replace_state(StateFun, Chs) -> + NChs = StateFun(Chs), + {ok, NChs, NChs}. + write_debug(Dev, Event, Name) -> io:format(Dev, "~p event = ~p~n", [Name, Event]).</pre> <p>Example on how the simple debugging functions in <c>sys</c> can @@ -281,10 +289,10 @@ sys:handle_debug(Deb, Func, Info, Event) => Deb1</code> <p><c>Deb</c> is the debug structure.</p> </item> <item> - <p><c>Func</c> is a tuple <c>{Module, Name}</c> (or a fun) and - should specify a (user defined) function used to format + <p><c>Func</c> is a fun specifying + a (user defined) function used to format trace output. For each system event, the format function is - called as <c>Module:Name(Dev, Event, Info)</c>, where:</p> + called as <c>Func(Dev, Event, Info)</c>, where:</p> <list type="bulleted"> <item> <p><c>Dev</c> is the IO device to which the output should @@ -319,15 +327,15 @@ sys:handle_debug(Deb, Func, Info, Event) => Deb1</code> loop(Chs, Parent, Deb) -> receive {From, alloc} -> - Deb2 = sys:handle_debug(Deb, {ch4, write_debug}, + Deb2 = sys:handle_debug(Deb, fun ch4:write_debug/3, ch4, {in, alloc, From}), {Ch, Chs2} = alloc(Chs), From ! {ch4, Ch}, - Deb3 = sys:handle_debug(Deb2, {ch4, write_debug}, + Deb3 = sys:handle_debug(Deb2, fun ch4:write_debug/3, ch4, {out, {ch4, Ch}, From}), loop(Chs2, Parent, Deb3); {free, Ch} -> - Deb2 = sys:handle_debug(Deb, {ch4, write_debug}, + Deb2 = sys:handle_debug(Deb, fun ch4:write_debug/3, ch4, {in, {free, Ch}}), Chs2 = free(Ch, Chs), loop(Chs2, Parent, Deb2); @@ -366,8 +374,15 @@ Module:system_terminate(Reason, Parent, Deb, State)</code> <item><c>Module</c> is the name of the module.</item> <item><c>Deb</c> is the debug structure.</item> <item><c>State</c> is a term describing the internal state and - is passed to <c>system_continue</c>/<c>system_terminate</c>.</item> + is passed to <c>system_continue</c>/<c>system_terminate</c>/ + <c>system_get_state</c>/<c>system_replace_state</c>.</item> </list> + <p>If the process should return its state <c>handle_system_msg</c> will call:</p> + <code type="none"> +Module:system_get_state(State)</code> + <p>or if the process should replace its state using the fun <c>StateFun</c>:</p> + <code type="none"> +Module:system_replace_state(StateFun, State)</code> <p>In the example:</p> <code type="none"> loop(Chs, Parent, Deb) -> @@ -383,7 +398,15 @@ system_continue(Parent, Deb, Chs) -> loop(Chs, Parent, Deb). system_terminate(Reason, Parent, Deb, Chs) -> - exit(Reason).</code> + exit(Reason). + +system_get_state(Chs) -> + {ok, Chs, Chs}. + +system_replace_state(StateFun, Chs) -> + NChs = StateFun(Chs), + {ok, NChs, NChs}. +</code> <p>If the special process is set to trap exits, note that if the parent process terminates, the expected behavior is to terminate with the same reason:</p> diff --git a/system/doc/installation_guide/otp_version.xml b/system/doc/installation_guide/otp_version.xml deleted file mode 100644 index 0ce5144160..0000000000 --- a/system/doc/installation_guide/otp_version.xml +++ /dev/null @@ -1,96 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2014</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - </legalnotice> - - <title>OTP version</title> - <prepared></prepared> - <responsible></responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date>2014-02-19</date> - <rev></rev> - <file>otp_version.xml</file> - </header> - <p>As of OTP release 17, the OTP release number corresponds to - the major part of the OTP version. The OTP version as a concept was - introduced in OTP 17. In the normal case, the OTP version will be - constructed as - <c><Major>.<Minor>.<Patch></c> where <c><Major></c> - is the most significant part. However, more dot separated parts than - this may exist. The dot separated parts consists of integers. If all - parts less significant than <c><Minor></c> equals <c>0</c>, they - are omitted. The three normal parts - <c><Major>.<Minor>.<Patch></c> will be changed as - follows:</p> - <taglist> - <tag><c><Major></c></tag><item>Increased when major changes, - including incompatibilities, have been made.</item> - <tag><c><Minor></c></tag><item>Increased when new functionality - has been added.</item> - <tag><c><Patch></c></tag><item>Increased when pure bug fixes - have been made.</item> - </taglist> - <p>When a part in the version number is increased, all less significant - parts are set to <c>0</c>. Release candidates have an <c>-rc<N></c> - suffix. The suffix <c>-rc0</c> will be used during development up to - the first release candidate.</p> - - <p>OTP of a specific version is a set of applications of - specific versions. The application versions identified by - an OTP version corresponds to application versions that - have been tested together by the Erlang/OTP team at Ericsson AB. - An OTP system can however be put together with applications from - different OTP versions. Such a combination of application versions - has not been tested by the Erlang/OTP team. It is therefore - <em>always preferred to use OTP applications from one single OTP - version</em>.</p> - - <p>Application versions will be managed the same way as the OTP version. - Application versions part of a release candidate will however not have an - <c>-rc<N></c> suffix as the OTP version. Also note that a major - increment in an application version does not necessarily imply a major - increment of the OTP version. This depends on whether the - major change in the application is considered as a major change for - OTP as a whole or not.</p> - - <p>In an OTP source code tree as well as in an installed OTP - development system, the OTP version can be read from the text - file <c>OTP_VERSION</c> in the OTP installation root directory - (<seealso marker="kernel:code#root_dir/0"><c>code:root_dir()</c></seealso>).</p> - <p>If the version read from the <c>OTP_VERSION</c> file in a - development system has a <c>**</c> suffix, the system has been - patched using the <c>$ERL_TOP/otp_build patch_app</c> tool. In - this case, the system consists of application versions from - multiple OTP versions. The version preceding the <c>**</c> - suffix corresponds to the OTP version of the base system that - has been patched. Note that if a development system is updated by - other means than <c>$ERL_TOP/otp_build patch_app</c>, the - <c>OTP_VERSION</c> file may identify wrong OTP version.</p> - - <p>On a target system (see the - <seealso marker="doc/system_principles:create_target">system principles</seealso> - documentation) no <c>OTP_VERSION</c> file will exist. This since - one easily can create a target system where it is hard to even - determine the base OTP version.</p> -</chapter> - diff --git a/system/doc/installation_guide/part.xml b/system/doc/installation_guide/part.xml index 150df39512..19808fd165 100644 --- a/system/doc/installation_guide/part.xml +++ b/system/doc/installation_guide/part.xml @@ -31,7 +31,6 @@ <description> <p>How to install Erlang/OTP on UNIX or Windows.</p> </description> - <xi:include href="otp_version.xml"/> <xi:include href="install-binary.xml"/> <xi:include href="verification.xml"/> <xi:include href="INSTALL.xml"/> diff --git a/system/doc/installation_guide/xmlfiles.mk b/system/doc/installation_guide/xmlfiles.mk index 245491ab94..3995c607af 100644 --- a/system/doc/installation_guide/xmlfiles.mk +++ b/system/doc/installation_guide/xmlfiles.mk @@ -17,7 +17,6 @@ # %CopyrightEnd% # INST_GUIDE_CHAPTER_FILES = \ - otp_version.xml \ install-binary.xml \ verification.xml \ INSTALL.xml \ diff --git a/system/doc/reference_manual/character_set.xml b/system/doc/reference_manual/character_set.xml new file mode 100644 index 0000000000..884898eb34 --- /dev/null +++ b/system/doc/reference_manual/character_set.xml @@ -0,0 +1,132 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2014</year><year>2014</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Character Set and Source File Encoding</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>character_set.xml</file> + </header> + + <section> + <title>Character Set</title> + <p>In Erlang 4.8/OTP R5A the syntax of Erlang tokens was extended to + allow the use of the full ISO-8859-1 (Latin-1) character set. This + is noticeable in the following ways:</p> + <list type="bulleted"> + <item> + <p>All the Latin-1 printable characters can be used and are + shown without the escape backslash convention.</p> + </item> + <item> + <p>Atoms and variables can use all Latin-1 letters.</p> + </item> + </list> + <table> + <row> + <cell align="left" valign="middle"><em>Octal</em></cell> + <cell align="left" valign="middle"><em>Decimal</em></cell> + <cell align="left" valign="middle"> </cell> + <cell align="left" valign="middle"><em>Class</em></cell> + </row> + <row> + <cell align="left" valign="middle">200 - 237</cell> + <cell align="left" valign="middle">128 - 159</cell> + <cell align="left" valign="middle"> </cell> + <cell align="left" valign="middle">Control characters</cell> + </row> + <row> + <cell align="left" valign="middle">240 - 277</cell> + <cell align="left" valign="middle">160 - 191</cell> + <cell align="right" valign="middle">- ¿</cell> + <cell align="left" valign="middle">Punctuation characters</cell> + </row> + <row> + <cell align="left" valign="middle">300 - 326</cell> + <cell align="left" valign="middle">192 - 214</cell> + <cell align="center" valign="middle">À - Ö</cell> + <cell align="left" valign="middle">Uppercase letters</cell> + </row> + <row> + <cell align="center" valign="middle">327</cell> + <cell align="center" valign="middle">215</cell> + <cell align="center" valign="middle">×</cell> + <cell align="left" valign="middle">Punctuation character</cell> + </row> + <row> + <cell align="left" valign="middle">330 - 336</cell> + <cell align="left" valign="middle">216 - 222</cell> + <cell align="center" valign="middle">Ø - Þ</cell> + <cell align="left" valign="middle">Uppercase letters</cell> + </row> + <row> + <cell align="left" valign="middle">337 - 366</cell> + <cell align="left" valign="middle">223 - 246</cell> + <cell align="center" valign="middle">ß - ö</cell> + <cell align="left" valign="middle">Lowercase letters</cell> + </row> + <row> + <cell align="center" valign="middle">367</cell> + <cell align="center" valign="middle">247</cell> + <cell align="center" valign="middle">÷</cell> + <cell align="left" valign="middle">Punctuation character</cell> + </row> + <row> + <cell align="left" valign="middle">370 - 377</cell> + <cell align="left" valign="middle">248 - 255</cell> + <cell align="center" valign="middle">ø - ÿ</cell> + <cell align="left" valign="middle">Lowercase letters</cell> + </row> + <tcaption>Character Classes.</tcaption> + </table> + <p>In Erlang/OTP R16B the syntax of Erlang tokens was extended to + handle Unicode. To begin with the support is limited to + strings, but Erlang/OTP 18 is expected to handle Unicode atoms + as well. More about the usage of Unicode in Erlang source files + can be found in <seealso + marker="stdlib:unicode_usage#unicode_in_erlang">STDLIB's User's + Guide</seealso>.</p> + </section> + <section> + <title>Source File Encoding</title> + <p>The Erlang source file <marker + id="encoding">encoding</marker> is selected by a + comment in one of the first two lines of the source file. The + first string that matches the regular expression + <c>coding\s*[:=]\s*([-a-zA-Z0-9])+</c> selects the encoding. If + the matching string is not a valid encoding it is ignored. The + valid encodings are <c>Latin-1</c> and <c>UTF-8</c> where the + case of the characters can be chosen freely.</p> + <p>The following example selects UTF-8 as default encoding:</p> + <pre> +%% coding: utf-8</pre> + <p>Two more examples, both selecting Latin-1 as default encoding:</p> + <pre> +%% For this file we have chosen encoding = Latin-1</pre> + <pre> +%% -*- coding: latin-1 -*-</pre> + <p>The default encoding for Erlang source files was changed from + Latin-1 to UTF-8 in Erlang OTP 17.0.</p> + </section> +</chapter> diff --git a/system/doc/reference_manual/introduction.xml b/system/doc/reference_manual/introduction.xml index aa42967625..36bec17825 100644 --- a/system/doc/reference_manual/introduction.xml +++ b/system/doc/reference_manual/introduction.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2013</year> + <year>2003</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -79,88 +79,5 @@ when xor</p> </section> - <section> - <title>Character Set</title> - <p>In Erlang 4.8/OTP R5A the syntax of Erlang tokens was extended to - allow the use of the full ISO-8859-1 (Latin-1) character set. This - is noticeable in the following ways:</p> - <list type="bulleted"> - <item> - <p>All the Latin-1 printable characters can be used and are - shown without the escape backslash convention.</p> - </item> - <item> - <p>Atoms and variables can use all Latin-1 letters.</p> - </item> - </list> - <table> - <row> - <cell align="left" valign="middle"><em>Octal</em></cell> - <cell align="left" valign="middle"><em>Decimal</em></cell> - <cell align="left" valign="middle"> </cell> - <cell align="left" valign="middle"><em>Class</em></cell> - </row> - <row> - <cell align="left" valign="middle">200 - 237</cell> - <cell align="left" valign="middle">128 - 159</cell> - <cell align="left" valign="middle"> </cell> - <cell align="left" valign="middle">Control characters</cell> - </row> - <row> - <cell align="left" valign="middle">240 - 277</cell> - <cell align="left" valign="middle">160 - 191</cell> - <cell align="right" valign="middle">- ¿</cell> - <cell align="left" valign="middle">Punctuation characters</cell> - </row> - <row> - <cell align="left" valign="middle">300 - 326</cell> - <cell align="left" valign="middle">192 - 214</cell> - <cell align="center" valign="middle">À - Ö</cell> - <cell align="left" valign="middle">Uppercase letters</cell> - </row> - <row> - <cell align="center" valign="middle">327</cell> - <cell align="center" valign="middle">215</cell> - <cell align="center" valign="middle">×</cell> - <cell align="left" valign="middle">Punctuation character</cell> - </row> - <row> - <cell align="left" valign="middle">330 - 336</cell> - <cell align="left" valign="middle">216 - 222</cell> - <cell align="center" valign="middle">Ø - Þ</cell> - <cell align="left" valign="middle">Uppercase letters</cell> - </row> - <row> - <cell align="left" valign="middle">337 - 366</cell> - <cell align="left" valign="middle">223 - 246</cell> - <cell align="center" valign="middle">ß - ö</cell> - <cell align="left" valign="middle">Lowercase letters</cell> - </row> - <row> - <cell align="center" valign="middle">367</cell> - <cell align="center" valign="middle">247</cell> - <cell align="center" valign="middle">÷</cell> - <cell align="left" valign="middle">Punctuation character</cell> - </row> - <row> - <cell align="left" valign="middle">370 - 377</cell> - <cell align="left" valign="middle">248 - 255</cell> - <cell align="center" valign="middle">ø - ÿ</cell> - <cell align="left" valign="middle">Lowercase letters</cell> - </row> - <tcaption>Character Classes.</tcaption> - </table> - <p>In Erlang/OTP R16 the syntax of Erlang tokens was extended to - handle Unicode. To begin with the support is limited to strings, - but Erlang/OTP R18 is expected to handle Unicode atoms as well. - More about the usage of Unicode in Erlang source files can be - found in <seealso - marker="stdlib:unicode_usage#unicode_in_erlang">STDLIB's User'S - Guide</seealso>. The default encoding for Erlang source files - is still Latin-1, but in Erlang/OTP R17 the default encoding - will be UTF-8. The details on how to state the encoding of an - Erlang source file can be found in <seealso - marker="stdlib:epp#encoding">epp(3)</seealso>.</p> - </section> </chapter> diff --git a/system/doc/reference_manual/part.xml b/system/doc/reference_manual/part.xml index b4f114c268..ee8f3dd7eb 100644 --- a/system/doc/reference_manual/part.xml +++ b/system/doc/reference_manual/part.xml @@ -4,7 +4,7 @@ <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2013</year> + <year>2003</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -28,6 +28,7 @@ <rev></rev> </header> <xi:include href="introduction.xml"/> + <xi:include href="character_set.xml"/> <xi:include href="data_types.xml"/> <xi:include href="patterns.xml"/> <xi:include href="modules.xml"/> diff --git a/system/doc/system_principles/part.xml b/system/doc/system_principles/part.xml index 915d5aca9b..811428baae 100644 --- a/system/doc/system_principles/part.xml +++ b/system/doc/system_principles/part.xml @@ -31,5 +31,6 @@ <xi:include href="system_principles.xml"/> <xi:include href="error_logging.xml"/> <xi:include href="create_target.xml"/> + <xi:include href="versions.xml"/> </part> diff --git a/system/doc/system_principles/versions.xml b/system/doc/system_principles/versions.xml new file mode 100644 index 0000000000..2bf0d18010 --- /dev/null +++ b/system/doc/system_principles/versions.xml @@ -0,0 +1,268 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2014</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Versions</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date>2014-02-19</date> + <rev></rev> + <file>versions.xml</file> + </header> + <section><title>OTP Version</title> + <p>As of OTP release 17, the OTP release number corresponds to + the major part of the OTP version. The OTP version as a concept was + introduced in OTP 17. The <seealso marker="#version_scheme">version + scheme</seealso> used is described in more detail below.</p> + + <p>OTP of a specific version is a set of applications of specific + versions. The application versions identified by an OTP version + corresponds to application versions that have been tested together + by the Erlang/OTP team at Ericsson AB. An OTP system can however be + put together with applications from different OTP versions. Such a + combination of application versions has not been tested by the + Erlang/OTP team. It is therefore <em>always preferred to use OTP + applications from one single OTP version</em>.</p> + + <p>Release candidates have an <c>-rc<N></c> + suffix. The suffix <c>-rc0</c> will be used during development up to + the first release candidate.</p> + + <section><title>Retrieving Current OTP Version</title> + <p>In an OTP source code tree, the OTP version can be read from + the text file <c><OTP source root>/OTP_VERSION</c>. The + absolute path to the file can be constructed by calling + <c>filename:join([<seealso marker="kernel:code#root_dir/0">code:root_dir()</seealso>, "OTP_VERSION"])</c>.</p> + <p>In an installed OTP development system, the OTP version can be read + from the text file <c><OTP installation root>/releases/<OTP release number>/OTP_VERSION</c>. + The absolute path to the file can by constructed by calling + <c>filename:join([<seealso marker="kernel:code#root_dir/0">code:root_dir()</seealso>, "releases", <seealso marker="erts:erlang#system_info_otp_release">erlang:system_info(otp_release)</seealso>, "OTP_VERSION"]).</c></p> + <p>If the version read from the <c>OTP_VERSION</c> file in a + development system has a <c>**</c> suffix, the system has been + patched using the + <seealso marker="doc/installation_guide:PATCH-APP"><c>$ERL_TOP/otp_build patch_app</c></seealso> + tool. In this case, the system consists of application versions from + multiple OTP versions. The version preceding the <c>**</c> + suffix corresponds to the OTP version of the base system that + has been patched. Note that if a development system is updated by + other means than <c>$ERL_TOP/otp_build patch_app</c>, the + <c>OTP_VERSION</c> file may identify wrong OTP version.</p> + + <p>No <c>OTP_VERSION</c> file will be placed in a + <seealso marker="create_target">target system</seealso> created + by OTP tools. This since one easily can create a target system + where it is hard to even determine the base OTP version. You may, + however, place such a file there yourself if you know the OTP + version.</p> + </section> + + <section><title>OTP Versions Table</title> + <p>The text file <c><OTP source root>/otp_versions.table</c> that + is part of the source code contains information about all OTP versions + from OTP 17.0 up to current OTP version. Each line contains information + about application versions that are part of a specific OTP version, and + is on the format:</p> +<pre> +<OtpVersion> : <ChangedAppVersions> # <UnchangedAppVersions> : +</pre> + <p><c><OtpVersion></c> is on the format <c>OTP-<VSN></c>, i.e., + the same as the git tag used to identify the source. + <c><ChangedAppVersions></c> and <c><UnchangedAppVersions></c> + are space separated lists of application versions on the + format <c><application>-<vsn></c>. + <c><ChangedAppVersions></c> corresponds to changed applications + with new version numbers in this OTP version, and + <c><UnchangedAppVersions></c> corresponds to unchanged application + versions in this OTP version. Both of them might be empty, although + not at the same time. If <ChangedAppVersions> is empty, no changes + has been made that change the build result of any application. This could + for example be a pure bug fix of the build system. The order of lines + is undefined. All white space characters in this file are either space + (character 32) or line-break (character 10).</p> + <p>Using ordinary UNIX tools like <c>sed</c> and <c>grep</c> one + can easily find answers to various questions like:</p> + <taglist> + <tag>Which OTP versions are <c>kernel-3.0</c> part of?</tag> + <item><p><c> $ grep ' kernel-3\.0 ' otp_versions.table</c></p></item> + <tag>In which OTP version was <c>kernel-3.0</c> introduced?</tag> + <item><p><c> $ sed 's/#.*//;/ kernel-3\.0 /!d' otp_versions.table</c></p></item> + </taglist> + <p>The above commands give a bit more information than the exact answers, + but adequate information when manually searching for answers to these + questions.</p> + <warning><p>The format of the <c>otp_versions.table</c> might be subject + to changes during the OTP 17 release.</p></warning> + </section> + </section> + + <section><title>Application Version</title> + <p>As of OTP 17.0 application versions will use the same + <seealso marker="#version_scheme">version scheme</seealso> as the + OTP version. Application versions part of a release candidate will + however not have an <c>-rc<N></c> suffix as the OTP version. + Also note that a major increment in an application version does not + necessarily imply a major increment of the OTP version. This depends + on whether the major change in the application is considered as a + major change for OTP as a whole or not.</p> + </section> + + <marker id="version_scheme"/> + <section><title>Version Scheme</title> + <note>Note that the version scheme was changed as of OTP 17.0. This implies + that application versions used prior to OTP 17.0 do not adhere to this + version scheme. <seealso marker="#otp_17_0_app_versions">A list of + application versions used in OTP 17.0</seealso> can be found + at the end of this document.</note> + + <p>In the normal case, a version will be constructed as + <c><Major>.<Minor>.<Patch></c> where <c><Major></c> + is the most significant part. However, more dot separated parts than + this may exist. The dot separated parts consists of non-negative integers. + If all parts less significant than <c><Minor></c> equals <c>0</c>, + they are omitted. The three normal parts + <c><Major>.<Minor>.<Patch></c> will be changed as + follows:</p> + <taglist> + <tag><c><Major></c></tag><item>Increased when major changes, + including incompatibilities, have been made.</item> + <tag><c><Minor></c></tag><item>Increased when new functionality + has been added.</item> + <tag><c><Patch></c></tag><item>Increased when pure bug fixes + have been made.</item> + </taglist> + <p>When a part in the version number is increased, all less significant + parts are set to <c>0</c>.</p> + + <p>An application version or an OTP version identifies source code + versions. That is, it does not imply anything about how the application + or OTP has been built.</p> + + <section><title>Order of Versions</title> + <p>Version numbers in general are only partially ordered. However, + normal version numbers (with three parts) as of OTP 17.0 have a total + or linear order. This applies both to normal OTP versions and + normal application versions.</p> + + <p>When comparing two version numbers that have an order, one + compare each part as ordinary integers from the most + significant part towards less significant parts. The order is + defined by the first parts of the same significance that + differ. An OTP version with a larger version include all + changes that that are part of a smaller OTP version. The same + goes for application versions.</p> + + <p>In the general case, versions may have more than three parts. In + this case the versions are only partially ordered. Note that such + versions are only used in exceptional cases. When an extra + part (out of the normal three parts) is added to a version number, + a new branch of versions is made. The new branch has a linear + order against the base version. However, versions on different + branches have no order. Since they have no order, we + only know that they all include what is included in their + closest common ancestor. When branching multiple times from the + same base version, <c>0</c> parts are added between the base + version and the least significant <c>1</c> part until a unique + version is found. Versions that have an order can be compared + as described in the paragraph above.</p> + + <p>An example of branched versions: The version <c>6.0.2.1</c> + is a branched version from the base version <c>6.0.2</c>. + Versions on the form <c>6.0.2.<X></c> can be compared + with normal versions smaller than or equal to <c>6.0.2</c>, + and other versions on the form <c>6.0.2.<X></c>. The + version <c>6.0.2.1</c> will include all changes in + <c>6.0.2</c>. However, <c>6.0.3</c> will most likely + <em>not</em> include all changes in <c>6.0.2.1</c> (note that + these versions have no order). A second branched version from the base + version <c>6.0.2</c> will be version <c>6.0.2.0.1</c>, and a + third branched version will be <c>6.0.2.0.0.1</c>.</p> + </section> + </section> + + <marker id="otp_17_0_app_versions"/> + <section><title>OTP 17.0 Application Versions</title> + <p>The following application versions were part of OTP 17.0. If + the normal part of an applications version number compares + as smaller than the corresponding application version in this list, + the version number does not adhere to the version scheme introduced + in OTP 17.0 and should be considered as not having an order against + versions used as of OTP 17.0.</p> + <list> + <item><c>asn1-3.0</c></item> + <item><c>common_test-1.8</c></item> + <item><c>compiler-5.0</c></item> + <item><c>cosEvent-2.1.15</c></item> + <item><c>cosEventDomain-1.1.14</c></item> + <item><c>cosFileTransfer-1.1.16</c></item> + <item><c>cosNotification-1.1.21</c></item> + <item><c>cosProperty-1.1.17</c></item> + <item><c>cosTime-1.1.14</c></item> + <item><c>cosTransactions-1.2.14</c></item> + <item><c>crypto-3.3</c></item> + <item><c>debugger-4.0</c></item> + <item><c>dialyzer-2.7</c></item> + <item><c>diameter-1.6</c></item> + <item><c>edoc-0.7.13</c></item> + <item><c>eldap-1.0.3</c></item> + <item><c>erl_docgen-0.3.5</c></item> + <item><c>erl_interface-3.7.16</c></item> + <item><c>erts-6.0</c></item> + <item><c>et-1.5</c></item> + <item><c>eunit-2.2.7</c></item> + <item><c>gs-1.5.16</c></item> + <item><c>hipe-3.10.3</c></item> + <item><c>ic-4.3.5</c></item> + <item><c>inets-5.10</c></item> + <item><c>jinterface-1.5.9</c></item> + <item><c>kernel-3.0</c></item> + <item><c>megaco-3.17.1</c></item> + <item><c>mnesia-4.12</c></item> + <item><c>observer-2.0</c></item> + <item><c>odbc-2.10.20</c></item> + <item><c>orber-3.6.27</c></item> + <item><c>os_mon-2.2.15</c></item> + <item><c>ose-1.0</c></item> + <item><c>otp_mibs-1.0.9</c></item> + <item><c>parsetools-2.0.11</c></item> + <item><c>percept-0.8.9</c></item> + <item><c>public_key-0.22</c></item> + <item><c>reltool-0.6.5</c></item> + <item><c>runtime_tools-1.8.14</c></item> + <item><c>sasl-2.4</c></item> + <item><c>snmp-4.25.1</c></item> + <item><c>ssh-3.0.1</c></item> + <item><c>ssl-5.3.4</c></item> + <item><c>stdlib-2.0</c></item> + <item><c>syntax_tools-1.6.14</c></item> + <item><c>test_server-3.7</c></item> + <item><c>tools-2.6.14</c></item> + <item><c>typer-0.9.6</c></item> + <item><c>webtool-0.8.10</c></item> + <item><c>wx-1.2</c></item> + <item><c>xmerl-1.3.7</c></item> + </list> + </section> +</chapter> + diff --git a/system/doc/system_principles/xmlfiles.mk b/system/doc/system_principles/xmlfiles.mk index 4cbc00ed52..9743949798 100644 --- a/system/doc/system_principles/xmlfiles.mk +++ b/system/doc/system_principles/xmlfiles.mk @@ -19,4 +19,5 @@ SYSTEM_PRINCIPLES_CHAPTER_FILES = \ system_principles.xml \ error_logging.xml \ - create_target.xml + create_target.xml \ + versions.xml diff --git a/xcomp/erl-xcomp-arm-android.conf b/xcomp/erl-xcomp-arm-android.conf new file mode 100644 index 0000000000..4fd1f270d1 --- /dev/null +++ b/xcomp/erl-xcomp-arm-android.conf @@ -0,0 +1,264 @@ +## -*-shell-script-*- +## +## %CopyrightBegin% +## +## Copyright Ericsson AB 2009-2010. All Rights Reserved. +## +## The contents of this file are subject to the Erlang Public License, +## Version 1.1, (the "License"); you may not use this file except in +## compliance with the License. You should have received a copy of the +## Erlang Public License along with this software. If not, it can be +## retrieved online at http://www.erlang.org/. +## +## Software distributed under the License is distributed on an "AS IS" +## basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +## the License for the specific language governing rights and limitations +## under the License. +## +## %CopyrightEnd% +## +## File: erl-xcomp.conf.template +## Author: +## +## ----------------------------------------------------------------------------- +## When cross compiling Erlang/OTP using `otp_build', copy this file and set +## the variables needed below. Then pass the path to the copy of this file as +## an argument to `otp_build' in the configure stage: +## `otp_build configure --xcomp-conf=<FILE>' +## ----------------------------------------------------------------------------- + +## Note that you cannot define arbitrary variables in a cross compilation +## configuration file. Only the ones listed below will be guaranteed to be +## visible throughout the whole execution of all `configure' scripts. Other +## variables needs to be defined as arguments to `configure' or exported in +## the environment. + +## -- Variables for `otp_build' Only ------------------------------------------- + +## Variables in this section are only used, when configuring Erlang/OTP for +## cross compilation using `$ERL_TOP/otp_build configure'. + +## *NOTE*! These variables currently have *no* effect if you configure using +## the `configure' script directly. + +# * `erl_xcomp_build' - The build system used. This value will be passed as +# `--build=$erl_xcomp_build' argument to the `configure' script. It does +# not have to be a full `CPU-VENDOR-OS' triplet, but can be. The full +# `CPU-VENDOR-OS' triplet will be created by +# `$ERL_TOP/erts/autoconf/config.sub $erl_xcomp_build'. If set to `guess', +# the build system will be guessed using +# `$ERL_TOP/erts/autoconf/config.guess'. +erl_xcomp_build=guess + +# * `erl_xcomp_host' - Cross host/target system to build for. This value will +# be passed as `--host=$erl_xcomp_host' argument to the `configure' script. +# It does not have to be a full `CPU-VENDOR-OS' triplet, but can be. The +# full `CPU-VENDOR-OS' triplet will be created by +# `$ERL_TOP/erts/autoconf/config.sub $erl_xcomp_host'. +erl_xcomp_host=arm-linux-androideabi + +# * `erl_xcomp_configure_flags' - Extra configure flags to pass to the +# `configure' script. +erl_xcomp_configure_flags="--disable-hipe --without-termcap" + + +## -- Cross Compiler and Other Tools ------------------------------------------- + +## +## +NDK_SYSROOT=$NDK_ROOT/platforms/$NDK_PLAT/arch-arm + +## If the cross compilation tools are prefixed by `<HOST>-' you probably do +## not need to set these variables (where `<HOST>' is what has been passed as +## `--host=<HOST>' argument to `configure'). + +## All variables in this section can also be used when native compiling. + +# * `CC' - C compiler. +CC="arm-linux-androideabi-gcc --sysroot=$NDK_SYSROOT" + +# * `CFLAGS' - C compiler flags. +CFLAGS="-static -march=armv7-a -msoft-float -mthumb" + +# * `STATIC_CFLAGS' - Static C compiler flags. +#STATIC_CFLAGS= + +# * `CFLAG_RUNTIME_LIBRARY_PATH' - This flag should set runtime library +# search path for the shared libraries. Note that this actually is a +# linker flag, but it needs to be passed via the compiler. +#CFLAG_RUNTIME_LIBRARY_PATH= + +# * `CPP' - C pre-processor. +CPP="arm-linux-androideabi-cpp --sysroot=$NDK_SYSROOT" + +# * `CPPFLAGS' - C pre-processor flags. +CPPFLAGS="-static -march=armv7-a -msoft-float -mthumb" + +# * `CXX' - C++ compiler. +CXX="arm-linux-androideabi-c++ --sysroot=$NDK_SYSROOT" + +# * `CXXFLAGS' - C++ compiler flags. +CXXFLAGS="-static -march=armv7-a -msoft-float -mthumb" + +# * `LD' - Linker. +#LD= + +# * `LDFLAGS' - Linker flags. +LDFLAGS="-static -march=armv7-a -msoft-float -mthumb" + +# * `LIBS' - Libraries. +#LIBS= + +## -- *D*ynamic *E*rlang *D*river Linking -- + +## *NOTE*! Either set all or none of the `DED_LD*' variables. + +# * `DED_LD' - Linker for Dynamically loaded Erlang Drivers. +#DED_LD= + +# * `DED_LDFLAGS' - Linker flags to use with `DED_LD'. +#DED_LDFLAGS= + +# * `DED_LD_FLAG_RUNTIME_LIBRARY_PATH' - This flag should set runtime library +# search path for shared libraries when linking with `DED_LD'. +#DED_LD_FLAG_RUNTIME_LIBRARY_PATH= + +## -- Large File Support -- + +## *NOTE*! Either set all or none of the `LFS_*' variables. + +# * `LFS_CFLAGS' - Large file support C compiler flags. +#LFS_CFLAGS= + +# * `LFS_LDFLAGS' - Large file support linker flags. +#LFS_LDFLAGS= + +# * `LFS_LIBS' - Large file support libraries. +#LFS_LIBS= + +## -- Other Tools -- + +# * `RANLIB' - `ranlib' archive index tool. +#RANLIB= + +# * `AR' - `ar' archiving tool. +#AR= + +# * `GETCONF' - `getconf' system configuration inspection tool. `getconf' is +# currently used for finding out large file support flags to use, and +# on Linux systems for finding out if we have an NPTL thread library or +# not. +#GETCONF= + +## -- Cross System Root Locations ---------------------------------------------- + +# * `erl_xcomp_sysroot' - The absolute path to the system root of the cross +# compilation environment. Currently, the `crypto', `odbc', `ssh' and +# `ssl' applications need the system root. These applications will be +# skipped if the system root has not been set. The system root might be +# needed for other things too. If this is the case and the system root +# has not been set, `configure' will fail and request you to set it. +erl_xcomp_sysroot="$NDK_SYSROOT" + + +# * `erl_xcomp_isysroot' - The absolute path to the system root for includes +# of the cross compilation environment. If not set, this value defaults +# to `$erl_xcomp_sysroot', i.e., only set this value if the include system +# root path is not the same as the system root path. +#erl_xcomp_isysroot= + +## -- Optional Feature, and Bug Tests ------------------------------------------ + +## These tests cannot (always) be done automatically when cross compiling. You +## usually do not need to set these variables. Only set these if you really +## know what you are doing. + +## Note that some of these values will override results of tests performed +## by `configure', and some will not be used until `configure' is sure that +## it cannot figure the result out. + +## The `configure' script will issue a warning when a default value is used. +## When a variable has been set, no warning will be issued. + +# * `erl_xcomp_after_morecore_hook' - `yes|no'. Defaults to `no'. If `yes', +# the target system must have a working `__after_morecore_hook' that can be +# used for tracking used `malloc()' implementations core memory usage. +# This is currently only used by unsupported features. +#erl_xcomp_after_morecore_hook= + +# * `erl_xcomp_bigendian' - `yes|no'. No default. If `yes', the target system +# must be big endian. If `no', little endian. This can often be +# automatically detected, but not always. If not automatically detected, +# `configure' will fail unless this variable is set. Since no default +# value is used, `configure' will try to figure this out automatically. +#erl_xcomp_bigendian= + +# * `erl_xcomp_clock_gettime_cpu_time' - `yes|no'. Defaults to `no'. If `yes', +# the target system must have a working `clock_gettime()' implementation +# that can be used for retrieving process CPU time. +#erl_xcomp_clock_gettime_cpu_time= + +# * `erl_xcomp_getaddrinfo' - `yes|no'. Defaults to `no'. If `yes', the target +# system must have a working `getaddrinfo()' implementation that can +# handle both IPv4 and IPv6. +#erl_xcomp_getaddrinfo= + +# * `erl_xcomp_gethrvtime_procfs_ioctl' - `yes|no'. Defaults to `no'. If `yes', +# the target system must have a working `gethrvtime()' implementation and +# is used with procfs `ioctl()'. +#erl_xcomp_gethrvtime_procfs_ioctl= + +# * `erl_xcomp_dlsym_brk_wrappers' - `yes|no'. Defaults to `no'. If `yes', the +# target system must have a working `dlsym(RTLD_NEXT, <S>)' implementation +# that can be used on `brk' and `sbrk' symbols used by the `malloc()' +# implementation in use, and by this track the `malloc()' implementations +# core memory usage. This is currently only used by unsupported features. +#erl_xcomp_dlsym_brk_wrappers= + +# * `erl_xcomp_kqueue' - `yes|no'. Defaults to `no'. If `yes', the target +# system must have a working `kqueue()' implementation that returns a file +# descriptor which can be used by `poll()' and/or `select()'. If `no' and +# the target system has not got `epoll()' or `/dev/poll', the kernel-poll +# feature will be disabled. +#erl_xcomp_kqueue= + +# * `erl_xcomp_linux_clock_gettime_correction' - `yes|no'. Defaults to `yes' on +# Linux; otherwise, `no'. If `yes', `clock_gettime(CLOCK_MONOTONIC, _)' on +# the target system must work. This variable is recommended to be set to +# `no' on Linux systems with kernel versions less than 2.6. +#erl_xcomp_linux_clock_gettime_correction= + +# * `erl_xcomp_linux_nptl' - `yes|no'. Defaults to `yes' on Linux; otherwise, +# `no'. If `yes', the target system must have NPTL (Native POSIX Thread +# Library). Older Linux systems have LinuxThreads instead of NPTL (Linux +# kernel versions typically less than 2.6). +#erl_xcomp_linux_nptl= + +# * `erl_xcomp_linux_usable_sigaltstack' - `yes|no'. Defaults to `yes' on Linux; +# otherwise, `no'. If `yes', `sigaltstack()' must be usable on the target +# system. `sigaltstack()' on Linux kernel versions less than 2.4 are +# broken. +#erl_xcomp_linux_usable_sigaltstack= + +# * `erl_xcomp_linux_usable_sigusrx' - `yes|no'. Defaults to `yes'. If `yes', +# the `SIGUSR1' and `SIGUSR2' signals must be usable by the ERTS. Old +# LinuxThreads thread libraries (Linux kernel versions typically less than +# 2.2) used these signals and made them unusable by the ERTS. +#erl_xcomp_linux_usable_sigusrx= + +# * `erl_xcomp_poll' - `yes|no'. Defaults to `no' on Darwin/MacOSX; otherwise, +# `yes'. If `yes', the target system must have a working `poll()' +# implementation that also can handle devices. If `no', `select()' will be +# used instead of `poll()'. +#erl_xcomp_poll= + +# * `erl_xcomp_putenv_copy' - `yes|no'. Defaults to `no'. If `yes', the target +# system must have a `putenv()' implementation that stores a copy of the +# key/value pair. +#erl_xcomp_putenv_copy= + +# * `erl_xcomp_reliable_fpe' - `yes|no'. Defaults to `no'. If `yes', the target +# system must have reliable floating point exceptions. +#erl_xcomp_reliable_fpe= + +## ----------------------------------------------------------------------------- |