diff options
481 files changed, 8552 insertions, 5094 deletions
diff --git a/bootstrap/bin/no_dot_erlang.boot b/bootstrap/bin/no_dot_erlang.boot Binary files differindex c8ea423483..a94d524db7 100644 --- a/bootstrap/bin/no_dot_erlang.boot +++ b/bootstrap/bin/no_dot_erlang.boot diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex c8ea423483..a94d524db7 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex c8ea423483..a94d524db7 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_a.beam b/bootstrap/lib/compiler/ebin/beam_a.beam Binary files differindex 57affa6b30..72f044da26 100644 --- a/bootstrap/lib/compiler/ebin/beam_a.beam +++ b/bootstrap/lib/compiler/ebin/beam_a.beam diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 5e94a4b715..cca4e97485 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_block.beam b/bootstrap/lib/compiler/ebin/beam_block.beam Binary files differindex 082458338e..18f5dc6967 100644 --- a/bootstrap/lib/compiler/ebin/beam_block.beam +++ b/bootstrap/lib/compiler/ebin/beam_block.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bs.beam b/bootstrap/lib/compiler/ebin/beam_bs.beam Binary files differindex b624c01c96..1361b0b08d 100644 --- a/bootstrap/lib/compiler/ebin/beam_bs.beam +++ b/bootstrap/lib/compiler/ebin/beam_bs.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bsm.beam b/bootstrap/lib/compiler/ebin/beam_bsm.beam Binary files differindex 9a3a7623af..6bf3b576b2 100644 --- a/bootstrap/lib/compiler/ebin/beam_bsm.beam +++ b/bootstrap/lib/compiler/ebin/beam_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_clean.beam b/bootstrap/lib/compiler/ebin/beam_clean.beam Binary files differindex 959137f06f..ab18a32fa8 100644 --- a/bootstrap/lib/compiler/ebin/beam_clean.beam +++ b/bootstrap/lib/compiler/ebin/beam_clean.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dead.beam b/bootstrap/lib/compiler/ebin/beam_dead.beam Binary files differindex 7294026d65..33ee4dea87 100644 --- a/bootstrap/lib/compiler/ebin/beam_dead.beam +++ b/bootstrap/lib/compiler/ebin/beam_dead.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dict.beam b/bootstrap/lib/compiler/ebin/beam_dict.beam Binary files differindex 9c66f6d9f3..2e9c7d0f00 100644 --- a/bootstrap/lib/compiler/ebin/beam_dict.beam +++ b/bootstrap/lib/compiler/ebin/beam_dict.beam diff --git a/bootstrap/lib/compiler/ebin/beam_disasm.beam b/bootstrap/lib/compiler/ebin/beam_disasm.beam Binary files differindex 256efdb750..5983f341f5 100644 --- a/bootstrap/lib/compiler/ebin/beam_disasm.beam +++ b/bootstrap/lib/compiler/ebin/beam_disasm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_except.beam b/bootstrap/lib/compiler/ebin/beam_except.beam Binary files differindex 19b30319ab..6eed82587a 100644 --- a/bootstrap/lib/compiler/ebin/beam_except.beam +++ b/bootstrap/lib/compiler/ebin/beam_except.beam diff --git a/bootstrap/lib/compiler/ebin/beam_flatten.beam b/bootstrap/lib/compiler/ebin/beam_flatten.beam Binary files differindex 7fcbf5dcde..8758e98a71 100644 --- a/bootstrap/lib/compiler/ebin/beam_flatten.beam +++ b/bootstrap/lib/compiler/ebin/beam_flatten.beam diff --git a/bootstrap/lib/compiler/ebin/beam_jump.beam b/bootstrap/lib/compiler/ebin/beam_jump.beam Binary files differindex e51f1b179b..5551ba9f90 100644 --- a/bootstrap/lib/compiler/ebin/beam_jump.beam +++ b/bootstrap/lib/compiler/ebin/beam_jump.beam diff --git a/bootstrap/lib/compiler/ebin/beam_opcodes.beam b/bootstrap/lib/compiler/ebin/beam_opcodes.beam Binary files differindex 04305d7896..98b2c19b48 100644 --- a/bootstrap/lib/compiler/ebin/beam_opcodes.beam +++ b/bootstrap/lib/compiler/ebin/beam_opcodes.beam diff --git a/bootstrap/lib/compiler/ebin/beam_peep.beam b/bootstrap/lib/compiler/ebin/beam_peep.beam Binary files differindex 58c0b1c623..d2b10e030b 100644 --- a/bootstrap/lib/compiler/ebin/beam_peep.beam +++ b/bootstrap/lib/compiler/ebin/beam_peep.beam diff --git a/bootstrap/lib/compiler/ebin/beam_receive.beam b/bootstrap/lib/compiler/ebin/beam_receive.beam Binary files differindex c662b603a4..8d16d7d39d 100644 --- a/bootstrap/lib/compiler/ebin/beam_receive.beam +++ b/bootstrap/lib/compiler/ebin/beam_receive.beam diff --git a/bootstrap/lib/compiler/ebin/beam_record.beam b/bootstrap/lib/compiler/ebin/beam_record.beam Binary files differindex 2901d6c33f..55cac5310d 100644 --- a/bootstrap/lib/compiler/ebin/beam_record.beam +++ b/bootstrap/lib/compiler/ebin/beam_record.beam diff --git a/bootstrap/lib/compiler/ebin/beam_reorder.beam b/bootstrap/lib/compiler/ebin/beam_reorder.beam Binary files differindex d3d0b89a35..05562e19ea 100644 --- a/bootstrap/lib/compiler/ebin/beam_reorder.beam +++ b/bootstrap/lib/compiler/ebin/beam_reorder.beam diff --git a/bootstrap/lib/compiler/ebin/beam_split.beam b/bootstrap/lib/compiler/ebin/beam_split.beam Binary files differindex 88b3f96d8c..8165aab002 100644 --- a/bootstrap/lib/compiler/ebin/beam_split.beam +++ b/bootstrap/lib/compiler/ebin/beam_split.beam diff --git a/bootstrap/lib/compiler/ebin/beam_trim.beam b/bootstrap/lib/compiler/ebin/beam_trim.beam Binary files differindex 9143b1615d..75fcff822e 100644 --- a/bootstrap/lib/compiler/ebin/beam_trim.beam +++ b/bootstrap/lib/compiler/ebin/beam_trim.beam diff --git a/bootstrap/lib/compiler/ebin/beam_type.beam b/bootstrap/lib/compiler/ebin/beam_type.beam Binary files differindex d3ca9d07de..0c8341a6c9 100644 --- a/bootstrap/lib/compiler/ebin/beam_type.beam +++ b/bootstrap/lib/compiler/ebin/beam_type.beam diff --git a/bootstrap/lib/compiler/ebin/beam_utils.beam b/bootstrap/lib/compiler/ebin/beam_utils.beam Binary files differindex 24aaf40e10..1e9c30d00a 100644 --- a/bootstrap/lib/compiler/ebin/beam_utils.beam +++ b/bootstrap/lib/compiler/ebin/beam_utils.beam diff --git a/bootstrap/lib/compiler/ebin/beam_validator.beam b/bootstrap/lib/compiler/ebin/beam_validator.beam Binary files differindex e87c1ed4ae..25f8772409 100644 --- a/bootstrap/lib/compiler/ebin/beam_validator.beam +++ b/bootstrap/lib/compiler/ebin/beam_validator.beam diff --git a/bootstrap/lib/compiler/ebin/beam_z.beam b/bootstrap/lib/compiler/ebin/beam_z.beam Binary files differindex 03685931e3..57698dbb3b 100644 --- a/bootstrap/lib/compiler/ebin/beam_z.beam +++ b/bootstrap/lib/compiler/ebin/beam_z.beam diff --git a/bootstrap/lib/compiler/ebin/cerl.beam b/bootstrap/lib/compiler/ebin/cerl.beam Binary files differindex c49aac27a0..61515da20a 100644 --- a/bootstrap/lib/compiler/ebin/cerl.beam +++ b/bootstrap/lib/compiler/ebin/cerl.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_clauses.beam b/bootstrap/lib/compiler/ebin/cerl_clauses.beam Binary files differindex fe1a83018e..fdd67aebde 100644 --- a/bootstrap/lib/compiler/ebin/cerl_clauses.beam +++ b/bootstrap/lib/compiler/ebin/cerl_clauses.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_inline.beam b/bootstrap/lib/compiler/ebin/cerl_inline.beam Binary files differindex 7ae6b03424..36888f6363 100644 --- a/bootstrap/lib/compiler/ebin/cerl_inline.beam +++ b/bootstrap/lib/compiler/ebin/cerl_inline.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_trees.beam b/bootstrap/lib/compiler/ebin/cerl_trees.beam Binary files differindex fc525300df..3deb60c450 100644 --- a/bootstrap/lib/compiler/ebin/cerl_trees.beam +++ b/bootstrap/lib/compiler/ebin/cerl_trees.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex 27e0802959..82d601c128 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/core_lint.beam b/bootstrap/lib/compiler/ebin/core_lint.beam Binary files differindex a6f6a886ca..f3b420121a 100644 --- a/bootstrap/lib/compiler/ebin/core_lint.beam +++ b/bootstrap/lib/compiler/ebin/core_lint.beam diff --git a/bootstrap/lib/compiler/ebin/core_parse.beam b/bootstrap/lib/compiler/ebin/core_parse.beam Binary files differindex d3085e14fb..cd4a22c6d8 100644 --- a/bootstrap/lib/compiler/ebin/core_parse.beam +++ b/bootstrap/lib/compiler/ebin/core_parse.beam diff --git a/bootstrap/lib/compiler/ebin/core_pp.beam b/bootstrap/lib/compiler/ebin/core_pp.beam Binary files differindex 57ed874762..59503a2933 100644 --- a/bootstrap/lib/compiler/ebin/core_pp.beam +++ b/bootstrap/lib/compiler/ebin/core_pp.beam diff --git a/bootstrap/lib/compiler/ebin/core_scan.beam b/bootstrap/lib/compiler/ebin/core_scan.beam Binary files differindex 6195c3f759..bc0190fdb1 100644 --- a/bootstrap/lib/compiler/ebin/core_scan.beam +++ b/bootstrap/lib/compiler/ebin/core_scan.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_alias.beam b/bootstrap/lib/compiler/ebin/sys_core_alias.beam Binary files differindex 3757572a10..52e46da7ad 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_alias.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_alias.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_bsm.beam b/bootstrap/lib/compiler/ebin/sys_core_bsm.beam Binary files differindex 22d045a8a7..451b2ff3b5 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_bsm.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_dsetel.beam b/bootstrap/lib/compiler/ebin/sys_core_dsetel.beam Binary files differindex b2ca054d45..c189549b16 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_dsetel.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_dsetel.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold.beam b/bootstrap/lib/compiler/ebin/sys_core_fold.beam Binary files differindex df1daa74d7..62e56d92b0 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam Binary files differindex 346ddcb2a7..0ad7ab7f9c 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_inline.beam b/bootstrap/lib/compiler/ebin/sys_core_inline.beam Binary files differindex b64e1be0ad..40f6ebf167 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_inline.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_inline.beam diff --git a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam Binary files differindex 2dd8388c18..91a29c5266 100644 --- a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam +++ b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam diff --git a/bootstrap/lib/compiler/ebin/v3_codegen.beam b/bootstrap/lib/compiler/ebin/v3_codegen.beam Binary files differindex 2e88f9c333..bf1bcaafd2 100644 --- a/bootstrap/lib/compiler/ebin/v3_codegen.beam +++ b/bootstrap/lib/compiler/ebin/v3_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/v3_core.beam b/bootstrap/lib/compiler/ebin/v3_core.beam Binary files differindex 01055394ce..b4672b1315 100644 --- a/bootstrap/lib/compiler/ebin/v3_core.beam +++ b/bootstrap/lib/compiler/ebin/v3_core.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel.beam b/bootstrap/lib/compiler/ebin/v3_kernel.beam Binary files differindex 1a5f4eb193..082bd6b75d 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam Binary files differindex 57f30e63d8..fb780c2a43 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam diff --git a/bootstrap/lib/kernel/ebin/application_controller.beam b/bootstrap/lib/kernel/ebin/application_controller.beam Binary files differindex f7b9037688..5e63aa4e8a 100644 --- a/bootstrap/lib/kernel/ebin/application_controller.beam +++ b/bootstrap/lib/kernel/ebin/application_controller.beam diff --git a/bootstrap/lib/kernel/ebin/application_master.beam b/bootstrap/lib/kernel/ebin/application_master.beam Binary files differindex 9547b5e159..4bafc43619 100644 --- a/bootstrap/lib/kernel/ebin/application_master.beam +++ b/bootstrap/lib/kernel/ebin/application_master.beam diff --git a/bootstrap/lib/kernel/ebin/application_starter.beam b/bootstrap/lib/kernel/ebin/application_starter.beam Binary files differindex 354b96eae5..ca7b77abcb 100644 --- a/bootstrap/lib/kernel/ebin/application_starter.beam +++ b/bootstrap/lib/kernel/ebin/application_starter.beam diff --git a/bootstrap/lib/kernel/ebin/auth.beam b/bootstrap/lib/kernel/ebin/auth.beam Binary files differindex 41b47131c4..e8904c142a 100644 --- a/bootstrap/lib/kernel/ebin/auth.beam +++ b/bootstrap/lib/kernel/ebin/auth.beam diff --git a/bootstrap/lib/kernel/ebin/code.beam b/bootstrap/lib/kernel/ebin/code.beam Binary files differindex 108b562bea..102ac41ac2 100644 --- a/bootstrap/lib/kernel/ebin/code.beam +++ b/bootstrap/lib/kernel/ebin/code.beam diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam Binary files differindex bdc9dea570..922c5e6308 100644 --- a/bootstrap/lib/kernel/ebin/code_server.beam +++ b/bootstrap/lib/kernel/ebin/code_server.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log.beam b/bootstrap/lib/kernel/ebin/disk_log.beam Binary files differindex 48af9c8383..76264b975b 100644 --- a/bootstrap/lib/kernel/ebin/disk_log.beam +++ b/bootstrap/lib/kernel/ebin/disk_log.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_1.beam b/bootstrap/lib/kernel/ebin/disk_log_1.beam Binary files differindex bd20da839e..05c002ba67 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_1.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_1.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_server.beam b/bootstrap/lib/kernel/ebin/disk_log_server.beam Binary files differindex c323958241..f3ca0687dd 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_server.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_server.beam diff --git a/bootstrap/lib/kernel/ebin/dist_ac.beam b/bootstrap/lib/kernel/ebin/dist_ac.beam Binary files differindex c59c245f66..7419ba7c27 100644 --- a/bootstrap/lib/kernel/ebin/dist_ac.beam +++ b/bootstrap/lib/kernel/ebin/dist_ac.beam diff --git a/bootstrap/lib/kernel/ebin/dist_util.beam b/bootstrap/lib/kernel/ebin/dist_util.beam Binary files differindex 0c1804c2ca..3a4d230a7f 100644 --- a/bootstrap/lib/kernel/ebin/dist_util.beam +++ b/bootstrap/lib/kernel/ebin/dist_util.beam diff --git a/bootstrap/lib/kernel/ebin/erl_boot_server.beam b/bootstrap/lib/kernel/ebin/erl_boot_server.beam Binary files differindex 226052b9ac..d921ffabd9 100644 --- a/bootstrap/lib/kernel/ebin/erl_boot_server.beam +++ b/bootstrap/lib/kernel/ebin/erl_boot_server.beam diff --git a/bootstrap/lib/kernel/ebin/erl_epmd.beam b/bootstrap/lib/kernel/ebin/erl_epmd.beam Binary files differindex da09ca6122..d06aa1add3 100644 --- a/bootstrap/lib/kernel/ebin/erl_epmd.beam +++ b/bootstrap/lib/kernel/ebin/erl_epmd.beam diff --git a/bootstrap/lib/kernel/ebin/erl_signal_handler.beam b/bootstrap/lib/kernel/ebin/erl_signal_handler.beam Binary files differindex bbb5e7fcb4..881e36e6fb 100644 --- a/bootstrap/lib/kernel/ebin/erl_signal_handler.beam +++ b/bootstrap/lib/kernel/ebin/erl_signal_handler.beam diff --git a/bootstrap/lib/kernel/ebin/error_handler.beam b/bootstrap/lib/kernel/ebin/error_handler.beam Binary files differindex f1a466ee6c..1ce96ca432 100644 --- a/bootstrap/lib/kernel/ebin/error_handler.beam +++ b/bootstrap/lib/kernel/ebin/error_handler.beam diff --git a/bootstrap/lib/kernel/ebin/error_logger.beam b/bootstrap/lib/kernel/ebin/error_logger.beam Binary files differindex bc11d2eb54..4490480184 100644 --- a/bootstrap/lib/kernel/ebin/error_logger.beam +++ b/bootstrap/lib/kernel/ebin/error_logger.beam diff --git a/bootstrap/lib/kernel/ebin/erts_debug.beam b/bootstrap/lib/kernel/ebin/erts_debug.beam Binary files differindex 0145b2e3c9..d59aed49df 100644 --- a/bootstrap/lib/kernel/ebin/erts_debug.beam +++ b/bootstrap/lib/kernel/ebin/erts_debug.beam diff --git a/bootstrap/lib/kernel/ebin/file.beam b/bootstrap/lib/kernel/ebin/file.beam Binary files differindex 7fa8c2a5c5..df612743bd 100644 --- a/bootstrap/lib/kernel/ebin/file.beam +++ b/bootstrap/lib/kernel/ebin/file.beam diff --git a/bootstrap/lib/kernel/ebin/file_io_server.beam b/bootstrap/lib/kernel/ebin/file_io_server.beam Binary files differindex 00c19eb71c..d37ed85781 100644 --- a/bootstrap/lib/kernel/ebin/file_io_server.beam +++ b/bootstrap/lib/kernel/ebin/file_io_server.beam diff --git a/bootstrap/lib/kernel/ebin/global.beam b/bootstrap/lib/kernel/ebin/global.beam Binary files differindex 7597ed4a09..8b057dff1a 100644 --- a/bootstrap/lib/kernel/ebin/global.beam +++ b/bootstrap/lib/kernel/ebin/global.beam diff --git a/bootstrap/lib/kernel/ebin/global_group.beam b/bootstrap/lib/kernel/ebin/global_group.beam Binary files differindex 9d119f0c96..b072f96e5d 100644 --- a/bootstrap/lib/kernel/ebin/global_group.beam +++ b/bootstrap/lib/kernel/ebin/global_group.beam diff --git a/bootstrap/lib/kernel/ebin/group.beam b/bootstrap/lib/kernel/ebin/group.beam Binary files differindex 99bd4bf343..43c35f0a3a 100644 --- a/bootstrap/lib/kernel/ebin/group.beam +++ b/bootstrap/lib/kernel/ebin/group.beam diff --git a/bootstrap/lib/kernel/ebin/group_history.beam b/bootstrap/lib/kernel/ebin/group_history.beam Binary files differindex 5e8390f36d..97f98ebe2f 100644 --- a/bootstrap/lib/kernel/ebin/group_history.beam +++ b/bootstrap/lib/kernel/ebin/group_history.beam diff --git a/bootstrap/lib/kernel/ebin/heart.beam b/bootstrap/lib/kernel/ebin/heart.beam Binary files differindex bf849fb229..0c3caaf969 100644 --- a/bootstrap/lib/kernel/ebin/heart.beam +++ b/bootstrap/lib/kernel/ebin/heart.beam diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam Binary files differindex 8ca0b915a2..16841cc417 100644 --- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam +++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam diff --git a/bootstrap/lib/kernel/ebin/inet.beam b/bootstrap/lib/kernel/ebin/inet.beam Binary files differindex 7790401681..c1f7d2e8b0 100644 --- a/bootstrap/lib/kernel/ebin/inet.beam +++ b/bootstrap/lib/kernel/ebin/inet.beam diff --git a/bootstrap/lib/kernel/ebin/inet_config.beam b/bootstrap/lib/kernel/ebin/inet_config.beam Binary files differindex 257bfe46de..5b50b553d7 100644 --- a/bootstrap/lib/kernel/ebin/inet_config.beam +++ b/bootstrap/lib/kernel/ebin/inet_config.beam diff --git a/bootstrap/lib/kernel/ebin/inet_db.beam b/bootstrap/lib/kernel/ebin/inet_db.beam Binary files differindex 155536239d..e86ce51920 100644 --- a/bootstrap/lib/kernel/ebin/inet_db.beam +++ b/bootstrap/lib/kernel/ebin/inet_db.beam diff --git a/bootstrap/lib/kernel/ebin/inet_dns.beam b/bootstrap/lib/kernel/ebin/inet_dns.beam Binary files differindex 12b7468fa6..c437ed85d7 100644 --- a/bootstrap/lib/kernel/ebin/inet_dns.beam +++ b/bootstrap/lib/kernel/ebin/inet_dns.beam diff --git a/bootstrap/lib/kernel/ebin/inet_gethost_native.beam b/bootstrap/lib/kernel/ebin/inet_gethost_native.beam Binary files differindex 387f03b8df..ccd9f21cd4 100644 --- a/bootstrap/lib/kernel/ebin/inet_gethost_native.beam +++ b/bootstrap/lib/kernel/ebin/inet_gethost_native.beam diff --git a/bootstrap/lib/kernel/ebin/inet_hosts.beam b/bootstrap/lib/kernel/ebin/inet_hosts.beam Binary files differindex 209d86690d..601a6704ee 100644 --- a/bootstrap/lib/kernel/ebin/inet_hosts.beam +++ b/bootstrap/lib/kernel/ebin/inet_hosts.beam diff --git a/bootstrap/lib/kernel/ebin/inet_parse.beam b/bootstrap/lib/kernel/ebin/inet_parse.beam Binary files differindex 5949536932..561e53dcbd 100644 --- a/bootstrap/lib/kernel/ebin/inet_parse.beam +++ b/bootstrap/lib/kernel/ebin/inet_parse.beam diff --git a/bootstrap/lib/kernel/ebin/inet_res.beam b/bootstrap/lib/kernel/ebin/inet_res.beam Binary files differindex a2d5924dd0..8c18d0d1ad 100644 --- a/bootstrap/lib/kernel/ebin/inet_res.beam +++ b/bootstrap/lib/kernel/ebin/inet_res.beam diff --git a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam Binary files differindex 4575ad2d9b..2aa0a7270a 100644 --- a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam +++ b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam diff --git a/bootstrap/lib/kernel/ebin/inet_udp.beam b/bootstrap/lib/kernel/ebin/inet_udp.beam Binary files differindex 317a0d4b51..d358ee15dc 100644 --- a/bootstrap/lib/kernel/ebin/inet_udp.beam +++ b/bootstrap/lib/kernel/ebin/inet_udp.beam diff --git a/bootstrap/lib/kernel/ebin/kernel.app b/bootstrap/lib/kernel/ebin/kernel.app index 4979bf69b5..d027eb6bcb 100644 --- a/bootstrap/lib/kernel/ebin/kernel.app +++ b/bootstrap/lib/kernel/ebin/kernel.app @@ -22,7 +22,7 @@ {application, kernel, [ {description, "ERTS CXC 138 10"}, - {vsn, "5.4.1"}, + {vsn, "5.4.2"}, {modules, [application, application_controller, application_master, diff --git a/bootstrap/lib/kernel/ebin/net_adm.beam b/bootstrap/lib/kernel/ebin/net_adm.beam Binary files differindex 79dad6cfa5..05aa46dcb8 100644 --- a/bootstrap/lib/kernel/ebin/net_adm.beam +++ b/bootstrap/lib/kernel/ebin/net_adm.beam diff --git a/bootstrap/lib/kernel/ebin/net_kernel.beam b/bootstrap/lib/kernel/ebin/net_kernel.beam Binary files differindex eea9fde3c6..d9abceade5 100644 --- a/bootstrap/lib/kernel/ebin/net_kernel.beam +++ b/bootstrap/lib/kernel/ebin/net_kernel.beam diff --git a/bootstrap/lib/kernel/ebin/os.beam b/bootstrap/lib/kernel/ebin/os.beam Binary files differindex 76894e9bc9..f2a003d44f 100644 --- a/bootstrap/lib/kernel/ebin/os.beam +++ b/bootstrap/lib/kernel/ebin/os.beam diff --git a/bootstrap/lib/kernel/ebin/pg2.beam b/bootstrap/lib/kernel/ebin/pg2.beam Binary files differindex 8039120d20..4661129bc9 100644 --- a/bootstrap/lib/kernel/ebin/pg2.beam +++ b/bootstrap/lib/kernel/ebin/pg2.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam b/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam Binary files differindex 9060a2bffe..222817d912 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam b/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam Binary files differindex 4c1df2bb57..228a8f28a5 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam diff --git a/bootstrap/lib/kernel/ebin/rpc.beam b/bootstrap/lib/kernel/ebin/rpc.beam Binary files differindex 7be3630e48..e4ed087628 100644 --- a/bootstrap/lib/kernel/ebin/rpc.beam +++ b/bootstrap/lib/kernel/ebin/rpc.beam diff --git a/bootstrap/lib/kernel/ebin/user.beam b/bootstrap/lib/kernel/ebin/user.beam Binary files differindex 58bd43a18e..e57d6896bd 100644 --- a/bootstrap/lib/kernel/ebin/user.beam +++ b/bootstrap/lib/kernel/ebin/user.beam diff --git a/bootstrap/lib/kernel/ebin/wrap_log_reader.beam b/bootstrap/lib/kernel/ebin/wrap_log_reader.beam Binary files differindex b38ddd7288..a0f24625b4 100644 --- a/bootstrap/lib/kernel/ebin/wrap_log_reader.beam +++ b/bootstrap/lib/kernel/ebin/wrap_log_reader.beam diff --git a/bootstrap/lib/stdlib/ebin/array.beam b/bootstrap/lib/stdlib/ebin/array.beam Binary files differindex 68bf4904ce..1dd8846725 100644 --- a/bootstrap/lib/stdlib/ebin/array.beam +++ b/bootstrap/lib/stdlib/ebin/array.beam diff --git a/bootstrap/lib/stdlib/ebin/beam_lib.beam b/bootstrap/lib/stdlib/ebin/beam_lib.beam Binary files differindex 9b70424aae..7ffab34254 100644 --- a/bootstrap/lib/stdlib/ebin/beam_lib.beam +++ b/bootstrap/lib/stdlib/ebin/beam_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/binary.beam b/bootstrap/lib/stdlib/ebin/binary.beam Binary files differindex 85fc484c32..e712319947 100644 --- a/bootstrap/lib/stdlib/ebin/binary.beam +++ b/bootstrap/lib/stdlib/ebin/binary.beam diff --git a/bootstrap/lib/stdlib/ebin/c.beam b/bootstrap/lib/stdlib/ebin/c.beam Binary files differindex fb24f24b98..b3f62c467d 100644 --- a/bootstrap/lib/stdlib/ebin/c.beam +++ b/bootstrap/lib/stdlib/ebin/c.beam diff --git a/bootstrap/lib/stdlib/ebin/calendar.beam b/bootstrap/lib/stdlib/ebin/calendar.beam Binary files differindex 3b38655345..89a9168805 100644 --- a/bootstrap/lib/stdlib/ebin/calendar.beam +++ b/bootstrap/lib/stdlib/ebin/calendar.beam diff --git a/bootstrap/lib/stdlib/ebin/dets.beam b/bootstrap/lib/stdlib/ebin/dets.beam Binary files differindex 7ffca2b7ee..9384a4379f 100644 --- a/bootstrap/lib/stdlib/ebin/dets.beam +++ b/bootstrap/lib/stdlib/ebin/dets.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_server.beam b/bootstrap/lib/stdlib/ebin/dets_server.beam Binary files differindex cfe0257a9f..31f0f2701a 100644 --- a/bootstrap/lib/stdlib/ebin/dets_server.beam +++ b/bootstrap/lib/stdlib/ebin/dets_server.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_utils.beam b/bootstrap/lib/stdlib/ebin/dets_utils.beam Binary files differindex 76b65ad011..265facaa57 100644 --- a/bootstrap/lib/stdlib/ebin/dets_utils.beam +++ b/bootstrap/lib/stdlib/ebin/dets_utils.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_v9.beam b/bootstrap/lib/stdlib/ebin/dets_v9.beam Binary files differindex adcb52b007..f53f9fcf85 100644 --- a/bootstrap/lib/stdlib/ebin/dets_v9.beam +++ b/bootstrap/lib/stdlib/ebin/dets_v9.beam diff --git a/bootstrap/lib/stdlib/ebin/dict.beam b/bootstrap/lib/stdlib/ebin/dict.beam Binary files differindex b04ad73e70..345df4c87f 100644 --- a/bootstrap/lib/stdlib/ebin/dict.beam +++ b/bootstrap/lib/stdlib/ebin/dict.beam diff --git a/bootstrap/lib/stdlib/ebin/digraph.beam b/bootstrap/lib/stdlib/ebin/digraph.beam Binary files differindex 606cf5c08b..c51ecf54fc 100644 --- a/bootstrap/lib/stdlib/ebin/digraph.beam +++ b/bootstrap/lib/stdlib/ebin/digraph.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin.beam b/bootstrap/lib/stdlib/ebin/edlin.beam Binary files differindex 807c7df5c7..c8d4cac351 100644 --- a/bootstrap/lib/stdlib/ebin/edlin.beam +++ b/bootstrap/lib/stdlib/ebin/edlin.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin_expand.beam b/bootstrap/lib/stdlib/ebin/edlin_expand.beam Binary files differindex 68515c00a0..62e209dcc7 100644 --- a/bootstrap/lib/stdlib/ebin/edlin_expand.beam +++ b/bootstrap/lib/stdlib/ebin/edlin_expand.beam diff --git a/bootstrap/lib/stdlib/ebin/epp.beam b/bootstrap/lib/stdlib/ebin/epp.beam Binary files differindex 66ddbaea09..58e29d0c3b 100644 --- a/bootstrap/lib/stdlib/ebin/epp.beam +++ b/bootstrap/lib/stdlib/ebin/epp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_bits.beam b/bootstrap/lib/stdlib/ebin/erl_bits.beam Binary files differindex 5b2a46aec1..b9b75ecaef 100644 --- a/bootstrap/lib/stdlib/ebin/erl_bits.beam +++ b/bootstrap/lib/stdlib/ebin/erl_bits.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_compile.beam b/bootstrap/lib/stdlib/ebin/erl_compile.beam Binary files differindex d40909b3cf..a317ae28a1 100644 --- a/bootstrap/lib/stdlib/ebin/erl_compile.beam +++ b/bootstrap/lib/stdlib/ebin/erl_compile.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_eval.beam b/bootstrap/lib/stdlib/ebin/erl_eval.beam Binary files differindex 95de15156d..eb913d070d 100644 --- a/bootstrap/lib/stdlib/ebin/erl_eval.beam +++ b/bootstrap/lib/stdlib/ebin/erl_eval.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_expand_records.beam b/bootstrap/lib/stdlib/ebin/erl_expand_records.beam Binary files differindex 9032984049..1869788f7b 100644 --- a/bootstrap/lib/stdlib/ebin/erl_expand_records.beam +++ b/bootstrap/lib/stdlib/ebin/erl_expand_records.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_lint.beam b/bootstrap/lib/stdlib/ebin/erl_lint.beam Binary files differindex c6c62b3657..b8350e8975 100644 --- a/bootstrap/lib/stdlib/ebin/erl_lint.beam +++ b/bootstrap/lib/stdlib/ebin/erl_lint.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_parse.beam b/bootstrap/lib/stdlib/ebin/erl_parse.beam Binary files differindex 6ac2e29240..bfc5843912 100644 --- a/bootstrap/lib/stdlib/ebin/erl_parse.beam +++ b/bootstrap/lib/stdlib/ebin/erl_parse.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_pp.beam b/bootstrap/lib/stdlib/ebin/erl_pp.beam Binary files differindex 16843acd17..ca3d18a5db 100644 --- a/bootstrap/lib/stdlib/ebin/erl_pp.beam +++ b/bootstrap/lib/stdlib/ebin/erl_pp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_tar.beam b/bootstrap/lib/stdlib/ebin/erl_tar.beam Binary files differindex f2c935dad1..ba7382f874 100644 --- a/bootstrap/lib/stdlib/ebin/erl_tar.beam +++ b/bootstrap/lib/stdlib/ebin/erl_tar.beam diff --git a/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam b/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam Binary files differindex 7cc374d4b6..2f2e54de1a 100644 --- a/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam +++ b/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam diff --git a/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam b/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam Binary files differindex a4b3321379..d504f97f9f 100644 --- a/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam +++ b/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam diff --git a/bootstrap/lib/stdlib/ebin/escript.beam b/bootstrap/lib/stdlib/ebin/escript.beam Binary files differindex 5721d41a52..5ae2752a6d 100644 --- a/bootstrap/lib/stdlib/ebin/escript.beam +++ b/bootstrap/lib/stdlib/ebin/escript.beam diff --git a/bootstrap/lib/stdlib/ebin/ets.beam b/bootstrap/lib/stdlib/ebin/ets.beam Binary files differindex 06576e533a..f2a649e0d1 100644 --- a/bootstrap/lib/stdlib/ebin/ets.beam +++ b/bootstrap/lib/stdlib/ebin/ets.beam diff --git a/bootstrap/lib/stdlib/ebin/eval_bits.beam b/bootstrap/lib/stdlib/ebin/eval_bits.beam Binary files differindex 76b1a2a2ae..0a3c0b47fb 100644 --- a/bootstrap/lib/stdlib/ebin/eval_bits.beam +++ b/bootstrap/lib/stdlib/ebin/eval_bits.beam diff --git a/bootstrap/lib/stdlib/ebin/file_sorter.beam b/bootstrap/lib/stdlib/ebin/file_sorter.beam Binary files differindex 22e7f2c2b4..8a9e501e1b 100644 --- a/bootstrap/lib/stdlib/ebin/file_sorter.beam +++ b/bootstrap/lib/stdlib/ebin/file_sorter.beam diff --git a/bootstrap/lib/stdlib/ebin/filelib.beam b/bootstrap/lib/stdlib/ebin/filelib.beam Binary files differindex e632631389..7fc4d0b06a 100644 --- a/bootstrap/lib/stdlib/ebin/filelib.beam +++ b/bootstrap/lib/stdlib/ebin/filelib.beam diff --git a/bootstrap/lib/stdlib/ebin/filename.beam b/bootstrap/lib/stdlib/ebin/filename.beam Binary files differindex 7fff3a33cc..42e8e24580 100644 --- a/bootstrap/lib/stdlib/ebin/filename.beam +++ b/bootstrap/lib/stdlib/ebin/filename.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_sets.beam b/bootstrap/lib/stdlib/ebin/gb_sets.beam Binary files differindex 7b3b19efb8..4b524f17a3 100644 --- a/bootstrap/lib/stdlib/ebin/gb_sets.beam +++ b/bootstrap/lib/stdlib/ebin/gb_sets.beam diff --git a/bootstrap/lib/stdlib/ebin/gen.beam b/bootstrap/lib/stdlib/ebin/gen.beam Binary files differindex 14d8f04cba..0c36dfe05a 100644 --- a/bootstrap/lib/stdlib/ebin/gen.beam +++ b/bootstrap/lib/stdlib/ebin/gen.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_event.beam b/bootstrap/lib/stdlib/ebin/gen_event.beam Binary files differindex f4b87ab26d..21806df360 100644 --- a/bootstrap/lib/stdlib/ebin/gen_event.beam +++ b/bootstrap/lib/stdlib/ebin/gen_event.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_fsm.beam b/bootstrap/lib/stdlib/ebin/gen_fsm.beam Binary files differindex 566aa94342..fccfecd7d8 100644 --- a/bootstrap/lib/stdlib/ebin/gen_fsm.beam +++ b/bootstrap/lib/stdlib/ebin/gen_fsm.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_server.beam b/bootstrap/lib/stdlib/ebin/gen_server.beam Binary files differindex 6ec210dada..3f2d31465f 100644 --- a/bootstrap/lib/stdlib/ebin/gen_server.beam +++ b/bootstrap/lib/stdlib/ebin/gen_server.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_statem.beam b/bootstrap/lib/stdlib/ebin/gen_statem.beam Binary files differindex b2c88ce783..c79b4a16f9 100644 --- a/bootstrap/lib/stdlib/ebin/gen_statem.beam +++ b/bootstrap/lib/stdlib/ebin/gen_statem.beam diff --git a/bootstrap/lib/stdlib/ebin/io.beam b/bootstrap/lib/stdlib/ebin/io.beam Binary files differindex 40109e2a13..e1841e184c 100644 --- a/bootstrap/lib/stdlib/ebin/io.beam +++ b/bootstrap/lib/stdlib/ebin/io.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib.beam b/bootstrap/lib/stdlib/ebin/io_lib.beam Binary files differindex 41fa869dde..adb2ac56b0 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_format.beam b/bootstrap/lib/stdlib/ebin/io_lib_format.beam Binary files differindex e564659aa8..713f6236aa 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_format.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_format.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam Binary files differindex b2f489b702..bf9e95e6be 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam diff --git a/bootstrap/lib/stdlib/ebin/lib.beam b/bootstrap/lib/stdlib/ebin/lib.beam Binary files differindex 649d4452ad..7d342a59a9 100644 --- a/bootstrap/lib/stdlib/ebin/lib.beam +++ b/bootstrap/lib/stdlib/ebin/lib.beam diff --git a/bootstrap/lib/stdlib/ebin/lists.beam b/bootstrap/lib/stdlib/ebin/lists.beam Binary files differindex 5b91ffeca9..bfcebe04a4 100644 --- a/bootstrap/lib/stdlib/ebin/lists.beam +++ b/bootstrap/lib/stdlib/ebin/lists.beam diff --git a/bootstrap/lib/stdlib/ebin/ms_transform.beam b/bootstrap/lib/stdlib/ebin/ms_transform.beam Binary files differindex 3113f71b58..8228a076ba 100644 --- a/bootstrap/lib/stdlib/ebin/ms_transform.beam +++ b/bootstrap/lib/stdlib/ebin/ms_transform.beam diff --git a/bootstrap/lib/stdlib/ebin/pool.beam b/bootstrap/lib/stdlib/ebin/pool.beam Binary files differindex bc3b2a861c..b056cf44ec 100644 --- a/bootstrap/lib/stdlib/ebin/pool.beam +++ b/bootstrap/lib/stdlib/ebin/pool.beam diff --git a/bootstrap/lib/stdlib/ebin/proc_lib.beam b/bootstrap/lib/stdlib/ebin/proc_lib.beam Binary files differindex 191dc94be7..6689783d5d 100644 --- a/bootstrap/lib/stdlib/ebin/proc_lib.beam +++ b/bootstrap/lib/stdlib/ebin/proc_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/proplists.beam b/bootstrap/lib/stdlib/ebin/proplists.beam Binary files differindex 3750c1ddb3..a9768d2929 100644 --- a/bootstrap/lib/stdlib/ebin/proplists.beam +++ b/bootstrap/lib/stdlib/ebin/proplists.beam diff --git a/bootstrap/lib/stdlib/ebin/qlc.beam b/bootstrap/lib/stdlib/ebin/qlc.beam Binary files differindex 5b2d4bad4c..6c2895ac79 100644 --- a/bootstrap/lib/stdlib/ebin/qlc.beam +++ b/bootstrap/lib/stdlib/ebin/qlc.beam diff --git a/bootstrap/lib/stdlib/ebin/qlc_pt.beam b/bootstrap/lib/stdlib/ebin/qlc_pt.beam Binary files differindex c9d329fe86..d40c197029 100644 --- a/bootstrap/lib/stdlib/ebin/qlc_pt.beam +++ b/bootstrap/lib/stdlib/ebin/qlc_pt.beam diff --git a/bootstrap/lib/stdlib/ebin/queue.beam b/bootstrap/lib/stdlib/ebin/queue.beam Binary files differindex c4cce1f88d..8c6b611247 100644 --- a/bootstrap/lib/stdlib/ebin/queue.beam +++ b/bootstrap/lib/stdlib/ebin/queue.beam diff --git a/bootstrap/lib/stdlib/ebin/re.beam b/bootstrap/lib/stdlib/ebin/re.beam Binary files differindex 3f316c9a44..784cf2fe7a 100644 --- a/bootstrap/lib/stdlib/ebin/re.beam +++ b/bootstrap/lib/stdlib/ebin/re.beam diff --git a/bootstrap/lib/stdlib/ebin/sets.beam b/bootstrap/lib/stdlib/ebin/sets.beam Binary files differindex 2337a8a8d9..3f90d78b8f 100644 --- a/bootstrap/lib/stdlib/ebin/sets.beam +++ b/bootstrap/lib/stdlib/ebin/sets.beam diff --git a/bootstrap/lib/stdlib/ebin/shell.beam b/bootstrap/lib/stdlib/ebin/shell.beam Binary files differindex ca29384984..e2dfa3c636 100644 --- a/bootstrap/lib/stdlib/ebin/shell.beam +++ b/bootstrap/lib/stdlib/ebin/shell.beam diff --git a/bootstrap/lib/stdlib/ebin/slave.beam b/bootstrap/lib/stdlib/ebin/slave.beam Binary files differindex f72a207e3e..596dda4ed5 100644 --- a/bootstrap/lib/stdlib/ebin/slave.beam +++ b/bootstrap/lib/stdlib/ebin/slave.beam diff --git a/bootstrap/lib/stdlib/ebin/sofs.beam b/bootstrap/lib/stdlib/ebin/sofs.beam Binary files differindex ef540df4f2..bfefd0c3a9 100644 --- a/bootstrap/lib/stdlib/ebin/sofs.beam +++ b/bootstrap/lib/stdlib/ebin/sofs.beam diff --git a/bootstrap/lib/stdlib/ebin/string.beam b/bootstrap/lib/stdlib/ebin/string.beam Binary files differindex 670cd530e9..feeeec6a84 100644 --- a/bootstrap/lib/stdlib/ebin/string.beam +++ b/bootstrap/lib/stdlib/ebin/string.beam diff --git a/bootstrap/lib/stdlib/ebin/supervisor.beam b/bootstrap/lib/stdlib/ebin/supervisor.beam Binary files differindex 61ae3c3de0..61e8fca92c 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor.beam diff --git a/bootstrap/lib/stdlib/ebin/sys.beam b/bootstrap/lib/stdlib/ebin/sys.beam Binary files differindex abf6451cb0..0cec9f4be8 100644 --- a/bootstrap/lib/stdlib/ebin/sys.beam +++ b/bootstrap/lib/stdlib/ebin/sys.beam diff --git a/bootstrap/lib/stdlib/ebin/unicode_util.beam b/bootstrap/lib/stdlib/ebin/unicode_util.beam Binary files differindex 12280f5e58..ab18c78028 100644 --- a/bootstrap/lib/stdlib/ebin/unicode_util.beam +++ b/bootstrap/lib/stdlib/ebin/unicode_util.beam diff --git a/bootstrap/lib/stdlib/ebin/uri_string.beam b/bootstrap/lib/stdlib/ebin/uri_string.beam Binary files differindex 5880b38385..95b7a2e6ae 100644 --- a/bootstrap/lib/stdlib/ebin/uri_string.beam +++ b/bootstrap/lib/stdlib/ebin/uri_string.beam diff --git a/bootstrap/lib/stdlib/ebin/win32reg.beam b/bootstrap/lib/stdlib/ebin/win32reg.beam Binary files differindex 93f6458657..3194d09463 100644 --- a/bootstrap/lib/stdlib/ebin/win32reg.beam +++ b/bootstrap/lib/stdlib/ebin/win32reg.beam diff --git a/bootstrap/lib/stdlib/ebin/zip.beam b/bootstrap/lib/stdlib/ebin/zip.beam Binary files differindex e6ba80cc7d..3e19c83090 100644 --- a/bootstrap/lib/stdlib/ebin/zip.beam +++ b/bootstrap/lib/stdlib/ebin/zip.beam diff --git a/bootstrap/lib/stdlib/include/assert.hrl b/bootstrap/lib/stdlib/include/assert.hrl index 2fbaeba0b2..2ec89e7d8a 100644 --- a/bootstrap/lib/stdlib/include/assert.hrl +++ b/bootstrap/lib/stdlib/include/assert.hrl @@ -309,7 +309,7 @@ {unexpected_success, __V}]}) catch Class:Term -> ok; - __C:__T -> + __C:__T:__S -> erlang:error({assertException, [{module, ?MODULE}, {line, ?LINE}, @@ -318,8 +318,7 @@ "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace()}}]}) + {__C, __T, __S}}]}) end end)()) end). @@ -338,7 +337,7 @@ {unexpected_success, __V}]}) catch Class:Term -> ok; - __C:__T -> + __C:__T:__S -> erlang:error({assertException, [{module, ?MODULE}, {line, ?LINE}, @@ -348,8 +347,7 @@ "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace()}}]}) + {__C, __T, __S}}]}) end end)()) end). @@ -378,7 +376,7 @@ try (Expr) of _ -> ok catch - __C:__T -> + __C:__T:__S -> case __C of Class -> case __T of @@ -391,9 +389,7 @@ "{ "++(??Class)++" , " ++(??Term)++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace() - }}]}); + {__C, __T, __S}}]}); _ -> ok end; _ -> ok @@ -407,7 +403,7 @@ try (Expr) of _ -> ok catch - __C:__T -> + __C:__T:__S -> case __C of Class -> case __T of @@ -421,9 +417,7 @@ "{ "++(??Class)++" , " ++(??Term)++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace() - }}]}); + {__C, __T, __S}}]}); _ -> ok end; _ -> ok diff --git a/erts/configure.in b/erts/configure.in index f15bb56435..b2435f1ac9 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -1760,6 +1760,25 @@ if test $ac_cv_sizeof_void_p = 8; then fi AC_SUBST(BITS64) +AC_MSG_CHECKING([for C compiler 'restrict' support]) +restrict_keyword="" +for x in restrict __restrict; do + AC_TRY_COMPILE([int * $x foo(int * $x arg); + int * $x foo(int * $x arg) + { int * $x var=arg; return var;} + ],[], + [restrict_keyword=$x],[]) + if test "x$restrict_keyword" != "x"; then + break + fi +done +AC_DEFINE_UNQUOTED(ERTS_RESTRICT,[$restrict_keyword],[Type qualifier restrict]) +if test "x$restrict_keyword" != "x"; then + AC_MSG_RESULT(yes) +else + AC_MSG_RESULT(no) +fi + if test "x$ac_compiler_gnu" = "xyes"; then AC_MSG_CHECKING([if we should add -fno-tree-copyrename to CFLAGS for computed gotos to work properly]) AC_TRY_COMPILE([],[ @@ -1779,8 +1798,6 @@ else AC_MSG_RESULT(no) fi - - AC_MSG_CHECKING([for broken gcc-4.3.0 compiler]) AC_TRY_RUN([ /* pr36339.c */ @@ -2749,23 +2766,6 @@ if test "$cross_compiling" != "yes" && test X${enable_hipe} != Xno; then fi fi -dnl Check to disable -fPIE and friends for HiPE on amd64 -if test X${enable_hipe} = Xyes && test X$ARCH = Xamd64; then - AC_TRY_COMPILE(, [#if defined(__pie__) || defined(__PIE__) - #error -fPIE is enabled by default - #endif], - [AC_MSG_NOTICE([No -fPIE enabled by default])], - [AC_MSG_WARN([Security feature -fPIE will be disabled for HiPE]) - STATIC_CFLAGS="-fno-PIE $STATIC_CFLAGS" - saved_LDFLAGS=$LDFLAGS - LDFLAGS="-no-pie $LDFLAGS" - AC_TRY_LINK(,, [], - [LDFLAGS="-fno-PIE $saved_LDFLAGS" - AC_TRY_LINK(,, [], - [AC_MSG_WARN([Linked does not accept option -no-pie nor -fno-PIE]) - LDFLAGS=$saved_LDFLAGS])])]) -fi - if test X${enable_hipe} = Xyes; then case $OPSYS in linux) diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index e49c8c32e9..2ada903edb 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2001</year><year>2017</year> + <year>2001</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -614,26 +614,58 @@ <item> <p>If C is a catch clause <c>P -> B</c>, where <c>P</c> is a pattern and <c>B</c> is a body, then - Rep(C) = <c>{clause,LINE,[Rep({throw,P,_})],[],Rep(B)}</c>.</p> + Rep(C) = <c>{clause,LINE,[Rep({throw,P,_})],[],Rep(B)}</c>, + that is, a catch clause with an explicit exception class + <c>throw</c> and with or without an explicit stacktrace + variable <c>_</c> cannot be distinguished from a catch clause + without an explicit exception class and without an explicit + stacktrace variable.</p> </item> <item> <p>If C is a catch clause <c>X : P -> B</c>, where <c>X</c> is an atomic literal or a variable pattern, <c>P</c> is a pattern, and <c>B</c> is a body, then - Rep(C) = <c>{clause,LINE,[Rep({X,P,_})],[],Rep(B)}</c>.</p> + Rep(C) = <c>{clause,LINE,[Rep({X,P,_})],[],Rep(B)}</c>, + that is, a catch clause with an explicit exception class and + with an explicit stacktrace variable <c>_</c> cannot be + distinguished from a catch clause with an explicit exception + class and without an explicit stacktrace variable.</p> + </item> + <item> + <p>If C is a catch clause <c>X : P : S -> B</c>, + where <c>X</c> is an atomic literal or a variable pattern, + <c>P</c> is a pattern, <c>S</c> is a variable, and <c>B</c> + is a body, then + Rep(C) = <c>{clause,LINE,[Rep({X,P,S})],[],Rep(B)}</c>.</p> </item> <item> <p>If C is a catch clause <c>P when Gs -> B</c>, where <c>P</c> is a pattern, <c>Gs</c> is a guard sequence, and <c>B</c> is a body, then - Rep(C) = <c>{clause,LINE,[Rep({throw,P,_})],Rep(Gs),Rep(B)}</c>.</p> + Rep(C) = <c>{clause,LINE,[Rep({throw,P,_})],Rep(Gs),Rep(B)}</c>, + that is, a catch clause with an explicit exception class + <c>throw</c> and with or without an explicit stacktrace + variable <c>_</c> cannot be distinguished from a catch clause + without an explicit exception class and without an explicit + stacktrace variable.</p> </item> <item> <p>If C is a catch clause <c>X : P when Gs -> B</c>, where <c>X</c> is an atomic literal or a variable pattern, <c>P</c> is a pattern, <c>Gs</c> is a guard sequence, and <c>B</c> is a body, then - Rep(C) = <c>{clause,LINE,[Rep({X,P,_})],Rep(Gs),Rep(B)}</c>.</p> + Rep(C) = <c>{clause,LINE,[Rep({X,P,_})],Rep(Gs),Rep(B)}</c>, + that is, a catch clause with an explicit exception class and + with an explicit stacktrace variable <c>_</c> cannot be + distinguished from a catch clause with an explicit exception + class and without an explicit stacktrace variable.</p> + </item> + <item> + <p>If C is a catch clause <c>X : P : S when Gs -> B</c>, + where <c>X</c> is an atomic literal or a variable pattern, + <c>P</c> is a pattern, <c>Gs</c> is a guard sequence, + <c>S</c> is a variable, and <c>B</c> is a body, then + Rep(C) = <c>{clause,LINE,[Rep({X,P,S})],Rep(Gs),Rep(B)}</c>.</p> </item> <item> <p>If C is a function clause <c>( Ps ) -> B</c>, diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index bd824b3405..99f0421080 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -360,11 +360,12 @@ </item> <tag><c><![CDATA[-mode interactive | embedded]]></c></tag> <item> - <p>Indicates if the system is to load code dynamically - (<c><![CDATA[interactive]]></c>), or if all code is to be loaded - during system initialization (<c><![CDATA[embedded]]></c>); see - <seealso marker="kernel:code"><c>code(3)</c></seealso>. - Defaults to <c><![CDATA[interactive]]></c>.</p> + <p>Modules are auto loaded when they are first referenced if the + runtime system runs in <c><![CDATA[interactive]]></c> mode, which is + the default. In <c><![CDATA[embedded]]></c> mode modules are not auto + loaded. The latter is recommended when the boot script preloads all + modules, as conventionally happens in OTP releases. See + <seealso marker="kernel:code"><c>code(3)</c></seealso></p>. </item> <tag><c><![CDATA[-name Name]]></c></tag> <item> @@ -1693,4 +1694,3 @@ code:load_abs("..../user_default"). ]]></code> <seealso marker="tools:make"><c>make(3)</c></seealso></p> </section> </comref> - diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 23f0c66429..f2f380ca4e 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -974,6 +974,10 @@ typedef struct { <seealso marker="#enif_make_binary"><c>enif_make_binary</c></seealso>. An allocated (and owned) <c>ErlNifBinary</c> can be kept between NIF calls.</p> + <p>If you do not need to reallocate or keep the data alive across NIF + calls, consider using <seealso marker="#enif_make_new_binary"> + <c>enif_make_new_binary</c></seealso> instead as it will allocate + small binaries on the process heap when possible.</p> <p>Returns <c>true</c> on success, or <c>false</c> if allocation fails.</p> </desc> @@ -3060,9 +3064,10 @@ if (retval & ERL_NIF_SELECT_STOP_CALLED) { </name> <fsummary>Get the pid of the calling process.</fsummary> <desc> - <p>Initializes the pid variable <c>*pid</c> to represent the - calling process.</p> - <p>Returns <c>pid</c>.</p> + <p>Initializes the <seealso marker="#ErlNifPid"><c>ErlNifPid</c></seealso> + variable at <c>*pid</c> to represent the calling process.</p> + <p>Returns <c>pid</c> if successful, or NULL if <c>caller_env</c> is not + a <seealso marker="#ErlNifEnv">process-bound environment</seealso>.</p> </desc> </func> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 24e0773e41..771897ba94 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -7116,6 +7116,23 @@ ok <func> <name name="system_flag" arity="2" clause_i="4"/> + <fsummary>Set system flag for erts_alloc.</fsummary> + <desc> + <p>Sets system flags for + <seealso marker="erts:erts_alloc"><c>erts_alloc(3)</c></seealso>. + <c><anno>Alloc</anno></c> is the allocator to affect, for example + <c>binary_alloc</c>. <c><anno>F</anno></c> is the flag to change and + <c><anno>V</anno></c> is the new value.</p> + <p>Only a subset of all <c>erts_alloc</c> flags can be changed + at run time. This subset is currently only the flag + <seealso marker="erts:erts_alloc#M_sbct"><c>sbct</c></seealso>.</p> + <p>Returns <c>ok</c> if the flag was set or <c>notsup</c> if not + supported by <c>erts_alloc</c>.</p> + </desc> + </func> + + <func> + <name name="system_flag" arity="2" clause_i="5"/> <fsummary>Set system flag fullsweep_after.</fsummary> <desc> <p>Sets system flag <c>fullsweep_after</c>. @@ -7134,7 +7151,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="5" + <name name="system_flag" arity="2" clause_i="6" anchor="system_flag_microstate_accounting"/> <fsummary>Set system flag microstate_accounting.</fsummary> <desc> @@ -7148,7 +7165,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="6"/> + <name name="system_flag" arity="2" clause_i="7"/> <fsummary>Set system flag min_heap_size.</fsummary> <desc> <p>Sets the default minimum heap size for processes. The size @@ -7163,7 +7180,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="7"/> + <name name="system_flag" arity="2" clause_i="8"/> <fsummary>Set system flag min_bin_vheap_size.</fsummary> <desc> <p>Sets the default minimum binary virtual heap size for @@ -7180,7 +7197,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="8" + <name name="system_flag" arity="2" clause_i="9" anchor="system_flag_max_heap_size"/> <fsummary>Set system flag max_heap_size.</fsummary> <type name="max_heap_size"/> @@ -7198,7 +7215,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="9" + <name name="system_flag" arity="2" clause_i="10" anchor="system_flag_multi_scheduling"/> <fsummary>Set system flag multi_scheduling.</fsummary> <desc> @@ -7254,7 +7271,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="10" + <name name="system_flag" arity="2" clause_i="11" anchor="system_flag_scheduler_bind_type"/> <fsummary>Set system flag scheduler_bind_type.</fsummary> <type name="scheduler_bind_type"/> @@ -7381,7 +7398,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="11" + <name name="system_flag" arity="2" clause_i="12" anchor="system_flag_scheduler_wall_time"/> <fsummary>Set system flag scheduler_wall_time.</fsummary> <desc> @@ -7394,7 +7411,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="12" + <name name="system_flag" arity="2" clause_i="13" anchor="system_flag_schedulers_online"/> <fsummary>Set system flag schedulers_online.</fsummary> <desc> @@ -7423,7 +7440,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="13"/> + <name name="system_flag" arity="2" clause_i="14"/> <fsummary>Set system flag trace_control_word.</fsummary> <desc> <p>Sets the value of the node trace control word to @@ -7437,7 +7454,7 @@ ok </func> <func> - <name name="system_flag" arity="2" clause_i="14" + <name name="system_flag" arity="2" clause_i="15" anchor="system_flag_time_offset"/> <fsummary>Finalize the time offset.</fsummary> <desc> diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index 580780e73b..049ffe560a 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -225,6 +225,33 @@ used. The time complexity is proportional to log N, where N is the number of free blocks.</p> </item> + <tag>Age order first fit carrier address order first fit</tag> + <item> + <p>Strategy: Find the <em>oldest carrier</em> that + can satisfy the requested block size, then find a block within + that carrier using the "address order first fit" strategy.</p> + <p>Implementation: A balanced binary search tree is + used. The time complexity is proportional to log N, where + N is the number of free blocks.</p> + </item> + <tag>Age order first fit carrier best fit</tag> + <item> + <p>Strategy: Find the <em>oldest carrier</em> that + can satisfy the requested block size, then find a block within + that carrier using the "best fit" strategy.</p> + <p>Implementation: Balanced binary search trees are + used. The time complexity is proportional to log N, where + N is the number of free blocks.</p> + </item> + <tag>Age order first fit carrier address order best fit</tag> + <item> + <p>Strategy: Find the <em>oldest carrier</em> that + can satisfy the requested block size, then find a block within + that carrier using the "address order best fit" strategy.</p> + <p>Implementation: Balanced binary search trees are + used. The time complexity is proportional to log N, where + N is the number of free blocks.</p> + </item> <tag>Good fit</tag> <item> <p>Strategy: Try to find the best fit, but settle for the best fit @@ -467,7 +494,8 @@ fetched, it will function as an ordinary carrier. This feature has special requirements on the <seealso marker="#M_as">allocation strategy</seealso> used. Only - the strategies <c>aoff</c>, <c>aoffcbf</c>, and <c>aoffcaobf</c> + the strategies <c>aoff</c>, <c>aoffcbf</c>, <c>aoffcaobf</c>, + <c>ageffcaoff</c>m, <c>ageffcbf</c> and <c>ageffcaobf</c> support abandoned carriers.</p> <p>This feature also requires <seealso marker="#M_t">multiple thread specific instances</seealso> @@ -478,8 +506,30 @@ allocators based on the <c>alloc_util</c> framework, except <c>temp_alloc</c> (which would be pointless).</p> </item> + + <tag><marker id="M_acfml"/><c><![CDATA[+M<S>acfml <bytes>]]></c> + </tag> + <item> + <p>Abandon carrier free block min limit. A valid <c><![CDATA[<bytes>]]></c> + is a positive integer representing a block size limit. The largest + free block in a carrier must be at least <c>bytes</c> large, for the + carrier to be abandoned. The default is zero but can be changed + in the future.</p> + <p>See also <seealso marker="#M_acul"><c>acul</c></seealso>.</p> + </item> + + <tag><marker id="M_acnl"/><c><![CDATA[+M<S>acnl <amount>]]></c> + </tag> + <item> + <p>Abandon carrier number limit. A valid <c><![CDATA[<amount>]]></c> + is a positive integer representing max number of abandoned carriers per + allocator instance. Defaults to 1000 which will practically disable + the limit, but this can be changed in the future.</p> + <p>See also <seealso marker="#M_acul"><c>acul</c></seealso>.</p> + </item> + <tag><marker id="M_as"/> - <c><![CDATA[+M<S>as bf|aobf|aoff|aoffcbf|aoffcaobf|gf|af]]></c></tag> + <c><![CDATA[+M<S>as bf|aobf|aoff|aoffcbf|aoffcaobf|ageffcaoff|ageffcbf|ageffcaobf|gf|af]]></c></tag> <item> <p>Allocation strategy. The following strategies are valid:</p> <list type="bulleted"> @@ -490,6 +540,11 @@ </item> <item><c>aoffcaobf</c> (address order first fit carrier address order best fit)</item> + <item><c>ageffcaoff</c> (age order first fit carrier address order first fit)</item> + <item><c>ageffcbf</c> (age order first fit carrier best fit) + </item> + <item><c>ageffcaobf</c> (age order first fit carrier address + order best fit)</item> <item><c>gf</c> (good fit)</item> <item><c>af</c> (a fit)</item> </list> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index e91e56e581..3109da6738 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,48 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 9.2.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Improve search algorithm of abandoned memory carriers. + Instead of limited linear search, each allocator instance + maintain a balanced search tree of all its abandoned + carriers for faster and more exhaustive search.</p> + <p> + Own Id: OTP-14915 Aux Id: ERIERL-88 </p> + </item> + <item> + <p> + New erts_alloc command line options <c>+M_acnl</c> and + <c>+M_acfml</c> to limit carrier abandonment.</p> + <p> + Own Id: OTP-14916 Aux Id: ERIERL-88 </p> + </item> + <item> + <p> + New family of <c>erts_alloc</c> strategies: Age Order + First Fit. Similar to "address order", but instead the + oldest possible carrier is always chosen for allocation.</p> + <p> + Own Id: OTP-14917 Aux Id: ERIERL-88 </p> + </item> + <item> + <p> + Add possibility to change allocator options at runtime + with <c>system_info(erts_alloc, ...)</c>. Only option + <c>sbct</c> (single block carrier threshold) is currently + supported via this interface.</p> + <p> + Own Id: OTP-14918 Aux Id: ERIERL-88 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 9.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index cb4fab51f1..42a368cdd8 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -124,7 +124,6 @@ atom big atom bif_return_trap atom bif_timer_server atom binary -atom binary_bin_to_list_trap atom binary_copy_trap atom binary_find_trap atom binary_longest_prefix_trap @@ -577,7 +576,8 @@ atom running_procs atom runtime atom safe atom save_calls -atom scheduler +atom sbct +atom scheduler atom scheduler_id atom scheduler_wall_time atom scheduler_wall_time_all diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index f086c434ea..d68ccc3028 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4805,6 +4805,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) "scheduled for removal in Erlang/OTP 18. For more\n" "information see the erlang:system_flag/2 documentation.\n"); return erts_bind_schedulers(BIF_P, BIF_ARG_2); + } else if (ERTS_IS_ATOM_STR("erts_alloc", BIF_ARG_1)) { + return erts_alloc_set_dyn_param(BIF_P, BIF_ARG_2); } error: BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index c0d5e8ce74..0d1166f6ed 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -551,9 +551,6 @@ bif binary:last/1 bif binary:at/2 bif binary:part/2 binary_binary_part_2 bif binary:part/3 binary_binary_part_3 -bif binary:bin_to_list/1 -bif binary:bin_to_list/2 -bif binary:bin_to_list/3 bif binary:list_to_bin/1 bif binary:copy/1 bif binary:copy/2 @@ -696,3 +693,4 @@ bif erlang:iolist_to_iovec/1 bif erts_internal:new_connection/1 bif erts_internal:abort_connection/2 bif erts_internal:map_next/3 +bif ets:whereis/1 diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index d4db1a4188..95d324d2c1 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -953,7 +953,10 @@ HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { if (is_binary(BIF_ARG_1)) { - BIF_RET(BIF_ARG_1); + if (binary_bitsize(BIF_ARG_1) == 0) { + BIF_RET(BIF_ARG_1); + } + BIF_ERROR(BIF_P, BADARG); } return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index e7acea0c5f..b9b70cd8ef 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -336,6 +336,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); + erts_print(to, to_arg, "BinVHeap: %b64u\n", p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap: %b64u\n", BIN_OLD_VHEAP(p)); + erts_print(to, to_arg, "BinVHeap unused: %b64u\n", + BIN_VHEAP_SZ(p) - p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap unused: %b64u\n", + BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)); erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0)); if (garbing) { @@ -891,6 +897,21 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)), erts_cbprintf(to, to_arg, "** crashed **\n")); } + for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_CPU_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_cpu_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_CPU_RUNQ); + + for (i = 0; i < erts_no_dirty_io_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_IO_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_io_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_IO_RUNQ); #endif diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index b11903a47b..94e0000c8b 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -990,6 +990,9 @@ ctx_to_bin.execute() { Uint hole_size; Uint orig = mb->orig; ErlSubBin* sb = (ErlSubBin *) boxed_val(context); + /* Since we're going to overwrite the match state with the result, an + * ErlBinMatchState must be at least as large as an ErlSubBin. */ + ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 10bf197405..7769a914db 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -611,7 +611,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint Eterm* htop; Eterm* hbot; Eterm* hp; - Eterm* objp; + Eterm* ERTS_RESTRICT objp; Eterm* tp; Eterm res; Eterm elem; @@ -1821,7 +1821,8 @@ all_clean: * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, + ErlOffHeap* off_heap) { Eterm* tp = ptr; Eterm* hp = *hpp; @@ -1985,7 +1986,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - move_boxed(&ptr, val, &hp, &dummy_ref); + ptr = move_boxed(ptr, val, &hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REF_SUBTAG: if (is_ordinary_ref_thing(hdr)) @@ -2002,7 +2003,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - move_cons(&ptr, val, &hp, &dummy_ref); + move_cons(ptr, val, &hp, &dummy_ref); ptr += 2; } } diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 88285d8be6..239dda6bcf 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -160,7 +160,7 @@ enum allctr_type { GOODFIT, BESTFIT, AFIT, - AOFIRSTFIT + FIRSTFIT }; struct au_init { @@ -500,8 +500,9 @@ set_default_test_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = 0; /* Disabled by default */ ip->thr_spec = -1 * erts_no_schedulers; - ip->atype = AOFIRSTFIT; - ip->init.aoff.flavor = AOFF_BF; + ip->atype = FIRSTFIT; + ip->init.aoff.crr_order = FF_AOFF; + ip->init.aoff.blk_order = FF_BF; ip->init.util.name_prefix = "test_"; ip->init.util.alloc_no = ERTS_ALC_A_TEST; ip->init.util.mmbcs = 0; /* Main carrier size */ @@ -599,10 +600,10 @@ static ERTS_INLINE int strategy_support_carrier_migration(struct au_init *auip) { /* - * Currently only aoff, aoffcbf and aoffcaobf support carrier + * Currently only aoff* and ageff* support carrier * migration, i.e, type AOFIRSTFIT. */ - return auip->atype == AOFIRSTFIT; + return auip->atype == FIRSTFIT; } static ERTS_INLINE void @@ -617,8 +618,9 @@ adjust_carrier_migration_support(struct au_init *auip) */ if (!strategy_support_carrier_migration(auip)) { /* Default to aoffcbf */ - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_BF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_BF; } } } @@ -1132,7 +1134,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, &init->init.af, &init->init.util); break; - case AOFIRSTFIT: + case FIRSTFIT: as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); @@ -1226,22 +1228,32 @@ get_bool_value(char *param_end, char** argv, int* ip) return -1; } +static Uint kb_to_bytes(Sint kb, Uint *bytes) +{ + const Uint max = ((~((Uint) 0))/1024) + 1; + + if (kb < 0 || (Uint)kb > max) + return 0; + if ((Uint)kb == max) + *bytes = ~((Uint) 0); + else + *bytes = ((Uint) kb)*1024; + return 1; +} + static Uint get_kb_value(char *param_end, char** argv, int* ip) { Sint tmp; - Uint max = ((~((Uint) 0))/1024) + 1; + Uint bytes = 0; char *rest; char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); errno = 0; tmp = (Sint) ErtsStrToSint(value, &rest, 10); - if (errno != 0 || rest == value || tmp < 0 || max < ((Uint) tmp)) + if (errno != 0 || rest == value || !kb_to_bytes(tmp, &bytes)) bad_value(param, param_end, value); - if (max == (Uint) tmp) - return ~((Uint) 0); - else - return ((Uint) tmp)*1024; + return bytes; } static UWord @@ -1328,18 +1340,30 @@ handle_au_arg(struct au_init *auip, switch (sub_param[0]) { case 'a': - if (has_prefix("acul", sub_param)) { - if (!auip->carrier_migration_allowed) { - if (!u_switch) - goto bad_switch; - else { - /* ignore */ - (void) get_acul_value(auip, sub_param + 4, argv, ip); - break; - } - } - auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip); - } + if (sub_param[1] == 'c') { /* Migration parameters "ac*" */ + UWord value; + UWord* wp; + if (!auip->carrier_migration_allowed && !u_switch) + goto bad_switch; + + if (has_prefix("acul", sub_param)) { + value = get_acul_value(auip, sub_param + 4, argv, ip); + wp = &auip->init.util.acul; + } + else if (has_prefix("acnl", sub_param)) { + value = get_amount_value(sub_param + 4, argv, ip); + wp = &auip->init.util.acnl; + } + else if (has_prefix("acfml", sub_param)) { + value = get_amount_value(sub_param + 5, argv, ip); + wp = &auip->init.util.acfml; + } + else + goto bad_switch; + + if (auip->carrier_migration_allowed) + *wp = value; + } else if(has_prefix("asbcst", sub_param)) { auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip); } @@ -1360,17 +1384,35 @@ handle_au_arg(struct au_init *auip, auip->atype = AFIT; } else if (strcmp("aoff", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_AOFF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_AOFF; } else if (strcmp("aoffcbf", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_BF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_BF; } else if (strcmp("aoffcaobf", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_AOBF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_AOBF; } + else if (strcmp("ageffcaoff", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_AOFF; + } + else if (strcmp("ageffcbf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_BF; + } + else if (strcmp("ageffcaobf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_AOBF; + } else { bad_value(param, sub_param + 1, alg); } @@ -3392,6 +3434,65 @@ erts_request_alloc_info(struct process *c_p, return 1; } +Eterm erts_alloc_set_dyn_param(Process* c_p, Eterm tuple) +{ + ErtsAllocatorThrSpec_t *tspec; + ErtsAlcType_t ai; + Allctr_t* allctr; + Eterm* tp; + Eterm res; + + if (!is_tuple_arity(tuple, 3)) + goto badarg; + + tp = tuple_val(tuple); + + /* + * Ex: {ets_alloc, sbct, 256000} + */ + if (!is_atom(tp[1]) || !is_atom(tp[2]) || !is_integer(tp[3])) + goto badarg; + + for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) + if (erts_is_atom_str(erts_alc_a2ad[ai], tp[1], 0)) + break; + + if (ai > ERTS_ALC_A_MAX) + goto badarg; + + if (!erts_allctrs_info[ai].enabled || + !erts_allctrs_info[ai].alloc_util) { + return am_notsup; + } + + if (tp[2] == am_sbct) { + Uint sbct; + int i, ok; + + if (!term_to_Uint(tp[3], &sbct)) + goto badarg; + + tspec = &erts_allctr_thr_spec[ai]; + if (tspec->enabled) { + ok = 0; + for (i = 0; i < tspec->size; i++) { + allctr = tspec->allctr[i]; + ok |= allctr->try_set_dyn_param(allctr, am_sbct, sbct); + } + } + else { + allctr = erts_allctrs_info[ai].extra; + ok = allctr->try_set_dyn_param(allctr, am_sbct, sbct); + } + return ok ? am_ok : am_notsup; + } + return am_notsup; + +badarg: + ERTS_BIF_PREP_ERROR(res, c_p, EXC_BADARG); + return res; +} + /* * The allocator wrapper prelocking stuff below is about the locking order. * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks @@ -3528,7 +3629,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) &init.init.af, &init.init.util); break; - case AOFIRSTFIT: + case FIRSTFIT: allctr = erts_aoffalc_start((AOFFAllctr_t *) erts_alloc(ERTS_ALC_T_UNDEF, sizeof(AOFFAllctr_t)), @@ -3622,7 +3723,9 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; - case 0xf16: { + case 0xf16: return (UWord) erts_realloc(ERTS_ALC_T_TEST, (void*)a1, (Uint)a2); + + case 0xf17: { Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; Uint offset = ts->allctr[0]->mbc_header_size; @@ -3649,7 +3752,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) *(void**)a3 = orig_destroying_mbc; return offset; } - case 0xf17: { + case 0xf18: { ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; return ts->allctr[0]->largest_mbc_size; } diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 117f96a4ad..174bf7a80a 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -126,8 +126,10 @@ typedef struct { void *extra; } ErtsAllocatorFunctions_t; -extern ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; -extern ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; +extern ErtsAllocatorFunctions_t + ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]); +extern ErtsAllocatorInfo_t + ERTS_WRITE_UNLIKELY(erts_allctrs_info[ERTS_ALC_A_MAX+1]); typedef struct { int enabled; @@ -144,7 +146,7 @@ typedef struct ErtsAllocatorWrapper_t_ { void (*unlock)(void); struct ErtsAllocatorWrapper_t_* next; }ErtsAllocatorWrapper_t; -ErtsAllocatorWrapper_t *erts_allctr_wrappers; +extern ErtsAllocatorWrapper_t *erts_allctr_wrappers; extern int erts_allctr_wrapper_prelocked; extern erts_tsd_key_t erts_allctr_prelock_tsd_key; void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper); @@ -169,6 +171,8 @@ __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint) __decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...) __noreturn; +Eterm erts_alloc_set_dyn_param(struct process*, Eterm); + #undef ERTS_HAVE_IS_IN_LITERAL_RANGE #if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) # define ERTS_HAVE_IS_IN_LITERAL_RANGE diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 4d4bddb93f..fa97ead908 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -362,8 +362,10 @@ do { \ #define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) #define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) +#define ERTS_CRR_ALCTR_FLG_HOMECOMING (((erts_aint_t) 1) << 2) #define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \ - ERTS_CRR_ALCTR_FLG_BUSY) + ERTS_CRR_ALCTR_FLG_BUSY | \ + ERTS_CRR_ALCTR_FLG_HOMECOMING) #define SBC_HEADER_SIZE \ (UNIT_CEILING(offsetof(Carrier_t, cpool) \ @@ -563,7 +565,7 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_CPOOL_INSERT(AP, CRR) \ +#define STAT_MBC_ABANDON(AP, CRR) \ do { \ UWord csz__ = CARRIER_SZ((CRR)); \ if (IS_MSEG_CARRIER((CRR))) \ @@ -1153,89 +1155,23 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) ASSERT(crr->next); crr->next->prev = crr->prev; } -} - - #ifdef DEBUG -static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* p; - - ASSERT(node != sentinel); - for (p = sentinel->next; p != sentinel; p = p->next) { - if (p == node) - return 1; - } - return 0; -} -#endif /* DEBUG */ - -static ERTS_INLINE void -link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* before_me = after_me->next; - ASSERT(node != after_me && node != before_me); - node->next = before_me; - node->prev = after_me; - before_me->prev = node; - after_me->next = node; -} - -static ERTS_INLINE void -link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* after_me = before_me->prev; - ASSERT(node != before_me && node != after_me); - node->next = before_me; - node->prev = after_me; - before_me->prev = node; - after_me->next = node; -} - -static ERTS_INLINE void -unlink_edl(ErtsDoubleLink_t* node) -{ - node->next->prev = node->prev; - node->prev->next = node->next; -} - -static ERTS_INLINE void -relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) -{ - if (node != before_me && node != before_me->prev) { - unlink_edl(node); - link_edl_before(before_me, node); - } + crr->next = crr; + crr->prev = crr; +#endif } static ERTS_INLINE int is_abandoned(Carrier_t *crr) { - return crr->cpool.abandoned.next != NULL; -} - -static ERTS_INLINE void -link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr) -{ - ASSERT(!is_abandoned(crr)); - - link_edl_after(list, &crr->cpool.abandoned); - - ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned); - ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned); + return crr->cpool.state != ERTS_MBC_IS_HOME; } static ERTS_INLINE void unlink_abandoned_carrier(Carrier_t *crr) { - ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list, - &crr->cpool.abandoned) || - is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list, - &crr->cpool.abandoned)); - - unlink_edl(&crr->cpool.abandoned); - - crr->cpool.abandoned.next = NULL; - crr->cpool.abandoned.prev = NULL; + if (crr->cpool.state == ERTS_MBC_WAS_POOLED) { + aoff_remove_pooled_mbc(crr->cpool.orig_allctr, crr); + } } static ERTS_INLINE void @@ -1243,24 +1179,19 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) { if (crr) { erts_aint_t max_size; - erts_aint_t new_val; + erts_aint_t iallctr; max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); erts_atomic_set_nob(&crr->cpool.max_size, max_size); - new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + iallctr = erts_atomic_read_nob(&crr->allctr); + ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == ((erts_aint_t)allctr | + ERTS_CRR_ALCTR_FLG_IN_POOL | + ERTS_CRR_ALCTR_FLG_BUSY)); -#ifdef ERTS_ALC_CPOOL_DEBUG - { - erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; - - ERTS_ALC_CPOOL_ASSERT(old_val - == erts_atomic_xchg_relb(&crr->allctr, - new_val)); - } -#else - erts_atomic_set_relb(&crr->allctr, new_val); -#endif + iallctr &= ~ERTS_CRR_ALCTR_FLG_BUSY; + erts_atomic_set_relb(&crr->allctr, iallctr); } } @@ -1658,6 +1589,11 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) } +static void set_new_allctr_abandon_limit(Allctr_t*); +static void abandon_carrier(Allctr_t*, Carrier_t*); +static void poolify_my_carrier(Allctr_t*, Carrier_t*); +static void enqueue_homecoming(Allctr_t*, Carrier_t*); + static ERTS_INLINE Allctr_t* get_pref_allctr(void *extra) { @@ -1724,9 +1660,23 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_aint_t act; ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); - act = erts_atomic_cmpxchg_ddrb(&crr->allctr, - iallctr|ERTS_CRR_ALCTR_FLG_BUSY, - iallctr); + if (iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING) { + /* + * This carrier has just been given back to us by writing + * to crr->allctr with a write barrier (see abandon_carrier). + * + * We need a mathing read barrier to guarantee a correct view + * of the carrier for deallocation work. + */ + act = erts_atomic_cmpxchg_rb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + } + else { + act = erts_atomic_cmpxchg_ddrb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + } if (act == iallctr) { *busy_pcrr_pp = crr; break; @@ -1742,13 +1692,6 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_mtx_unlock(&pref_allctr->mutex); } } - - ERTS_ALC_CPOOL_ASSERT( - (((iallctr & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t) pref_allctr) - ? (((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) - || ((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == 0)) - : 1)); - return used_allctr; } } @@ -2000,9 +1943,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) /* Carrier migrated; need to redirect block to new owner... */ int cinit = used_allctr->dd.ix - allctr->dd.ix; - ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - DEC_CC(allctr->calls.this_free); + DEC_CC(allctr->calls.this_free); ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) @@ -2011,8 +1954,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) } } -static void -schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr); +static void schedule_dealloc_carrier(Allctr_t*, Carrier_t*); +static void dealloc_my_carrier(Allctr_t*, Carrier_t*); + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, @@ -2074,39 +2018,61 @@ handle_delayed_dealloc(Allctr_t *allctr, res = 1; blk = UMEM2BLK(ptr); - if (IS_FREE_LAST_MBC_BLK(blk)) { + if (blk->bhdr == HOMECOMING_MBC_BLK_HDR) { /* * A multiblock carrier that previously has been migrated away - * from us and now is back to be deallocated. For more info - * see schedule_dealloc_carrier(). - * - * Note that we cannot use FBLK_TO_MBC(blk) since it - * data has been overwritten by the queue. + * from us, was sent back to us either because + * - it became empty and we need to deallocated it, or + * - it was inserted into the pool and we need to update our pooled_tree */ - Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); - - /* Restore word overwritten by the dd-queue as it will be read - * if this carrier is pulled from dc_list by cpool_fetch() - */ - ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr); - ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); -#ifdef MBC_ABLK_OFFSET_BITS - blk->u.carrier = crr; -#else - blk->carrier = crr; -#endif + Carrier_t *crr = ErtsContainerStruct(blk, Carrier_t, + cpool.homecoming_dd.blk); + Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr); + erts_aint_t iallctr; ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); - ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - != (erts_atomic_read_nob(&crr->allctr) - & ~ERTS_CRR_ALCTR_FLG_MASK)); - - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); - schedule_dealloc_carrier(allctr, crr); + iallctr = erts_atomic_read_nob(&crr->allctr); + ASSERT(iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING); + while (1) { + if ((iallctr & (~ERTS_CRR_ALCTR_FLG_MASK | + ERTS_CRR_ALCTR_FLG_IN_POOL)) + == (erts_aint_t)allctr) { + /* + * Carrier is home (mine and not in pool) + */ + ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + erts_atomic_set_nob(&crr->allctr, (erts_aint_t)allctr); + if (IS_FREE_LAST_MBC_BLK(first_blk)) + dealloc_my_carrier(allctr, crr); + else + ASSERT(crr->cpool.state == ERTS_MBC_IS_HOME); + } + else { + erts_aint_t exp = iallctr; + erts_aint_t want = iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING; + + iallctr = erts_atomic_cmpxchg_nob(&crr->allctr, + want, + exp); + if (iallctr != exp) + continue; /* retry */ + + ASSERT(crr->cpool.state != ERTS_MBC_IS_HOME); + unlink_abandoned_carrier(crr); + if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) + poolify_my_carrier(allctr, crr); + else + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; + } + break; + } } else { + ASSERT(IS_SBC_BLK(blk) || (ABLK_TO_MBC(blk) != + ErtsContainerStruct(blk, Carrier_t, + cpool.homecoming_dd.blk))); INC_CC(allctr->calls.this_free); @@ -2148,20 +2114,26 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, erts_alloc_notify_delayed_dealloc(allctr->ix); } - -static void -set_new_allctr_abandon_limit(Allctr_t *allctr); -static void -abandon_carrier(Allctr_t *allctr, Carrier_t *crr); - +static ERTS_INLINE void +update_pooled_tree(Allctr_t *allctr, Carrier_t *crr, Uint blk_sz) +{ + if (allctr == crr->cpool.orig_allctr && crr->cpool.state == ERTS_MBC_WAS_POOLED) { + /* + * Update pooled_tree with a potentially new (larger) max_sz + */ + AOFF_RBTree_t* crr_node = &crr->cpool.pooled; + if (blk_sz > crr_node->hdr.bhdr) { + crr_node->hdr.bhdr = blk_sz; + erts_aoff_larger_max_size(crr_node); + } + } +} static ERTS_INLINE void check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) { Carrier_t *crr; - - if (busy_pcrr_pp && *busy_pcrr_pp) - return; + UWord ncrr_in_pool, largest_fblk; if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) return; @@ -2170,8 +2142,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (--allctr->cpool.check_limit_count <= 0) set_new_allctr_abandon_limit(allctr); - if (!erts_thr_progress_is_managed_thread()) - return; + ASSERT(erts_thr_progress_is_managed_thread()); if (allctr->cpool.disable_abandon) return; @@ -2179,6 +2150,9 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) return; + ncrr_in_pool = erts_atomic_read_nob(&allctr->cpool.stat.no_carriers); + if (ncrr_in_pool >= allctr->cpool.in_pool_limit) + return; crr = FBLK_TO_MBC(fblk); @@ -2189,9 +2163,14 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) return; if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID - && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) - return; + && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) + return; + + largest_fblk = allctr->largest_fblk_in_mbc(allctr, crr); + if (largest_fblk < allctr->cpool.fblk_min_limit) + return; + erts_atomic_set_nob(&crr->cpool.max_size, largest_fblk); abandon_carrier(allctr, crr); } @@ -2237,6 +2216,7 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ else { Carrier_t *busy_pcrr_p; Allctr_t *used_allctr; + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, NULL, &busy_pcrr_p); if (used_allctr == allctr) { @@ -2253,10 +2233,10 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ /* Carrier migrated; need to redirect block to new owner... */ int cinit = used_allctr->dd.ix - allctr->dd.ix; - ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - if (dec_cc_on_redirect) - DEC_CC(allctr->calls.this_free); + if (dec_cc_on_redirect) + DEC_CC(allctr->calls.this_free); if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(used_allctr->ix); } @@ -2500,15 +2480,16 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(IS_MBC_BLK(blk)); - if (is_first_blk - && is_last_blk - && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { - destroy_carrier(allctr, blk, busy_pcrr_pp); + if (is_first_blk && is_last_blk && crr != allctr->main_carrier) { + destroy_carrier(allctr, blk, busy_pcrr_pp); } else { (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - check_abandon_carrier(allctr, blk, busy_pcrr_pp); + if (busy_pcrr_pp && *busy_pcrr_pp) + update_pooled_tree(allctr, crr, blk_sz); + else + check_abandon_carrier(allctr, blk, busy_pcrr_pp); } } @@ -2542,8 +2523,19 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ - if (busy_pcrr_pp && *busy_pcrr_pp) - goto realloc_move; /* Don't want to use carrier in pool */ + if (busy_pcrr_pp && *busy_pcrr_pp) { + /* + * Don't want to use carrier in pool + */ + new_p = mbc_alloc(allctr, size); + if (!new_p) + return NULL; + new_blk = UMEM2BLK(new_p); + ASSERT(!(IS_MBC_BLK(new_blk) && ABLK_TO_MBC(new_blk) == *busy_pcrr_pp)); + sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); + mbc_free(allctr, p, busy_pcrr_pp); + return new_p; + } get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); @@ -2776,7 +2768,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ - realloc_move: + #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); @@ -2880,8 +2872,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 -#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 -#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 100 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 @@ -3045,19 +3036,18 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) ErtsAlcCPoolData_t *cpd1p, *cpd2p; erts_aint_t val; ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + Allctr_t *orig_allctr = crr->cpool.orig_allctr; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); - ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) - == (erts_aint_t) allctr); - erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size, (erts_aint_t) crr->cpool.blocks_size); - erts_atomic_add_nob(&allctr->cpool.stat.no_blocks, + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks, (erts_aint_t) crr->cpool.blocks); - erts_atomic_add_nob(&allctr->cpool.stat.carriers_size, + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, (erts_aint_t) CARRIER_SZ(crr)); - erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); + erts_atomic_inc_nob(&orig_allctr->cpool.stat.no_carriers); /* * We search in 'next' direction and begin by passing @@ -3118,8 +3108,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) (erts_aint_t) &crr->cpool, (erts_aint_t) cpd1p); - erts_atomic_set_wb(&crr->allctr, - ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr)); } @@ -3221,130 +3209,126 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) static Carrier_t * cpool_fetch(Allctr_t *allctr, UWord size) { - int i, i_stop, has_passed_sentinel; + enum { IGNORANT, HAS_SEEN_SENTINEL, THE_LAST_ONE } loop_state; + int i; Carrier_t *crr; + Carrier_t *reinsert_crr = NULL; ErtsAlcCPoolData_t *cpdp; - ErtsAlcCPoolData_t *cpool_entrance; + ErtsAlcCPoolData_t *cpool_entrance = NULL; ErtsAlcCPoolData_t *sentinel; - ErtsDoubleLink_t* dl; - ErtsDoubleLink_t* first_old_traitor; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; - first_old_traitor = allctr->cpool.traitor_list.next; - cpool_entrance = NULL; LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size); /* - * Search my own pooled_list, + * Search my own pooled_tree, * i.e my abandoned carriers that were in the pool last time I checked. */ + do { + erts_aint_t exp, act; + + crr = aoff_lookup_pooled_mbc(allctr, size); + if (!crr) + break; + + ASSERT(crr->cpool.state == ERTS_MBC_WAS_POOLED); + ASSERT(crr->cpool.orig_allctr == allctr); + + aoff_remove_pooled_mbc(allctr, crr); + + exp = erts_atomic_read_nob(&crr->allctr); + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + ASSERT((exp & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t)allctr); + if (erts_atomic_read_nob(&crr->cpool.max_size) < size) { + /* + * This carrier has been fetched and inserted back again + * by a foreign allocator. That's why it has a stale search size. + */ + ASSERT(exp & ERTS_CRR_ALCTR_FLG_HOMECOMING); + crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size); + aoff_add_pooled_mbc(allctr, crr); + INC_CC(allctr->cpool.stat.skip_size); + continue; + } + else if (exp & ERTS_CRR_ALCTR_FLG_BUSY) { + /* + * This must be our own carrier as part of a realloc call. + * Skip it to make things simpler. + * Must wait to re-insert to not be found again by lookup. + */ + ASSERT(!reinsert_crr); + reinsert_crr = crr; + INC_CC(allctr->cpool.stat.skip_busy); + continue; + } - dl = allctr->cpool.pooled_list.next; - while(dl != &allctr->cpool.pooled_list) { - erts_aint_t exp, act; - crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); - - ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); - ASSERT(crr->cpool.orig_allctr == allctr); - dl = dl->next; - exp = erts_atomic_read_rb(&crr->allctr); - if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL - && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); - if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); - unlink_abandoned_carrier(crr); - - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.pooled_list); - return crr; - } - exp = act; - } - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!cpool_entrance) - cpool_entrance = &crr->cpool; - } - else { /* Not in pool, move to traitor_list */ - unlink_abandoned_carrier(crr); - link_abandoned_carrier(&allctr->cpool.traitor_list, crr); - } - if (--i <= 0) { - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.pooled_list); - return NULL; - } - } + /* Try to fetch it... */ + act = erts_atomic_cmpxchg_mb(&crr->allctr, + exp & ~ERTS_CRR_ALCTR_FLG_IN_POOL, + exp); + if (act == exp) { + cpool_delete(allctr, allctr, crr); + crr->cpool.state = ERTS_MBC_IS_HOME; + + if (reinsert_crr) + aoff_add_pooled_mbc(allctr, reinsert_crr); + return crr; + } + exp = act; + INC_CC(allctr->cpool.stat.skip_race); + } + else + INC_CC(allctr->cpool.stat.skip_not_pooled); - /* Now search traitor_list. - * i.e carriers employed by other allocators last time I checked. - * They might have been abandoned since then. - */ + /* Not in pool anymore */ + ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY)); + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; - i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ? - 0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT); - dl = first_old_traitor; - while(dl != &allctr->cpool.traitor_list) { - erts_aint_t exp, act; - crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); - ASSERT(dl != &allctr->cpool.pooled_list); - ASSERT(crr->cpool.orig_allctr == allctr); - dl = dl->next; - exp = erts_atomic_read_rb(&crr->allctr); - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) - && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); - if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); - unlink_abandoned_carrier(crr); + }while (--i > 0); - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.traitor_list); - return crr; - } - exp = act; - } - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!cpool_entrance) - cpool_entrance = &crr->cpool; + if (reinsert_crr) + aoff_add_pooled_mbc(allctr, reinsert_crr); - /* Move to pooled_list */ - unlink_abandoned_carrier(crr); - link_abandoned_carrier(&allctr->cpool.pooled_list, crr); - } - } - if (--i <= i_stop) { - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.traitor_list); - if (i > 0) - break; - else - return NULL; - } + /* + * Try find a nice cpool_entrance + */ + while (allctr->cpool.pooled_tree) { + erts_aint_t iallctr; + + crr = ErtsContainerStruct(allctr->cpool.pooled_tree, Carrier_t, cpool.pooled); + iallctr = erts_atomic_read_nob(&crr->allctr); + if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) { + cpool_entrance = &crr->cpool; + break; + } + /* Not in pool anymore */ + ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + aoff_remove_pooled_mbc(allctr, crr); + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; + + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_pooled); + return NULL; + } } + /* * Finally search the shared pool and try employ foreign carriers */ - sentinel = &carrier_pool[allctr->alloc_no].sentinel; if (cpool_entrance) { - /* We saw a pooled carried above, use it as entrance into the pool + /* + * We saw a pooled carried above, use it as entrance into the pool */ cpdp = cpool_entrance; } else { - /* No pooled carried seen above. Start search at cpool sentinel, + /* + * No pooled carried seen above. Start search at cpool sentinel, * but begin by passing one element before trying to fetch. * This in order to avoid contention with threads inserting elements. */ @@ -3354,8 +3338,8 @@ cpool_fetch(Allctr_t *allctr, UWord size) goto check_dc_list; } - has_passed_sentinel = 0; - while (1) { + loop_state = IGNORANT; + do { erts_aint_t exp; cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == cpool_entrance) { @@ -3364,38 +3348,52 @@ cpool_fetch(Allctr_t *allctr, UWord size) if (cpdp == sentinel) break; } - i = 0; /* Last one to inspect */ + loop_state = THE_LAST_ONE; } else if (cpdp == sentinel) { - if (has_passed_sentinel) { + if (loop_state == HAS_SEEN_SENTINEL) { /* We been here before. cpool_entrance must have been removed */ + INC_CC(allctr->cpool.stat.entrance_removed); break; } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) break; - has_passed_sentinel = 1; + loop_state = HAS_SEEN_SENTINEL; } - crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); + crr = ErtsContainerStruct(cpdp, Carrier_t, cpool); exp = erts_atomic_read_rb(&crr->allctr); - if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) - && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { + + if (erts_atomic_read_nob(&cpdp->max_size) < size) { + INC_CC(allctr->cpool.stat.skip_size); + } + else if ((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL | ERTS_CRR_ALCTR_FLG_BUSY)) + == ERTS_CRR_ALCTR_FLG_IN_POOL) { erts_aint_t act; - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); + erts_aint_t want = (((erts_aint_t) allctr) + | (exp & ERTS_CRR_ALCTR_FLG_HOMECOMING)); + /* Try to fetch it... */ + act = erts_atomic_cmpxchg_mb(&crr->allctr, want, exp); if (act == exp) { cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); if (crr->cpool.orig_allctr == allctr) { unlink_abandoned_carrier(crr); - } + crr->cpool.state = ERTS_MBC_IS_HOME; + } return crr; } } - if (--i <= 0) + + if (exp & ERTS_CRR_ALCTR_FLG_BUSY) + INC_CC(allctr->cpool.stat.skip_busy); + else + INC_CC(allctr->cpool.stat.skip_race); + + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_shared); return NULL; - } + } + }while (loop_state != THE_LAST_ONE); check_dc_list: /* Last; check our own pending dealloc carrier list... */ @@ -3404,23 +3402,23 @@ check_dc_list: if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { Block_t* blk; unlink_carrier(&allctr->cpool.dc_list, crr); -#ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT(erts_atomic_xchg_nob(&crr->allctr, - ((erts_aint_t) allctr)) - == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); -#else - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); -#endif + ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) + == ((erts_aint_t) allctr)); blk = MBC_TO_FIRST_BLK(allctr, crr); ASSERT(FBLK_TO_MBC(blk) == crr); allctr->link_free_block(allctr, blk); return crr; } crr = crr->prev; - if (--i <= 0) + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_pend_dealloc); return NULL; + } } + if (i != ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) + INC_CC(allctr->cpool.stat.fail); + return NULL; } @@ -3475,9 +3473,6 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; - Block_t *blk; - int check_pending_dealloc; - erts_aint_t max_size; ASSERT(IS_MB_CARRIER(crr)); @@ -3488,9 +3483,17 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) orig_allctr = crr->cpool.orig_allctr; - if (allctr != orig_allctr) { - int cinit = orig_allctr->dd.ix - allctr->dd.ix; - + if (allctr == orig_allctr) { + if (!(erts_atomic_read_nob(&crr->allctr) & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + dealloc_my_carrier(allctr, crr); + } + /*else + * Carrier was abandoned earlier by other thread and + * is still waiting for us in dd-queue. + * handle_delayed_dealloc() will handle it when crr is dequeued. + */ + } + else { /* * We send the carrier to its origin for deallocation. * This in order: @@ -3499,29 +3502,39 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * - to ensure that we always only reuse empty carriers * originating from our own thread specific mseg_alloc * instance which is beneficial on NUMA systems. - * - * The receiver will recognize that this is a carrier to - * deallocate (and not a block which is the common case) - * since the block is an mbc block that is free and last - * in the carrier. */ - blk = MBC_TO_FIRST_BLK(allctr, crr); - ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); - - ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); - ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - == (erts_atomic_read_nob(&crr->allctr) - & ~ERTS_CRR_ALCTR_FLG_MASK)); + erts_aint_t iallctr; +#ifdef ERTS_ALC_CPOOL_DEBUG + Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr); + ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(first_blk)); + + ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, first_blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(first_blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, first_blk)); + ERTS_ALC_CPOOL_ASSERT((erts_atomic_read_nob(&crr->allctr) + & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == (erts_aint_t) allctr); +#endif - if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) - erts_alloc_notify_delayed_dealloc(orig_allctr->ix); - return; + iallctr = (erts_aint_t)orig_allctr | ERTS_CRR_ALCTR_FLG_HOMECOMING; + if (!(erts_atomic_xchg_nob(&crr->allctr, iallctr) + & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + enqueue_homecoming(allctr, crr); + } } +} + +static void dealloc_my_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + Block_t *blk; + int check_pending_dealloc; + erts_aint_t max_size; - if (is_abandoned(crr)) - unlink_abandoned_carrier(crr); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + if (is_abandoned(crr)) { + unlink_abandoned_carrier(crr); + crr->cpool.state = ERTS_MBC_IS_HOME; + } if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { @@ -3553,6 +3566,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) static ERTS_INLINE void cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) { + crr->cpool.homecoming_dd.blk.bhdr = HOMECOMING_MBC_BLK_HDR; erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL); erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL); crr->cpool.orig_allctr = allctr; @@ -3571,8 +3585,7 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) limit = (csz/100)*allctr->cpool.util_limit; crr->cpool.abandon_limit = limit; } - crr->cpool.abandoned.next = NULL; - crr->cpool.abandoned.prev = NULL; + crr->cpool.state = ERTS_MBC_IS_HOME; } static void @@ -3598,23 +3611,62 @@ set_new_allctr_abandon_limit(Allctr_t *allctr) static void abandon_carrier(Allctr_t *allctr, Carrier_t *crr) { - erts_aint_t max_size; + erts_aint_t iallctr; - STAT_MBC_CPOOL_INSERT(allctr, crr); + STAT_MBC_ABANDON(allctr, crr); unlink_carrier(&allctr->mbc_list, crr); - if (crr->cpool.orig_allctr == allctr) { - link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + allctr->remove_mbc(allctr, crr); + set_new_allctr_abandon_limit(allctr); + + cpool_insert(allctr, crr); + + + iallctr = erts_atomic_read_nob(&crr->allctr); + if (allctr == crr->cpool.orig_allctr) { + /* preserve HOMECOMING flag */ + ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr); + erts_atomic_set_wb(&crr->allctr, iallctr | ERTS_CRR_ALCTR_FLG_IN_POOL); + poolify_my_carrier(allctr, crr); } + else { + ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr); + iallctr = ((erts_aint_t)crr->cpool.orig_allctr | + ERTS_CRR_ALCTR_FLG_HOMECOMING | + ERTS_CRR_ALCTR_FLG_IN_POOL); + if (!(erts_atomic_xchg_wb(&crr->allctr, iallctr) + & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + + enqueue_homecoming(allctr, crr); + } + } +} - allctr->remove_mbc(allctr, crr); +static void +enqueue_homecoming(Allctr_t* allctr, Carrier_t* crr) +{ + Allctr_t* orig_allctr = crr->cpool.orig_allctr; + const int cinit = orig_allctr->dd.ix - allctr->dd.ix; + Block_t* dd_blk = &crr->cpool.homecoming_dd.blk; - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); - erts_atomic_set_nob(&crr->cpool.max_size, max_size); + /* + * The receiver will recognize this as a carrier + * (and not a block which is the common case) + * since the block header is HOMECOMING_MBC_BLK_HDR. + */ + ASSERT(dd_blk->bhdr == HOMECOMING_MBC_BLK_HDR); + if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(dd_blk), cinit)) + erts_alloc_notify_delayed_dealloc(orig_allctr->ix); +} - cpool_insert(allctr, crr); +static void +poolify_my_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); - set_new_allctr_abandon_limit(allctr); + crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size); + aoff_add_pooled_mbc(allctr, crr); + crr->cpool.state = ERTS_MBC_WAS_POOLED; } static void @@ -3771,6 +3823,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) crr = cpool_fetch(allctr, blk_sz); if (crr) { STAT_MBC_CPOOL_FETCH(allctr, crr); + INC_CC(allctr->cpool.stat.fetch); link_carrier(&allctr->mbc_list, crr); (*allctr->add_mbc)(allctr, crr); blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0); @@ -4128,13 +4181,18 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) #endif if (busy_pcrr_pp && *busy_pcrr_pp) { + erts_aint_t iallctr = erts_atomic_read_nob(&crr->allctr); ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); - *busy_pcrr_pp = NULL; - ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) - == (((erts_aint_t) allctr) - | ERTS_CRR_ALCTR_FLG_IN_POOL - | ERTS_CRR_ALCTR_FLG_BUSY)); - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == (((erts_aint_t) allctr) + | ERTS_CRR_ALCTR_FLG_IN_POOL + | ERTS_CRR_ALCTR_FLG_BUSY)); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + + *busy_pcrr_pp = NULL; + erts_atomic_set_nob(&crr->allctr, + (iallctr & ~(ERTS_CRR_ALCTR_FLG_IN_POOL | + ERTS_CRR_ALCTR_FLG_BUSY))); cpool_delete(allctr, allctr, crr); } else @@ -4184,7 +4242,6 @@ static struct { Eterm e; Eterm t; Eterm ramv; - Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; Eterm rsbcst; @@ -4201,6 +4258,8 @@ static struct { Eterm smbcs; Eterm mbcgs; Eterm acul; + Eterm acnl; + Eterm acfml; #if HAVE_ERTS_MSEG Eterm mmc; @@ -4212,6 +4271,17 @@ static struct { Eterm mbcs; Eterm mbcs_pool; + Eterm fetch; + Eterm fail_pooled; + Eterm fail_shared; + Eterm fail_pend_dealloc; + Eterm fail; + Eterm skip_size; + Eterm skip_busy; + Eterm skip_not_pooled; + Eterm skip_homecoming; + Eterm skip_race; + Eterm entrance_removed; Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -4272,7 +4342,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); - AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); AM_INIT(rsbcst); @@ -4289,6 +4358,8 @@ init_atoms(Allctr_t *allctr) AM_INIT(smbcs); AM_INIT(mbcgs); AM_INIT(acul); + AM_INIT(acnl); + AM_INIT(acfml); #if HAVE_ERTS_MSEG AM_INIT(mmc); @@ -4300,6 +4371,17 @@ init_atoms(Allctr_t *allctr) AM_INIT(mbcs); AM_INIT(mbcs_pool); + AM_INIT(fetch); + AM_INIT(fail_pooled); + AM_INIT(fail_shared); + AM_INIT(fail_pend_dealloc); + AM_INIT(fail); + AM_INIT(skip_size); + AM_INIT(skip_busy); + AM_INIT(skip_not_pooled); + AM_INIT(skip_homecoming); + AM_INIT(skip_race); + AM_INIT(entrance_removed); AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -4583,9 +4665,56 @@ info_cpool(Allctr_t *allctr, if (hpp || szp) { res = NIL; + + if (!sz_only) { + add_3tup(hpp, szp, &res, am.fail_pooled, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pooled)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pooled))); + + add_3tup(hpp, szp, &res, am.fail_shared, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_shared)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_shared))); + + add_3tup(hpp, szp, &res, am.fail_pend_dealloc, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pend_dealloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pend_dealloc))); + + add_3tup(hpp, szp, &res, am.fail, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail))); + + add_3tup(hpp, szp, &res, am.fetch, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fetch)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fetch))); + + add_3tup(hpp, szp, &res, am.skip_size, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_size)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_size))); + + add_3tup(hpp, szp, &res, am.skip_busy, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_busy)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_busy))); + + add_3tup(hpp, szp, &res, am.skip_not_pooled, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_not_pooled)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_not_pooled))); + + add_3tup(hpp, szp, &res, am.skip_homecoming, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_homecoming)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_homecoming))); + + add_3tup(hpp, szp, &res, am.skip_race, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_race)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_race))); + + add_3tup(hpp, szp, &res, am.entrance_removed, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.entrance_removed)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.entrance_removed))); + add_2tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, csz)); + } if (!sz_only) add_2tup(hpp, szp, &res, am.carriers, @@ -4844,7 +4973,7 @@ info_options(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - int acul; + UWord acul, acnl, acfml; if (!allctr) { if (print_to_p) @@ -4857,6 +4986,8 @@ info_options(Allctr_t *allctr, } acul = allctr->cpool.util_limit; + acnl = allctr->cpool.in_pool_limit; + acfml = allctr->cpool.fblk_min_limit; if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ @@ -4884,7 +5015,7 @@ info_options(Allctr_t *allctr, "option lmbcs: %beu\n" "option smbcs: %beu\n" "option mbcgs: %beu\n" - "option acul: %d\n", + "option acul: %bpu\n", topt, allctr->ramv ? "true" : "false", allctr->sbc_threshold, @@ -4909,9 +5040,15 @@ info_options(Allctr_t *allctr, hpp, szp); if (hpp || szp) { + add_2tup(hpp, szp, &res, + am.acfml, + bld_uint(hpp, szp, acfml)); + add_2tup(hpp, szp, &res, + am.acnl, + bld_uint(hpp, szp, acnl)); add_2tup(hpp, szp, &res, am.acul, - bld_uint(hpp, szp, (UWord) acul)); + bld_uint(hpp, szp, acul)); add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); @@ -4947,7 +5084,7 @@ info_options(Allctr_t *allctr, bld_uint(hpp, szp, allctr->mseg_opt.abs_shrink_th)); #endif add_2tup(hpp, szp, &res, - am.sbct, + am_sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); @@ -5481,12 +5618,13 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) pref_allctr = get_pref_allctr(extra); used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED, p, NULL, &busy_pcrr_p); - if (pref_allctr != used_allctr) + if (pref_allctr != used_allctr) { enqueue_dealloc_other_instance(type, - used_allctr, - p, - (used_allctr->dd.ix - - pref_allctr->dd.ix)); + used_allctr, + p, + (used_allctr->dd.ix + - pref_allctr->dd.ix)); + } else { ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); @@ -5854,6 +5992,37 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, +static Uint adjust_sbct(Allctr_t* allctr, Uint sbct) +{ +#ifndef ARCH_64 + if (sbct > 0) { + Uint max_mbc_block_sz = UNIT_CEILING(sbct - 1 + ABLK_HDR_SZ); + if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK + || max_mbc_block_sz < sbct) { /* wrap around */ + /* + * By limiting sbc_threshold to (hard limit - min_block_size) + * we avoid having to split off free "residue blocks" + * smaller than min_block_size. + */ + max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); + sbct = max_mbc_block_sz - ABLK_HDR_SZ + 1; + } + } +#endif + return sbct; +} + +int erts_alcu_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value) +{ + const Uint MIN_DYN_SBCT = 4000; /* a lame catastrophe prevention */ + + if (param == am_sbct && value >= MIN_DYN_SBCT) { + allctr->sbc_threshold = adjust_sbct(allctr, value); + return 1; + } + return 0; +} + /* ------------------------------------------------------------------------- */ int @@ -5941,10 +6110,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = sz; } - allctr->cpool.pooled_list.next = &allctr->cpool.pooled_list; - allctr->cpool.pooled_list.prev = &allctr->cpool.pooled_list; - allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list; - allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list; + allctr->cpool.pooled_tree = NULL; allctr->cpool.dc_list.first = NULL; allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; @@ -5954,24 +6120,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; - allctr->cpool.util_limit = init->ts ? 0 : init->acul; - - allctr->sbc_threshold = init->sbct; -#ifndef ARCH_64 - if (allctr->sbc_threshold > 0) { - Uint max_mbc_block_sz = UNIT_CEILING(allctr->sbc_threshold - 1 + ABLK_HDR_SZ); - if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK - || max_mbc_block_sz < allctr->sbc_threshold) { /* wrap around */ - /* - * By limiting sbc_threshold to (hard limit - min_block_size) - * we avoid having to split off free "residue blocks" - * smaller than min_block_size. - */ - max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); - allctr->sbc_threshold = max_mbc_block_sz - ABLK_HDR_SZ + 1; - } + if (!init->ts && init->acul && init->acnl) { + allctr->cpool.util_limit = init->acul; + allctr->cpool.in_pool_limit = init->acnl; + allctr->cpool.fblk_min_limit = init->acfml; } -#endif + else { + allctr->cpool.util_limit = 0; + allctr->cpool.in_pool_limit = 0; + allctr->cpool.fblk_min_limit = 0; + } + + allctr->sbc_threshold = adjust_sbct(allctr, init->sbct); #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) @@ -6022,6 +6182,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->sys_realloc = &erts_alcu_sys_realloc; allctr->sys_dealloc = &erts_alcu_sys_dealloc; } + + allctr->try_set_dyn_param = &erts_alcu_try_set_dyn_param; + #if HAVE_ERTS_MSEG if (init->mseg_alloc) { ASSERT(init->mseg_realloc && init->mseg_dealloc); @@ -6036,6 +6199,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mseg_realloc = &erts_alcu_mseg_realloc; allctr->mseg_dealloc = &erts_alcu_mseg_dealloc; } + /* If a custom carrier alloc function is specified, make sure it's used */ if (init->mseg_alloc && !init->sys_alloc) { allctr->crr_set_flgs = CFLG_FORCE_MSEG; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index faeb5ef368..9a6de2bb75 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -61,7 +61,9 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; - int acul; + UWord acul; + UWord acnl; + UWord acfml; void *fix; size_t *fix_type_size; @@ -116,6 +118,8 @@ typedef struct { 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -149,6 +153,8 @@ typedef struct { 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -216,6 +222,8 @@ void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe void erts_lcnt_update_allocator_locks(int enable); #endif +int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); + #endif /* !ERL_ALLOC_UTIL__ */ #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) @@ -296,41 +304,7 @@ void erts_lcnt_update_allocator_locks(int enable); typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; - -typedef struct ErtsDoubleLink_t_ { - struct ErtsDoubleLink_t_ *next; - struct ErtsDoubleLink_t_ *prev; -}ErtsDoubleLink_t; - -typedef struct { - erts_atomic_t next; - erts_atomic_t prev; - Allctr_t *orig_allctr; /* read-only while carrier is alive */ - ErtsThrPrgrVal thr_prgr; - erts_atomic_t max_size; - UWord abandon_limit; - UWord blocks; - UWord blocks_size; - ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ -} ErtsAlcCPoolData_t; - - typedef struct Carrier_t_ Carrier_t; -struct Carrier_t_ { - UWord chdr; - Carrier_t *next; - Carrier_t *prev; - erts_atomic_t allctr; - ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ -}; - -#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ - ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) - -typedef struct { - Carrier_t *first; - Carrier_t *last; -} CarrierList_t; typedef struct { UWord bhdr; @@ -344,6 +318,22 @@ typedef struct { #endif } Block_t; +typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; + +union ErtsAllctrDDBlock_t_ { + erts_atomic_t atmc_next; + ErtsAllctrDDBlock_t *ptr_next; +}; + +typedef struct { + Block_t blk; +#if !MBC_ABLK_OFFSET_BITS + ErtsAllctrDDBlock_t umem_; +#endif +} ErtsFakeDDBlock_t; + + + #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) @@ -352,14 +342,13 @@ typedef struct { (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) /* - * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for - * distinguishing empty mbc's from allocated blocks in - * handle_delayed_dealloc(). + * HOMECOMING_MBC_BLK_HDR is a special block header combo used for + * distinguishing MBC's from allocated blocks in handle_delayed_dealloc(). */ -#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) +#define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) #define IS_FREE_LAST_MBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS) + (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) #define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) @@ -383,6 +372,57 @@ typedef struct { typedef UWord FreeBlkFtr_t; /* Footer of a free block */ +/* This AOFF stuff really belong in erl_ao_firstfit_alloc.h */ +typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; +struct AOFF_RBTree_t_ { + Block_t hdr; + AOFF_RBTree_t *parent; + AOFF_RBTree_t *left; + AOFF_RBTree_t *right; + Uint32 flags; + Uint32 max_sz; /* of all blocks in this sub-tree */ +}; + +void aoff_add_pooled_mbc(Allctr_t*, Carrier_t*); +void aoff_remove_pooled_mbc(Allctr_t*, Carrier_t*); +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t*, Uint size); +void erts_aoff_larger_max_size(AOFF_RBTree_t *node); + +typedef struct { + ErtsFakeDDBlock_t homecoming_dd; + erts_atomic_t next; + erts_atomic_t prev; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ + ErtsThrPrgrVal thr_prgr; + erts_atomic_t max_size; + UWord abandon_limit; + UWord blocks; + UWord blocks_size; + enum { + ERTS_MBC_IS_HOME, + ERTS_MBC_WAS_POOLED, + ERTS_MBC_WAS_TRAITOR + } state; + AOFF_RBTree_t pooled; /* node in pooled_tree */ +} ErtsAlcCPoolData_t; + +struct Carrier_t_ { + UWord chdr; + Carrier_t *next; + Carrier_t *prev; + erts_atomic_t allctr; + ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ +}; + +#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ + ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + +typedef struct { + Carrier_t *first; + Carrier_t *last; +} CarrierList_t; + + typedef Uint64 CallCounter_t; typedef struct { @@ -419,13 +459,6 @@ typedef struct { #endif -typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; - -union ErtsAllctrDDBlock_t_ { - erts_atomic_t atmc_next; - ErtsAllctrDDBlock_t *ptr_next; -}; - typedef struct { ErtsAllctrDDBlock_t marker; erts_atomic_t last; @@ -537,25 +570,37 @@ struct Allctr_t_ { UWord crr_set_flgs; UWord crr_clr_flgs; - /* Carriers */ + /* Carriers *employed* by this allocator */ CarrierList_t mbc_list; CarrierList_t sbc_list; struct { - /* pooled_list, traitor list and dc_list contain only - carriers _created_ by this allocator */ - ErtsDoubleLink_t pooled_list; - ErtsDoubleLink_t traitor_list; + /* pooled_tree and dc_list contain only + carriers *created* by this allocator */ + AOFF_RBTree_t* pooled_tree; CarrierList_t dc_list; UWord abandon_limit; int disable_abandon; int check_limit_count; - int util_limit; + UWord util_limit; /* acul */ + UWord in_pool_limit; /* acnl */ + UWord fblk_min_limit; /* acmfl */ struct { erts_atomic_t blocks_size; erts_atomic_t no_blocks; erts_atomic_t carriers_size; erts_atomic_t no_carriers; + CallCounter_t fail_pooled; + CallCounter_t fail_shared; + CallCounter_t fail_pend_dealloc; + CallCounter_t fail; + CallCounter_t fetch; + CallCounter_t skip_size; + CallCounter_t skip_busy; + CallCounter_t skip_not_pooled; + CallCounter_t skip_homecoming; + CallCounter_t skip_race; + CallCounter_t entrance_removed; } stat; } cpool; @@ -589,6 +634,8 @@ struct Allctr_t_ { void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); + int (*try_set_dyn_param)(Allctr_t*, Eterm param, Uint value); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 05ba1f9891..f8a6101b95 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -20,7 +20,7 @@ /* - * Description: An "address order first fit" allocator + * Description: A family of "first fit" allocator strategies * based on a Red-Black (binary search) Tree. The search, * insert, and delete operations are all O(log n) operations * on a Red-Black Tree. @@ -40,6 +40,10 @@ * sorting order. Blocks within the same carrier are sorted * wrt size instead of address. The 'max_sz' field is maintained * in order to dismiss entire carriers with too small blocks. + * Age Order: + * Carriers are ordered by creation time instead of address. + * Oldest carrier with a large enough free block is chosen. + * No age order supported for blocks. * * Authors: Rickard Green/Sverker Eriksson */ @@ -53,10 +57,12 @@ #include "erl_ao_firstfit_alloc.h" #ifdef DEBUG +# define IS_DEBUG 1 #if 0 #define HARD_DEBUG #endif #else +# define IS_DEBUG 0 #undef HARD_DEBUG #endif @@ -92,18 +98,6 @@ #define RBT_ASSERT(x) #endif - -/* Types... */ -typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; - -struct AOFF_RBTree_t_ { - Block_t hdr; - AOFF_RBTree_t *parent; - AOFF_RBTree_t *left; - AOFF_RBTree_t *right; - Uint32 flags; - Uint32 max_sz; /* of all blocks in this sub-tree */ -}; #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) /* BF block nodes keeps list of all with equal size @@ -121,6 +115,7 @@ typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; struct AOFF_Carrier_t_ { Carrier_t crr; AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + Sint64 birth_time; AOFF_RBTree_t* root; /* Root of my block tree */ }; #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) @@ -136,12 +131,12 @@ struct AOFF_Carrier_t_ { */ #ifdef HARD_DEBUG -# define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ) -static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint); +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) ASSERT(rbt_is_member(ROOT,NODE)) +# define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) check_tree(CRR, ORDER, ROOT, SZ) +static AOFF_RBTree_t * check_tree(Carrier_t*, enum AOFFSortOrder, AOFF_RBTree_t*, Uint); #else # define HARD_CHECK_IS_MEMBER(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) +# define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) #endif @@ -179,25 +174,63 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, else ASSERT(new_max == old_max); } -static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor, +/* + * Set possibly new larger 'max_sz' of node and propagate change toward root + */ +void erts_aoff_larger_max_size(AOFF_RBTree_t *node) +{ + AOFF_RBTree_t* x = node; + const Uint new_sz = node->hdr.bhdr; + + ASSERT(!x->left || x->left->max_sz <= x->max_sz); + ASSERT(!x->right || x->right->max_sz <= x->max_sz); + + while (new_sz > x->max_sz) { + x->max_sz = new_sz; + x = x->parent; + if (!x) + break; + } +} + +/* Compare nodes for both carrier and block trees */ +static ERTS_INLINE SWord cmp_blocks(enum AOFFSortOrder order, AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs) { ASSERT(lhs != rhs); - ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); - if (flavor != AOFF_AOFF) { - SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); - if (diff || flavor == AOFF_BF) return diff; + if (order == FF_AGEFF) { + AOFF_Carrier_t* lc = RBT_NODE_TO_MBC(lhs); + AOFF_Carrier_t* rc = RBT_NODE_TO_MBC(rhs); + Sint64 diff = lc->birth_time - rc->birth_time; + #ifdef ARCH_64 + if (diff) + return diff; + #else + if (diff < 0) + return -1; + else if (diff > 0) + return 1; + #endif + } + else { + ASSERT(order == FF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); + if (order != FF_AOFF) { + SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); + if (diff || order == FF_BF) return diff; + } } return (char*)lhs - (char*)rhs; } -static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor, +/* Compare candidate block. Only for block tree */ +static ERTS_INLINE SWord cmp_cand_blk(enum AOFFSortOrder order, Block_t* cand_blk, AOFF_RBTree_t* rhs) { - if (flavor != AOFF_AOFF) { + ASSERT(order != FF_AGEFF); + if (order != FF_AOFF) { if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) { SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr); - if (diff || flavor == AOFF_BF) return diff; + if (diff || order == FF_BF) return diff; } } return (char*)cand_blk - (char*)rhs; @@ -218,11 +251,8 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); /* Generic tree functions used by both carrier and block trees. */ static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del); -static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); +static void rbt_insert(enum AOFFSortOrder, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); -#ifdef HARD_DEBUG -static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); -#endif static Eterm info_options(Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms(void); @@ -230,10 +260,17 @@ static void init_atoms(void); static int atoms_initialized = 0; +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT +static erts_atomic64_t birth_time_counter; +#endif + void erts_aoffalc_init(void) { atoms_initialized = 0; +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + erts_atomic64_init_nob(&birth_time_counter, 0); +#endif } Allctr_t * @@ -254,11 +291,12 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); - alc->flavor = aoffinit->flavor; + alc->blk_order = aoffinit->blk_order; + alc->crr_order = aoffinit->crr_order; allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = (aoffinit->flavor == AOFF_BF ? + allctr->min_block_size = (aoffinit->blk_order == FF_BF ? sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t)); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -487,9 +525,9 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); - if (alc->flavor == AOFF_BF) { + if (alc->blk_order == FF_BF) { ASSERT(del->flags & IS_BF_FLG); if (IS_LIST_ELEM(del)) { /* Remove from list */ @@ -510,14 +548,14 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); return; } } rbt_delete(&crr->root, (AOFF_RBTree_t*)del); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); /* Update the carrier tree with a potentially new (lower) max_sz */ @@ -715,32 +753,33 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block) ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); - HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + HARD_CHECK_TREE(&blk_crr->crr, alc->blk_order, blk_crr->root, 0); - rbt_insert(alc->flavor, &blk_crr->root, blk); + rbt_insert(alc->blk_order, &blk_crr->root, blk); - /* Update the carrier tree with a potentially new (larger) max_sz - */ + /* + * Update carrier tree with a potentially new (larger) max_sz + */ crr_node = &blk_crr->rbt_node; if (blk_sz > crr_node->hdr.bhdr) { - ASSERT(blk_sz == blk_crr->root->max_sz); - crr_node->hdr.bhdr = blk_sz; - while (blk_sz > crr_node->max_sz) { - crr_node->max_sz = blk_sz; - crr_node = crr_node->parent; - if (!crr_node) break; - } + ASSERT(blk_sz == blk_crr->root->max_sz); + crr_node->hdr.bhdr = blk_sz; + while (blk_sz > crr_node->max_sz) { + crr_node->max_sz = blk_sz; + crr_node = crr_node->parent; + if (!crr_node) break; + } } - HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, alc->mbc_root, 0); } static void -rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) +rbt_insert(enum AOFFSortOrder order, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) { Uint blk_sz = AOFF_BLK_SZ(blk); #ifdef DEBUG - blk->flags = (flavor == AOFF_BF) ? IS_BF_FLG : 0; + blk->flags = (order == FF_BF) ? IS_BF_FLG : 0; #else blk->flags = 0; #endif @@ -760,7 +799,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (x->max_sz < blk_sz) { x->max_sz = blk_sz; } - diff = cmp_blocks(flavor, blk, x); + diff = cmp_blocks(order, blk, x); if (diff < 0) { if (!x->left) { blk->parent = x; @@ -778,7 +817,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) x = x->right; } else { - ASSERT(flavor == AOFF_BF); + ASSERT(order == FF_BF); ASSERT(blk->flags & IS_BF_FLG); ASSERT(x->flags & IS_BF_FLG); SET_LIST_ELEM(blk); @@ -798,7 +837,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (IS_RED(blk->parent)) tree_insert_fixup(root, blk); } - if (flavor == AOFF_BF) { + if (order == FF_BF) { SET_TREE_NODE(blk); LIST_NEXT(blk) = NULL; } @@ -826,6 +865,16 @@ rbt_search(AOFF_RBTree_t* root, Uint size) } } +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t* allctr, Uint size) +{ + AOFF_RBTree_t* node; + + if (!allctr->cpool.pooled_tree) + return NULL; + node = rbt_search(allctr->cpool.pooled_tree, size); + return node ? ErtsContainerStruct(node, Carrier_t, cpool.pooled) : NULL; +} + static Block_t * aoff_get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) @@ -850,7 +899,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, /* Get block within carrier tree */ #ifdef HARD_DEBUG - dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size); + dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, size); #endif blk = rbt_search(crr->root, size); @@ -863,7 +912,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) { + if (cand_blk && cmp_cand_blk(alc->blk_order, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } @@ -872,23 +921,32 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, return (Block_t *) blk; } +static ERTS_INLINE Sint64 get_birth_time(void) +{ +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return (Sint64) erts_os_monotonic_time(); +#else + return (Sint64) erts_atomic64_inc_read_nob(&birth_time_counter); +#endif +} + static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) { AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); - /* Link carrier in address order tree - */ crr->rbt_node.hdr.bhdr = 0; - rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + if (alc->crr_order == FF_AGEFF || IS_DEBUG) + crr->birth_time = get_birth_time(); + rbt_insert(alc->crr_order, root, &crr->rbt_node); /* aoff_link_free_block will add free block later */ crr->root = NULL; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); } #define IS_CRR_IN_TREE(CRR,ROOT) \ @@ -911,27 +969,38 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_RBTree_t **root = &alc->mbc_root; ASSERT(!IS_CRR_IN_TREE(crr, *root)); - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); + + rbt_insert(alc->crr_order, root, &crr->rbt_node); + + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); +} + +void aoff_add_pooled_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + AOFF_RBTree_t **root = &allctr->cpool.pooled_tree; + + ASSERT(allctr == crr->cpool.orig_allctr); + HARD_CHECK_TREE(NULL, 0, *root, 0); /* Link carrier in address order tree */ - rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + rbt_insert(FF_AOFF, root, &crr->cpool.pooled); HARD_CHECK_TREE(NULL, 0, *root, 0); } static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; - AOFF_RBTree_t **root = &alc->mbc_root; + AOFF_RBTree_t **root = &((AOFFAllctr_t*)allctr)->mbc_root; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*)carrier; ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); if (!IS_CRR_IN_TREE(crr,*root)) - return; + return; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); rbt_delete(root, &crr->rbt_node); crr->rbt_node.parent = NULL; @@ -939,9 +1008,27 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) crr->rbt_node.right = NULL; crr->rbt_node.max_sz = crr->rbt_node.hdr.bhdr; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); +} + +void aoff_remove_pooled_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(allctr == crr->cpool.orig_allctr); + + HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0); + + rbt_delete(&allctr->cpool.pooled_tree, &crr->cpool.pooled); +#ifdef DEBUG + crr->cpool.pooled.parent = NULL; + crr->cpool.pooled.left = NULL; + crr->cpool.pooled.right = NULL; + crr->cpool.pooled.max_sz = 0; +#endif + HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0); + } + static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) { AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; @@ -955,17 +1042,17 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) * info_options() */ +static const char* flavor_str[2][3] = { + {"ageffcaoff", "ageffcaobf", "ageffcbf"}, + { "aoff", "aoffcaobf", "aoffcbf"} +}; +static Eterm flavor_atoms[2][3]; + static struct { Eterm as; - Eterm aoff; - Eterm aoffcaobf; - Eterm aoffcbf; -#ifdef DEBUG - Eterm end_of_atoms; -#endif } am; -static void ERTS_INLINE atom_init(Eterm *atom, char *name) +static void ERTS_INLINE atom_init(Eterm *atom, const char *name) { *atom = am_atom_put(name, strlen(name)); } @@ -974,28 +1061,16 @@ static void ERTS_INLINE atom_init(Eterm *atom, char *name) static void init_atoms(void) { -#ifdef DEBUG - Eterm *atom; -#endif + int i, j; if (atoms_initialized) return; -#ifdef DEBUG - for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) { - *atom = THE_NON_VALUE; - } -#endif AM_INIT(as); - AM_INIT(aoff); - AM_INIT(aoffcaobf); - AM_INIT(aoffcbf); -#ifdef DEBUG - for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { - ASSERT(*atom != THE_NON_VALUE); - } -#endif + for (i = 0; i < 2; i++) + for (j = 0; j < 3; j++) + atom_init(&flavor_atoms[i][j], flavor_str[i][j]); atoms_initialized = 1; } @@ -1021,15 +1096,16 @@ info_options(Allctr_t *allctr, { AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; Eterm res = THE_NON_VALUE; - const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"}; - Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf}; + + ASSERT(alc->crr_order >= 0 && alc->crr_order <= 1); + ASSERT(alc->blk_order >= 1 && alc->blk_order <= 3); if (print_to_p) { erts_print(*print_to_p, print_to_arg, "%sas: %s\n", prefix, - flavor_str[alc->flavor]); + flavor_str[alc->crr_order][alc->blk_order-1]); } if (hpp || szp) { @@ -1039,7 +1115,8 @@ info_options(Allctr_t *allctr, __FILE__, __LINE__);; res = NIL; - add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]); + add_2tup(hpp, szp, &res, am.as, + flavor_atoms[alc->crr_order][alc->blk_order-1]); } return res; @@ -1057,7 +1134,7 @@ UWord erts_aoffalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF; + case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_AOBF; case 0x501: { AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; Uint size = (Uint) a2; @@ -1072,7 +1149,7 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) case 0x507: return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1); case 0x508: return (UWord) 0; /* IS_BF_ALGO */ case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz; - case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF; + case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_BF; case 0x50b: return (UWord) LIST_PREV(a1); default: ASSERT(0); return ~((UWord) 0); } @@ -1085,12 +1162,13 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) #ifdef HARD_DEBUG - -static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) +static int rbt_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) { while (node != root) { - ASSERT(node->parent); - ASSERT(node->parent->left == node || node->parent->right == node); + if (!node->parent || (node->parent->left != node && + node->parent->right != node)) { + return 0; + } node = node->parent; } return 1; @@ -1132,7 +1210,7 @@ static void print_tree(AOFF_RBTree_t*); */ static AOFF_RBTree_t * -check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size) +check_tree(Carrier_t* within_crr, enum AOFFSortOrder order, AOFF_RBTree_t* root, Uint size) { AOFF_RBTree_t *res = NULL; Sint blacks; @@ -1144,7 +1222,8 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, #ifdef PRINT_TREE print_tree(root); #endif - ASSERT(within_crr || flavor == AOFF_AOFF); + ASSERT((within_crr && order >= FF_AOFF) || + (!within_crr && order <= FF_AOFF)); if (!root) return res; @@ -1202,7 +1281,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr))); } - if (flavor == AOFF_BF) { + if (order == FF_BF) { AOFF_RBTree_t* y = x; AOFF_RBTree_t* nxt = LIST_NEXT(y); ASSERT(IS_TREE_NODE(x)); @@ -1225,13 +1304,13 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, if (x->left) { ASSERT(x->left->parent == x); - ASSERT(cmp_blocks(flavor, x->left, x) < 0); + ASSERT(cmp_blocks(order, x->left, x) < 0); ASSERT(x->left->max_sz <= x->max_sz); } if (x->right) { ASSERT(x->right->parent == x); - ASSERT(cmp_blocks(flavor, x->right, x) > 0); + ASSERT(cmp_blocks(order, x->right, x) > 0); ASSERT(x->right->max_sz <= x->max_sz); } ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); @@ -1240,7 +1319,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, || x->max_sz == (x->right ? x->right->max_sz : 0)); if (size && AOFF_BLK_SZ(x) >= size) { - if (!res || cmp_blocks(flavor, x, res) < 0) { + if (!res || cmp_blocks(order, x, res) < 0) { res = x; } } diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 7349c6ab19..9cf4fc81a8 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -28,14 +28,16 @@ typedef struct AOFFAllctr_t_ AOFFAllctr_t; -enum AOFF_Flavor { - AOFF_AOFF = 0, - AOFF_AOBF = 1, - AOFF_BF = 2 +enum AOFFSortOrder { + FF_AGEFF = 0, + FF_AOFF = 1, + FF_AOBF = 2, + FF_BF = 3 }; typedef struct { - enum AOFF_Flavor flavor; + enum AOFFSortOrder blk_order; + enum AOFFSortOrder crr_order; } AOFFAllctrInit_t; #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} @@ -53,12 +55,12 @@ Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *); #define GET_ERL_ALLOC_UTIL_IMPL #include "erl_alloc_util.h" - struct AOFFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ struct AOFF_RBTree_t_* mbc_root; - enum AOFF_Flavor flavor; + enum AOFFSortOrder blk_order; + enum AOFFSortOrder crr_order; }; UWord erts_aoffalc_test(UWord, UWord, UWord); diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 41c2ae08d3..33bc189182 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -61,8 +61,6 @@ static Export binary_longest_prefix_trap_export; static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); static Export binary_longest_suffix_trap_export; static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3); -static Export binary_bin_to_list_trap_export; -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3); static Export binary_copy_trap_export; static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; @@ -86,10 +84,6 @@ void erts_init_bif_binary(void) am_erlang, am_binary_longest_suffix_trap, 3, &binary_longest_suffix_trap); - erts_init_trap_export(&binary_bin_to_list_trap_export, - am_erlang, am_binary_bin_to_list_trap, 3, - &binary_bin_to_list_trap); - erts_init_trap_export(&binary_copy_trap_export, am_erlang, am_binary_copy_trap, 2, &binary_copy_trap); @@ -2440,191 +2434,6 @@ BIF_RETTYPE binary_at_2(BIF_ALIST_2) BIF_ERROR(BIF_P,BADARG); } -#define BIN_TO_LIST_OK 0 -#define BIN_TO_LIST_TRAP 1 -/* No badarg, checked before call */ - -#define BIN_TO_LIST_LOOP_FACTOR 10 - -static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs, - Uint start, Sint *lenp, Eterm *termp) -{ - Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */ - Uint len = *lenp; - Uint loops; - Eterm *hp; - Eterm term = *termp; - Uint n; - - ASSERT(reds > 0); - - loops = MIN(reds,len); - - BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR); - - hp = HAlloc(p,2*loops); - while (loops--) { - --len; - if (bit_offs) { - n = ((((Uint) bytes[start+len]) << bit_offs) | - (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF; - } else { - n = bytes[start+len]; - } - - term = CONS(hp,make_small(n),term); - hp +=2; - } - *termp = term; - *lenp = len; - if (len) { - BUMP_ALL_REDS(p); - return BIN_TO_LIST_TRAP; - } - return BIN_TO_LIST_OK; -} - - -static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary, - Uint start, Sint len, Eterm sofar) -{ - Eterm *hp; - Eterm blob; - - hp = HAlloc(p,3); - hp[0] = make_pos_bignum_header(2); - hp[1] = start; - hp[2] = (Uint) len; - blob = make_big(hp); - BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar); -} - -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3) -{ - Eterm *ptr; - Uint start; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = BIF_ARG_3; - - ptr = big_val(BIF_ARG_2); - start = ptr[1]; - len = (Sint) ptr[2]; - - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res); -} - -static BIF_RETTYPE binary_bin_to_list_common(Process *p, - Eterm bin, - Eterm epos, - Eterm elen) -{ - Uint pos; - Sint len; - size_t sz; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(bin)) { - goto badarg; - } - if (!term_to_Uint(epos, &pos)) { - goto badarg; - } - if (!term_to_Sint(elen, &len)) { - goto badarg; - } - if (len < 0) { - Uint lentmp = -(Uint)len; - /* overflow */ - if ((Sint)lentmp < 0) { - goto badarg; - } - len = lentmp; - if (len > pos) { - goto badarg; - } - pos -= len; - } - /* overflow */ - if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { - goto badarg; - } - sz = binary_size(bin); - - if (pos+len > sz) { - goto badarg; - } - ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(p,bin,pos,len,res); - - badarg: - BIF_ERROR(p,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3) -{ - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3); -} - -BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2) -{ - Eterm *tp; - - if (is_not_tuple(BIF_ARG_2)) { - goto badarg; - } - tp = tuple_val(BIF_ARG_2); - if (arityval(*tp) != 2) { - goto badarg; - } - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) -{ - Uint pos = 0; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(BIF_ARG_1)) { - goto badarg; - } - len = binary_size(BIF_ARG_1); - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1) BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 9f0c90ff7b..b184adedee 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -639,6 +639,27 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) BIF_RET(res); } +Eterm erts_port_data_read(Port* prt) +{ + Eterm res; + erts_aint_t data; + + data = erts_atomic_read_ddrb(&prt->data); + if (data == (erts_aint_t)NULL) + return am_undefined; /* Port terminated by racing thread */ + + if ((data & 0x3) != 0) { + res = (Eterm) (UWord) data; + ASSERT(is_immed(res)); + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + res = pdhp->data; + } + return res; +} + + /* * Open a port. Most of the work is not done here but rather in * the file io.c. diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 3ba0886464..1ab1c4a363 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1753,6 +1753,28 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_RET(ret); } +/* +** Retrieves the tid() of a named ets table. +*/ +BIF_RETTYPE ets_whereis_1(BIF_ALIST_1) +{ + DbTable* tb; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + + if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { + BIF_RET(am_undefined); + } + + res = make_tid(BIF_P, tb); + db_unlock(tb, LCK_READ); + + BIF_RET(res); +} + /* ** The lookup BIF */ @@ -3126,7 +3148,8 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed, am_write_concurrency, - am_read_concurrency}; + am_read_concurrency, + am_id}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -4016,7 +4039,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = is_table_named(tb) ? am_true : am_false; } else if (What == am_compressed) { ret = tb->common.compress ? am_true : am_false; + } else if (What == am_id) { + ret = make_tid(p, tb); } + /* * For debugging purposes */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 0da4468f9c..1c64644efc 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -116,6 +116,7 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); static Eterm *full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -142,7 +143,7 @@ static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop, static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* heap, Eterm* htop, Eterm* objv, int nobj); + Eterm* htop); static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); @@ -917,6 +918,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc) htop = heap; htop = full_sweep_heaps(p, + ERTS_INVALID_HFRAG_PTR, 1, heap, htop, @@ -1161,7 +1163,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { Eterm gval = *g_ptr; switch (primary_tag(gval)) { @@ -1170,26 +1172,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (ErtsInArea(ptr, area, area_size)) { - move_boxed(&ptr,val,&old_htop,g_ptr++); - } else { - g_ptr++; + move_boxed(ptr,val,&old_htop,g_ptr); } break; case TAG_PRIMARY_LIST: ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (ErtsInArea(ptr, area, area_size)) { - move_cons(&ptr,val,&old_htop,g_ptr++); - } else { - g_ptr++; - } + move_cons(ptr,val,&old_htop,g_ptr); + } break; default: - g_ptr++; break; } } @@ -1479,25 +1476,29 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); + n = setup_rootset(p, objv, nobj, &rootset); + roots = rootset.roots; + + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { /* * Move heap frags that we know are completely live * directly into the new young heap generation. */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); } - n = setup_rootset(p, objv, nobj, &rootset); - roots = rootset.roots; - GENSWEEP_NSTACK(p, old_htop, n_htop); while (n--) { Eterm* g_ptr = roots->v; Uint g_sz = roots->sz; roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { gval = *g_ptr; switch (primary_tag(gval)) { @@ -1507,14 +1508,12 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,g_ptr++); + move_boxed(ptr,val,&old_htop,g_ptr); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; - } + move_boxed(ptr,val,&n_htop,g_ptr); + } break; } @@ -1522,19 +1521,15 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr,val,&old_htop,g_ptr++); + move_cons(ptr,val,&old_htop,g_ptr); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_cons(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; - } + move_cons(ptr,val,&n_htop,g_ptr); + } break; } - default: - g_ptr++; break; } } @@ -1568,9 +1563,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,n_hp++); + move_boxed(ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,n_hp++); + move_boxed(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1582,9 +1577,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr,val,&old_htop,n_hp++); + move_cons(ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_cons(&ptr,val,&n_htop,n_hp++); + move_cons(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1604,10 +1599,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, *origptr = val; mb->base = binary_bytes(val); } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,origptr); + move_boxed(ptr,val,&old_htop,origptr); mb->base = binary_bytes(mb->orig); } else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,origptr); + move_boxed(ptr,val,&n_htop,origptr); mb->base = binary_bytes(mb->orig); } } @@ -1733,16 +1728,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { - /* - * Move heap frags that we know are completely live - * directly into the heap. - */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); - } - - n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj); + n_htop = full_sweep_heaps(p, live_hf_end, 0, n_heap, n_htop, oh, oh_size, + objv, nobj); /* Move the stack to the end of the heap */ stk_sz = HEAP_END(p) - p->stop; @@ -1789,6 +1776,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, static Eterm * full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -1805,6 +1793,19 @@ full_sweep_heaps(Process *p, n = setup_rootset(p, objv, nobj, &rootset); + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the heap. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); + } + #ifdef HIPE if (hibernate) hipe_empty_nstack(p); @@ -1818,7 +1819,7 @@ full_sweep_heaps(Process *p, Eterm g_sz = roots->sz; roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { Eterm* ptr; Eterm val; Eterm gval = *g_ptr; @@ -1830,32 +1831,26 @@ full_sweep_heaps(Process *p, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (!erts_is_literal(gval, ptr)) { - move_boxed(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; + move_boxed(ptr,val,&n_htop,g_ptr); } - continue; + break; } case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { - move_cons(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; + move_cons(ptr,val,&n_htop,g_ptr); } - continue; + break; } - default: { - g_ptr++; - continue; - } + default: + break; } } } @@ -2134,7 +2129,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - move_boxed(&ptr,val,&n_htop,n_hp++); + move_boxed(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2146,7 +2141,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - move_cons(&ptr,val,&n_htop,n_hp++); + move_cons(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2167,7 +2162,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) { - move_boxed(&ptr,val,&n_htop,origptr); + move_boxed(ptr,val,&n_htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2230,7 +2225,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, ASSERT(is_boxed(val)); *heap_ptr++ = val; } else if (ErtsInArea(ptr, src, src_size)) { - move_boxed(&ptr,val,&htop,heap_ptr++); + move_boxed(ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2242,7 +2237,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; } else if (ErtsInArea(ptr, src, src_size)) { - move_cons(&ptr,val,&htop,heap_ptr++); + move_cons(ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2263,7 +2258,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ErtsInArea(ptr, src, src_size)) { - move_boxed(&ptr,val,&htop,origptr); + move_boxed(ptr,val,&htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2296,11 +2291,11 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) ASSERT(val != ERTS_HOLE_MARKER); if (is_header(val)) { ASSERT(ptr + header_arity(val) < end); - move_boxed(&ptr, val, &n_htop, &dummy_ref); + ptr = move_boxed(ptr, val, &n_htop, &dummy_ref); } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - move_cons(&ptr, val, &n_htop, &dummy_ref); + move_cons(ptr, val, &n_htop, &dummy_ref); ptr += 2; } } @@ -2313,9 +2308,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) */ static Eterm* -collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* n_hstart, Eterm* n_htop, - Eterm* objv, int nobj) +collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop) { ErlHeapFragment* qb; char* frag_begin; @@ -3281,8 +3274,7 @@ reply_gc_info(void *vgcirp) gcireq_free(vgcirp); } -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { - Eterm *ptr = *pp; +Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig) { Eterm *htop = *hpp; Eterm gval; ErlSubBin *sb = (ErlSubBin *)ptr; @@ -3310,7 +3302,7 @@ void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { htop += heap_bin_size(sb->size); *hpp = htop; - *pp = ptr; + return ptr; } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 6a529b8443..dec0ab1143 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -33,14 +33,15 @@ #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig); +Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig); -ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp, + Eterm *orig); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) +ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp, + Eterm *orig) { - Eterm *ptr = *pp; - Eterm *htop = *hpp; + Eterm *ERTS_RESTRICT htop = *hpp; Eterm gval; htop[0] = car; /* copy car */ @@ -53,14 +54,15 @@ ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) } #endif -ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp, + Eterm *orig); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) +ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp, + Eterm *orig) { Eterm gval; Sint nelts; - Eterm *ptr = *pp; - Eterm *htop = *hpp; + Eterm *ERTS_RESTRICT htop = *hpp; ASSERT(is_header(hdr)); nelts = header_arity(hdr); @@ -71,8 +73,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) /* convert sub-binary to heap-binary if applicable */ if (sb->bitsize == 0 && sb->bitoffs == 0 && sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) { - erts_sub_binary_to_heap_binary(pp, hpp, orig); - return; + return erts_sub_binary_to_heap_binary(ptr, hpp, orig); } } nelts++; @@ -90,7 +91,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) while (nelts--) *htop++ = *ptr++; *hpp = htop; - *pp = ptr; + return ptr; } #endif diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 50aa41b4d2..a38f6c7daf 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -170,6 +170,7 @@ static void unlink_free_block (Allctr_t *, Block_t *); static void update_last_aux_mbc (Allctr_t *, Carrier_t *); static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); +static int gfalc_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); static void init_atoms (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -250,6 +251,8 @@ erts_gfalc_start(GFAllctr_t *gfallctr, if (!erts_alcu_start(allctr, init)) return NULL; + allctr->try_set_dyn_param = gfalc_try_set_dyn_param; + if (allctr->min_block_size != MIN_BLK_SZ) return NULL; @@ -584,6 +587,15 @@ info_options(Allctr_t *allctr, return res; } +static int gfalc_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value) +{ + if (param == am_sbct) { + /* Cannot change 'sbct' without rearranging buckets */ + return 0; + } + return erts_alcu_try_set_dyn_param(allctr, param, value); +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * NOTE: erts_gfalc_test() is only supposed to be used for testing. * * * diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 4846ccd2d3..e8048cfdfc 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1746,6 +1746,7 @@ erl_start(int argc, char **argv) } else if (has_prefix("ecio", sub_param)) { /* ignore argument, eager check io no longer used */ + arg = get_arg(sub_param+4, argv[i+1], &i); } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c index c93b5248d9..d779d1031a 100644 --- a/erts/emulator/beam/erl_io_queue.c +++ b/erts/emulator/beam/erl_io_queue.c @@ -801,12 +801,11 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term, ERTS_GET_REAL_BIN(bin_term, orig_pb_term, byte_offset, bit_offset, bit_size); - (void)bit_offset; - (void)bit_size; + ASSERT(bit_size == 0); sb->thing_word = HEADER_SUB_BIN; + sb->bitoffs = bit_offset; sb->bitsize = 0; - sb->bitoffs = 0; sb->orig = orig_pb_term; sb->is_writable = 0; @@ -975,7 +974,7 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { parent_header = binary_val(parent_binary); binary_size = binary_size(bin_term); - if (bit_offset != 0 || bit_size != 0) { + if (bit_size != 0) { return 0; } else if (binary_size == 0) { state->bytereds_spent += 1; @@ -1017,8 +1016,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { * then just copy it into the accumulator. */ iol2v_expand_acc(state, binary_size); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, binary_size); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, binary_size); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + binary_size * 8); + } state->acc_size += binary_size; } else { @@ -1029,8 +1036,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { iol2v_expand_acc(state, spill); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, spill); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, spill); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + spill * 8); + } state->acc_size += spill; @@ -1179,7 +1194,10 @@ BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) { if (is_nil(BIF_ARG_1)) { BIF_RET(NIL); } else if (is_binary(BIF_ARG_1)) { - if (binary_size(BIF_ARG_1) != 0) { + if (binary_bitsize(BIF_ARG_1) != 0) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } else if (binary_size(BIF_ARG_1) != 0) { Eterm *hp = HAlloc(BIF_P, 2); BIF_RET(CONS(hp, BIF_ARG_1, NIL)); diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 2588dec903..895b1ae319 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -159,12 +159,12 @@ struct erl_msacc_t_ { #ifdef ERTS_ENABLE_MSACC -extern erts_tsd_key_t erts_msacc_key; +extern erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key); #ifdef ERTS_MSACC_ALWAYS_ON #define erts_msacc_enabled 1 #else -extern int erts_msacc_enabled; +extern int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); #endif #define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key) diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 94ebf88b56..d2000aa71e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -237,9 +237,11 @@ static void cache_env(ErlNifEnv* env); static void full_flush_env(ErlNifEnv *env); static void flush_env(ErlNifEnv* env); -/* Temporary object header, auto-deallocated when NIF returns - * or when independent environment is cleared. - */ +/* Temporary object header, auto-deallocated when NIF returns or when + * independent environment is cleared. + * + * The payload can be accessed with &tmp_obj_ptr[1] but keep in mind that its + * first element must not require greater alignment than `next`. */ struct enif_tmp_obj_t { struct enif_tmp_obj_t* next; void (*dtor)(struct enif_tmp_obj_t*); @@ -256,6 +258,46 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) } } +/* Whether the given environment is bound to a process and will be cleaned up + * when the NIF returns. It's safe to use temp_alloc for objects in + * env->tmp_obj_list when this is true. */ +static ERTS_INLINE int is_proc_bound(ErlNifEnv *env) +{ + return env->mod_nif != NULL; +} + +/* Allocates and attaches an object to the given environment, running its + * destructor when the environment is cleared. To avoid temporary variables the + * address of the allocated object is returned instead of the enif_tmp_obj_t. + * + * The destructor *must* call `erts_free(tmp_obj->allocator, tmp_obj)` to free + * the object. If the destructor needs to refer to the allocated object its + * address will be &tmp_obj[1]. */ +static ERTS_INLINE void *alloc_tmp_obj(ErlNifEnv *env, size_t size, + void (*dtor)(struct enif_tmp_obj_t*)) { + struct enif_tmp_obj_t *tmp_obj; + ErtsAlcType_t allocator; + + allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + + tmp_obj = erts_alloc(allocator, sizeof(struct enif_tmp_obj_t) + MAX(1, size)); + + tmp_obj->next = env->tmp_obj_list; + tmp_obj->allocator = allocator; + tmp_obj->dtor = dtor; + + env->tmp_obj_list = tmp_obj; + + return (void*)&tmp_obj[1]; +} + +/* Generic destructor for objects allocated through alloc_tmp_obj that don't + * care about their payload. */ +static void tmp_alloc_dtor(struct enif_tmp_obj_t *tmp_obj) +{ + erts_free(tmp_obj->allocator, tmp_obj); +} + void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); @@ -446,6 +488,7 @@ static void cache_env(ErlNifEnv* env) env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } } + void* enif_priv_data(ErlNifEnv* env) { return env->mod_nif->priv_data; @@ -1019,11 +1062,6 @@ int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) return is_number(term); } -static ERTS_INLINE int is_proc_bound(ErlNifEnv* env) -{ - return env->mod_nif != NULL; -} - static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator); @@ -1058,22 +1096,14 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) u.tmp->dtor = &aligned_binary_dtor; env->tmp_obj_list = u.tmp; } - bin->bin_term = bin_term; bin->size = binary_size(bin_term); bin->ref_bin = NULL; ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } -static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) -{ - erts_free(obj->allocator, obj); -} - int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { - struct enif_tmp_obj_t* tobj; - ErtsAlcType_t allocator; ErlDrvSizeT sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); @@ -1081,7 +1111,6 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) if (is_nil(term)) { bin->data = (unsigned char*) &bin->data; /* dummy non-NULL */ bin->size = 0; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; return 1; } @@ -1089,16 +1118,8 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) return 0; } - allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t)); - tobj->allocator = allocator; - tobj->next = env->tmp_obj_list; - tobj->dtor = &tmp_alloc_dtor; - env->tmp_obj_list = tobj; - - bin->data = (unsigned char*) &tobj[1]; + bin->data = alloc_tmp_obj(env, sz, &tmp_alloc_dtor); bin->size = sz; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; erts_iolist_to_buf(term, (char*) bin->data, sz); ADD_READONLY_CHECK(env, bin->data, bin->size); @@ -1116,7 +1137,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = refbin; return 1; } @@ -1150,12 +1170,10 @@ void enif_release_binary(ErlNifBinary* bin) { if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; - ASSERT(bin->bin_term == THE_NON_VALUE); erts_bin_release(refbin); } #ifdef DEBUG bin->data = NULL; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; #endif } @@ -1312,29 +1330,51 @@ int enif_get_string(ErlNifEnv *env, ERL_NIF_TERM list, char* buf, unsigned len, Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) { - if (bin->bin_term != THE_NON_VALUE) { - return bin->bin_term; - } - else if (bin->ref_bin != NULL) { - Binary* bptr = bin->ref_bin; - Eterm bin_term; - - bin_term = erts_build_proc_bin(&MSO(env->proc), - alloc_heap(env, PROC_BIN_SIZE), - bptr); - if (erts_refc_read(&bptr->intern.refc, 1) == 1) { - /* Total ownership transfer */ - bin->ref_bin = NULL; - bin->bin_term = bin_term; - } - return bin_term; - } - else { - flush_env(env); - bin->bin_term = new_binary(env->proc, bin->data, bin->size); - cache_env(env); - return bin->bin_term; + Eterm bin_term; + + if (bin->ref_bin != NULL) { + Binary* binary = bin->ref_bin; + + /* If the binary is smaller than the heap binary limit we'll return a + * heap binary to reduce the number of small refc binaries in the + * system. We can't simply release the refc binary right away however; + * the documentation states that the binary should be considered + * read-only from this point on, which implies that it should still be + * readable. + * + * We could keep it alive until we return by adding it to the temporary + * object list, but that requires an off-heap allocation which is + * potentially quite slow, so we create a dummy ProcBin instead and + * rely on the next minor GC to get rid of it. */ + if (bin->size <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin* hb; + + hb = (ErlHeapBin*)alloc_heap(env, heap_bin_size(bin->size)); + hb->thing_word = header_heap_bin(bin->size); + hb->size = bin->size; + + sys_memcpy(hb->data, bin->data, bin->size); + + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + + bin_term = make_binary(hb); + } else { + bin_term = erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + } + + /* Our (possibly shared) ownership has been transferred to the term. */ + bin->ref_bin = NULL; + } else { + flush_env(env); + bin_term = new_binary(env->proc, bin->data, bin->size); + cache_env(env); } + + return bin_term; } Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, @@ -3284,8 +3324,8 @@ typedef struct { Eterm sublist_start; Eterm sublist_end; - UWord offheap_size; - UWord onheap_size; + UWord referenced_size; + UWord copied_size; UWord iovec_len; } iovec_slice_t; @@ -3295,16 +3335,16 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul result->sublist_start = list; result->sublist_length = 0; - result->offheap_size = 0; - result->onheap_size = 0; + result->referenced_size = 0; + result->copied_size = 0; result->iovec_len = 0; lookahead = result->sublist_start; while (is_list(lookahead)) { - Eterm *binary_header, binary; + UWord byte_size; + Eterm binary; Eterm *cell; - UWord size; cell = list_val(lookahead); binary = CAR(cell); @@ -3313,35 +3353,36 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 0; } - size = binary_size(binary); - binary_header = binary_val(binary); + byte_size = binary_size(binary); + + if (byte_size > 0) { + int bit_offset, bit_size; + Eterm parent_binary; + UWord byte_offset; - if (size > 0) { - /* If we're a sub-binary we'll need to check our underlying binary - * to determine whether we're on-heap or not. */ - if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { - ErlSubBin *sb = (ErlSubBin*)binary_header; + int requires_copying; - /* Reject bitstrings */ - if((sb->bitoffs + sb->bitsize) > 0) { - return 0; - } + ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, + bit_offset, bit_size); - ASSERT(size <= binary_size(sb->orig)); - binary_header = binary_val(sb->orig); + (void)byte_offset; + + if (bit_size != 0) { + return 0; } - if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { - ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); + /* If we're unaligned or an on-heap binary we'll need to copy + * ourselves over to a temporary buffer. */ + requires_copying = (bit_offset != 0) || + thing_subtag(*binary_val(parent_binary)) == HEAP_BINARY_SUBTAG; - result->iovec_len += 1; - result->onheap_size += size; + if (requires_copying) { + result->copied_size += byte_size; } else { - ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); - - result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; - result->offheap_size += size; + result->referenced_size += byte_size; } + + result->iovec_len += 1 + byte_size / MAX_SYSIOVEC_IOVLEN; } result->sublist_length += 1; @@ -3361,7 +3402,9 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 1; } -static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { +static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer, + UWord *copy_offset, ErlNifBinary *result) { + Eterm *parent_header; Eterm parent_binary; @@ -3372,10 +3415,11 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size); + ASSERT(bit_size == 0); + parent_header = binary_val(parent_binary); result->size = binary_size(binary); - result->bin_term = binary; if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { ProcBin *pb = (ProcBin*)parent_header; @@ -3398,24 +3442,48 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { result->data = &((unsigned char*)&hb->data)[byte_offset]; result->ref_bin = NULL; } + + /* If this isn't an *aligned* refc binary, copy its contents to the buffer + * and reference that instead. */ + + if (result->ref_bin == NULL || bit_offset != 0) { + ASSERT(copy_buffer->ref_bin != NULL && copy_buffer->data != NULL); + ASSERT(result->size <= (copy_buffer->size - *copy_offset)); + + if (bit_offset == 0) { + sys_memcpy(©_buffer->data[*copy_offset], + result->data, result->size); + } else { + erts_copy_bits(result->data, bit_offset, 1, + (byte*)©_buffer->data[*copy_offset], 0, 1, + result->size * 8); + } + + result->data = ©_buffer->data[*copy_offset]; + result->ref_bin = copy_buffer->ref_bin; + + *copy_offset += result->size; + } } static int fill_iovec_with_slice(ErlNifEnv *env, iovec_slice_t *slice, ErlNifIOVec *iovec) { - UWord onheap_offset, iovec_idx; - ErlNifBinary onheap_data; + ErlNifBinary copy_buffer = {0}; + UWord copy_offset, iovec_idx; Eterm sublist_iterator; - /* Set up a common refc binary for all on-heap binaries. */ - if (slice->onheap_size > 0) { - if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) { + /* Set up a common refc binary for all on-heap and unaligned binaries. */ + if (slice->copied_size > 0) { + if (!enif_alloc_binary(slice->copied_size, ©_buffer)) { return 0; } + + ASSERT(copy_buffer.ref_bin != NULL); } sublist_iterator = slice->sublist_start; - onheap_offset = 0; + copy_offset = 0; iovec_idx = 0; while (sublist_iterator != slice->sublist_end) { @@ -3423,20 +3491,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env, Eterm *cell; cell = list_val(sublist_iterator); - inspect_raw_binary_data(CAR(cell), &raw_data); - - /* If this isn't a refc binary, copy its contents to the onheap buffer - * and reference that instead. */ - if (raw_data.size > 0 && raw_data.ref_bin == NULL) { - ASSERT(onheap_offset < onheap_data.size); - ASSERT(slice->onheap_size > 0); - - sys_memcpy(&onheap_data.data[onheap_offset], - raw_data.data, raw_data.size); - - raw_data.data = &onheap_data.data[onheap_offset]; - raw_data.ref_bin = onheap_data.ref_bin; - } + marshal_iovec_binary(CAR(cell), ©_buffer, ©_offset, &raw_data); while (raw_data.size > 0) { UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN); @@ -3467,16 +3522,18 @@ static int fill_iovec_with_slice(ErlNifEnv *env, erts_refc_inc(&refc_binary->intern.refc, 1); } - if (slice->onheap_size > 0) { + if (slice->copied_size > 0) { /* Transfer ownership to the iovec; we've taken references to it in * the above loop. */ - enif_release_binary(&onheap_data); + enif_release_binary(©_buffer); } } else { - if (slice->onheap_size > 0) { - /* Attach the binary to our environment and let the GC take care of - * it after returning. */ - enif_make_binary(env, &onheap_data); + if (slice->copied_size > 0) { + /* Attach the binary to our environment and let the next minor GC + * get rid of it. This is slightly faster than using the tmp object + * list since it avoids off-heap allocations. */ + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), copy_buffer.ref_bin); } } @@ -3502,19 +3559,14 @@ static int create_iovec_from_slice(ErlNifEnv *env, alloc_size = binv_offset; alloc_size += slice->iovec_len * sizeof(Binary*); - /* If we have an environment we'll attach the allocated data to it. The - * GC will take care of releasing it later on. */ + /* When the user passes an environment, we attach the iovec to it so + * the user won't have to bother managing it (similar to + * enif_inspect_binary). It'll disappear once the environment is + * cleaned up. */ if (env != NULL) { - ErlNifBinary gc_bin; - - if (!enif_alloc_binary(alloc_size, &gc_bin)) { - return 0; - } - - alloc_base = (char*)gc_bin.data; - enif_make_binary(env, &gc_bin); + alloc_base = alloc_tmp_obj(env, alloc_size, &tmp_alloc_dtor); } else { - alloc_base = enif_alloc(alloc_size); + alloc_base = erts_alloc(ERTS_ALC_T_NIF, alloc_size); } iovec = (ErlNifIOVec*)alloc_base; @@ -3523,12 +3575,12 @@ static int create_iovec_from_slice(ErlNifEnv *env, iovec->flags = 0; } - iovec->size = slice->offheap_size + slice->onheap_size; + iovec->size = slice->referenced_size + slice->copied_size; iovec->iovcnt = slice->iovec_len; if(!fill_iovec_with_slice(env, slice, iovec)) { if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { - enif_free(iovec); + erts_free(ERTS_ALC_T_NIF, iovec); } return 0; @@ -4218,34 +4270,31 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size); struct readonly_check_t { - struct enif_tmp_obj_t hdr; unsigned char* ptr; unsigned size; unsigned checksum; }; static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) { - ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - struct readonly_check_t* obj = erts_alloc(allocator, - sizeof(struct readonly_check_t)); - obj->hdr.allocator = allocator; - obj->hdr.next = env->tmp_obj_list; - env->tmp_obj_list = &obj->hdr; - obj->hdr.dtor = &readonly_check_dtor; + struct readonly_check_t* obj; + + obj = alloc_tmp_obj(env, sizeof(struct readonly_check_t), + &readonly_check_dtor); + obj->ptr = ptr; obj->size = sz; - obj->checksum = calc_checksum(ptr, sz); + obj->checksum = calc_checksum(ptr, sz); } -static void readonly_check_dtor(struct enif_tmp_obj_t* o) +static void readonly_check_dtor(struct enif_tmp_obj_t* tmp_obj) { - struct readonly_check_t* obj = (struct readonly_check_t*) o; - unsigned chksum = calc_checksum(obj->ptr, obj->size); - if (chksum != obj->checksum) { + struct readonly_check_t* ro_check = (struct readonly_check_t*)&tmp_obj[1]; + unsigned chksum = calc_checksum(ro_check->ptr, ro_check->size); + if (chksum != ro_check->checksum) { fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ" - " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); + " %x != %x\r\nABORTING\r\n", chksum, ro_check->checksum); abort(); } - erts_free(obj->hdr.allocator, obj); + erts_free(tmp_obj->allocator, tmp_obj); } static unsigned calc_checksum(unsigned char* ptr, unsigned size) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 053f7673c4..a99b4db705 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -137,8 +137,9 @@ typedef struct unsigned char* data; /* Internals (avert your eyes) */ - ERL_NIF_TERM bin_term; void* ref_bin; + /* for future additions to be ABI compatible (same struct size) */ + void* __spare__[2]; }ErlNifBinary; #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 9117eb1f72..6b90187d60 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -180,6 +180,7 @@ void erts_init_port_data(Port *); void erts_cleanup_port_data(Port *); Uint erts_port_data_size(Port *); ErlOffHeap *erts_port_data_offheap(Port *); +Eterm erts_port_data_read(Port* prt); #define ERTS_PORT_GET_CONNECTED(PRT) \ ((Eterm) erts_atomic_read_nob(&(PRT)->connected)) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a807d60ec7..3baca6ce79 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1538,6 +1538,20 @@ erts_proclist_destroy(ErtsProcList *plp) proclist_destroy(plp); } +void +erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList *plp) +{ + ErtsProcList *first = plp; + + while (plp) { + erts_print(to, to_arg, "%T", plp->pid); + plp = plp->next; + if (plp == first) + break; + } + erts_print(to, to_arg, "\n"); +} + void * erts_psd_set_init(Process *p, int ix, void *data) { @@ -13398,16 +13412,33 @@ stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) return yreg; } +static void print_current_process_info(fmtfn_t, void *to_arg, ErtsSchedulerData*); + /* * Print scheduler information */ void -erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { +erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) +{ int i; erts_aint32_t flg; - Process *p; - erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + switch (esdp->type) { + case ERTS_SCHED_NORMAL: + erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + break; + case ERTS_SCHED_DIRTY_CPU: + erts_print(to, to_arg, "=dirty_cpu_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers)); + break; + case ERTS_SCHED_DIRTY_IO: + erts_print(to, to_arg, "=dirty_io_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers + erts_no_dirty_cpu_schedulers)); + break; + default: + erts_print(to, to_arg, "=unknown_scheduler_type:%u\n", esdp->type); + break; + } flg = erts_atomic32_read_dirty(&esdp->ssi->flags); erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); @@ -13453,10 +13484,24 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } erts_print(to, to_arg, "\n"); - erts_print(to, to_arg, "Current Port: "); - if (esdp->current_port) - erts_print(to, to_arg, "%T", esdp->current_port->common.id); - erts_print(to, to_arg, "\n"); + if (esdp->type == ERTS_SCHED_NORMAL) { + erts_print(to, to_arg, "Current Port: "); + if (esdp->current_port) + erts_print(to, to_arg, "%T", esdp->current_port->common.id); + erts_print(to, to_arg, "\n"); + + erts_print_run_queue_info(to, to_arg, esdp->run_queue); + } + + /* This *MUST* to be the last information in scheduler block */ + print_current_process_info(to, to_arg, esdp); +} + +void erts_print_run_queue_info(fmtfn_t to, void *to_arg, + ErtsRunQueue *run_queue) +{ + erts_aint32_t flg; + int i; for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { erts_print(to, to_arg, "Run Queue "); @@ -13478,12 +13523,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { break; } erts_print(to, to_arg, "Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + erts_atomic32_read_dirty(&run_queue->procs.prio_info[i].len)); } erts_print(to, to_arg, "Run Queue Port Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + erts_atomic32_read_dirty(&run_queue->ports.info.len)); - flg = erts_atomic32_read_dirty(&esdp->run_queue->flags); + flg = erts_atomic32_read_dirty(&run_queue->flags); erts_print(to, to_arg, "Run Queue Flags: "); for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { erts_aint32_t chk = (1 << i); @@ -13550,9 +13595,15 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } } erts_print(to, to_arg, "\n"); +} + + +static void print_current_process_info(fmtfn_t to, void *to_arg, + ErtsSchedulerData* esdp) +{ + Process *p = esdp->current_process; + erts_aint32_t flg; - /* This *MUST* to be the last information in scheduler block */ - p = esdp->current_process; erts_print(to, to_arg, "Current Process: "); if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { flg = erts_atomic32_read_dirty(&p->state); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 66d7848f89..4f200b13ec 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -105,13 +105,13 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; -extern int erts_sched_compact_load; -extern int erts_sched_balance_util; -extern Uint erts_no_schedulers; -extern Uint erts_no_total_schedulers; -extern Uint erts_no_dirty_cpu_schedulers; -extern Uint erts_no_dirty_io_schedulers; -extern Uint erts_no_run_queues; +extern int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); +extern int ERTS_WRITE_UNLIKELY(erts_sched_balance_util); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues); extern int erts_sched_thread_suggested_stack_size; extern int erts_dcpu_sched_thread_suggested_stack_size; extern int erts_dio_sched_thread_suggested_stack_size; @@ -522,7 +522,7 @@ typedef union { char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))]; } ErtsAlignedRunQueue; -extern ErtsAlignedRunQueue *erts_aligned_run_queues; +extern ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues); #define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\ do { \ @@ -675,9 +675,9 @@ typedef union { char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; } ErtsAlignedSchedulerData; -extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; -extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; -extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data); +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data); +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data); #if defined(ERTS_ENABLE_LOCK_CHECK) @@ -1270,7 +1270,7 @@ void erts_check_for_holes(Process* p); #define SPO_OFF_HEAP_MSGQ 16 #define SPO_ON_HEAP_MSGQ 32 -extern int erts_default_spo_flags; +extern int ERTS_WRITE_UNLIKELY(erts_default_spo_flags); /* * The following struct contains options for a process to be spawned. @@ -1326,10 +1326,10 @@ extern erts_rwmtx_t erts_cpu_bind_rwmtx; ** erts_system_monitor must be != NIL, to allow testing on just ** the erts_system_monitor_* variables. */ -extern Eterm erts_system_monitor; -extern Uint erts_system_monitor_long_gc; -extern Uint erts_system_monitor_long_schedule; -extern Uint erts_system_monitor_large_heap; +extern Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap); struct erts_system_monitor_flags_t { unsigned int busy_port : 1; unsigned int busy_dist_port : 1; @@ -1546,6 +1546,7 @@ Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); +void erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList*); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *); @@ -1792,6 +1793,7 @@ void erts_stack_dump(fmtfn_t to, void *to_arg, Process *); void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *); void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *); void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp); +void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*); void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index f562fc961b..0d76a4ef53 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -78,8 +78,17 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) Process *p = erts_pix2proc(i); if (p && p->i != ENULL) { erts_aint32_t state = erts_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC))) - dump_process_info(to, to_arg, p); + if (state & ERTS_PSFLG_EXITING) + continue; + if (state & ERTS_PSFLG_GC) { + ErtsSchedulerData *sdp = erts_get_scheduler_data(); + if (!sdp || p != sdp->current_process) + continue; + + /* We want to dump the garbing process that caused the dump */ + } + + dump_process_info(to, to_arg, p); } } @@ -135,9 +144,12 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) ErtsMessage* mp; int yreg = -1; + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) + return; + ERTS_MSGQ_MV_INQ2PRIVQ(p); - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) { + if (p->msg.first) { erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id); for (mp = p->msg.first; mp != NULL; mp = mp->next) { Eterm mesg = ERL_MESSAGE_TERM(mp); @@ -152,38 +164,34 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) } } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - if (p->dictionary) { - erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); - erts_deep_dictionary_dump(to, to_arg, - p->dictionary, dump_element_nl); - } + if (p->dictionary) { + erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); + erts_deep_dictionary_dump(to, to_arg, + p->dictionary, dump_element_nl); } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, sp, yreg); - } + erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + yreg = stack_element_dump(to, to_arg, sp, yreg); + } - erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - Eterm term = *sp; - - if (!is_catch(term) && !is_CP(term)) { - heap_dump(to, to_arg, term); - } - } - for (mp = p->msg.first; mp != NULL; mp = mp->next) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) - heap_dump(to, to_arg, mesg); - mesg = ERL_MESSAGE_TOKEN(mp); - heap_dump(to, to_arg, mesg); - } - if (p->dictionary) { - erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); - } + erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + Eterm term = *sp; + + if (!is_catch(term) && !is_CP(term)) { + heap_dump(to, to_arg, term); + } + } + for (mp = p->msg.first; mp != NULL; mp = mp->next) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) + heap_dump(to, to_arg, mesg); + mesg = ERL_MESSAGE_TOKEN(mp); + heap_dump(to, to_arg, mesg); + } + if (p->dictionary) { + erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); } } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4b996d8fc2..d2495479ab 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2783,24 +2783,28 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, ASSERT(0); } - /* Only remove tracer on self() and ports */ + /* Only remove tracer on (self() or ports) AND we are on a normal scheduler */ if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsProcLocks c_p_xlocks = 0; - if (is_internal_pid(t_p->id)) { - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); - if (c_p_locks != ERTS_PROC_LOCKS_ALL) { - c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; - if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { - erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); - erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (is_internal_pid(t_p->id)) { + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + if (c_p_locks != ERTS_PROC_LOCKS_ALL) { + c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; + if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { + erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } } } - } - erts_tracer_replace(t_p, erts_tracer_nil); - t_p->trace_flags &= ~TRACEE_FLAGS; - if (c_p_xlocks) - erts_proc_unlock(c_p, c_p_xlocks); + erts_tracer_replace(t_p, erts_tracer_nil); + t_p->trace_flags &= ~TRACEE_FLAGS; + + if (c_p_xlocks) + erts_proc_unlock(c_p, c_p_xlocks); + } } return 0; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 86e2c351af..d757651933 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -948,8 +948,8 @@ void erts_update_ranges(BeamInstr* code, Uint size); void erts_remove_from_ranges(BeamInstr* code); UWord erts_ranges_sz(void); void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); -ErtsLiteralArea** erts_dump_lit_areas; -Uint erts_dump_num_lit_areas; +extern ErtsLiteralArea** erts_dump_lit_areas; +extern Uint erts_dump_num_lit_areas; /* break.c */ void init_break_handler(void); @@ -1066,7 +1066,7 @@ Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_ #define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) -Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); +Eterm copy_shallow(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs, int literals); diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index f19027b1ec..6a531fcc09 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -322,6 +322,16 @@ get_list(Src, Hd, Tl) { $Tl = tl; } +get_hd(Src, Hd) { + Eterm* tmp_ptr = list_val($Src); + $Hd = CAR(tmp_ptr); +} + +get_tl(Src, Tl) { + Eterm* tmp_ptr = list_val($Src); + $Tl = CDR(tmp_ptr); +} + i_get(Src, Dst) { $Dst = erts_pd_hash_get(c_p, $Src); } diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 2c1b7871c4..4bb2c0cb01 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -5075,6 +5075,93 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd) erts_print(prtd->to, prtd->arg, "%T", lnk->pid); } +static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state) +{ + erts_aint32_t rest; + int unknown = 0; + char delim = ' '; + + erts_print(to, arg, "State:"); + + rest = state; + while (rest) { + erts_aint32_t chk = (rest ^ (rest-1)) & rest; /* lowest set bit */ + char* s; + + rest &= ~chk; + switch (chk) { + case ERTS_PORT_SFLG_CONNECTED: s = "CONNECTED"; break; + case ERTS_PORT_SFLG_EXITING: s = "EXITING"; break; + case ERTS_PORT_SFLG_DISTRIBUTION: s = "DISTR"; break; + case ERTS_PORT_SFLG_BINARY_IO: s = "BINARY_IO"; break; + case ERTS_PORT_SFLG_SOFT_EOF: s = "SOFT_EOF"; break; + case ERTS_PORT_SFLG_CLOSING: s = "CLOSING"; break; + case ERTS_PORT_SFLG_SEND_CLOSED: s = "SEND_CLOSED"; break; + case ERTS_PORT_SFLG_LINEBUF_IO: s = "LINEBUF_IO"; break; + case ERTS_PORT_SFLG_FREE: s = "FREE"; break; + case ERTS_PORT_SFLG_INITIALIZING: s = "INITIALIZING"; break; + case ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK: s = "PORT_LOCK"; break; + case ERTS_PORT_SFLG_INVALID: s = "INVALID"; break; + case ERTS_PORT_SFLG_HALT: s = "HALT"; break; +#ifdef DEBUG + case ERTS_PORT_SFLG_PORT_DEBUG: s = "DEBUG"; break; +#endif + default: + unknown = 1; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown || !state) + erts_print(to, arg, "%c0x%x\n", delim, state); + else + erts_print(to, arg, "\n"); +} + +static void dump_port_task_flags(fmtfn_t to, void *arg, Port* p) +{ + erts_aint32_t flags = erts_atomic32_read_nob(&p->sched.flags); + erts_aint32_t unknown = 0; + char delim = ' '; + + if (!flags) + return; + + erts_print(to, arg, "Task Flags:"); + + while (flags) { + erts_aint32_t chk = (flags ^ (flags-1)) & flags; /* lowest set bit */ + char* s; + + flags &= ~chk; + switch (chk) { + case ERTS_PTS_FLG_IN_RUNQ: s = "IN_RUNQ"; break; + case ERTS_PTS_FLG_EXEC: s = "EXEC"; break; + case ERTS_PTS_FLG_HAVE_TASKS: s = "HAVE_TASKS"; break; + case ERTS_PTS_FLG_EXIT: s = "EXIT"; break; + case ERTS_PTS_FLG_BUSY_PORT: s = "BUSY_PORT"; break; + case ERTS_PTS_FLG_BUSY_PORT_Q: s = "BUSY_Q"; break; + case ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q: s = "CHK_UNSET_BUSY_Q"; break; + case ERTS_PTS_FLG_HAVE_BUSY_TASKS: s = "BUSY_TASKS"; break; + case ERTS_PTS_FLG_HAVE_NS_TASKS: s = "NS_TASKS"; break; + case ERTS_PTS_FLG_PARALLELISM: s = "PARALLELISM"; break; + case ERTS_PTS_FLG_FORCE_SCHED: s = "FORCE_SCHED"; break; + case ERTS_PTS_FLG_EXITING: s = "EXITING"; break; + case ERTS_PTS_FLG_EXEC_IMM: s = "EXEC_IMM"; break; + default: + unknown |= chk; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown) + erts_print(to, arg, "%cUNKNOWN(0x%x)\n", delim, unknown); + else + erts_print(to, arg, "\n"); +} + void print_port_info(Port *p, fmtfn_t to, void *arg) { @@ -5084,6 +5171,8 @@ print_port_info(Port *p, fmtfn_t to, void *arg) return; erts_print(to, arg, "=port:%T\n", p->common.id); + dump_port_state(to, arg, state); + dump_port_task_flags(to, arg, p); erts_print(to, arg, "Slot: %d\n", internal_port_index(p->common.id)); if (state & ERTS_PORT_SFLG_CONNECTED) { erts_print(to, arg, "Connected: %T", ERTS_PORT_GET_CONNECTED(p)); @@ -5106,6 +5195,10 @@ print_port_info(Port *p, fmtfn_t to, void *arg) erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd); erts_print(to, arg, "\n"); } + if (p->suspended) { + erts_print(to, arg, "Suspended: "); + erts_proclist_dump(to, arg, p->suspended); + } if (p->common.u.alive.reg != NULL) erts_print(to, arg, "Registered as: %T\n", p->common.u.alive.reg->name); @@ -5123,6 +5216,14 @@ print_port_info(Port *p, fmtfn_t to, void *arg) } else { erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name); } + erts_print(to, arg, "Input: %beu\n", p->bytes_in); + erts_print(to, arg, "Output: %beu\n", p->bytes_out); + erts_print(to, arg, "Queue: %beu\n", erts_ioq_size(&p->ioq)); + { + Eterm port_data = erts_port_data_read(p); + if (port_data != am_undefined) + erts_print(to, arg, "Port Data: %T\n", port_data); + } } void diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 1f4a8eadb0..77e375f2c0 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -182,6 +182,9 @@ get_list r x y get_list r y r get_list r x r +get_hd xy xy +get_tl xy xy + # Old-style catch. catch y f catch_end y diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index fe9f1c7606..4bf60619ba 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3152,6 +3152,9 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; + if (eq_only && a_size != b_size) { + RETURN_NEQ(a_size - b_size); + } ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index e3cff4a4ba..f23f341e6d 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -28,6 +28,7 @@ #include "error.h" #include "bif.h" #include "big.h" /* term_to_Sint() */ +#include "erl_binary.h" #include "hipe_arch.h" #include "hipe_bif0.h" @@ -38,6 +39,8 @@ #undef ERL_FUN_SIZE #include "hipe_literals.h" +static void patch_trampoline(void *trampoline, void *destAddress); + const Uint sse2_fnegate_mask[2] = {0x8000000000000000,0}; void hipe_patch_load_fe(Uint64 *address, Uint64 value) @@ -52,9 +55,9 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) switch (type) { case am_closure: case am_constant: + case am_c_const: *(Uint64*)address = value; break; - case am_c_const: case am_atom: /* check that value fits in an unsigned imm32 */ /* XXX: are we sure it's not really a signed imm32? */ @@ -71,14 +74,18 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { - Sint rel32; + Sint64 destOffset = (Sint64)destAddress - (Sint64)callAddress - 4; - ASSERT(trampoline == NULL); + if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) { + destOffset = (Sint64)trampoline - (Sint64)callAddress - 4; - rel32 = (Sint)destAddress - (Sint)callAddress - 4; - if ((Sint)(Sint32)rel32 != rel32) - return -1; - *(Uint32*)callAddress = (Uint32)rel32; + if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) + return -1; + + patch_trampoline(trampoline, destAddress); + } + + *(Uint32*)callAddress = (Uint32)destOffset; hipe_flush_icache_word(callAddress); return 0; } @@ -96,12 +103,80 @@ static void *alloc_code(unsigned int alloc_bytes) return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } +static int check_callees(Eterm callees) +{ + Eterm *tuple; + Uint arity; + Uint i; + + if (is_not_tuple(callees)) + return -1; + tuple = tuple_val(callees); + arity = arityval(tuple[0]); + for (i = 1; i <= arity; ++i) { + Eterm mfa = tuple[i]; + if (is_atom(mfa)) + continue; + if (is_not_tuple(mfa) || + tuple_val(mfa)[0] != make_arityval(3) || + is_not_atom(tuple_val(mfa)[1]) || + is_not_atom(tuple_val(mfa)[2]) || + is_not_small(tuple_val(mfa)[3]) || + unsigned_val(tuple_val(mfa)[3]) > 255) + return -1; + } + return arity; +} + +#define TRAMPOLINE_BYTES 12 + +static void generate_trampolines(unsigned char *address, + int nrcallees, Eterm callees, + unsigned char **trampvec) +{ + unsigned char *trampoline = address; + int i; + + for(i = 0; i < nrcallees; ++i) { + trampoline[0] = 0x48; /* movabsq $..., %rax; */ + trampoline[1] = 0xb8; + *(void**)(trampoline+2) = NULL; /* callee's address */ + trampoline[10] = 0xff; /* jmpq *%rax */ + trampoline[11] = 0xe0; + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_BYTES; + } + hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_BYTES); +} + +static void patch_trampoline(void *trampoline, void *destAddress) +{ + unsigned char *tp = (unsigned char*) trampoline; + + ASSERT(tp[0] == 0x48 && tp[1] == 0xb8); + + *(void**)(tp+2) = destAddress; /* callee's address */ + hipe_flush_icache_word(tp+2); +} + void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - if (is_not_nil(callees)) + int nrcallees; + Eterm trampvecbin; + unsigned char **trampvec; + unsigned char *address; + + nrcallees = check_callees(callees); + if (nrcallees < 0) return NULL; - *trampolines = NIL; - return alloc_code(nrbytes); + + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned char*)); + trampvec = (unsigned char **)binary_bytes(trampvecbin); + + address = alloc_code(nrbytes + nrcallees*TRAMPOLINE_BYTES); + generate_trampolines(address + nrbytes, nrcallees, callees, trampvec); + *trampolines = trampvecbin; + return address; } void hipe_free_code(void* code, unsigned int bytes) @@ -129,10 +204,9 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) */ unsigned int codeSize; unsigned char *code, *codep; - unsigned int callEmuOffset; - codeSize = /* 23, 26, 29, or 32 bytes */ - 23 + /* 23 when all offsets are 8-bit */ + codeSize = /* 30, 33, 36, or 39 bytes */ + 30 + /* 30 when all offsets are 8-bit */ (P_CALLEE_EXP >= 128 ? 3 : 0) + ((P_CALLEE_EXP + 4) >= 128 ? 3 : 0) + (P_ARITY >= 128 ? 3 : 0); @@ -197,14 +271,15 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) codep[0] = beamArity; codep += 1; - /* jmp callemu; 5 bytes */ - callEmuOffset = (unsigned char*)nbif_callemu - (code + codeSize); - codep[0] = 0xe9; - codep[1] = callEmuOffset & 0xFF; - codep[2] = (callEmuOffset >> 8) & 0xFF; - codep[3] = (callEmuOffset >> 16) & 0xFF; - codep[4] = (callEmuOffset >> 24) & 0xFF; - codep += 5; + /* jmp callemu; 12 bytes */ + codep[0] = 0x48; + codep[1] = 0xb8; + codep += 2; + *(Uint64*)codep = (Uint64)nbif_callemu; + codep += 8; + codep[0] = 0xff; + codep[1] = 0xe0; + codep += 2; ASSERT(codep == code + codeSize); diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index e477c4cdea..a8be64e08d 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1112,7 +1112,7 @@ static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) struct hipe_ref { struct hipe_ref_head head; /* list of refs to same calleee */ void *address; -#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) void *trampoline; #endif unsigned int flags; @@ -1543,7 +1543,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) ref = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(struct hipe_ref)); ref->address = address; -#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; #endif ref->flags = flags; @@ -1819,7 +1819,7 @@ void hipe_redirect_to_module(Module* modp) if (ref->flags & REF_FLAG_IS_LOAD_MFA) res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); else { -#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) void* trampoline = ref->trampoline; #else void* trampoline = NULL; diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 1a4a4c7952..aaedba1afd 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -91,7 +91,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) ASSERT(is_boxed(val)); *nsp_i = val; } else if (!erts_is_literal(gval, ptr)) { - move_boxed(&ptr, val, &n_htop, nsp_i); + move_boxed(ptr, val, &n_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -100,7 +100,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) *nsp_i = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_cons(&ptr, val, &n_htop, nsp_i); + move_cons(ptr, val, &n_htop, nsp_i); } } } @@ -206,10 +206,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) ASSERT(is_boxed(val)); *nsp_i = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr, val, &old_htop, nsp_i); + move_boxed(ptr, val, &old_htop, nsp_i); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_boxed(&ptr, val, &n_htop, nsp_i); + move_boxed(ptr, val, &n_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -217,10 +217,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr, val, &old_htop, nsp_i); + move_cons(ptr, val, &old_htop, nsp_i); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_cons(&ptr, val, &n_htop, nsp_i); + move_cons(ptr, val, &n_htop, nsp_i); } } } @@ -278,7 +278,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, ASSERT(is_boxed(val)); *nsp_i = val; } else if (ErtsInArea(ptr, area, area_size)) { - move_boxed(&ptr, val, &old_htop, nsp_i); + move_boxed(ptr, val, &old_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -286,7 +286,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (ErtsInArea(ptr, area, area_size)) { - move_cons(&ptr, val, &old_htop, nsp_i); + move_cons(ptr, val, &old_htop, nsp_i); } } } diff --git a/erts/emulator/internal_doc/CarrierMigration.md b/erts/emulator/internal_doc/CarrierMigration.md index 2a9594db25..3a796d11b7 100644 --- a/erts/emulator/internal_doc/CarrierMigration.md +++ b/erts/emulator/internal_doc/CarrierMigration.md @@ -3,17 +3,17 @@ Carrier Migration The ERTS memory allocators manage memory blocks in two types of raw memory chunks. We call these chunks of raw memory -*carriers*. Singleblock carriers which only contain one large block, -and multiblock carriers which contain multiple blocks. A carrier is +*carriers*. Single-block carriers which only contain one large block, +and multi-block carriers which contain multiple blocks. A carrier is typically created using `mmap()` on unix systems. However, how a carrier is created is of minor importance. An allocator instance -typically manages a mixture of single- and multiblock carriers. +typically manages a mixture of single- and multi-block carriers. Problem ------- When a carrier is empty, i.e. contains only one large free block, it -is deallocated. Since multiblock carriers can contain both allocated +is deallocated. Since multi-block carriers can contain both allocated blocks and free blocks at the same time, an allocator instance might be stuck with a large amount of poorly utilized carriers if the memory load decreases. After a peak in memory usage it is expected that not @@ -23,9 +23,9 @@ can usually be reused if the memory load increases again. However, since each scheduler thread manages its own set of allocator instances, and memory load is not necessarily correlated to CPU load, we might get into a situation where there are lots of poorly utilized -multiblock carriers on some allocator instances while we need to -allocate new multiblock carriers on other allocator instances. In -scenarios like this, the demand for multiblock carriers in the system +multi-block carriers on some allocator instances while we need to +allocate new multi-block carriers on other allocator instances. In +scenarios like this, the demand for multi-block carriers in the system might increase at the same time as the actual memory demand in the system has decreased which is both unwanted and quite unexpected for the end user. @@ -34,7 +34,7 @@ Solution -------- In order to prevent scenarios like this we've implemented support for -migration of multiblock carriers between allocator instances of the +migration of multi-block carriers between allocator instances of the same type. ### Management of Free Blocks ### @@ -44,7 +44,7 @@ and add it to another we need to be able to move references to the free blocks of the carrier between the allocator instances. The allocator instance specific data structure referring to the free blocks it manages often refers to the same carrier from multiple -places. For example, when the address order bestfit strategy is used +places. For example, when the address order best-fit strategy is used this data structure is a binary search tree spanning all carriers that the allocator instance manages. Free blocks in one specific carrier can be referred to from potentially every other carrier that is @@ -135,7 +135,7 @@ carriers between scheduler specific allocator instances of the same allocator type. Each allocator instance keeps track of the current utilization of its -multiblock carriers. When the total utilization falls below the "abandon +multi-block carriers. When the total utilization falls below the "abandon carrier utilization limit" it starts to inspect the utilization of the current carrier when deallocations are made. If also the utilization of the carrier falls below the "abandon carrier utilization limit" it @@ -144,31 +144,45 @@ and inserts the carrier into the pool. Since the carrier has been unlinked from the data structure of available free blocks, no more allocations will be made in the -carrier. The allocator instance putting the carrier into the pool, -however, still has the responsibility of performing deallocations in -it while it remains in the pool. The allocator instance with this -deallocation responsibility is here called the **employer**. - -Each carrier has a flag field containing information about the -employing allocator instance, a flag indicating if the carrier is in -the pool or not, and a flag indicating if it is busy or not. When the -carrier is in the pool, the employing allocator instance needs to mark it -as busy while operating on it. If another thread inspects it in order -to try to fetch it from the pool, it will skip it if it is busy. When -fetching the carrier from the pool, employment will change and further +carrier. + +The allocator instance that created a carrier is called its **owner**. +Ownership never changes. + +The allocator instance that has the responsibility to perform deallocations in a +carrier is called its **employer**. The employer may also perform allocations if +the carrier is not in the pool. Employment may change when a carrier is fetched from +or inserted into the pool. + +Deallocations in a carrier, while it remains in the pool, is always performed +the owner. That is, all pooled carriers are employed by their owners. + +Each carrier has an atomic word containing a pointer to the employing allocator +instance and three bit flags; IN_POOL, BUSY and HOMECOMING. + +When fetching a carrier from the pool, employment may change and further deallocations in the carrier will be redirected to the new employer using the delayed dealloc functionality. -If a carrier in the pool becomes empty, it will be withdrawn from the -pool. All carriers that become empty are also always passed to its -**owning** allocator instance for deallocation using the delayed -dealloc functionality. Since carriers this way always will be -deallocated by the owner that allocated the carrier, the +When a foreign allocator instance abandons a carrier back into the pool, it will +also pass it back to its **owner** using the delayed dealloc queue. When doing +this it will set the HOMECOMING bit flag to mark it as "enqueued". The owner +will later clear the HOMECOMING bit when the carrier is dequeued. This mechanism +prevents a carrier from being enqueued again before it has been dequeued. + +When a carrier becomes empty, it will be deallocated. Carrier deallocation is +always done by the owner that allocated the carrier. By doing this, the underlying functionality of allocating and deallocating carriers can remain simple and doesn't have to bother about multiple threads. In a NUMA system we will also not mix carriers originating from multiple NUMA nodes. +If a carrier in the pool becomes empty, it will be withdrawn from the +pool and be deallocated by the owner which already employs it. + +If a carrier employed by a foreign allocator becomes empty, it will be passed +back to the owner for deallocation using the delayed dealloc functionality. + In short: * The allocator instance that created a carrier **owns** it. @@ -177,34 +191,31 @@ In short: * The allocator instance that uses a carrier **employs** it. * An **employer** can abandon a carrier into the pool. * Pooled carriers are not allocated from. -* Deallocation in a pooled carrier is still performed by its **employer**. -* **Employment** can only change when a carrier is fetched from the pool. +* Pooled carriers are always **employed** by their **owner**. +* **Employment** can only change from **owner** to a foreign allocator + when a carrier is fetched from the pool. + ### Searching the pool ### +When an allocator instance needs more carrier space, it inspects the pool. If no +carrier could be fetched from the pool, it will allocate a new +carrier. Regardless of where the allocator instance gets the carrier from, it +just links in the carrier into its data structure of free blocks. + To harbor real time characteristics, searching the pool is limited. We only inspect a limited number of carriers. If none of those carriers had a free block large enough to satisfy the allocation -request, the search will fail. A carrier in the pool can also be busy +request, the search will fail. A carrier in the pool can also be BUSY if another thread is currently doing block deallocation work on the -carrier. A busy carrier will also be skipped by the search as it can +carrier. A BUSY carrier will also be skipped by the search as it can not satisfy the request. The pool is lock-free and we do not want to block, waiting for the other thread to finish. -#### Before OTP 17.4 #### +### The bad cluster problem ### -When an allocator instance needs more carrier space, it always begins -by inspecting its own carriers that are waiting for thread progress -before they can be deallocated. If no such carrier could be found, it -then inspects the pool. If no carrier could be fetched from the pool, -it will allocate a new carrier. Regardless of where the allocator -instance gets the carrier from it the just links in the carrier into -its data structure of free blocks. - -#### After OTP 17.4 #### - -The old search algorithm had a problem as the search always started at -the same position in the pool, the sentinel. This could lead to +Before OTP-17.4 the search algorithm had a problem as the search always started +at the same position in the pool, the sentinel. This could lead to contention from concurrent searching processes. But even worse, it could lead to a "bad" state when searches fail with a high rate leading to new carriers instead being allocated. These new carriers @@ -236,26 +247,27 @@ The result is that we prefer carriers created by the thread itself, which is good for NUMA performance. And we get more entry points when searching the pool, which will ease contention and clustering. +### Our own pooled tree ### + To do the first search among own carriers, every allocator instance -has two new lists: `pooled_list` and `traitor_list`. These lists are only -accessed by the allocator itself and they only contain the allocator's -own carriers. When an owned carrier is abandoned and put in the -pool, it is also linked into `pooled_list`. When we search our -`pooled_list` and find a carrier that is no longer in the pool, we -move that carrier from `pooled_list` to `traitor_list` as it is now -employed by another allocator. If searching `pooled_list` fails, we -also do a limited search of `traitor_list`. When finding an abandoned -carrier in `traitor_list` it is either employed or moved back to -`pooled_list` if it could not satisfy the allocation request. - -When searching `pooled_list` and `traitor_list` we always start at the -point where the last search ended. This to avoid clustering -problems and increase the probability to find a "good" carrier. As -`pooled_list` and `traitor_list` are only accessed by the owning -allocator instance, they need no thread synchronization at all. +has a `pooled_tree` of carriers. This tree is only accessed by the allocator +itself and can only contain its own carriers. When a carrier is +abandoned and put in the pool, it is also inserted into `pooled_tree`. This is +either done direct, if the carrier was already employed by its owner, or by +first passing it back to the owner via the delayed dealloc queue. + +When we search our `pooled_tree` and find a carrier that is no longer in the +pool, we remove that carrier from `pooled_tree` and mark it as TRAITOR, as it is +now employed by a foreign allocator. We will not find any carriers in +`pooled_tree` that are marked as BUSY by other threads. + +If no carrier in `pooled_tree` had a large enough free block, we search it again +to find any carrier that may act as an entry point into the shared list of all +pooled carriers. This in order to, if possible, avoid starting at the sentinel +and thereby ease the "bad clustering" problem. Furthermore, the search for own carriers that are scheduled -for deallocation is now done as the last search option. The idea is +for deallocation is done as the last search option. The idea is that it is better to reuse a poorly utilized carrier than to resurrect an empty carrier that was just about to be released back to the OS. @@ -271,14 +283,14 @@ load did not. When using the `aoffcaobf` or `aoff` strategies compared to `gf` or `bf`, we loose some performance since we get more modifications in the data structure of free blocks. This performance penalty is however -reduced using the `aoffcbf` strategy. A tradeoff between memory +reduced using the `aoffcbf` strategy. A trade off between memory consumption and performance is however inevitable, and it is up to the user to decide what is most important. Further work ------------ -It would be quite easy to extend this to allow migration of multiblock +It would be quite easy to extend this to allow migration of multi-block carriers between all allocator types. More or less the only obstacle is maintenance of the statistics information. diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c index 57c8ef62e1..4a6c476882 100644 --- a/erts/emulator/nifs/unix/unix_prim_file.c +++ b/erts/emulator/nifs/unix/unix_prim_file.c @@ -125,24 +125,11 @@ static int open_file_type_check(const efile_path_t *path, int fd) { * immediately in a read within the call, but the new implementation * never does that. */ return 1; - } else { - /* The old driver tolerated opening /dev/null despite the "no devices" - * limitation. It provided no explanation for this but we still need - * to match the behavior. We're checking through stat(2) instead of - * comparing the name to account for links. */ - struct stat null_device_info; - int is_dev_null; - - is_dev_null = (stat("/dev/null", &null_device_info) == 0); - is_dev_null &= (file_info.st_ino == null_device_info.st_ino); - is_dev_null &= (file_info.st_dev == null_device_info.st_dev); - - if(is_dev_null) { - return 1; - } } - if(!S_ISREG(file_info.st_mode)) { + /* Allow everything that isn't a directory, and error out on the next call + * if it's unsupported. */ + if(S_ISDIR(file_info.st_mode)) { return 0; } diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 5e5cd6f578..88ff2a7a92 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -31,6 +31,7 @@ mseg_clear_cache/1, erts_mmap/1, cpool/1, + set_dyn_param/1, migration/1]). -include_lib("common_test/include/ct.hrl"). @@ -41,6 +42,7 @@ suite() -> all() -> [basic, coalesce, threads, realloc_copy, bucket_index, + set_dyn_param, bucket_mask, rbtree, mseg_clear_cache, erts_mmap, cpool, migration]. init_per_testcase(Case, Config) when is_list(Config) -> @@ -65,7 +67,11 @@ mseg_clear_cache(Cfg) -> drv_case(Cfg). cpool(Cfg) -> drv_case(Cfg). migration(Cfg) -> - drv_case(Cfg, concurrent, "+MZe true"). + %% Enable test_alloc. + %% Disable driver_alloc to avoid recursive alloc_util calls + %% through enif_mutex_create() in my_creating_mbc(). + drv_case(Cfg, concurrent, "+MZe true +MRe false"), + drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas ageffcbf"). erts_mmap(Config) when is_list(Config) -> case {os:type(), mmsc_flags()} of @@ -110,7 +116,7 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) -> 0 -> O1; _ -> O1 ++ " +MMscrfsd"++integer_to_list(SCRFSD) end, - {ok, Node} = start_node(Config, Opts), + {ok, Node} = start_node(Config, Opts, []), Self = self(), Ref = make_ref(), F = fun() -> @@ -140,6 +146,82 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) -> Result. +%% Test erlang:system_flag(erts_alloc, ...) +set_dyn_param(_Config) -> + {_, _, _, AlcList} = erlang:system_info(allocator), + + {Enabled, Disabled, Others} = + lists:foldl(fun({sys_alloc,_}, {Es, Ds, Os}) -> + {Es, [sys_alloc | Ds], Os}; + + ({AT, Opts}, {Es, Ds, Os}) when is_list(Opts) -> + case lists:keyfind(e, 1, Opts) of + {e, true} -> + {[AT | Es], Ds, Os}; + {e, false} -> + {Es, [AT | Ds], Os}; + false -> + {Es, Ds, [AT | Os]} + end; + + (_, Acc) -> Acc + end, + {[], [], []}, + AlcList), + + Param = sbct, + lists:foreach(fun(AT) -> set_dyn_param_enabled(AT, Param) end, + Enabled), + + lists:foreach(fun(AT) -> + Tpl = {AT, Param, 12345}, + io:format("~p\n", [Tpl]), + notsup = erlang:system_flag(erts_alloc, Tpl) + end, + Disabled), + + lists:foreach(fun(AT) -> + Tpl = {AT, Param, 12345}, + io:format("~p\n", [Tpl]), + {'EXIT',{badarg,_}} = + (catch erlang:system_flag(erts_alloc, Tpl)) + end, + Others), + ok. + +set_dyn_param_enabled(AT, Param) -> + OldVal = get_alc_param(AT, Param), + + Val1 = OldVal div 2, + Tuple = {AT, Param, Val1}, + io:format("~p\n", [Tuple]), + ok = erlang:system_flag(erts_alloc, Tuple), + Val1 = get_alc_param(AT, Param), + + ok = erlang:system_flag(erts_alloc, {AT, Param, OldVal}), + OldVal = get_alc_param(AT, Param), + ok. + +get_alc_param(AT, Param) -> + lists:foldl(fun({instance,_,Istats}, Acc) -> + {options,Opts} = lists:keyfind(options, 1, Istats), + {Param,Val} = lists:keyfind(Param, 1, Opts), + {as,Strategy} = lists:keyfind(as, 1, Opts), + + case {param_for_strat(Param, Strategy), Acc} of + {false, _} -> Acc; + {true, undefined} -> Val; + {true, _} -> + Val = Acc + end + end, + undefined, + erlang:system_info({allocator, AT})). + +param_for_strat(sbct, gf) -> false; +param_for_strat(_, _) -> true. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% %% Internal functions %% @@ -151,7 +233,9 @@ drv_case(Config) -> drv_case(Config, Mode, NodeOpts) when is_list(Config) -> case os:type() of {Family, _} when Family == unix; Family == win32 -> - {ok, Node} = start_node(Config, NodeOpts), + %%Prog = {prog,"/my/own/otp/bin/cerl -debug"}, + Prog = [], + {ok, Node} = start_node(Config, NodeOpts, Prog), Self = self(), Ref = make_ref(), spawn_link(Node, @@ -217,19 +301,35 @@ wait_for_memory_deallocations() -> end. print_stats(migration) -> - {Btot,Ctot} = lists:foldl(fun({instance,Inr,Istats}, {Bacc,Cacc}) -> - {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats), - Btup = lists:keyfind(blocks, 1, MBCS), - Ctup = lists:keyfind(carriers, 1, MBCS), - io:format("{instance,~p,~p,~p}\n", [Inr, Btup, Ctup]), - {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup)}; - (_, Acc) -> Acc - end, - {{blocks,0,0,0},{carriers,0,0,0}}, - erlang:system_info({allocator,test_alloc})), - + IFun = fun({instance,Inr,Istats}, {Bacc,Cacc,Pacc}) -> + {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats), + Btup = lists:keyfind(blocks, 1, MBCS), + Ctup = lists:keyfind(carriers, 1, MBCS), + + Ptup = case lists:keyfind(mbcs_pool, 1, Istats) of + {mbcs_pool,POOL} -> + {blocks, Bpool} = lists:keyfind(blocks, 1, POOL), + {carriers, Cpool} = lists:keyfind(carriers, 1, POOL), + {pool, Bpool, Cpool}; + false -> + {pool, 0, 0} + end, + io:format("{instance,~p,~p,~p,~p}}\n", + [Inr, Btup, Ctup, Ptup]), + {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup), + tuple_add(Pacc,Ptup)}; + (_, Acc) -> Acc + end, + + {Btot,Ctot,Ptot} = lists:foldl(IFun, + {{blocks,0,0,0},{carriers,0,0,0},{pool,0,0}}, + erlang:system_info({allocator,test_alloc})), + + {pool, PBtot, PCtot} = Ptot, io:format("Number of blocks : ~p\n", [Btot]), - io:format("Number of carriers: ~p\n", [Ctot]); + io:format("Number of carriers: ~p\n", [Ctot]), + io:format("Number of pooled blocks : ~p\n", [PBtot]), + io:format("Number of pooled carriers: ~p\n", [PCtot]); print_stats(_) -> ok. tuple_add(T1, T2) -> @@ -326,13 +426,13 @@ handle_result(_State, Result0) -> continue end. -start_node(Config, Opts) when is_list(Config), is_list(Opts) -> +start_node(Config, Opts, Prog) when is_list(Config), is_list(Opts) -> case proplists:get_value(debug,Config) of true -> {ok, node()}; - _ -> start_node_1(Config, Opts) + _ -> start_node_1(Config, Opts, Prog) end. -start_node_1(Config, Opts) -> +start_node_1(Config, Opts, Prog) -> Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) ++ "-" @@ -341,7 +441,11 @@ start_node_1(Config, Opts) -> ++ integer_to_list(erlang:system_time(second)) ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, slave, [{args, Opts++" -pa "++Pa}]). + ErlArg = case Prog of + [] -> []; + _ -> [{erl,[Prog]}] + end, + test_server:start_node(Name, slave, [{args, Opts++" -pa "++Pa} | ErlArg]). stop_node(Node) when Node =:= node() -> ok; stop_node(Node) -> diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h index 97ee58cdad..5272f86c98 100644 --- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h +++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h @@ -156,7 +156,8 @@ typedef void* erts_cond; #define IS_SMP_ENABLED ((int) ALC_TEST0(0xf13)) #define ALLOC_TEST(S) ((void*) ALC_TEST1(0xf14, (S))) #define FREE_TEST(P) ((void) ALC_TEST1(0xf15, (P))) -#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf16, (SZ), (CMBC), (DMBC))) -#define GET_TEST_MBC_SIZE() ((int) ALC_TEST0(0xf17)) +#define REALLOC_TEST(P,S) ((void*) ALC_TEST2(0xf16, (P), (S))) +#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf17, (SZ), (CMBC), (DMBC))) +#define GET_TEST_MBC_SIZE() ((int) ALC_TEST0(0xf18)) #endif diff --git a/erts/emulator/test/alloc_SUITE_data/migration.c b/erts/emulator/test/alloc_SUITE_data/migration.c index b9a4de03b3..1d974225fc 100644 --- a/erts/emulator/test/alloc_SUITE_data/migration.c +++ b/erts/emulator/test/alloc_SUITE_data/migration.c @@ -223,6 +223,42 @@ static int rand_int(MigrationState* state, int low, int high) return low + (x % (high+1-low)); } +enum Operation +{ + ALLOCATE_OP, + FREE_OP, + REALLOC_OP, + CLEANUP_OP +}; + +static enum Operation rand_op(MigrationState* state) +{ + int r = rand_int(state, 1, 100); + switch (state->phase) { + case GROWING: + FATAL_ASSERT(state->nblocks < state->max_nblocks); + if (r > 10 || state->nblocks == 0) + return ALLOCATE_OP; + else if (r > 5) + return FREE_OP; + else + return REALLOC_OP; + + case SHRINKING: + FATAL_ASSERT(state->nblocks > 0); + if (r > 10 || state->nblocks == state->max_nblocks) + return FREE_OP; + else if (r > 5) + return ALLOCATE_OP; + else + return REALLOC_OP; + + case CLEANUP: + return CLEANUP_OP; + default: + FATAL_ASSERT(!"Invalid op phase"); + } +} static void do_cleanup(TestCaseState_t *tcs, MigrationState* state) { @@ -275,53 +311,75 @@ testcase_run(TestCaseState_t *tcs) state->goal_nblocks = rand_int(state, 1, state->max_nblocks); } - switch (state->phase) { - case GROWING: { + switch (rand_op(state)) { + case ALLOCATE_OP: { MyBlock* p; FATAL_ASSERT(!state->blockv[state->nblocks]); - p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size)); + p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size)); FATAL_ASSERT(p); add_block(p, state); - state->blockv[state->nblocks] = p; - if (++state->nblocks >= state->goal_nblocks) { - /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/ - state->phase = SHRINKING; - state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1); - } - else - FATAL_ASSERT(!state->blockv[state->nblocks]); + state->blockv[state->nblocks++] = p; break; } - case SHRINKING: { + case FREE_OP: { int ix = rand_int(state, 0, state->nblocks-1); FATAL_ASSERT(state->blockv[ix]); remove_block(state->blockv[ix]); FREE_TEST(state->blockv[ix]); state->blockv[ix] = state->blockv[--state->nblocks]; state->blockv[state->nblocks] = NULL; - - if (state->nblocks <= state->goal_nblocks) { - /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/ - if (++state->round >= MAX_ROUNDS) { - state->phase = CLEANUP; - } else { - state->phase = GROWING; - state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks); - } - } break; } + case REALLOC_OP: { + int ix = rand_int(state, 0, state->nblocks-1); + MyBlock* p; + FATAL_ASSERT(state->blockv[ix]); + remove_block(state->blockv[ix]); + p = REALLOC_TEST(state->blockv[ix], rand_int(state, state->block_size/2, state->block_size)); + FATAL_ASSERT(p); + add_block(p, state); + state->blockv[ix] = p; + break; + } + case CLEANUP_OP: + do_cleanup(tcs, state); + break; + default: + FATAL_ASSERT(!"Invalid operation"); + } + + switch (state->phase) { + case GROWING: { + if (state->nblocks >= state->goal_nblocks) { + /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/ + state->phase = SHRINKING; + state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1); + } + else + FATAL_ASSERT(!state->blockv[state->nblocks]); + break; + } + case SHRINKING: { + if (state->nblocks <= state->goal_nblocks) { + /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/ + if (++state->round >= MAX_ROUNDS) { + state->phase = CLEANUP; + } else { + state->phase = GROWING; + state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks); + } + } + break; + } case CLEANUP: - do_cleanup(tcs, state); - break; + case DONE: + break; default: FATAL_ASSERT(!"Invalid phase"); } - if (state->phase == DONE) { - } - else { + if (state->phase != DONE) { testcase_continue(tcs); } } diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 4bc1838139..a3c3daac15 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -262,6 +262,7 @@ test_deep_bitstr(List) -> {Bin,bitstring_to_list(Bin)}. bad_list_to_binary(Config) when is_list(Config) -> + test_bad_bin(<<1:1>>), test_bad_bin(atom), test_bad_bin(42), test_bad_bin([1|2]), diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl index 16d581a567..821381bf0d 100644 --- a/erts/emulator/test/efile_SUITE.erl +++ b/erts/emulator/test/efile_SUITE.erl @@ -46,14 +46,14 @@ iter_max_files_1(Config) -> DataDir = proplists:get_value(data_dir,Config), TestFile = filename:join(DataDir, "existing_file"), N = 10, - %% Run on a different node in order to set the max ports + %% Run on a different node in order to make the test more stable. Dir = filename:dirname(code:which(?MODULE)), {ok,Node} = test_server:start_node(test_iter_max_files,slave, - [{args,"+Q 1524 -pa " ++ Dir}]), + [{args,"-pa " ++ Dir}]), L = rpc:call(Node,?MODULE,do_iter_max_files,[N, TestFile]), test_server:stop_node(Node), io:format("Number of files opened in each test:~n~w\n", [L]), - all_equal(L), + verify_max_files(L), Head = hd(L), if Head >= 2 -> ok; true -> ct:fail(too_few_files) @@ -65,12 +65,15 @@ do_iter_max_files(N, Name) when N > 0 -> do_iter_max_files(_, _) -> []. -all_equal([E, E| T]) -> - all_equal([E| T]); -all_equal([_]) -> - ok; -all_equal([]) -> - ok. +%% The attempts shouldn't vary too much; we used to require that they were all +%% exactly equal, but after we reimplemented the file driver as a NIF we +%% noticed that the only reason it was stable on Darwin was because the port +%% limit was hit before ulimit. +verify_max_files(Attempts) -> + N = length(Attempts), + Mean = lists:sum(Attempts) / N, + Variance = lists:sum([(X - Mean) * (X - Mean) || X <- Attempts]) / N, + true = math:sqrt(Variance) =< 1 + (Mean / 1000). max_files(Name) -> Fds = open_files(Name), diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl index 49dc64b0d2..963b7e2501 100644 --- a/erts/emulator/test/iovec_SUITE.erl +++ b/erts/emulator/test/iovec_SUITE.erl @@ -25,7 +25,8 @@ -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1, mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1, sub_binary_lists/1, iolist_to_iovec_idempotence/1, - iolist_to_iovec_correctness/1]). + iolist_to_iovec_correctness/1, unaligned_sub_binaries/1, + direct_binary_arg/1]). -include_lib("common_test/include/ct.hrl"). @@ -36,7 +37,8 @@ suite() -> all() -> [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists, sub_binary_lists, illegal_lists, improper_lists, cons_bomb, - iolist_to_iovec_idempotence, iolist_to_iovec_correctness]. + iolist_to_iovec_idempotence, iolist_to_iovec_correctness, + unaligned_sub_binaries, direct_binary_arg]. init_per_suite(Config) -> Config. @@ -78,7 +80,7 @@ illegal_lists(Config) when is_list(Config) -> BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]), BadInts = gen_variations(["gurka", 890, "gaffel"]), Atoms = gen_variations([gurka, "gaffel"]), - BadTails = [["test" | 0], ["gurka", gaffel]], + BadTails = [["test" | 0], ["gurka" | gaffel], ["gaffel" | <<1:1>>]], Variations = BitStrs ++ BadInts ++ Atoms ++ BadTails, @@ -98,14 +100,7 @@ cons_bomb(Config) when is_list(Config) -> BinBase = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), MixBase = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), - Rounds = - case system_mem_size() of - Mem when Mem >= (16 bsl 30) -> 32; - Mem when Mem >= (3 bsl 30) -> 28; - _ -> 20 - end, - - Variations = gen_variations([IntBase, BinBase, MixBase], Rounds), + Variations = gen_variations([IntBase, BinBase, MixBase], 16), equivalence_test(fun erlang:iolist_to_iovec/1, Variations). iolist_to_iovec_idempotence(Config) when is_list(Config) -> @@ -130,6 +125,21 @@ iolist_to_iovec_correctness(Config) when is_list(Config) -> true = is_iolist_equal(Optimized, Variations), ok. +unaligned_sub_binaries(Config) when is_list(Config) -> + UnalignedBins = [gen_unaligned_binary(I) || I <- lists:seq(32, 4 bsl 10, 512)], + UnalignedVariations = gen_variations(UnalignedBins), + + Optimized = erlang:iolist_to_iovec(UnalignedVariations), + + true = is_iolist_equal(Optimized, UnalignedVariations), + ok. + +direct_binary_arg(Config) when is_list(Config) -> + {'EXIT',{badarg, _}} = (catch erlang:iolist_to_iovec(<<1:1>>)), + [<<1>>] = erlang:iolist_to_iovec(<<1>>), + [] = erlang:iolist_to_iovec(<<>>), + ok. + illegality_test(Fun, Variations) -> [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations], ok. @@ -145,11 +155,18 @@ equivalence_test(Fun, [Head | _] = Variations) -> is_iolist_equal(A, B) -> iolist_to_binary(A) =:= iolist_to_binary(B). +gen_unaligned_binary(Size) -> + Bin0 = << <<I>> || I <- lists:seq(1, Size) >>, + <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>), + Bin. + +id(I) -> I. + %% Generates a bunch of lists whose contents will be equal to Base repeated a %% few times. The lists only differ by their structure, so their reduction to %% a simpler format should yield the same result. gen_variations(Base) -> - gen_variations(Base, 16). + gen_variations(Base, 12). gen_variations(Base, N) -> [gen_flat_list(Base, N), gen_nested_list(Base, N), @@ -169,8 +186,3 @@ gen_nasty_list_1([Head | Base], Result) when is_list(Head) -> gen_nasty_list_1(Base, [[Result], [gen_nasty_list_1(Head, [])]]); gen_nasty_list_1([Head | Base], Result) -> gen_nasty_list_1(Base, [[Result], [Head]]). - -system_mem_size() -> - application:ensure_all_started(os_mon), - {Tot,_Used,_} = memsup:get_memory_data(), - Tot. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 85c302e310..b6e15ababb 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -3049,8 +3049,24 @@ nif_ioq(Config) -> {peek_head, a}, {peek, a}, - %% Test to enqueue a bunch of refc binaries + %% Ensure that enqueued refc binaries are intact after a roundtrip. + %% + %% This test and the ones immediately following it does not go through + %% erlang:iolist_to_iovec/1 {enqv, a, [nif_ioq_payload(refcbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... heap binaries + {enqv, a, [nif_ioq_payload(heapbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... plain sub-binaries + {enqv, a, [nif_ioq_payload(subbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... unaligned binaries + {enqv, a, [nif_ioq_payload(unaligned_bin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, %% Enq stuff to destroy with data in queue {enqv, a, 2, 100}, @@ -3206,13 +3222,16 @@ nif_ioq_payload(N) when is_integer(N) -> Tail = if N > 3 -> nif_ioq_payload(N-3); true -> [] end, Head = element(1, lists:split(N,[nif_ioq_payload(subbin), nif_ioq_payload(heapbin), - nif_ioq_payload(refcbin) | Tail])), + nif_ioq_payload(refcbin), + nif_ioq_payload(unaligned_bin) | Tail])), erlang:iolist_to_iovec(Head); nif_ioq_payload(subbin) -> Bin = nif_ioq_payload(refcbin), Sz = size(Bin) - 1, <<_:8,SubBin:Sz/binary,_/bits>> = Bin, SubBin; +nif_ioq_payload(unaligned_bin) -> + make_unaligned_binary(<< <<I>> || I <- lists:seq(1, 255) >>); nif_ioq_payload(heapbin) -> <<"a literal heap binary">>; nif_ioq_payload(refcbin) -> @@ -3220,6 +3239,13 @@ nif_ioq_payload(refcbin) -> nif_ioq_payload(Else) -> Else. +make_unaligned_binary(Bin0) -> + Size = byte_size(Bin0), + <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>), + Bin. + +id(I) -> I. + %% The NIFs: lib_version() -> undefined. call_history() -> ?nif_stub. diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index f7c8c27d53..f7774c6e2e 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -80,6 +80,8 @@ static char *plusM_au_alloc_switches[] = { "as", "asbcst", "acul", + "acnl", + "acfml", "e", "t", "lmbcs", diff --git a/erts/lib_src/common/erl_printf_format.c b/erts/lib_src/common/erl_printf_format.c index 3302083288..5a680d6f9d 100644 --- a/erts/lib_src/common/erl_printf_format.c +++ b/erts/lib_src/common/erl_printf_format.c @@ -326,7 +326,7 @@ static int fmt_double(fmtfn_t fn,void*arg,double val, { int res; int fi = 0; - char format_str[7]; + char format_str[8]; char sbuf[32]; char *bufp = sbuf; double dexp; diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex 2d86dd3843..02e77bfbb2 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex f0da8b5869..cf4b5c56ea 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/src/erl_prim_loader.erl b/erts/preloaded/src/erl_prim_loader.erl index d16fe5a0bc..b82b9f5b46 100644 --- a/erts/preloaded/src/erl_prim_loader.erl +++ b/erts/preloaded/src/erl_prim_loader.erl @@ -56,7 +56,7 @@ -export([purge_archive_cache/0]). %% Used by init and the code server. --export([get_modules/2,get_modules/3]). +-export([get_modules/2,get_modules/3, is_basename/1]). -include_lib("kernel/include/file.hrl"). diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 278a485a01..d352e11751 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -2403,6 +2403,10 @@ subtract(_,_) -> OldDirtyCPUSchedulersOnline when DirtyCPUSchedulersOnline :: pos_integer(), OldDirtyCPUSchedulersOnline :: pos_integer(); + (erts_alloc, {Alloc, F, V}) -> ok | notsup when + Alloc :: atom(), + F :: atom(), + V :: integer(); (fullsweep_after, Number) -> OldNumber when Number :: non_neg_integer(), OldNumber :: non_neg_integer(); @@ -3730,15 +3734,14 @@ memory_is_supported() -> get_blocks_size([{blocks_size, Sz, _, _} | Rest], Acc) -> get_blocks_size(Rest, Acc+Sz); -get_blocks_size([{_, _, _, _} | Rest], Acc) -> - get_blocks_size(Rest, Acc); get_blocks_size([{blocks_size, Sz} | Rest], Acc) -> get_blocks_size(Rest, Acc+Sz); -get_blocks_size([{_, _} | Rest], Acc) -> +get_blocks_size([_ | Rest], Acc) -> get_blocks_size(Rest, Acc); get_blocks_size([], Acc) -> Acc. + blocks_size([{Carriers, SizeList} | Rest], Acc) when Carriers == mbcs; Carriers == mbcs_pool; Carriers == sbcs -> diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 679a2241d2..e0ae6b1656 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -32,8 +32,8 @@ %% (Optional - default efile) %% -hosts [Node] : List of hosts from which we can boot. %% (Mandatory if -loader inet) -%% -mode embedded : Load all modules at startup, no automatic loading -%% -mode interactive : Auto load modules (default system behaviour). +%% -mode interactive : Auto load modules not needed at startup (default system behaviour). +%% -mode embedded : Load all modules in the boot script, disable auto loading. %% -path : Override path in bootfile. %% -pa Path+ : Add my own paths first. %% -pz Path+ : Add my own paths last. diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl index b3d0ba20bc..bb1afecafc 100644 --- a/erts/test/z_SUITE.erl +++ b/erts/test/z_SUITE.erl @@ -40,7 +40,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap, {minutes, 5}}]. + {timetrap, {minutes, 10}}]. all() -> [core_files]. diff --git a/erts/vsn.mk b/erts/vsn.mk index 8b22afea3b..50b77fdbc0 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 9.2 +VSN = 9.2.1 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/asn1/src/asn1ct_gen_per.erl b/lib/asn1/src/asn1ct_gen_per.erl index 82e9326294..c09b0f47d1 100644 --- a/lib/asn1/src/asn1ct_gen_per.erl +++ b/lib/asn1/src/asn1ct_gen_per.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -47,14 +47,20 @@ dialyzer_suppressions(#gen{erule=per,aligned=Aligned}) -> false -> uper; true -> per end, - case asn1ct_func:is_used({Mod,complete,1}) of + suppress({Mod,complete,1}), + suppress({per_common,to_bitstring,2}), + emit([" ok.",nl]). + +suppress({M,F,A}=MFA) -> + case asn1ct_func:is_used(MFA) of false -> ok; true -> - emit([" _ = complete(Arg),",nl]) - end, - emit([" ok.",nl]). - + Args = + [lists:concat(["element(",I,", Arg)"]) + || I <- lists:seq(1, A)], + emit([" ",{call,M,F,Args},com,nl]) + end. gen_encode(Erules,Type) when is_record(Type,typedef) -> gen_encode_user(Erules,Type). diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 69e371a30f..fd7fa07b81 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -872,8 +872,8 @@ fail(Reason) -> try exit({test_case_failed,Reason}) catch - Class:R -> - case erlang:get_stacktrace() of + Class:R:S -> + case S of [{?MODULE,fail,1,_}|Stk] -> ok; Stk -> ok end, @@ -894,8 +894,8 @@ fail(Format, Args) -> try exit({test_case_failed,lists:flatten(Str)}) catch - Class:R -> - case erlang:get_stacktrace() of + Class:R:S -> + case S of [{?MODULE,fail,2,_}|Stk] -> ok; Stk -> ok end, diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index f0592a40be..bb33f0243b 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -233,8 +233,7 @@ call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) -> Rest ++ [{NewId, call_init}, {NewId,NextFun}]} end, call(resort(NewRest,NewHooks,Meta), Config, Meta, NewHooks) - catch Error:Reason -> - Trace = erlang:get_stacktrace(), + catch Error:Reason:Trace -> ct_logs:log("Suite Hook","Failed to start a CTH: ~tp:~tp", [Error,{Reason,Trace}]), call([], {fail,"Failed to start CTH" @@ -422,8 +421,7 @@ catch_apply(M,F,A, Default, Fallback) -> catch_apply(M,F,A) -> try erlang:apply(M,F,A) - catch _:Reason -> - Trace = erlang:get_stacktrace(), + catch _:Reason:Trace -> ct_logs:log("Suite Hook","Call to CTH failed: ~w:~tp", [error,{Reason,Trace}]), throw({error_in_cth_call, diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 8c401cf3f5..afc5fee4f7 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -442,11 +442,9 @@ script_start2(Opts = #opts{vts = undefined, TestSpecData -> execute_all_specs(TestSpecData, Opts, Args, []) catch - throw:{error,Reason} -> - StackTrace = erlang:get_stacktrace(), + throw:{error,Reason}:StackTrace -> {error,{invalid_testspec,{Reason,StackTrace}}}; - _:Reason -> - StackTrace = erlang:get_stacktrace(), + _:Reason:StackTrace -> {error,{invalid_testspec,{Reason,StackTrace}}} end; [] -> @@ -1211,11 +1209,9 @@ run_spec_file(Relaxed, TestSpecData -> run_all_specs(TestSpecData, Opts, StartOpts, []) catch - throw:{error,CTReason} -> - StackTrace = erlang:get_stacktrace(), + throw:{error,CTReason}:StackTrace -> exit({error,{invalid_testspec,{CTReason,StackTrace}}}); - _:CTReason -> - StackTrace = erlang:get_stacktrace(), + _:CTReason:StackTrace -> exit({error,{invalid_testspec,{CTReason,StackTrace}}}) end. diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index bb445bb0d2..bd3755722f 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -1425,7 +1425,12 @@ skip_groups1(Suite,Groups,Cmt,Suites0) -> GrAndCases1 = GrAndCases0 ++ SkipGroups, insert_in_order({Suite,GrAndCases1},Suites0,replace); false -> - insert_in_order({Suite,SkipGroups},Suites0,replace) + case Suites0 of + [{all,_}=All|Skips]-> + [All|Skips++[{Suite,SkipGroups}]]; + _ -> + insert_in_order({Suite,SkipGroups},Suites0,replace) + end end. skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) -> diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index 10a06d5c88..e904bb1e7c 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -209,11 +209,10 @@ do_start(Parent, Mode, LogDir, Verbosity) -> self() ! {{stop,{self(),{user_error,CTHReason}}}, {Parent,make_ref()}} catch - _:CTHReason -> + _:CTHReason:StackTrace -> ErrorInfo = if is_atom(CTHReason) -> io_lib:format("{~tp,~tp}", - [CTHReason, - erlang:get_stacktrace()]); + [CTHReason, StackTrace]); true -> CTHReason end, diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 7250041e13..9469619aa9 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -1340,13 +1340,12 @@ do_init_per_testcase(Mod, Args) -> {skip,Reason}; exit:{Skip,Reason} when Skip =:= skip; Skip =:= skipped -> {skip,Reason}; - throw:Other -> - set_loc(erlang:get_stacktrace()), + throw:Other:Stk -> + set_loc(Stk), Line = get_loc(), print_init_conf_result(Line,"thrown",Other), {skip,{failed,{Mod,init_per_testcase,Other}}}; - _:Reason0 -> - Stk = erlang:get_stacktrace(), + _:Reason0:Stk -> Reason = {Reason0,Stk}, set_loc(Stk), Line = get_loc(), @@ -1395,20 +1394,19 @@ do_end_per_testcase(Mod,EndFunc,Func,Conf) -> _ -> ok catch - throw:Other -> + throw:Other:Stk -> Comment0 = case read_comment() of "" -> ""; Cmt -> Cmt ++ test_server_ctrl:xhtml("<br>", "<br />") end, - set_loc(erlang:get_stacktrace()), + set_loc(Stk), comment(io_lib:format("~ts<font color=\"red\">" "WARNING: ~w thrown!" "</font>\n",[Comment0,EndFunc])), print_end_tc_warning(EndFunc,Other,"thrown",get_loc()), {failed,{Mod,end_per_testcase,Other}}; - Class:Reason -> - Stk = erlang:get_stacktrace(), + Class:Reason:Stk -> set_loc(Stk), Why = case Class of exit -> {'EXIT',Reason}; @@ -1550,8 +1548,7 @@ ts_tc(M, F, A) -> throw:{skipped, Reason} -> {skip, Reason}; exit:{skip, Reason} -> {skip, Reason}; exit:{skipped, Reason} -> {skip, Reason}; - Type:Reason -> - Stk = erlang:get_stacktrace(), + Type:Reason:Stk -> set_loc(Stk), case Type of throw -> @@ -1740,8 +1737,8 @@ fail(Reason) -> try exit({suite_failed,Reason}) catch - Class:R -> - case erlang:get_stacktrace() of + Class:R:Stacktrace -> + case Stacktrace of [{?MODULE,fail,1,_}|Stk] -> ok; Stk -> ok end, @@ -1763,8 +1760,8 @@ fail() -> try exit(suite_failed) catch - Class:R -> - case erlang:get_stacktrace() of + Class:R:Stacktrace -> + case Stacktrace of [{?MODULE,fail,0,_}|Stk] -> ok; Stk -> ok end, @@ -2043,15 +2040,15 @@ call_user_timetrap(Func, Sup) when is_function(Func) -> try Func() of Result -> Sup ! {self(),Result} - catch _:Error -> - exit({Error,erlang:get_stacktrace()}) + catch _:Error:Stk -> + exit({Error,Stk}) end; call_user_timetrap({M,F,A}, Sup) -> try apply(M,F,A) of Result -> Sup ! {self(),Result} - catch _:Error -> - exit({Error,erlang:get_stacktrace()}) + catch _:Error:Stk -> + exit({Error,Stk}) end. save_user_timetrap(TCPid, UserTTSup, StartTime) -> diff --git a/lib/common_test/src/test_server_sup.erl b/lib/common_test/src/test_server_sup.erl index 6ddbf1ad27..139621141d 100644 --- a/lib/common_test/src/test_server_sup.erl +++ b/lib/common_test/src/test_server_sup.erl @@ -781,8 +781,8 @@ framework_call(Callback,Func,Args,DefaultReturn) -> catch exit:Why -> EH(Why); - error:Why -> - EH({Why,erlang:get_stacktrace()}); + error:Why:Stacktrace -> + EH({Why,Stacktrace}); throw:Why -> EH(Why) end; diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl index 63bf9be134..975f6cafcb 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl @@ -231,8 +231,7 @@ data_for_channel(CM, Ch, Data, State) -> {ok, NewState} end catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> error_logger:error_report([{?MODULE, data_for_channel}, {request, Data}, {buffer, State#session.buffer}, diff --git a/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl b/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl index 993452500e..041c13cdbd 100644 --- a/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl +++ b/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl @@ -77,7 +77,7 @@ all() -> 'fail_αβ_4'(_Config) -> ct:log("This is test case ~tw",[?FUNCTION_NAME]), - S = try throw(ok) catch throw:ok -> erlang:get_stacktrace() end, + S = try throw(ok) catch throw:ok:Stacktrace -> Stacktrace end, erlang:raise(error,{error,testcase,?FUNCTION_NAME},S), ok. diff --git a/lib/common_test/test_server/ts_install_cth.erl b/lib/common_test/test_server/ts_install_cth.erl index 5d325b1115..35a5ec916c 100644 --- a/lib/common_test/test_server/ts_install_cth.erl +++ b/lib/common_test/test_server/ts_install_cth.erl @@ -108,8 +108,7 @@ pre_init_per_suite(_Suite,Config,State) -> {add_node_name(Config, State), State} catch error:{badmatch,{error,enoent}} -> {add_node_name(Config, State), State}; - Error:Reason -> - Stack = erlang:get_stacktrace(), + Error:Reason:Stack -> ct:pal("~p failed! ~p:{~p,~p}",[?MODULE,Error,Reason,Stack]), {{fail,{?MODULE,{Error,Reason, Stack}}},State} end. diff --git a/lib/compiler/src/beam_a.erl b/lib/compiler/src/beam_a.erl index 7df2edd714..91acb19971 100644 --- a/lib/compiler/src/beam_a.erl +++ b/lib/compiler/src/beam_a.erl @@ -61,6 +61,14 @@ rename_instrs([{'%live',_}|Is]) -> %% Ignore old type of live annotation. Only happens when compiling %% from very old .S files. rename_instrs(Is); +rename_instrs([{get_list,S,D1,D2}|Is]) -> + %% Only happens when compiling from old .S files. + if + D1 =:= S -> + [{get_tl,S,D2},{get_hd,S,D1}|rename_instrs(Is)]; + true -> + [{get_hd,S,D1},{get_tl,S,D2}|rename_instrs(Is)] + end; rename_instrs([I|Is]) -> [rename_instr(I)|rename_instrs(Is)]; rename_instrs([]) -> []. diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index 453e00fce3..fa919ca862 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -407,7 +407,17 @@ encode_arg({atom, Atom}, Dict0) when is_atom(Atom) -> {Index, Dict} = beam_dict:atom(Atom, Dict0), {encode(?tag_a, Index), Dict}; encode_arg({integer, N}, Dict) -> - {encode(?tag_i, N), Dict}; + %% Conservatily assume that all integers whose absolute + %% value is greater than 1 bsl 128 will be bignums in + %% the runtime system. + if + N >= 1 bsl 128 -> + encode_arg({literal, N}, Dict); + N =< -(1 bsl 128) -> + encode_arg({literal, N}, Dict); + true -> + {encode(?tag_i, N), Dict} + end; encode_arg(nil, Dict) -> {encode(?tag_a, 0), Dict}; encode_arg({f, W}, Dict) -> diff --git a/lib/compiler/src/beam_block.erl b/lib/compiler/src/beam_block.erl index d0536e0669..47a2be8ab5 100644 --- a/lib/compiler/src/beam_block.erl +++ b/lib/compiler/src/beam_block.erl @@ -36,13 +36,11 @@ module({Mod,Exp,Attr,Fs0,Lc}, Opts) -> function({function,Name,Arity,CLabel,Is0}, Blockify) -> try %% Collect basic blocks and optimize them. - Is2 = case Blockify of - true -> - Is1 = blockify(Is0), - embed_lines(Is1); - false -> - Is0 + Is1 = case Blockify of + false -> Is0; + true -> blockify(Is0) end, + Is2 = embed_lines(Is1), Is3 = local_cse(Is2), Is4 = beam_utils:anno_defs(Is3), Is5 = move_allocates(Is4), @@ -110,7 +108,8 @@ collect({put_tuple,A,D}) -> {set,[D],[],{put_tuple,A}}; collect({put,S}) -> {set,[],[S],put}; collect({get_tuple_element,S,I,D}) -> {set,[D],[S],{get_tuple_element,I}}; collect({set_tuple_element,S,D,I}) -> {set,[],[S,D],{set_tuple_element,I}}; -collect({get_list,S,D1,D2}) -> {set,[D1,D2],[S],get_list}; +collect({get_hd,S,D}) -> {set,[D],[S],get_hd}; +collect({get_tl,S,D}) -> {set,[D],[S],get_tl}; collect(remove_message) -> {set,[],[],remove_message}; collect({put_map,F,Op,S,D,R,{list,Puts}}) -> {set,[D],[S|Puts],{alloc,R,{put_map,Op,F}}}; @@ -138,6 +137,11 @@ embed_lines([{block,B2},{line,_}=Line,{block,B1}|T], Acc) -> embed_lines([{block,B1},{line,_}=Line|T], Acc) -> B = {block,[{set,[],[],Line}|B1]}, embed_lines([B|T], Acc); +embed_lines([{block,B2},{block,B1}|T], Acc) -> + %% This can only happen when beam_block is run for + %% the second time. + B = {block,B1++B2}, + embed_lines([B|T], Acc); embed_lines([I|Is], Acc) -> embed_lines(Is, [I|Acc]); embed_lines([], Acc) -> Acc. @@ -205,7 +209,7 @@ move_allocates([]) -> []. move_allocates_1([{'%anno',_}|Is], Acc) -> move_allocates_1(Is, Acc); -move_allocates_1([I|Is], [{set,[],[],{alloc,Live0,Info}}|Acc]=Acc0) -> +move_allocates_1([I|Is], [{set,[],[],{alloc,Live0,Info0}}|Acc]=Acc0) -> case alloc_may_pass(I) of false -> move_allocates_1(Is, [I|Acc0]); @@ -214,6 +218,7 @@ move_allocates_1([I|Is], [{set,[],[],{alloc,Live0,Info}}|Acc]=Acc0) -> not_possible -> move_allocates_1(Is, [I|Acc0]); Live when is_integer(Live) -> + Info = safe_info(Info0), A = {set,[],[],{alloc,Live,Info}}, move_allocates_1(Is, [A,I|Acc]) end @@ -222,12 +227,20 @@ move_allocates_1([I|Is], Acc) -> move_allocates_1(Is, [I|Acc]); move_allocates_1([], Acc) -> Acc. +alloc_may_pass({set,_,[{fr,_}],fmove}) -> false; alloc_may_pass({set,_,_,{alloc,_,_}}) -> false; alloc_may_pass({set,_,_,{set_tuple_element,_}}) -> false; alloc_may_pass({set,_,_,put_list}) -> false; alloc_may_pass({set,_,_,put}) -> false; alloc_may_pass({set,_,_,_}) -> true. +safe_info({nozero,Stack,Heap,_}) -> + %% nozero is not safe if the allocation instruction is moved + %% upwards past an instruction that may throw an exception + %% (such as element/2). + {zero,Stack,Heap,[]}; +safe_info(Info) -> Info. + %% opt([Instruction]) -> [Instruction] %% Optimize the instruction stream inside a basic block. @@ -251,6 +264,16 @@ opt([{set,[D1],[{integer,Idx1},Reg],{bif,element,{f,L}}}=I1, {set,[D2],[{integer,Idx2},Reg],{bif,element,{f,L}}}=I2|Is]) when Idx1 < Idx2, D1 =/= D2, D1 =/= Reg, D2 =/= Reg -> opt([I2,I1|Is]); +opt([{set,Hd0,Cons,get_hd}=GetHd, + {set,Tl0,Cons,get_tl}=GetTl|Is0]) -> + case {opt_moves(Hd0, [GetTl|Is0]),opt_moves(Tl0, [GetHd|Is0])} of + {{Hd0,Is},{Tl0,_}} -> + [GetHd|opt(Is)]; + {{Hd,Is},{Tl0,_}} -> + [{set,Hd,Cons,get_hd}|opt(Is)]; + {{_,_},{Tl,Is}} -> + [{set,Tl,Cons,get_tl}|opt(Is)] + end; opt([{set,Ds0,Ss,Op}|Is0]) -> {Ds,Is} = opt_moves(Ds0, Is0), [{set,Ds,Ss,Op}|opt(Is)]; @@ -266,17 +289,6 @@ opt_moves([D0]=Ds, Is0) -> case opt_move(D0, Is0) of not_possible -> {Ds,Is0}; {D1,Is} -> {[D1],Is} - end; -opt_moves([X0,Y0], Is0) -> - {X,Is2} = case opt_move(X0, Is0) of - not_possible -> {X0,Is0}; - {Y0,_} -> {X0,Is0}; - {_X1,_Is1} = XIs1 -> XIs1 - end, - case opt_move(Y0, Is2) of - not_possible -> {[X,Y0],Is2}; - {X,_} -> {[X,Y0],Is2}; - {Y,Is} -> {[X,Y],Is} end. %% opt_move(Dest, [Instruction]) -> {UpdatedDest,[Instruction]} | not_possible @@ -619,7 +631,13 @@ cse_find(Expr, Es) -> end. cse_expr({set,[D],Ss,{bif,N,_}}) -> - {ok,D,{{bif,N},Ss}}; + case D of + {fr,_} -> + %% There are too many things that can go wrong. + none; + _ -> + {ok,D,{{bif,N},Ss}} + end; cse_expr({set,[D],Ss,{alloc,_,{gc_bif,N,_}}}) -> {ok,D,{{gc_bif,N},Ss}}; cse_expr({set,[D],Ss,put_list}) -> diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl index e094c2c320..7ddf9fa2e2 100644 --- a/lib/compiler/src/beam_clean.erl +++ b/lib/compiler/src/beam_clean.erl @@ -24,7 +24,7 @@ -export([module/2]). -export([bs_clean_saves/1]). -export([clean_labels/1]). --import(lists, [foldl/3,reverse/1,filter/2]). +-import(lists, [foldl/3,reverse/1]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -303,8 +303,21 @@ maybe_remove_lines(Fs, Opts) -> end. remove_lines([{function,N,A,Lbl,Is0}|T]) -> - Is = filter(fun({line,_}) -> false; - (_) -> true - end, Is0), + Is = remove_lines_fun(Is0), [{function,N,A,Lbl,Is}|remove_lines(T)]; remove_lines([]) -> []. + +remove_lines_fun([{line,_}|Is]) -> + remove_lines_fun(Is); +remove_lines_fun([{block,Bl0}|Is]) -> + Bl = remove_lines_block(Bl0), + [{block,Bl}|remove_lines_fun(Is)]; +remove_lines_fun([I|Is]) -> + [I|remove_lines_fun(Is)]; +remove_lines_fun([]) -> []. + +remove_lines_block([{set,_,_,{line,_}}|Is]) -> + remove_lines_block(Is); +remove_lines_block([I|Is]) -> + [I|remove_lines_block(Is)]; +remove_lines_block([]) -> []. diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index 50b76d7f29..a68c4b5367 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -1090,6 +1090,10 @@ resolve_inst({build_stacktrace,[]},_,_,_) -> build_stacktrace; resolve_inst({raw_raise,[]},_,_,_) -> raw_raise; +resolve_inst({get_hd,[Src,Dst]},_,_,_) -> + {get_hd,Src,Dst}; +resolve_inst({get_tl,[Src,Dst]},_,_,_) -> + {get_tl,Src,Dst}; %% %% Catches instructions that are not yet handled. diff --git a/lib/compiler/src/beam_flatten.erl b/lib/compiler/src/beam_flatten.erl index a4d45a4ca6..c60211f516 100644 --- a/lib/compiler/src/beam_flatten.erl +++ b/lib/compiler/src/beam_flatten.erl @@ -50,6 +50,9 @@ norm_block([{set,[],[],{alloc,R,Alloc}}|Is], Acc0) -> Acc -> norm_block(Is, Acc) end; +norm_block([{set,[D1],[S],get_hd},{set,[D2],[S],get_tl}|Is], Acc) -> + I = {get_list,S,D1,D2}, + norm_block(Is, [I|Acc]); norm_block([I|Is], Acc) -> norm_block(Is, [norm(I)|Acc]); norm_block([], Acc) -> Acc. @@ -64,12 +67,14 @@ norm({set,[D],[],{put_tuple,A}}) -> {put_tuple,A,D}; norm({set,[],[S],put}) -> {put,S}; norm({set,[D],[S],{get_tuple_element,I}}) -> {get_tuple_element,S,I,D}; norm({set,[],[S,D],{set_tuple_element,I}}) -> {set_tuple_element,S,D,I}; -norm({set,[D1,D2],[S],get_list}) -> {get_list,S,D1,D2}; +norm({set,[D],[S],get_hd}) -> {get_hd,S,D}; +norm({set,[D],[S],get_tl}) -> {get_tl,S,D}; norm({set,[D],[S|Puts],{alloc,R,{put_map,Op,F}}}) -> {put_map,F,Op,S,D,R,{list,Puts}}; norm({set,[],[],remove_message}) -> remove_message; norm({set,[],[],fclearerror}) -> fclearerror; -norm({set,[],[],fcheckerror}) -> {fcheckerror,{f,0}}. +norm({set,[],[],fcheckerror}) -> {fcheckerror,{f,0}}; +norm({set,[],[],{line,_}=Line}) -> Line. norm_allocate({_Zero,nostack,Nh,[]}, Regs) -> [{test_heap,Nh,Regs}]; diff --git a/lib/compiler/src/beam_split.erl b/lib/compiler/src/beam_split.erl index d041f18806..52dd89b5bb 100644 --- a/lib/compiler/src/beam_split.erl +++ b/lib/compiler/src/beam_split.erl @@ -50,8 +50,9 @@ split_block([{set,[R],[_,_,_]=As,{bif,is_record,{f,Lbl}}}|Is], Bl, Acc) -> split_block(Is, [], [{bif,is_record,{f,Lbl},As,R}|make_block(Bl, Acc)]); split_block([{set,[R],As,{bif,N,{f,Lbl}=Fail}}|Is], Bl, Acc) when Lbl =/= 0 -> split_block(Is, [], [{bif,N,Fail,As,R}|make_block(Bl, Acc)]); -split_block([{set,[R],As,{bif,raise,{f,_}=Fail}}|Is], Bl, Acc) -> - split_block(Is, [], [{bif,raise,Fail,As,R}|make_block(Bl, Acc)]); +split_block([{set,[],[],{line,_}=Line}, + {set,[R],As,{bif,raise,{f,_}=Fail}}|Is], Bl, Acc) -> + split_block(Is, [], [{bif,raise,Fail,As,R},Line|make_block(Bl, Acc)]); split_block([{set,[R],As,{alloc,Live,{gc_bif,N,{f,Lbl}=Fail}}}|Is], Bl, Acc) when Lbl =/= 0 -> split_block(Is, [], [{gc_bif,N,Fail,Live,As,R}|make_block(Bl, Acc)]); @@ -61,8 +62,6 @@ split_block([{set,[D],[S|Puts],{alloc,R,{put_map,Op,{f,Lbl}=Fail}}}|Is], make_block(Bl, Acc)]); split_block([{set,[R],[],{try_catch,Op,L}}|Is], Bl, Acc) -> split_block(Is, [], [{Op,R,L}|make_block(Bl, Acc)]); -split_block([{set,[],[],{line,_}=Line}|Is], Bl, Acc) -> - split_block(Is, [], [Line|make_block(Bl, Acc)]); split_block([I|Is], Bl, Acc) -> split_block(Is, [I|Bl], Acc); split_block([], Bl, Acc) -> make_block(Bl, Acc). diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index b2fabed2c5..b83ed17b55 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -477,8 +477,6 @@ update({set,[D],[S1,S2],{alloc,_,{gc_bif,Op,{f,0}}}}, Ts0) -> update({set,[],_Src,_Op}, Ts0) -> Ts0; update({set,[D],_Src,_Op}, Ts0) -> tdb_update([{D,kill}], Ts0); -update({set,[D1,D2],_Src,_Op}, Ts0) -> - tdb_update([{D1,kill},{D2,kill}], Ts0); update({kill,D}, Ts) -> tdb_update([{D,kill}], Ts); @@ -943,10 +941,10 @@ merge_type_info({tuple,SzKind1,Sz1,[]}, {tuple,_SzKind2,_Sz2,First}=Tuple2) -> merge_type_info({tuple,SzKind1,Sz1,First}, Tuple2); merge_type_info({tuple,_SzKind1,_Sz1,First}=Tuple1, {tuple,SzKind2,Sz2,_}) -> merge_type_info(Tuple1, {tuple,SzKind2,Sz2,First}); -merge_type_info(integer, {integer,_}=Int) -> - Int; -merge_type_info({integer,_}=Int, integer) -> - Int; +merge_type_info(integer, {integer,_}) -> + integer; +merge_type_info({integer,_}, integer) -> + integer; merge_type_info({integer,{Min1,Max1}}, {integer,{Min2,Max2}}) -> {integer,{max(Min1, Min2),min(Max1, Max2)}}; merge_type_info({binary,U1}, {binary,U2}) -> diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index 5333925589..814cfb8265 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -118,7 +118,7 @@ is_killed(R, Is, D) -> St = #live{lbl=D,res=gb_trees:empty()}, case check_liveness(R, Is, St) of {killed,_} -> true; - {exit_not_used,_} -> true; + {exit_not_used,_} -> false; {_,_} -> false end. @@ -131,7 +131,7 @@ is_killed_at(R, Lbl, D) when is_integer(Lbl) -> St0 = #live{lbl=D,res=gb_trees:empty()}, case check_liveness_at(R, Lbl, St0) of {killed,_} -> true; - {exit_not_used,_} -> true; + {exit_not_used,_} -> false; {_,_} -> false end. @@ -148,7 +148,7 @@ is_not_used(R, Is, D) -> St = #live{lbl=D,res=gb_trees:empty()}, case check_liveness(R, Is, St) of {used,_} -> false; - {exit_not_used,_} -> false; + {exit_not_used,_} -> true; {_,_} -> true end. @@ -440,8 +440,11 @@ check_liveness(R, [{bs_init,_,_,Live,Ss,Dst}|Is], St) -> case member(R, Ss) of true -> {used,St}; false -> + %% If the exception is taken, the stack may + %% be scanned. Therefore the register is not + %% guaranteed to be killed. if - R =:= Dst -> {killed,St}; + R =:= Dst -> {not_used,St}; true -> not_used(check_liveness(R, Is, St)) end end @@ -602,8 +605,11 @@ check_liveness(R, [{test_heap,N,Live}|Is], St) -> check_liveness(R, [{allocate_zero,N,Live}|Is], St) -> I = {block,[{set,[],[],{alloc,Live,{zero,N,0,[]}}}]}, check_liveness(R, [I|Is], St); -check_liveness(R, [{get_list,S,D1,D2}|Is], St) -> - I = {block,[{set,[D1,D2],[S],get_list}]}, +check_liveness(R, [{get_hd,S,D}|Is], St) -> + I = {block,[{set,[D],[S],get_hd}]}, + check_liveness(R, [I|Is], St); +check_liveness(R, [{get_tl,S,D}|Is], St) -> + I = {block,[{set,[D],[S],get_tl}]}, check_liveness(R, [I|Is], St); check_liveness(R, [remove_message|Is], St) -> check_liveness(R, Is, St); @@ -732,8 +738,8 @@ check_liveness_block_1(R, Ss, Ds, Op, Is, St0) -> end end. -check_liveness_block_2(R, {gc_bif,_Op,{f,Lbl}}, _Ss, St) -> - check_liveness_block_3(R, Lbl, St); +check_liveness_block_2(R, {gc_bif,Op,{f,Lbl}}, Ss, St) -> + check_liveness_block_3(R, Lbl, {Op,length(Ss)}, St); check_liveness_block_2(R, {bif,Op,{f,Lbl}}, Ss, St) -> Arity = length(Ss), case erl_internal:comp_op(Op, Arity) orelse @@ -741,16 +747,23 @@ check_liveness_block_2(R, {bif,Op,{f,Lbl}}, Ss, St) -> true -> {killed,St}; false -> - check_liveness_block_3(R, Lbl, St) + check_liveness_block_3(R, Lbl, {Op,length(Ss)}, St) end; check_liveness_block_2(R, {put_map,_Op,{f,Lbl}}, _Ss, St) -> - check_liveness_block_3(R, Lbl, St); + check_liveness_block_3(R, Lbl, {unsafe,0}, St); check_liveness_block_2(_, _, _, St) -> {killed,St}. -check_liveness_block_3(_, 0, St) -> +check_liveness_block_3({x,_}, 0, _FA, St) -> {killed,St}; -check_liveness_block_3(R, Lbl, St0) -> +check_liveness_block_3({y,_}, 0, {F,A}, St) -> + %% If the exception is thrown, the stack may be scanned, + %% thus implicitly using the y register. + case erl_bifs:is_safe(erlang, F, A) of + true -> {killed,St}; + false -> {used,St} + end; +check_liveness_block_3(R, Lbl, _FA, St0) -> check_liveness_at(R, Lbl, St0). index_labels_1([{label,Lbl}|Is0], Acc) -> @@ -991,47 +1004,52 @@ live_opt([{recv_mark,_}=I|Is], Regs, D, Acc) -> live_opt([], _, _, Acc) -> Acc. -live_opt_block([{set,Ds,Ss,Op0}|Is], Regs0, D, Acc) -> - Regs1 = x_live(Ss, x_dead(Ds, Regs0)), - {Op, Regs} = live_opt_block_op(Op0, Regs1, D), - I = {set, Ds, Ss, Op}, - - case Ds of - [{x,X}] -> - case (not is_live(X, Regs0)) andalso Op =:= move of - true -> - live_opt_block(Is, Regs0, D, Acc); - false -> - live_opt_block(Is, Regs, D, [I|Acc]) - end; - _ -> - live_opt_block(Is, Regs, D, [I|Acc]) +live_opt_block([{set,[{x,X}]=Ds,Ss,move}=I|Is], Regs0, D, Acc) -> + Regs = x_live(Ss, x_dead(Ds, Regs0)), + case is_live(X, Regs0) of + true -> + live_opt_block(Is, Regs, D, [I|Acc]); + false -> + %% Useless move, will never be used. + live_opt_block(Is, Regs, D, Acc) end; -live_opt_block([{'%anno',_}|Is], Regs, D, Acc) -> - live_opt_block(Is, Regs, D, Acc); -live_opt_block([], Regs, _, Acc) -> {Acc,Regs}. - -live_opt_block_op({alloc,Live0,AllocOp}, Regs0, D) -> - Regs = - case AllocOp of - {Kind, _N, Fail} when Kind =:= gc_bif; Kind =:= put_map -> - live_join_label(Fail, D, Regs0); - _ -> - Regs0 - end, +live_opt_block([{set,Ds,Ss,{alloc,Live0,AllocOp}}|Is], Regs0, D, Acc) -> + %% Calculate liveness from the point of view of the GC. + %% There will never be a GC if the instruction fails, so we should + %% ignore the failure branch. + GcRegs1 = x_dead(Ds, Regs0), + GcRegs = x_live(Ss, GcRegs1), + Live = live_regs(GcRegs), %% The life-time analysis used by the code generator is sometimes too %% conservative, so it may be possible to lower the number of live %% registers based on the exact liveness information. The main benefit is %% that more optimizations that depend on liveness information (such as the - %% beam_bool and beam_dead passes) may be applied. - Live = live_regs(Regs), - true = Live =< Live0, - {{alloc,Live,AllocOp}, live_call(Live)}; -live_opt_block_op({bif,_N,Fail} = Op, Regs, D) -> - {Op, live_join_label(Fail, D, Regs)}; -live_opt_block_op(Op, Regs, _D) -> - {Op, Regs}. + %% beam_dead pass) may be applied. + true = Live =< Live0, %Assertion. + I = {set,Ds,Ss,{alloc,Live,AllocOp}}, + + %% Calculate liveness from the point of view of the preceding instruction. + %% The liveness is the union of live registers in the GC and the live + %% registers at the failure label. + Regs1 = live_call(Live), + Regs = live_join_alloc(AllocOp, D, Regs1), + live_opt_block(Is, Regs, D, [I|Acc]); +live_opt_block([{set,Ds,Ss,{bif,_,Fail}}=I|Is], Regs0, D, Acc) -> + Regs1 = x_dead(Ds, Regs0), + Regs2 = x_live(Ss, Regs1), + Regs = live_join_label(Fail, D, Regs2), + live_opt_block(Is, Regs, D, [I|Acc]); +live_opt_block([{set,Ds,Ss,_}=I|Is], Regs0, D, Acc) -> + Regs = x_live(Ss, x_dead(Ds, Regs0)), + live_opt_block(Is, Regs, D, [I|Acc]); +live_opt_block([{'%anno',_}|Is], Regs, D, Acc) -> + live_opt_block(Is, Regs, D, Acc); +live_opt_block([], Regs, _, Acc) -> {Acc,Regs}. + +live_join_alloc({Kind,_Name,Fail}, D, Regs) when Kind =:= gc_bif; Kind =:= put_map -> + live_join_label(Fail, D, Regs); +live_join_alloc(_, _, Regs) -> Regs. live_join_labels([{f,L}|T], D, Regs0) when L =/= 0 -> Regs = gb_trees:get(L, D) bor Regs0, diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index f8bf935132..c30ab34ac7 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -591,6 +591,12 @@ valfun_4({get_list,Src,D1,D2}, Vst0) -> assert_type(cons, Src, Vst0), Vst = set_type_reg(term, D1, Vst0), set_type_reg(term, D2, Vst); +valfun_4({get_hd,Src,Dst}, Vst) -> + assert_type(cons, Src, Vst), + set_type_reg(term, Dst, Vst); +valfun_4({get_tl,Src,Dst}, Vst) -> + assert_type(cons, Src, Vst), + set_type_reg(term, Dst, Vst); valfun_4({get_tuple_element,Src,I,Dst}, Vst) -> assert_type({tuple_element,I+1}, Src, Vst), set_type_reg(term, Dst, Vst); @@ -1147,6 +1153,7 @@ set_type_y(Type, {y,Y}=Reg, #vst{current=#st{y=Ys0}=St}=Vst) {value,_} -> gb_trees:update(Y, Type, Ys0) end, + check_try_catch_tags(Type, Y, Ys0), Vst#vst{current=St#st{y=Ys}}; set_type_y(Type, Reg, #vst{}) -> error({invalid_store,Reg,Type}). @@ -1154,6 +1161,29 @@ set_catch_end({y,Y}, #vst{current=#st{y=Ys0}=St}=Vst) -> Ys = gb_trees:update(Y, initialized, Ys0), Vst#vst{current=St#st{y=Ys}}. +check_try_catch_tags(Type, LastY, Ys) -> + case is_try_catch_tag(Type) of + false -> + ok; + true -> + %% Every catch or try/catch must use a lower Y register + %% number than any enclosing catch or try/catch. That will + %% ensure that when the stack is scanned when an + %% exception occurs, the innermost try/catch tag is found + %% first. + Bad = [{{y,Y},Tag} || {Y,Tag} <- gb_trees:to_list(Ys), + Y < LastY, is_try_catch_tag(Tag)], + case Bad of + [] -> + ok; + [_|_] -> + error({bad_try_catch_nesting,{y,LastY},Bad}) + end + end. + +is_try_catch_tag({catchtag,_}) -> true; +is_try_catch_tag({trytag,_}) -> true; +is_try_catch_tag(_) -> false. is_reg_defined({x,_}=Reg, Vst) -> is_type_defined_x(Reg, Vst); is_reg_defined({y,_}=Reg, Vst) -> is_type_defined_y(Reg, Vst); @@ -1343,7 +1373,12 @@ branch_arities([Sz,{f,L}|T], Tuple, #vst{current=St}=Vst0) Vst = branch_state(L, Vst1), branch_arities(T, Tuple, Vst#vst{current=St}). -branch_state(0, #vst{}=Vst) -> Vst; +branch_state(0, #vst{}=Vst) -> + %% If the instruction fails, the stack may be scanned + %% looking for a catch tag. Therefore the Y registers + %% must be initialized at this point. + verify_y_init(Vst), + Vst; branch_state(L, #vst{current=St,branched=B}=Vst) -> Vst#vst{ branched=case gb_trees:is_defined(L, B) of diff --git a/lib/compiler/src/beam_z.erl b/lib/compiler/src/beam_z.erl index 1c56b95a9e..6c3a6995d7 100644 --- a/lib/compiler/src/beam_z.erl +++ b/lib/compiler/src/beam_z.erl @@ -24,18 +24,20 @@ -export([module/2]). --import(lists, [dropwhile/2]). +-import(lists, [dropwhile/2,map/2]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_asm:module_code()}. -module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> - Fs = [function(F) || F <- Fs0], +module({Mod,Exp,Attr,Fs0,Lc}, Opts) -> + NoGetHdTl = proplists:get_bool(no_get_hd_tl, Opts), + Fs = [function(F, NoGetHdTl) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. -function({function,Name,Arity,CLabel,Is0}) -> +function({function,Name,Arity,CLabel,Is0}, NoGetHdTl) -> try - Is = undo_renames(Is0), + Is1 = undo_renames(Is0), + Is = maybe_eliminate_get_hd_tl(Is1, NoGetHdTl), {function,Name,Arity,CLabel,Is} catch Class:Error:Stack -> @@ -65,6 +67,10 @@ undo_renames([{bif,raise,_,_,_}=I|Is0]) -> (_) -> true end, Is0), [I|undo_renames(Is)]; +undo_renames([{get_hd,Src,Dst1},{get_tl,Src,Dst2}|Is]) -> + [{get_list,Src,Dst1,Dst2}|undo_renames(Is)]; +undo_renames([{get_tl,Src,Dst2},{get_hd,Src,Dst1}|Is]) -> + [{get_list,Src,Dst1,Dst2}|undo_renames(Is)]; undo_renames([I|Is]) -> [undo_rename(I)|undo_renames(Is)]; undo_renames([]) -> []. @@ -107,3 +113,17 @@ undo_rename({get_map_elements,Fail,Src,{list,List}}) -> undo_rename({select,I,Reg,Fail,List}) -> {I,Reg,Fail,{list,List}}; undo_rename(I) -> I. + +%%% +%%% Eliminate get_hd/get_tl instructions if requested by +%%% the no_get_hd_tl option. +%%% + +maybe_eliminate_get_hd_tl(Is, true) -> + map(fun({get_hd,Cons,Hd}) -> + {get_list,Cons,Hd,{x,1022}}; + ({get_tl,Cons,Tl}) -> + {get_list,Cons,{x,1022},Tl}; + (I) -> I + end, Is); +maybe_eliminate_get_hd_tl(Is, false) -> Is. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 1409c358c2..c6a0056a70 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -219,13 +219,15 @@ expand_opt(report, Os) -> expand_opt(return, Os) -> [return_errors,return_warnings|Os]; expand_opt(r16, Os) -> - [no_record_opt,no_utf8_atoms|Os]; + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; expand_opt(r17, Os) -> - [no_record_opt,no_utf8_atoms|Os]; + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; expand_opt(r18, Os) -> - [no_record_opt,no_utf8_atoms|Os]; + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; expand_opt(r19, Os) -> - [no_record_opt,no_utf8_atoms|Os]; + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; +expand_opt(r20, Os) -> + [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; expand_opt({debug_info_key,_}=O, Os) -> [encrypt_debug_info,O|Os]; expand_opt(no_float_opt, Os) -> diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index d59bb241a8..a47d4e8cf7 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -564,3 +564,13 @@ BEAM_FORMAT_NUMBER=0 ## exception, but store the atom 'badarg' in x(0) and execute the ## next instruction. 161: raw_raise/0 + +## @spec get_hd Source Head +## @doc Get the head (or car) part of a list (a cons cell) from Source and +## put it into the register Head. +162: get_hd/2 + +## @spec get_tl Source Tail +## @doc Get the tail (or cdr) part of a list (a cons cell) from Source and +## put it into the register Tail. +163: get_tl/2 diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl index a96d58a903..a8f4926e55 100644 --- a/lib/compiler/src/v3_codegen.erl +++ b/lib/compiler/src/v3_codegen.erl @@ -1495,28 +1495,34 @@ select_extract_map(Src, Vs, Fail, I, Vdb, Bef, St) -> {Code, Aft, St}. -select_extract_cons(Src, [#k_var{name=Hd}, #k_var{name=Tl}], I, Vdb, Bef, St) -> - {Es,Aft} = case {vdb_find(Hd, Vdb), vdb_find(Tl, Vdb)} of - {{_,_,Lhd}, {_,_,Ltl}} when Lhd =< I, Ltl =< I -> - %% Both head and tail are dead. No need to generate - %% any instruction. - {[], Bef}; - _ -> - %% At least one of head and tail will be used, - %% but we must always fetch both. We will call - %% clear_dead/2 to allow reuse of the register - %% in case only of them is used. - - Reg0 = put_reg(Tl, put_reg(Hd, Bef#sr.reg)), - Int0 = Bef#sr{reg=Reg0}, - Rsrc = fetch_var(Src, Int0), - Rhd = fetch_reg(Hd, Reg0), - Rtl = fetch_reg(Tl, Reg0), - Int1 = clear_dead(Int0, I, Vdb), - {[{get_list,Rsrc,Rhd,Rtl}], Int1} - end, - {Es,Aft,St}. - +select_extract_cons(Src, [#k_var{name=Hd},#k_var{name=Tl}], I, Vdb, Bef, St) -> + Rsrc = fetch_var(Src, Bef), + Int = clear_dead(Bef, I, Vdb), + {{_,_,Lhd},{_,_,Ltl}} = {vdb_find(Hd, Vdb),vdb_find(Tl, Vdb)}, + case {Lhd =< I, Ltl =< I} of + {true,true} -> + %% Both dead. + {[],Bef,St}; + {true,false} -> + %% Head dead. + Reg0 = put_reg(Tl, Bef#sr.reg), + Aft = Int#sr{reg=Reg0}, + Rtl = fetch_reg(Tl, Reg0), + {[{get_tl,Rsrc,Rtl}],Aft,St}; + {false,true} -> + %% Tail dead. + Reg0 = put_reg(Hd, Bef#sr.reg), + Aft = Int#sr{reg=Reg0}, + Rhd = fetch_reg(Hd, Reg0), + {[{get_hd,Rsrc,Rhd}],Aft,St}; + {false,false} -> + %% Both used. + Reg0 = put_reg(Tl, put_reg(Hd, Bef#sr.reg)), + Aft = Bef#sr{reg=Reg0}, + Rhd = fetch_reg(Hd, Reg0), + Rtl = fetch_reg(Tl, Reg0), + {[{get_hd,Rsrc,Rhd},{get_tl,Rsrc,Rtl}],Aft,St} + end. guard_clause_cg(#k_guard_clause{anno=#l{vdb=Vdb},guard=G,body=B}, Fail, Bef, St0) -> {Gis,Int,St1} = guard_cg(G, Fail, Vdb, Bef, St0), diff --git a/lib/compiler/test/beam_block_SUITE.erl b/lib/compiler/test/beam_block_SUITE.erl index fac18789e0..38ead96cc8 100644 --- a/lib/compiler/test/beam_block_SUITE.erl +++ b/lib/compiler/test/beam_block_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, init_per_group/2,end_per_group/2, get_map_elements/1,otp_7345/1,move_opt_across_gc_bif/1, - erl_202/1,repro/1,local_cse/1]). + erl_202/1,repro/1,local_cse/1,second_block_pass/1]). %% The only test for the following functions is that %% the code compiles and is accepted by beam_validator. @@ -41,7 +41,8 @@ groups() -> move_opt_across_gc_bif, erl_202, repro, - local_cse + local_cse, + second_block_pass ]}]. init_per_suite(Config) -> @@ -295,6 +296,15 @@ local_cse_4() -> do_local_cse_4(X, Y, Z) -> {X,Y,Z}. +%% Tests previously found bugs when running beam_block the second time. + +second_block_pass(_Config) -> + [#{dts:=5.0}] = second_1([#{dts => 10.0}], 2.0), + ok. + +second_1(Fs, TS) -> + [F#{dts=>DTS / TS} || #{dts:=DTS} = F <- Fs]. + %%% %%% Common functions. %%% diff --git a/lib/compiler/test/beam_type_SUITE.erl b/lib/compiler/test/beam_type_SUITE.erl index dfbf2aa4a0..e33df809ff 100644 --- a/lib/compiler/test/beam_type_SUITE.erl +++ b/lib/compiler/test/beam_type_SUITE.erl @@ -67,6 +67,15 @@ integers(_Config) -> college = do_integers_3(), + zero = do_integers_4(<<0:1>>, 0), + one = do_integers_4(<<1:1>>, 0), + other = do_integers_4(<<1:1>>, 2), + + zero = do_integers_5(0, 0), + one = do_integers_5(0, 1), + two = do_integers_5(0, 2), + three = do_integers_5(0, 3), + ok. do_integers_1(B0) -> @@ -89,6 +98,30 @@ do_integers_3() -> 1 -> 0 end. +do_integers_4(<<X:1,T/bits>>, C) -> + %% Binary matching gives the range 0-1 for X. + %% The range for `X bor C` is unknown. It must not be inherited + %% from X. (`X bor C` will reuse the register used for X.) + case X bor C of + 0 -> do_integers_4(T, C, zero); + 1 -> do_integers_4(T, C, one); + _ -> do_integers_4(T, C, other) + end. + +do_integers_4(_, _, Res) -> + Res. + +do_integers_5(X0, Y0) -> + %% X and Y will use the same register. + X = X0 band 1, + Y = Y0 band 3, + case Y of + 0 -> zero; + 1 -> one; + 2 -> two; + 3 -> three + end. + coverage(_Config) -> {'EXIT',{badarith,_}} = (catch id(1) bsl 0.5), {'EXIT',{badarith,_}} = (catch id(2.0) bsl 2), diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index 63a13281a8..b8fff7b100 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -33,7 +33,7 @@ state_after_fault_in_catch/1,no_exception_in_catch/1, undef_label/1,illegal_instruction/1,failing_gc_guard_bif/1, map_field_lists/1,cover_bin_opt/1, - val_dsetel/1,bad_tuples/1]). + val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1]). -include_lib("common_test/include/ct.hrl"). @@ -62,7 +62,7 @@ groups() -> state_after_fault_in_catch,no_exception_in_catch, undef_label,illegal_instruction,failing_gc_guard_bif, map_field_lists,cover_bin_opt,val_dsetel, - bad_tuples]}]. + bad_tuples,bad_try_catch_nesting]}]. init_per_suite(Config) -> Config. @@ -523,6 +523,14 @@ bad_tuples(Config) -> ok. +bad_try_catch_nesting(Config) -> + Errors = do_val(bad_try_catch_nesting, Config), + [{{bad_try_catch_nesting,main,2}, + {{'try',{y,2},{f,3}}, + 7, + {bad_try_catch_nesting,{y,2},[{{y,1},{trytag,[5]}}]}}}] = Errors, + ok. + %%%------------------------------------------------------------------------- transform_remove(Remove, Module) -> diff --git a/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S b/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S new file mode 100644 index 0000000000..9f1b21a17b --- /dev/null +++ b/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S @@ -0,0 +1,64 @@ +{module, bad_try_catch_nesting}. %% version = 0 + +{exports, [{main,2},{module_info,0},{module_info,1}]}. + +{attributes, []}. + +{labels, 11}. + + +{function, main, 2, 2}. + {label,1}. + {line,[{location,"bad_try_catch_nesting.erl",4}]}. + {func_info,{atom,bad_try_catch_nesting},{atom,main},2}. + {label,2}. + {allocate_zero,3,2}. + {'try',{y,1},{f,5}}. + {move,{x,1},{y,0}}. + {'try',{y,2},{f,3}}. + {line,[{location,"bad_try_catch_nesting.erl",7}]}. + {call_fun,0}. + {try_end,{y,2}}. + {jump,{f,4}}. + {label,3}. + {try_case,{y,2}}. + {test,is_ne_exact,{f,4},[{x,0},{atom,error}]}. + {line,[]}. + {bif,raise,{f,0},[{x,2},{x,1}],{x,0}}. + {label,4}. + {move,{y,0},{x,0}}. + {kill,{y,0}}. + {line,[{location,"bad_try_catch_nesting.erl",12}]}. + {call_fun,0}. + {try_end,{y,1}}. + {deallocate,3}. + return. + {label,5}. + {try_case,{y,1}}. + {test,is_eq_exact,{f,6},[{x,0},{atom,throw}]}. + {deallocate,3}. + return. + {label,6}. + {line,[]}. + {bif,raise,{f,0},[{x,2},{x,1}],{x,0}}. + + +{function, module_info, 0, 8}. + {label,7}. + {line,[]}. + {func_info,{atom,bad_try_catch_nesting},{atom,module_info},0}. + {label,8}. + {move,{atom,bad_try_catch_nesting},{x,0}}. + {line,[]}. + {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. + + +{function, module_info, 1, 10}. + {label,9}. + {line,[]}. + {func_info,{atom,bad_try_catch_nesting},{atom,module_info},1}. + {label,10}. + {move,{x,0},{x,1}}. + {move,{atom,bad_try_catch_nesting},{x,0}}. + {line,[]}. + {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index 35c11d894d..eee5bc733f 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -1455,19 +1455,21 @@ env_compiler_options(_Config) -> bc_options(Config) -> DataDir = proplists:get_value(data_dir, Config), - 101 = highest_opcode(DataDir, small_float, [no_line_info]), + 101 = highest_opcode(DataDir, small_float, [no_get_hd_tl,no_line_info]), 103 = highest_opcode(DataDir, big, - [no_record_opt,no_line_info,no_stack_trimming]), + [no_get_hd_tl,no_record_opt, + no_line_info,no_stack_trimming]), - 125 = highest_opcode(DataDir, small_float, [no_line_info,no_float_opt]), + 125 = highest_opcode(DataDir, small_float, + [no_get_hd_tl,no_line_info,no_float_opt]), 132 = highest_opcode(DataDir, small, - [no_record_opt,no_float_opt,no_line_info]), + [no_get_hd_tl,no_record_opt,no_float_opt,no_line_info]), - 136 = highest_opcode(DataDir, big, [no_record_opt,no_line_info]), + 136 = highest_opcode(DataDir, big, [no_get_hd_tl,no_record_opt,no_line_info]), - 153 = highest_opcode(DataDir, big, [no_record_opt]), + 153 = highest_opcode(DataDir, big, [no_get_hd_tl,no_record_opt]), 153 = highest_opcode(DataDir, big, [r16]), 153 = highest_opcode(DataDir, big, [r17]), 153 = highest_opcode(DataDir, big, [r18]), @@ -1478,9 +1480,10 @@ bc_options(Config) -> 158 = highest_opcode(DataDir, small_maps, [r17]), 158 = highest_opcode(DataDir, small_maps, [r18]), 158 = highest_opcode(DataDir, small_maps, [r19]), + 158 = highest_opcode(DataDir, small_maps, [r20]), 158 = highest_opcode(DataDir, small_maps, []), - 159 = highest_opcode(DataDir, big, []), + 163 = highest_opcode(DataDir, big, []), ok. diff --git a/lib/compiler/test/compile_SUITE_data/big.erl b/lib/compiler/test/compile_SUITE_data/big.erl index 2e54ee8660..1db07755a1 100644 --- a/lib/compiler/test/compile_SUITE_data/big.erl +++ b/lib/compiler/test/compile_SUITE_data/big.erl @@ -741,3 +741,7 @@ snmp_access(suite) -> debug_support(suite) -> [ info, schema, schema, kill, lkill ]. +%% Cover translation of get_hd/2 to get_list/3 when option no_get_hd_tl +%% is given. +cover_get_hd([Hd|_]) -> + Hd. diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index d93c5dda1e..4e39f4663e 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -359,9 +359,7 @@ integer_encoding_1(Config) -> io:put_chars(Src, "t(Last) ->[\n"), io:put_chars(Data, "[\n"), - do_integer_encoding(-(id(1) bsl 10000), Src, Data), - do_integer_encoding(id(1) bsl 10000, Src, Data), - do_integer_encoding(1024, 0, Src, Data), + do_integer_encoding(137, 0, Src, Data), _ = [begin B = 1 bsl I, do_integer_encoding(-B-1, Src, Data), @@ -370,7 +368,7 @@ integer_encoding_1(Config) -> do_integer_encoding(B-1, Src, Data), do_integer_encoding(B, Src, Data), do_integer_encoding(B+1, Src, Data) - end || I <- lists:seq(1, 128)], + end || I <- lists:seq(1, 130)], io:put_chars(Src, "Last].\n\n"), ok = file:close(Src), io:put_chars(Data, "0].\n\n"), @@ -384,8 +382,6 @@ integer_encoding_1(Config) -> %% Compare lists. List = Mod:t(0), {ok,[List]} = file:consult(DataFile), - OneBsl10000 = id(1) bsl 10000, - [-(1 bsl 10000),OneBsl10000|_] = List, %% Cleanup. file:delete(SrcFile), @@ -404,7 +400,3 @@ do_integer_encoding(I, Src, Data) -> Str = integer_to_list(I), io:put_chars(Src, [Str,",\n"]), io:put_chars(Data, [Str,",\n"]). - - -id(I) -> I. - diff --git a/lib/crypto/test/engine_SUITE.erl b/lib/crypto/test/engine_SUITE.erl index f206f967c7..f410542f72 100644 --- a/lib/crypto/test/engine_SUITE.erl +++ b/lib/crypto/test/engine_SUITE.erl @@ -72,7 +72,12 @@ groups() -> init_per_suite(Config) -> try crypto:start() of ok -> - Config; + case crypto:info_lib() of + [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}] -> + {skip, "Problem with engine on OpenSSL 1.0.1s-freebsd"}; + _ -> + Config + end; {error,{already_started,crypto}} -> Config catch _:_ -> diff --git a/lib/debugger/src/dbg_debugged.erl b/lib/debugger/src/dbg_debugged.erl index e142af4ae0..8b64a60a9c 100644 --- a/lib/debugger/src/dbg_debugged.erl +++ b/lib/debugger/src/dbg_debugged.erl @@ -31,32 +31,25 @@ %% Called via the error handler. %%-------------------------------------------------------------------- eval(Mod, Func, Args) -> - SaveStacktrace = erlang:get_stacktrace(), Meta = dbg_ieval:eval(Mod, Func, Args), Mref = erlang:monitor(process, Meta), - msg_loop(Meta, Mref, SaveStacktrace). + msg_loop(Meta, Mref). %%==================================================================== %% Internal functions %%==================================================================== -msg_loop(Meta, Mref, SaveStacktrace) -> +msg_loop(Meta, Mref) -> receive %% Evaluated function has returned a value {sys, Meta, {ready, Val}} -> erlang:demonitor(Mref, [flush]), - - %% Restore original stacktrace and return the value - try erlang:raise(throw, stack, SaveStacktrace) - catch - throw:stack -> - case Val of - {dbg_apply,M,F,A} -> - apply(M, F, A); - _ -> - Val - end + case Val of + {dbg_apply,M,F,A} -> + apply(M, F, A); + _ -> + Val end; %% Evaluated function raised an (uncaught) exception @@ -74,32 +67,25 @@ msg_loop(Meta, Mref, SaveStacktrace) -> Meta ! {self(), rec_acked}, ok end, - msg_loop(Meta, Mref, SaveStacktrace); + msg_loop(Meta, Mref); %% Meta needs something evaluated within context of real process {sys, Meta, {command,Command}} -> Reply = handle_command(Command), Meta ! {sys, self(), Reply}, - msg_loop(Meta, Mref, SaveStacktrace); + msg_loop(Meta, Mref); %% Meta has terminated %% Must be due to int:stop() (or -heaven forbid- a debugger bug) {'DOWN', Mref, _, _, Reason} -> - - %% Restore original stacktrace and return a dummy value - try erlang:raise(throw, stack, SaveStacktrace) - catch - throw:stack -> - {interpreter_terminated, Reason} - end + {interpreter_terminated, Reason} end. handle_command(Command) -> try reply(Command) - catch Class:Reason -> - Stacktrace = stacktrace_f(erlang:get_stacktrace()), - {exception,{Class,Reason,Stacktrace}} + catch Class:Reason:Stacktrace -> + {exception,{Class,Reason,stacktrace_f(Stacktrace)}} end. reply({apply,M,F,As}) -> diff --git a/lib/debugger/src/dbg_ieval.erl b/lib/debugger/src/dbg_ieval.erl index 8009d62629..b0db8228e5 100644 --- a/lib/debugger/src/dbg_ieval.erl +++ b/lib/debugger/src/dbg_ieval.erl @@ -924,8 +924,7 @@ expr({dbg,Line,raise,As0}, Bs0, #ieval{level=Le}=Ieval0) -> trace(return, {Le,Error}), {value,Error,Bs} catch - _:_ -> - Stk = erlang:get_stacktrace(), %Possibly truncated. + _:_:Stk -> %Possibly truncated. StkFun = fun(_) -> Stk end, do_exception(Class, Reason, StkFun, Bs, Ieval) end; diff --git a/lib/debugger/src/dbg_wx_mon.erl b/lib/debugger/src/dbg_wx_mon.erl index a32a6894b8..00aee62a87 100644 --- a/lib/debugger/src/dbg_wx_mon.erl +++ b/lib/debugger/src/dbg_wx_mon.erl @@ -119,9 +119,9 @@ init(CallingPid, Mode, SFile) -> init2(CallingPid, Mode, SFile, GS) catch exit:stop -> stop; - Error:Reason -> + Error:Reason:Stacktrace -> io:format("~p: Crashed {~p,~p} in~n ~p", - [?MODULE, Error, Reason, erlang:get_stacktrace()]) + [?MODULE, Error, Reason, Stacktrace]) end end. diff --git a/lib/debugger/src/dbg_wx_trace.erl b/lib/debugger/src/dbg_wx_trace.erl index b1e0e03b4c..505d53005f 100644 --- a/lib/debugger/src/dbg_wx_trace.erl +++ b/lib/debugger/src/dbg_wx_trace.erl @@ -95,9 +95,9 @@ start(Pid, Env, Parent, TraceWin, BackTrace, Strings) -> catch _:stop -> exit(stop); - E:R -> + E:R:S -> io:format("TraceWin Crashed ~p~n",[E]), - io:format(" ~p in ~p~n",[R, erlang:get_stacktrace()]), + io:format(" ~p in ~p~n",[R, S]), exit(R) end; error -> diff --git a/lib/dialyzer/src/dialyzer_cl_parse.erl b/lib/dialyzer/src/dialyzer_cl_parse.erl index 80c10183cf..f21eaed087 100644 --- a/lib/dialyzer/src/dialyzer_cl_parse.erl +++ b/lib/dialyzer/src/dialyzer_cl_parse.erl @@ -41,8 +41,8 @@ start() -> Ret catch throw:{dialyzer_cl_parse_error, Msg} -> {error, Msg}; - _:R -> - Msg = io_lib:format("~tp\n~tp\n", [R, erlang:get_stacktrace()]), + _:R:S -> + Msg = io_lib:format("~tp\n~tp\n", [R, S]), {error, lists:flatten(Msg)} end. diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl index 384912f983..c5f93a3392 100644 --- a/lib/dialyzer/src/dialyzer_dataflow.erl +++ b/lib/dialyzer/src/dialyzer_dataflow.erl @@ -299,7 +299,7 @@ traverse(Tree, Map, State) -> match_fail -> t_none(); raise -> t_none(); bs_init_writable -> t_from_term(<<>>); - build_stacktrace -> t_list(); + build_stacktrace -> erl_bif_types:type(erlang, build_stacktrace, 0); Other -> erlang:error({'Unsupported primop', Other}) end, {State, Map, Type}; @@ -1236,6 +1236,13 @@ handle_tuple(Tree, Map, State) -> State2 = state__add_warning(State1, ?WARN_OPAQUE, Tree, Msg), {State2, Map1, t_none()}; + {error, record, ErrorPat, ErrorType, _} -> + Msg = {record_match, + [format_patterns(ErrorPat), + format_type(ErrorType, State1)]}, + State2 = state__add_warning(State1, ?WARN_MATCHING, + Tree, Msg), + {State2, Map1, t_none()}; {Map2, ETypes} -> {State1, Map2, t_tuple(ETypes)} end @@ -3437,19 +3444,19 @@ state__fun_info(Fun, #state{callgraph = CG, fun_tab = FunTab, plt = PLT}) -> {Fun, Sig, Contract, LocalRet}. forward_args(Fun, ArgTypes, #state{work = Work, fun_tab = FunTab} = State) -> - {OldArgTypes, OldOut, Fixpoint} = + {NewArgTypes, OldOut, Fixpoint} = case dict:find(Fun, FunTab) of - {ok, {not_handled, {OldArgTypes0, OldOut0}}} -> - {OldArgTypes0, OldOut0, false}; + {ok, {not_handled, {_OldArgTypesAreNone, OldOut0}}} -> + {ArgTypes, OldOut0, false}; {ok, {OldArgTypes0, OldOut0}} -> - {OldArgTypes0, OldOut0, - t_is_subtype(t_product(ArgTypes), t_product(OldArgTypes0))} + NewArgTypes0 = [t_sup(X, Y) || + {X, Y} <- lists:zip(ArgTypes, OldArgTypes0)], + {NewArgTypes0, OldOut0, + t_is_equal(t_product(NewArgTypes0), t_product(OldArgTypes0))} end, case Fixpoint of true -> State; false -> - NewArgTypes = [t_sup(X, Y) || - {X, Y} <- lists:zip(ArgTypes, OldArgTypes)], NewWork = add_work(Fun, Work), ?debug("~tw: forwarding args ~ts\n", [state__lookup_name(Fun, State), diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 95c8b5ebce..2af4534396 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -531,17 +531,19 @@ compute_md5_from_files(Files) -> lists:keysort(1, [{F, compute_md5_from_file(F)} || F <- Files]). compute_md5_from_file(File) -> - case filelib:is_regular(File) of - false -> + case beam_lib:all_chunks(File) of + {ok, _, Chunks} -> + %% We cannot use beam_lib:md5 because it does not consider + %% the debug_info chunk, where typespecs are likely stored. + %% So we consider almost all chunks except the useless ones. + Filtered = [[ID, Chunk] || {ID, Chunk} <- Chunks, ID =/= "CInf", ID =/= "Docs"], + erlang:md5(lists:sort(Filtered)); + {error, beam_lib, {file_error, _, enoent}} -> Msg = io_lib:format("Not a regular file: ~ts\n", [File]), throw({dialyzer_error, Msg}); - true -> - case dialyzer_utils:get_core_from_beam(File) of - {error, Error} -> - throw({dialyzer_error, Error}); - {ok, Core} -> - erlang:md5(term_to_binary(Core)) - end + {error, beam_lib, _} -> + Msg = io_lib:format("Could not compute MD5 for .beam: ~ts\n", [File]), + throw({dialyzer_error, Msg}) end. init_diff_list(RemoveFiles, AddFiles) -> diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index b1f6a54503..dede475f98 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -418,7 +418,11 @@ traverse(Tree, DefinedVars, State) -> match_fail -> throw(error); raise -> throw(error); bs_init_writable -> {State, t_from_term(<<>>)}; - build_stacktrace -> {State, t_list()}; + build_stacktrace -> + V = mk_var(Tree), + Type = erl_bif_types:type(erlang, build_stacktrace, 0), + State1 = state__store_conj(V, sub, Type, State), + {State1, V}; Other -> erlang:error({'Unsupported primop', Other}) end; 'receive' -> @@ -1896,9 +1900,8 @@ solver(Solver, SolveFun) -> ?debug("Solver ~w returned unexpected result:\n ~P\n", [Solver, _R, 60]), throw(error) - catch E:R -> - io:format("Solver ~w failed: ~w:~p\n ~tp\n", - [Solver, E, R, erlang:get_stacktrace()]), + catch E:R:S -> + io:format("Solver ~w failed: ~w:~p\n ~tp\n", [Solver, E, R, S]), throw(error) end. diff --git a/lib/dialyzer/src/typer.erl b/lib/dialyzer/src/typer.erl index 16b9c8a94a..9d3d9ce438 100644 --- a/lib/dialyzer/src/typer.erl +++ b/lib/dialyzer/src/typer.erl @@ -164,9 +164,9 @@ get_type_info(#analysis{callgraph = CallGraph, CodeServer), Analysis#analysis{callgraph = StrippedCallGraph, trust_plt = NewPlt} catch - error:What -> + error:What:Stacktrace -> fatal_error(io_lib:format("Analysis failed with message: ~tp", - [{What, erlang:get_stacktrace()}])); + [{What, Stacktrace}])); throw:{dialyzer_succ_typing_error, Msg} -> fatal_error(io_lib:format("Analysis failed with message: ~ts", [Msg])) end. diff --git a/lib/dialyzer/test/behaviour_SUITE_data/src/proper/proper_typeserver.erl b/lib/dialyzer/test/behaviour_SUITE_data/src/proper/proper_typeserver.erl index b16075763f..12f6532c0c 100644 --- a/lib/dialyzer/test/behaviour_SUITE_data/src/proper/proper_typeserver.erl +++ b/lib/dialyzer/test/behaviour_SUITE_data/src/proper/proper_typeserver.erl @@ -539,7 +539,7 @@ apply_spec_test({Mod,Fun,_Arity}=MFA, {_Domain,Range}, SpecTimeout, FalsePositiv try apply(Mod, Fun, Args) of X -> {ok, X} catch - X:Y -> {X, Y} + X:Y:S -> {{X, Y}, S} end, case Result of {ok, Z} -> @@ -551,15 +551,15 @@ apply_spec_test({Mod,Fun,_Arity}=MFA, {_Domain,Range}, SpecTimeout, FalsePositiv false -> false end; - Exception when is_function(FalsePositiveMFAs) -> + {Exception, S2} when is_function(FalsePositiveMFAs) -> case FalsePositiveMFAs(MFA, Args, Exception) of true -> true; false -> - error(Exception, erlang:get_stacktrace()) + error(Exception, S2) end; - Exception -> - error(Exception, erlang:get_stacktrace()) + {Exception, S3} -> + error(Exception, S3) end end). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl new file mode 100644 index 0000000000..44149f4199 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl @@ -0,0 +1,15 @@ +-module(same). + +-export([baz/1]). + +-record(bar, { + a :: same_type:st(integer()), + b :: same_type:st(atom()) + }). + +baz(Bar) -> + _ = wrap_find(0, Bar#bar.a), + wrap_find(0, Bar#bar.b). + +wrap_find(K, D) -> + same_type:t(K, D). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl new file mode 100644 index 0000000000..855a5d30be --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl @@ -0,0 +1,13 @@ +-module(same_type). + +-export([t/2]). + +-export_type([st/1]). + +%% When unopaqued all specializations of st/1 are equal. +-opaque st(_A) :: {st, tuple()}. + +-spec t(_, st(_)) -> _. + +t(K, V) -> + {K, V}. diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/proper/proper_typeserver.erl b/lib/dialyzer/test/opaque_SUITE_data/src/proper/proper_typeserver.erl index 1677b4efb8..529f9fba72 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/proper/proper_typeserver.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/proper/proper_typeserver.erl @@ -533,7 +533,7 @@ apply_spec_test({Mod,Fun,_Arity}=MFA, {_Domain,Range}, SpecTimeout, FalsePositiv try apply(Mod,Fun,Args) of X -> {ok, X} catch - X:Y -> {X, Y} + X:Y:S -> {{X, Y}, S} end, case Result of {ok, Z} -> @@ -545,15 +545,15 @@ apply_spec_test({Mod,Fun,_Arity}=MFA, {_Domain,Range}, SpecTimeout, FalsePositiv false -> false end; - Exception when is_function(FalsePositiveMFAs) -> + {Exception, S2} when is_function(FalsePositiveMFAs) -> case FalsePositiveMFAs(MFA, Args, Exception) of true -> true; false -> - error(Exception, erlang:get_stacktrace()) + error(Exception, S2) end; - Exception -> - error(Exception, erlang:get_stacktrace()) + {Exception, S3} -> + error(Exception, S3) end end). diff --git a/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl b/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl index 8fe43163f6..ea92613781 100644 --- a/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl +++ b/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl @@ -174,7 +174,7 @@ validate_error(Error, Name, Ar) -> -endif. validate_error_1(Error, Name, Ar) -> {{'_',Name,Ar}, - {internal_error,'_',{Error,erlang:get_stacktrace()}}}. + {internal_error,'_',{Error,[]}}}. -record(st, %Emulation state {x=init_regs(0, term), %x register info. diff --git a/lib/dialyzer/test/small_SUITE_data/results/record_match b/lib/dialyzer/test/small_SUITE_data/results/record_match new file mode 100644 index 0000000000..a0dd6f560a --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/record_match @@ -0,0 +1,3 @@ + +record_match.erl:16: Function select/0 has no local return +record_match.erl:17: Matching of pattern {'b_literal', 'undefined'} tagged with a record name violates the declared type of #b_local{} | #b_remote{} diff --git a/lib/dialyzer/test/small_SUITE_data/results/stacktrace b/lib/dialyzer/test/small_SUITE_data/results/stacktrace new file mode 100644 index 0000000000..fd60881953 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/stacktrace @@ -0,0 +1,5 @@ + +stacktrace.erl:11: The pattern {'a', 'b'} can never match the type [{atom(),atom(),[any()] | byte(),[{'file',string()} | {'line',pos_integer()}]}] +stacktrace.erl:19: The pattern ['a', 'b'] can never match the type [{atom(),atom(),[any()] | byte(),[{'file',string()} | {'line',pos_integer()}]}] +stacktrace.erl:44: The pattern {'a', 'b'} can never match the type [{atom(),atom(),[any()] | byte(),[{'file',string()} | {'line',pos_integer()}]}] +stacktrace.erl:53: The pattern ['a', 'b'] can never match the type [{atom(),atom(),[any()] | byte(),[{'file',string()} | {'line',pos_integer()}]}] diff --git a/lib/dialyzer/test/small_SUITE_data/src/record_match.erl b/lib/dialyzer/test/small_SUITE_data/src/record_match.erl new file mode 100644 index 0000000000..8e9b91937f --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/record_match.erl @@ -0,0 +1,17 @@ +-module(record_match). + +-export([select/0]). + +-record(b_literal, {val}). +-record(b_remote, {mod,name,arity}). +-record(b_local, {name,arity}). + +-type b_remote() :: #b_remote{}. +-type b_local() :: #b_local{}. + +-type argument() :: b_remote() | b_local(). + +-record(b_set, {args=[] :: [argument()]}). + +select() -> + #b_set{args=[#b_remote{},#b_literal{}]}. diff --git a/lib/dialyzer/test/small_SUITE_data/src/stacktrace.erl b/lib/dialyzer/test/small_SUITE_data/src/stacktrace.erl new file mode 100644 index 0000000000..de79e710e9 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/stacktrace.erl @@ -0,0 +1,73 @@ +-module(stacktrace). + +%% Check the stacktrace variable introduced in Erlang/OTP 21.0 + +-export([t1/0, t2/0, t3/0, t4/0, s1/0, s2/0, s3/0, s4/0]). + +t1() -> + try foo:bar() + catch + E:P:S -> + {a,b} = S, % can never match + {E, P} + end. + +t2() -> + try foo:bar() + catch + E:P:S -> + [a,b] = S, % can never match + {E, P} + end. + +t3() -> + try foo:bar() + catch + E:P:S -> + [{m,f,[],[]}] = S, + {E, P} + end. + +t4() -> + try foo:bar() + catch + E:P:S -> + [{m,f,1,[{file,"tjo"},{line,95}]}] = S, + {E, P} + end. + +s1() -> + try foo:bar() + catch + E:P -> + S = erlang:get_stacktrace(), + {a,b} = S, % can never match + {E, P} + end. + +s2() -> + try foo:bar() + catch + E:P -> + S = erlang:get_stacktrace(), + [a,b] = S, % can never match + {E, P} + end. + +s3() -> + try foo:bar() + catch + E:P -> + S = erlang:get_stacktrace(), + [{m,f,[],[]}] = S, + {E, P} + end. + +s4() -> + try foo:bar() + catch + E:P -> + S = erlang:get_stacktrace(), + [{m,f,1,[{file,"tjo"},{line,95}]}] = S, + {E, P} + end. diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 6b84b22eb5..6bc7d147c0 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -1865,8 +1865,8 @@ An example return value with for a client service with Origin-Host {raddr,{127,0,0,1}}, {rport,3868}, {reuseaddr,true}]}]}, - {watchdog,{<0.66.0>,{1346,171491,996448},okay}}, - {peer,{<0.67.0>,{1346,171491,999906}}}, + {watchdog,{<0.66.0>,-576460736368485571,okay}}, + {peer,{<0.67.0>,-576460736357885808}}, {apps,[{0,common}]}, {caps,[{origin_host,{"client.example.com","server.example.com"}}, {origin_realm,{"example.com","example.com"}}, @@ -1946,8 +1946,8 @@ connection might look as follows.</p> {transport_config,[{reuseaddr,true}, {ip,{127,0,0,1}}, {port,3868}]}]}, - {accept,[[{watchdog,{<0.56.0>,{1346,171481,226895},okay}}, - {peer,{<0.58.0>,{1346,171491,999511}}}, + {accept,[[{watchdog,{<0.56.0>,-576460739249514012,okay}}, + {peer,{<0.58.0>,-576460638229179167}}, {apps,[{0,common}]}, {caps,[{origin_host,{"server.example.com","client.example.com"}}, {origin_realm,{"example.com","example.com"}}, @@ -1976,7 +1976,7 @@ connection might look as follows.</p> {send_max,148}, {send_avg,87}, {send_pend,0}]}]}], - [{watchdog,{<0.72.0>,{1346,171491,998404},initial}}]]}, + [{watchdog,{<0.72.0>,-576460638229717546,initial}}]]}, {statistics,[{{{0,280,0},recv},7}, {{{0,280,1},send},7}, {{{0,280,0},recv,{'Result-Code',2001}},7}, @@ -2024,8 +2024,8 @@ A return value for the server above might look as follows.</p> {transport_config,[{reuseaddr,true}, {ip,{127,0,0,1}}, {port,3868}]}]}, - {watchdog,{<0.56.0>,{1346,171481,226895},okay}}, - {peer,{<0.58.0>,{1346,171491,999511}}}, + {watchdog,{<0.56.0>,-576460739249514012,okay}}, + {peer,{<0.58.0>,-576460638229179167}}, {apps,[{0,common}]}, {caps,[{origin_host,{"server.example.com","client.example.com"}}, {origin_realm,{"example.com","example.com"}}, diff --git a/lib/diameter/src/base/diameter_reg.erl b/lib/diameter/src/base/diameter_reg.erl index 5b7cfab31a..c1762a07e3 100644 --- a/lib/diameter/src/base/diameter_reg.erl +++ b/lib/diameter/src/base/diameter_reg.erl @@ -246,8 +246,11 @@ handle_call({add, Uniq, Key}, {Pid, _}, S) -> handle_call({remove, Key}, {Pid, _}, S) -> Rec = {Key, Pid}, - ets:delete_object(?TABLE, Rec), - {reply, true, notify(remove, Rec, S)}; + {reply, true, try + notify(remove, Rec, S) + after + ets:delete_object(?TABLE, Rec) + end}; handle_call({wait, Pat}, {Pid, _} = From, S) -> NS = add_monitor(Pid, S), @@ -370,10 +373,12 @@ send({_,_} = From, add, Rec) -> down(Pid, #state{monitors = Ps} = S) -> Recs = match('_', Pid), - ets:match_delete(?TABLE, {'_', Pid}), - lists:foldl(fun(R,NS) -> notify(remove, R, NS) end, - flush(Pid, S#state{monitors = sets:del_element(Pid, Ps)}), - Recs). + Acc0 = flush(Pid, S#state{monitors = sets:del_element(Pid, Ps)}), + try + lists:foldl(fun(R,NS) -> notify(remove, R, NS) end, Acc0, Recs) + after + ets:match_delete(?TABLE, {'_', Pid}) + end. %% flush/3 diff --git a/lib/diameter/src/base/diameter_service.erl b/lib/diameter/src/base/diameter_service.erl index 31dd92f878..cbe66ef27a 100644 --- a/lib/diameter/src/base/diameter_service.erl +++ b/lib/diameter/src/base/diameter_service.erl @@ -151,7 +151,7 @@ apps :: match([{0..16#FFFFFFFF, diameter:app_alias()}] %% {Id, Alias} | [diameter:app_alias()]), %% remote caps :: match(#diameter_caps{}), - started = diameter_lib:now(), %% at process start or sharing + started = diameter_lib:now(), %% at connection_up watchdog :: match(pid() %% key into watchdogT | undefined)}). %% undefined if remote @@ -554,15 +554,25 @@ terminate(Reason, #state{service_name = Name, local = {PeerT, _, _}} = S) -> %% wait for watchdog state changes to take care of if. That this %% takes place after deleting the state entry ensures that the %% resulting failover by request processes accomplishes nothing. - ets:foldl(fun(#peer{pid = TPid}, _) -> - diameter_traffic:peer_down(TPid) - end, - ok, - PeerT), + ets:foldl(fun peer_down/2, ok, PeerT), shutdown == Reason %% application shutdown andalso shutdown(application, S). +%% peer_down/1 +%% +%% Entries with watchdog state SUSPECT are already down: ignore the +%% expected failure. This assumes the current implementation, but +%% double the number of lookups (in the typical case) could be the +%% greater evil if there are many peer connections. + +peer_down(#peer{pid = TPid}, _) -> + try + diameter_traffic:peer_down(TPid) + catch + error: {badmatch, []} -> ok + end. + %% --------------------------------------------------------------------------- %% # code_change/3 %% --------------------------------------------------------------------------- diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 7da59f8b25..05a8c9378e 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -54,10 +54,10 @@ {"1.12.1", [{restart_application, diameter}]}, %% 19.1 {"1.12.2", [{restart_application, diameter}]}, %% 19.3 {"2.0", [{restart_application, diameter}]}, %% 20.0 - {"2.1", [{load_module, diameter_gen}, %% 20.1 - {update, diameter_reg, {advanced, "2.1"}}]}, - {"2.1.1", [{load_module, diameter_gen}]}, %% 20.1.2 - {"2.1.2", []} %% 20.1.3 + {"2.1", [{restart_application, diameter}]}, %% 20.1 + {"2.1.1", [{restart_application, diameter}]}, %% 20.1.2 + {"2.1.2", [{restart_application, diameter}]}, %% 20.1.3 + {"2.1.3", [{restart_application, diameter}]} %% 20.2 ], [ {"0.9", [{restart_application, diameter}]}, @@ -94,7 +94,8 @@ {"1.12.2", [{restart_application, diameter}]}, {"2.0", [{restart_application, diameter}]}, {"2.1", [{restart_application, diameter}]}, - {"2.1.1", [{load_module, diameter_gen}]}, - {"2.1.2", []} + {"2.1.1", [{restart_application, diameter}]}, + {"2.1.2", [{restart_application, diameter}]}, + {"2.1.3", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index 0c852d75cd..b0fb4ada28 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -1,6 +1,6 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2018. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,5 +17,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 2.1.3 +DIAMETER_VSN = 2.1.4 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/et/src/et_wx_contents_viewer.erl b/lib/et/src/et_wx_contents_viewer.erl index bca517317e..7ca41da3e8 100644 --- a/lib/et/src/et_wx_contents_viewer.erl +++ b/lib/et/src/et_wx_contents_viewer.erl @@ -93,8 +93,8 @@ start_link(Options) -> end, {ok, Pid} catch - error:Reason -> - {error, {'EXIT', Reason, erlang:get_stacktrace()}} + error:Reason:Stacktrace -> + {error, {'EXIT', Reason, Stacktrace}} end; {error, Reason} -> {error, Reason} diff --git a/lib/eunit/src/eunit_lib.erl b/lib/eunit/src/eunit_lib.erl index d1bd160ea1..771541354c 100644 --- a/lib/eunit/src/eunit_lib.erl +++ b/lib/eunit/src/eunit_lib.erl @@ -202,13 +202,13 @@ format_exception_test_() -> "\nymmud:rorre"++_, lists:reverse(lists:flatten( format_exception(try erlang:error(dummy) - catch C:R -> {C, R, erlang:get_stacktrace()} + catch C:R:S -> {C, R, S} end)))), ?_assertMatch( "\nymmud:rorre"++_, lists:reverse(lists:flatten( format_exception(try erlang:error(dummy, [a]) - catch C:R -> {C, R, erlang:get_stacktrace()} + catch C:R:S -> {C, R, S} end))))]. -endif. diff --git a/lib/eunit/src/eunit_listener.erl b/lib/eunit/src/eunit_listener.erl index e652c5b2f6..75aa05c543 100644 --- a/lib/eunit/src/eunit_listener.erl +++ b/lib/eunit/src/eunit_listener.erl @@ -137,8 +137,7 @@ call(F, As, St) when is_atom(F) -> try apply(St#state.callback, F, As) of Substate -> St#state{state = Substate} catch - Class:Term -> - Trace = erlang:get_stacktrace(), + Class:Term:Trace -> if F =/= terminate -> call(terminate, [{error, {Class, Term, Trace}}, St#state.state], St); diff --git a/lib/eunit/src/eunit_proc.erl b/lib/eunit/src/eunit_proc.erl index e075005238..96bdcf88b6 100644 --- a/lib/eunit/src/eunit_proc.erl +++ b/lib/eunit/src/eunit_proc.erl @@ -628,7 +628,7 @@ io_request({put_chars, M, F, As}, Buf) -> try apply(M, F, As) of Chars -> {ok, [Chars | Buf]} catch - C:T -> {{error, {C,T,erlang:get_stacktrace()}}, Buf} + C:T:S -> {{error, {C,T,S}}, Buf} end; io_request({put_chars, _Enc, Chars}, Buf) -> io_request({put_chars, Chars}, Buf); diff --git a/lib/eunit/src/eunit_test.erl b/lib/eunit/src/eunit_test.erl index 6036537178..6fe85ae70a 100644 --- a/lib/eunit/src/eunit_test.erl +++ b/lib/eunit/src/eunit_test.erl @@ -39,11 +39,11 @@ %% somewhat, but you can't have everything.) Note that we assume that %% this particular module is the boundary between eunit and user code. -get_stacktrace() -> - get_stacktrace([]). +get_stacktrace(Trace) -> + get_stacktrace(Trace, []). -get_stacktrace(Ts) -> - eunit_lib:uniq(prune_trace(erlang:get_stacktrace(), Ts)). +get_stacktrace(Trace, Ts) -> + eunit_lib:uniq(prune_trace(Trace, Ts)). -dialyzer({no_match, prune_trace/2}). prune_trace([{eunit_data, _, _} | Rest], Tail) -> @@ -75,8 +75,8 @@ run_testfun(F) -> {eunit_internal, Term} -> %% Internally generated: re-throw Term (lose the trace) throw(Term); - Class:Reason -> - {error, {Class, Reason, get_stacktrace()}} + Class:Reason:Trace -> + {error, {Class, Reason, Trace}} end. @@ -272,7 +272,7 @@ mf_wrapper(M, F) -> fun () -> try M:F() catch - error:undef -> + error:undef:Trace -> %% Check if it was M:F/0 that was undefined case erlang:module_loaded(M) of false -> @@ -282,14 +282,14 @@ mf_wrapper(M, F) -> false -> fail({no_such_function, {M,F,0}}); true -> - rethrow(error, undef, [{M,F,0}]) + rethrow(error, undef, Trace, [{M,F,0}]) end end end end. -rethrow(Class, Reason, Trace) -> - erlang:raise(Class, Reason, get_stacktrace(Trace)). +rethrow(Class, Reason, Trace, Ts) -> + erlang:raise(Class, Reason, get_stacktrace(Trace, Ts)). fail(Term) -> throw({eunit_internal, Term}). @@ -332,12 +332,14 @@ enter_context(Setup, Cleanup, Instantiate, Callback) -> T -> case eunit_lib:is_not_test(T) of true -> - catch throw(error), % generate a stack trace + {_, Stacktrace} = + erlang:process_info(self(), + current_stacktrace), {module,M} = erlang:fun_info(Instantiate, module), {name,N} = erlang:fun_info(Instantiate, name), {arity,A} = erlang:fun_info(Instantiate, arity), context_error({bad_instantiator, {{M,N,A},T}}, - error, badarg); + error, Stacktrace, badarg); false -> ok end, @@ -346,21 +348,22 @@ enter_context(Setup, Cleanup, Instantiate, Callback) -> %% Always run cleanup; client may be an idiot try Cleanup(R) catch - Class:Term -> - context_error(cleanup_failed, Class, Term) + Class:Term:Trace -> + context_error(cleanup_failed, + Class, Trace, Term) end end catch - Class:Term -> - context_error(instantiation_failed, Class, Term) + Class:Term:Trace -> + context_error(instantiation_failed, Class, Trace, Term) end catch - Class:Term -> - context_error(setup_failed, Class, Term) + Class:Term:Trace -> + context_error(setup_failed, Class, Trace, Term) end. -context_error(Type, Class, Term) -> - throw({context_error, Type, {Class, Term, get_stacktrace()}}). +context_error(Type, Class, Trace, Term) -> + throw({context_error, Type, {Class, Term, get_stacktrace(Trace)}}). %% This generates single setup/cleanup functions from a list of tuples %% on the form {Tag, Setup, Cleanup}, where the setup function always @@ -378,8 +381,8 @@ multi_setup([{Tag, S, C} | Es], CleanupPrev) -> try C(R) of _ -> CleanupPrev(Rs) catch - Class:Term -> - throw({Tag, {Class, Term, get_stacktrace()}}) + Class:Term:Trace -> + throw({Tag, {Class, Term, Trace}}) end end, {SetupRest, CleanupAll} = multi_setup(Es, Cleanup), @@ -388,9 +391,9 @@ multi_setup([{Tag, S, C} | Es], CleanupPrev) -> R -> SetupRest([R|Rs]) catch - Class:Term -> + Class:Term:Trace -> CleanupPrev(Rs), - throw({Tag, {Class, Term, get_stacktrace()}}) + throw({Tag, {Class, Term, Trace}}) end end, CleanupAll}; diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index 5fda857bf1..bfffb8db41 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -585,6 +585,13 @@ type(erlang, float, 1, Xs, Opaques) -> %% Guard bif, needs to be here. type(erlang, floor, 1, Xs, Opaques) -> strict(erlang, floor, 1, Xs, fun (_) -> t_integer() end, Opaques); +%% Primop, needs to be somewhere. +type(erlang, build_stacktrace, 0, _, _Opaques) -> + t_list(t_tuple([t_module(), + t_atom(), + t_sup([t_arity(),t_list()]), + t_list(t_sup([t_tuple([t_atom('file'),t_string()]), + t_tuple([t_atom('line'),t_pos_integer()])]))])); %% Guard bif, needs to be here. type(erlang, hd, 1, Xs, Opaques) -> strict(erlang, hd, 1, Xs, fun ([X]) -> t_cons_hd(X) end, Opaques); @@ -2336,6 +2343,9 @@ arg_types(erlang, float, 1) -> %% Guard bif, needs to be here. arg_types(erlang, floor, 1) -> [t_number()]; +%% Primop, needs to be somewhere. +arg_types(erlang, build_stacktrace, 0) -> + []; %% Guard bif, needs to be here. arg_types(erlang, hd, 1) -> [t_cons()]; diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index 6e66ec057c..f429d40272 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -605,6 +605,16 @@ trans_fun([{get_list,List,Head,Tail}|Instructions], Env) -> ?error_msg("hd and tl regs identical in get_list~n",[]), erlang:error(not_handled) end; +%%--- get_hd --- +trans_fun([{get_hd,List,Head}|Instructions], Env) -> + TransList = [trans_arg(List)], + I = hipe_icode:mk_primop([mk_var(Head)],unsafe_hd,TransList), + [I | trans_fun(Instructions,Env)]; +%%--- get_tl --- +trans_fun([{get_tl,List,Tail}|Instructions], Env) -> + TransList = [trans_arg(List)], + I = hipe_icode:mk_primop([mk_var(Tail)],unsafe_tl,TransList), + [I | trans_fun(Instructions,Env)]; %%--- get_tuple_element --- trans_fun([{get_tuple_element,Xreg,Index,Dst}|Instructions], Env) -> I = hipe_icode:mk_primop([mk_var(Dst)], diff --git a/lib/hipe/icode/hipe_icode_inline_bifs.erl b/lib/hipe/icode/hipe_icode_inline_bifs.erl index 7a6947f190..16a95991e7 100644 --- a/lib/hipe/icode/hipe_icode_inline_bifs.erl +++ b/lib/hipe/icode/hipe_icode_inline_bifs.erl @@ -24,8 +24,9 @@ %% Currently inlined BIFs: %% and, or, xor, not, <, >, >=, =<, ==, /=, =/=, =:= -%% is_atom, is_boolean, is_binary, is_float, is_function, -%% is_integer, is_list, is_pid, is_port, is_reference, is_tuple +%% is_atom, is_binary, is_bitstring, is_boolean, is_float, +%% is_function, is_integer, is_list, is_map, is_number, +%% is_pid, is_port, is_reference, is_tuple -module(hipe_icode_inline_bifs). @@ -116,17 +117,20 @@ try_type_tests(I) -> I. is_type_test(Name) -> case Name of - is_integer -> {true, integer}; + is_atom -> {true, atom}; + is_binary -> {true, binary}; + is_bitstring -> {true, bitstr}; + is_boolean -> {true, boolean}; is_float -> {true, float}; - is_tuple -> {true, tuple}; - is_binary -> {true, binary}; + is_function -> {true, function}; + is_integer -> {true, integer}; is_list -> {true, list}; + is_map -> {true, map}; + is_number -> {true, number}; is_pid -> {true, pid}; - is_atom -> {true, atom}; - is_boolean -> {true, boolean}; - is_function -> {true, function}; - is_reference -> {true, reference}; is_port -> {true, port}; + is_reference -> {true, reference}; + is_tuple -> {true, tuple}; _ -> false end. diff --git a/lib/hipe/main/hipe_main.erl b/lib/hipe/main/hipe_main.erl index 4b5eb4c63e..6e48f0cffd 100644 --- a/lib/hipe/main/hipe_main.erl +++ b/lib/hipe/main/hipe_main.erl @@ -410,9 +410,9 @@ icode_to_rtl(MFA, Icode, Options, Servers) -> hipe_llvm_liveness:analyze(RtlCfg4) end, pp(RtlCfg5, MFA, rtl, pp_rtl, Options, Servers), - case proplists:get_bool(verify_gcsafe, Options) of - false -> ok; - true -> + case proplists:get_bool(no_verify_gcsafe, Options) of + true -> ok; + false -> ok = hipe_rtl_verify_gcsafe:check(RtlCfg5) end, LinearRTL1 = hipe_rtl_cfg:linearize(RtlCfg5), diff --git a/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl b/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl index c3f20bfec1..01d7e89ccd 100644 --- a/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl +++ b/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl @@ -76,6 +76,7 @@ safe_primop(bs_allocate) -> true; safe_primop(bs_reallocate) -> true; safe_primop(bs_utf8_size) -> true; safe_primop(bs_get_utf8) -> true; +safe_primop(bs_put_utf8) -> true; safe_primop(bs_utf16_size) -> true; safe_primop(bs_get_utf16) -> true; safe_primop(bs_validate_unicode_retract) -> true; diff --git a/lib/hipe/x86/hipe_rtl_to_x86.erl b/lib/hipe/x86/hipe_rtl_to_x86.erl index 31e4f6e4ac..22947da148 100644 --- a/lib/hipe/x86/hipe_rtl_to_x86.erl +++ b/lib/hipe/x86/hipe_rtl_to_x86.erl @@ -646,7 +646,7 @@ conv_imm(Opnd, Map) -> is_imm64(Value) when is_integer(Value) -> (Value < -(1 bsl (32 - 1))) or (Value > (1 bsl (32 - 1)) - 1); is_imm64({_,atom}) -> false; % Atoms are 32 bits. -is_imm64({_,c_const}) -> false; % c_consts are 32 bits. +is_imm64({_,c_const}) -> true; % c_consts are 64 bits. is_imm64({_,_}) -> true . % Other relocs are 64 bits. -else. conv_imm(Opnd, Map) -> @@ -777,6 +777,18 @@ conv_fconv(Dst, Src) -> %%% Finalise the conversion of a 2-address FP operation. +-ifdef(HIPE_AMD64). +conv_fp_unary(Dst, Src, 'fchs') -> + Tmp = new_untagged_temp(), + case same_opnd(Dst, Src) of + true -> + []; + _ -> + [hipe_x86:mk_fmove(Src, Dst)] + end ++ + mk_load_address(c_const, hipe_x86:mk_imm({sse2_fnegate_mask, c_const}), Tmp) ++ + [hipe_x86:mk_fp_binop('xorpd', hipe_x86:mk_mem(Tmp, hipe_x86:mk_imm(0), double), Dst)]. +-else. conv_fp_unary(Dst, Src, FpUnOp) -> case same_opnd(Dst, Src) of true -> @@ -785,6 +797,7 @@ conv_fp_unary(Dst, Src, FpUnOp) -> [hipe_x86:mk_fmove(Src, Dst), hipe_x86:mk_fp_unop(FpUnOp, Dst)] end. +-endif. conv_fp_unop(RtlFpUnOp) -> case RtlFpUnOp of @@ -854,13 +867,8 @@ mk_jmp_switch(Index, JTabLab, Labels) -> %%% Finalise the translation of a load_address instruction. -ifdef(HIPE_AMD64). -mk_load_address(Type, Src, Dst) -> - case Type of - c_const -> % 32 bits - [hipe_x86:mk_move(Src, Dst)]; - _ -> - [hipe_x86:mk_move64(Src, Dst)] - end. +mk_load_address(_Type, Src, Dst) -> + [hipe_x86:mk_move64(Src, Dst)]. -else. mk_load_address(_Type, Src, Dst) -> [hipe_x86:mk_move(Src, Dst)]. diff --git a/lib/hipe/x86/hipe_x86_assemble.erl b/lib/hipe/x86/hipe_x86_assemble.erl index 50919bdf4e..9d2586a14d 100644 --- a/lib/hipe/x86/hipe_x86_assemble.erl +++ b/lib/hipe/x86/hipe_x86_assemble.erl @@ -735,6 +735,7 @@ resolve_sse2_op(Op) -> fdiv -> divsd; fmul -> mulsd; fsub -> subsd; + xorpd -> xorpd; _ -> exit({?MODULE, unknown_sse2_operator, Op}) end. diff --git a/lib/inets/doc/src/http_client.xml b/lib/inets/doc/src/http_client.xml index 212958f17f..15e383ec77 100644 --- a/lib/inets/doc/src/http_client.xml +++ b/lib/inets/doc/src/http_client.xml @@ -97,27 +97,32 @@ 7 > {ok, {{NewVersion, 200, NewReasonPhrase}, NewHeaders, NewBody}} = httpc:request(get, {"http://www.erlang.org", [{"connection", "close"}]}, [], []).</code> - + <p>This sends an HTTP request over a unix domain socket (experimental):</p> + <code type="erl"> + 8 > httpc:set_options([{ipfamily, local}, + {unix_socket,"/tmp/unix_socket/consul_http.sock"}]). + 9 > {ok, {{NewVersion, 200, NewReasonPhrase}, NewHeaders, NewBody}} = + httpc:request(put, {"http:///v1/kv/foo", [], [], "hello"}, [], []).</code> <p>Start an HTTP client profile:</p> <code><![CDATA[ - 8 > {ok, Pid} = inets:start(httpc, [{profile, foo}]). + 10 > {ok, Pid} = inets:start(httpc, [{profile, foo}]). {ok, <0.45.0>} ]]></code> <p>The new profile has no proxy settings, so the connection is refused:</p> <code type="erl"> - 9 > httpc:request("http://www.erlang.org", foo). + 11 > httpc:request("http://www.erlang.org", foo). {error, econnrefused}</code> <p>Stop the HTTP client profile:</p> <code type="erl"> - 10 > inets:stop(httpc, foo). + 12 > inets:stop(httpc, foo). ok</code> <p>Alternative way to stop the HTTP client profile:</p> <code type="erl"> - 10 > inets:stop(httpc, Pid). + 13 > inets:stop(httpc, Pid). ok</code> </section> diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 58714328c5..14662f257c 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -210,7 +210,8 @@ ip | port | socket_opts | - verbose</v> + verbose | + unix_socket</v> <v>Profile = profile() | pid()</v> <d>When started <c>stand_alone</c> only the pid can used.</d> <v>Values = [{option_item(), term()}]</v> @@ -297,8 +298,8 @@ {full_result, boolean()} | {headers_as_is, boolean() | {socket_opts, socket_opts()} | - {receiver, receiver()}, - {ipv6_host_with_brackets, boolean()}}</v> + {receiver, receiver()} | + {ipv6_host_with_brackets, boolean()}</v> <v>stream_to() = none | self | {self, once} | filename()</v> <v>socket_opts() = [socket_opt()]</v> <v>receiver() = pid() | function()/1 | {Module, Function, Args}</v> @@ -533,7 +534,8 @@ <v>| {ip, IpAddress}</v> <v>| {port, Port}</v> <v>| {socket_opts, socket_opts()}</v> - <v>| {verbose, VerboseMode}</v> + <v>| {verbose, VerboseMode}</v> + <v>| {unix_socket, UnixSocket}</v> <v>Proxy = {Hostname, Port}</v> <v>Hostname = string()</v> <d>Example: "localhost" or "foo.bar.se"</d> @@ -576,7 +578,7 @@ If option <c>verify</c> is used, function <c>store_cookies/2</c> has to be called for the cookies to be saved. Default is <c>disabled</c>.</d> - <v>IpFamily = inet | inet6 </v> + <v>IpFamily = inet | inet6 | local</v> <d>Default is <c>inet</c>.</d> <v>IpAddress = ip_address()</v> <d>If the host has several network interfaces, this option specifies @@ -601,6 +603,12 @@ It is a debug feature.</d> <v>Profile = profile() | pid()</v> <d>When started <c>stand_alone</c> only the pid can be used.</d> + <v>UnixSocket = path()</v> + <d> + Experimental option for sending HTTP requests over a unix domain socket. The value + of <c>unix_socket</c> shall be the full path to a unix domain socket file with read/write + permissions for the erlang process. Default is <c>undefined</c>. + </d> </type> <desc> <p>Sets options to be used for subsequent requests.</p> diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 821eb7f02f..a73503a5ce 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -171,6 +171,7 @@ request(Method, HTTPOptions, Options, Profile) when (Method =:= options) orelse (Method =:= get) orelse + (Method =:= put) orelse (Method =:= head) orelse (Method =:= delete) orelse (Method =:= trace) andalso @@ -531,6 +532,7 @@ handle_request(Method, Url, Stream = proplists:get_value(stream, Options), Receiver = proplists:get_value(receiver, Options), SocketOpts = proplists:get_value(socket_opts, Options), + UnixSocket = proplists:get_value(unix_socket, Options), BracketedHost = proplists:get_value(ipv6_host_with_brackets, Options), @@ -558,6 +560,7 @@ handle_request(Method, Url, headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, started = Started, + unix_socket = UnixSocket, ipv6_host_with_brackets = BracketedHost}, case httpc_manager:request(Request, profile_name(Profile)) of {ok, RequestId} -> @@ -823,7 +826,7 @@ request_options_defaults() -> error end, - VerifyBrackets = VerifyBoolean, + VerifyBrackets = VerifyBoolean, [ {sync, true, VerifySync}, @@ -894,11 +897,36 @@ request_options_sanity_check(Opts) -> end, ok. -validate_options(Options) -> - (catch validate_options(Options, [])). - -validate_options([], ValidateOptions) -> - {ok, lists:reverse(ValidateOptions)}; +validate_ipfamily_unix_socket(Options0) -> + IpFamily = proplists:get_value(ipfamily, Options0, inet), + UnixSocket = proplists:get_value(unix_socket, Options0, undefined), + Options1 = proplists:delete(ipfamily, Options0), + Options2 = proplists:delete(ipfamily, Options1), + validate_ipfamily_unix_socket(IpFamily, UnixSocket, Options2, + [{ipfamily, IpFamily}, {unix_socket, UnixSocket}]). +%% +validate_ipfamily_unix_socket(local, undefined, _Options, _Acc) -> + bad_option(unix_socket, undefined); +validate_ipfamily_unix_socket(IpFamily, UnixSocket, _Options, _Acc) + when IpFamily =/= local, UnixSocket =/= undefined -> + bad_option(ipfamily, IpFamily); +validate_ipfamily_unix_socket(IpFamily, UnixSocket, Options, Acc) -> + validate_ipfamily(IpFamily), + validate_unix_socket(UnixSocket), + {Options, Acc}. + + +validate_options(Options0) -> + try + {Options, Acc} = validate_ipfamily_unix_socket(Options0), + validate_options(Options, Acc) + catch + error:Reason -> + {error, Reason} + end. +%% +validate_options([], ValidOptions) -> + {ok, lists:reverse(ValidOptions)}; validate_options([{proxy, Proxy} = Opt| Tail], Acc) -> validate_proxy(Proxy), @@ -958,6 +986,10 @@ validate_options([{verbose, Value} = Opt| Tail], Acc) -> validate_verbose(Value), validate_options(Tail, [Opt | Acc]); +validate_options([{unix_socket, Value} = Opt| Tail], Acc) -> + validate_unix_socket(Value), + validate_options(Tail, [Opt | Acc]); + validate_options([{_, _} = Opt| _], _Acc) -> {error, {not_an_option, Opt}}. @@ -1026,7 +1058,8 @@ validate_ipv6(BadValue) -> bad_option(ipv6, BadValue). validate_ipfamily(Value) - when (Value =:= inet) orelse (Value =:= inet6) orelse (Value =:= inet6fb4) -> + when (Value =:= inet) orelse (Value =:= inet6) orelse + (Value =:= inet6fb4) orelse (Value =:= local) -> Value; validate_ipfamily(BadValue) -> bad_option(ipfamily, BadValue). @@ -1056,6 +1089,15 @@ validate_verbose(Value) validate_verbose(BadValue) -> bad_option(verbose, BadValue). +validate_unix_socket(Value) + when (Value =:= undefined) -> + Value; +validate_unix_socket(Value) + when is_list(Value) andalso length(Value) > 0 -> + Value; +validate_unix_socket(BadValue) -> + bad_option(unix_socket, BadValue). + bad_option(Option, BadValue) -> throw({error, {bad_option, Option, BadValue}}). diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 1482f4f922..9b09832eb8 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -711,9 +711,9 @@ do_handle_info({'EXIT', _, _}, State = #state{request = undefined}) -> %% can retry requests in the pipeline. do_handle_info({'EXIT', _, _}, State) -> {noreply, State#state{status = close}}. - + call(Msg, Pid) -> - try gen_server:call(Pid, Msg) + try gen_server:call(Pid, Msg, infinity) catch exit:{noproc, _} -> {error, closed}; @@ -754,6 +754,7 @@ connect(SocketType, ToAddress, #options{ipfamily = IpFamily, ip = FromAddress, port = FromPort, + unix_socket = UnixSocket, socket_opts = Opts0}, Timeout) -> Opts1 = case FromPort of @@ -789,6 +790,16 @@ connect(SocketType, ToAddress, OK -> OK end; + local -> + Opts3 = [IpFamily | Opts2], + SocketAddr = {local, UnixSocket}, + case http_transport:connect(SocketType, {SocketAddr, 0}, Opts3, Timeout) of + {error, Reason} -> + {error, {failed_connect, [{to_address, SocketAddr}, + {IpFamily, Opts3, Reason}]}}; + Else -> + Else + end; _ -> Opts3 = [IpFamily | Opts2], case http_transport:connect(SocketType, ToAddress, Opts3, Timeout) of @@ -800,9 +811,23 @@ connect(SocketType, ToAddress, end end. -connect_and_send_first_request(Address, Request, #state{options = Options} = State) -> +handle_unix_socket_options(#request{unix_socket = UnixSocket}, Options) + when UnixSocket =:= undefined -> + Options; + +handle_unix_socket_options(#request{unix_socket = UnixSocket}, + Options = #options{ipfamily = IpFamily}) -> + case IpFamily of + local -> + Options#options{unix_socket = UnixSocket}; + Else -> + error({badarg, [{ipfamily, Else}, {unix_socket, UnixSocket}]}) + end. + +connect_and_send_first_request(Address, Request, #state{options = Options0} = State) -> SocketType = socket_type(Request), ConnTimeout = (Request#request.settings)#http_options.connect_timeout, + Options = handle_unix_socket_options(Request, Options0), case connect(SocketType, Address, Options, ConnTimeout) of {ok, Socket} -> ClientClose = @@ -841,9 +866,10 @@ connect_and_send_first_request(Address, Request, #state{options = Options} = Sta {ok, State#state{request = Request}} end. -connect_and_send_upgrade_request(Address, Request, #state{options = Options} = State) -> +connect_and_send_upgrade_request(Address, Request, #state{options = Options0} = State) -> ConnTimeout = (Request#request.settings)#http_options.connect_timeout, SocketType = ip_comm, + Options = handle_unix_socket_options(Request, Options0), case connect(SocketType, Address, Options, ConnTimeout) of {ok, Socket} -> SessionType = httpc_manager:session_type(Options), @@ -1685,9 +1711,8 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> insert_session(Session2, ProfileName); error:badarg -> {stop, normal}; - T:E -> + T:E:Stacktrace -> %% Unexpected this must be an error! - Stacktrace = erlang:get_stacktrace(), error_logger:error_msg("Failed updating session: " "~n ProfileName: ~p" "~n SessionId: ~p" diff --git a/lib/inets/src/http_client/httpc_internal.hrl b/lib/inets/src/http_client/httpc_internal.hrl index 5f8c70f28d..c5fe439722 100644 --- a/lib/inets/src/http_client/httpc_internal.hrl +++ b/lib/inets/src/http_client/httpc_internal.hrl @@ -83,10 +83,11 @@ max_sessions = ?HTTP_MAX_TCP_SESSIONS, cookies = disabled, % enabled | disabled | verify verbose = false, % boolean(), - ipfamily = inet, % inet | inet6 | inet6fb4 + ipfamily = inet, % inet | inet6 | inet6fb4 | local ip = default, % specify local interface port = default, % specify local port - socket_opts = [] % other socket options + socket_opts = [], % other socket options + unix_socket = undefined % Local unix socket } ). -type options() :: #options{}. @@ -115,6 +116,7 @@ % request timer :: undefined | reference(), socket_opts, % undefined | [socket_option()] + unix_socket, % undefined | string() ipv6_host_with_brackets % boolean() } ). diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index ffdf1603b3..7b8d7875de 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -553,7 +553,8 @@ handle_cast({set_options, Options}, State = #state{options = OldOptions}) -> ip = get_ip(Options, OldOptions), port = get_port(Options, OldOptions), verbose = get_verbose(Options, OldOptions), - socket_opts = get_socket_opts(Options, OldOptions) + socket_opts = get_socket_opts(Options, OldOptions), + unix_socket = get_unix_socket_opts(Options, OldOptions) }, case {OldOptions#options.verbose, NewOptions#options.verbose} of {Same, Same} -> @@ -963,7 +964,10 @@ get_option(ip, #options{ip = IP}) -> get_option(port, #options{port = Port}) -> Port; get_option(socket_opts, #options{socket_opts = SocketOpts}) -> - SocketOpts. + SocketOpts; +get_option(unix_socket, #options{unix_socket = UnixSocket}) -> + UnixSocket. + get_proxy(Opts, #options{proxy = Default}) -> proplists:get_value(proxy, Opts, Default). @@ -1016,6 +1020,8 @@ get_verbose(Opts, #options{verbose = Default}) -> get_socket_opts(Opts, #options{socket_opts = Default}) -> proplists:get_value(socket_opts, Opts, Default). +get_unix_socket_opts(Opts, #options{unix_socket = Default}) -> + proplists:get_value(unix_socket, Opts, Default). handle_verbose(debug) -> dbg:p(self(), [call]), diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index 6b9053fda6..57ce162922 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -84,14 +84,14 @@ traverse_modules(ModData,[Module|Rest]) -> {proceed, NewData} -> traverse_modules(ModData#mod{data = NewData}, Rest) catch - T:E -> + T:E:Stacktrace -> String = lists:flatten( io_lib:format("module traverse failed: ~p:do => " "~n Error Type: ~p" "~n Error: ~p" "~n Stack trace: ~p", - [Module, T, E, ?STACK()])), + [Module, T, E, Stacktrace])), httpd_util:error_log(ModData#mod.config_db, String), send_status(ModData, 500, none), done diff --git a/lib/inets/src/inets_app/inets.appup.src b/lib/inets/src/inets_app/inets.appup.src index fdf4cc6e07..a86413147c 100644 --- a/lib/inets/src/inets_app/inets.appup.src +++ b/lib/inets/src/inets_app/inets.appup.src @@ -18,14 +18,10 @@ %% %CopyrightEnd% {"%VSN%", [ - {<<"6.4.3">>, [{load_module, httpd_esi, - soft_purge, soft_purge, []}]}, {<<"6\\..*">>,[{restart_application, inets}]}, {<<"5\\..*">>,[{restart_application, inets}]} ], [ - {<<"6.4.3">>, [{load_module, httpd_esi, - soft_purge, soft_purge, []}]}, {<<"6\\..*">>,[{restart_application, inets}]}, {<<"5\\..*">>,[{restart_application, inets}]} ] diff --git a/lib/inets/src/inets_app/inets_internal.hrl b/lib/inets/src/inets_app/inets_internal.hrl index 079b415b56..e0f59bba5f 100644 --- a/lib/inets/src/inets_app/inets_internal.hrl +++ b/lib/inets/src/inets_app/inets_internal.hrl @@ -22,8 +22,6 @@ -ifndef(inets_internal_hrl). -define(inets_internal_hrl, true). --define(STACK(), erlang:get_stacktrace()). - %% Various trace macros -define(report(Severity, Label, Service, Content), diff --git a/lib/inets/test/http_test_lib.erl b/lib/inets/test/http_test_lib.erl index 38e9e4976e..4e119cce04 100644 --- a/lib/inets/test/http_test_lib.erl +++ b/lib/inets/test/http_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% Copyright Ericsson AB 2015-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -55,6 +55,25 @@ dummy_server_init(Caller, ip_comm, Inet, Extra) -> ]]}, [], ContentCb, Conf, ListenSocket); +dummy_server_init(Caller, unix_socket, Inet, Extra) -> + ContentCb = proplists:get_value(content_cb, Extra), + UnixSocket = proplists:get_value(unix_socket, Extra), + SocketAddr = {local, UnixSocket}, + BaseOpts = [binary, {packet, 0}, {reuseaddr,true}, {active, false}, {nodelay, true}, + {ifaddr, SocketAddr}], + Conf = proplists:get_value(conf, Extra), + {ok, ListenSocket} = gen_tcp:listen(0, [Inet | BaseOpts]), + {ok, Port} = inet:port(ListenSocket), + Caller ! {port, Port}, + dummy_ipcomm_server_loop({httpd_request, parse, [[{max_uri, ?HTTP_MAX_URI_SIZE}, + {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}, + [], ContentCb, Conf, ListenSocket); + dummy_server_init(Caller, ssl, Inet, Extra) -> ContentCb = proplists:get_value(content_cb, Extra), SSLOptions = proplists:get_value(ssl, Extra), diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 0533b9ab70..38705372c9 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -37,6 +37,10 @@ -define(TLS_URL_START, "https://"). -define(NOT_IN_USE_PORT, 8997). +%% Using hardcoded file path to keep it below 107 charaters +%% (maximum length supported by erlang) +-define(UNIX_SOCKET, "/tmp/inets_httpc_SUITE.sock"). + -record(sslsocket, {fd = nil, pid = nil}). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- @@ -50,6 +54,8 @@ all() -> [ {group, http}, {group, sim_http}, + {group, http_internal}, + {group, http_unix_socket}, {group, https}, {group, sim_https}, {group, misc} @@ -62,6 +68,8 @@ groups() -> %% and it shall be the last test case in the suite otherwise cookie %% will fail. {sim_http, [], only_simulated() ++ [process_leak_on_keepalive]}, + {http_internal, [], real_requests_esi()}, + {http_unix_socket, [], simulated_unix_socket()}, {https, [], real_requests()}, {sim_https, [], only_simulated()}, {misc, [], misc()} @@ -97,6 +105,12 @@ real_requests()-> invalid_body ]. +real_requests_esi() -> + [slow_connection]. + +simulated_unix_socket() -> + [unix_domain_socket]. + only_simulated() -> [ cookie, @@ -182,15 +196,29 @@ init_per_group(Group, Config0) when Group =:= sim_https; Group =:= https-> _:_ -> {skip, "Crypto did not start"} end; - +init_per_group(http_unix_socket = Group, Config0) -> + case os:type() of + {win32,_} -> + {skip, "Unix Domain Sockets are not supported on Windows"}; + _ -> + file:delete(?UNIX_SOCKET), + start_apps(Group), + Config = proplists:delete(port, Config0), + Port = server_start(Group, server_config(Group, Config)), + [{port, Port} | Config] + end; init_per_group(Group, Config0) -> start_apps(Group), Config = proplists:delete(port, Config0), Port = server_start(Group, server_config(Group, Config)), [{port, Port} | Config]. +end_per_group(http_unix_socket,_Config) -> + file:delete(?UNIX_SOCKET), + ok; end_per_group(_, _Config) -> ok. + do_init_per_group(Group, Config0) -> Config = proplists:delete(port, Config0), Port = server_start(Group, server_config(Group, Config)), @@ -1245,7 +1273,40 @@ stream_fun_server_close(Config) when is_list(Config) -> after 13000 -> ct:fail(did_not_receive_close) end. - + +%%-------------------------------------------------------------------- +slow_connection() -> + [{doc, "Test that a request on a slow keep-alive connection won't crash the httpc_manager"}]. +slow_connection(Config) when is_list(Config) -> + BodyFun = fun(0) -> eof; + (LenLeft) -> timer:sleep(1000), + {ok, lists:duplicate(10, "1"), LenLeft - 10} + end, + Request = {url(group_name(Config), "/httpc_SUITE:esi_post", Config), + [{"content-length", "100"}], + "text/plain", + {BodyFun, 100}}, + {ok, _} = httpc:request(post, Request, [], []), + %% Second request causes a crash if gen_server timeout is not set to infinity + %% in httpc_handler. + {ok, _} = httpc:request(post, Request, [], []). + +%%------------------------------------------------------------------------- +unix_domain_socket() -> + [{"doc, Test HTTP requests over unix domain sockets"}]. +unix_domain_socket(Config) when is_list(Config) -> + + URL = "http:///v1/kv/foo", + + {ok,[{unix_socket,?UNIX_SOCKET}]} = + httpc:get_options([unix_socket]), + {ok, {{_,200,_}, [_ | _], _}} + = httpc:request(put, {URL, [], [], ""}, [], []), + {ok, {{_,200,_}, [_ | _], _}} + = httpc:request(get, {URL, []}, [], []). + + + %%-------------------------------------------------------------------- %% Internal Functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -1339,6 +1400,8 @@ url(https, End, Config) -> ?TLS_URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ End; url(sim_http, End, Config) -> url(http, End, Config); +url(http_internal, End, Config) -> + url(http, End, Config); url(sim_https, End, Config) -> url(https, End, Config). url(http, UserInfo, End, Config) -> @@ -1358,19 +1421,28 @@ group_name(Config) -> server_start(sim_http, _) -> Inet = inet_version(), - ok = httpc:set_options([{ipfamily, Inet}]), + ok = httpc:set_options([{ipfamily, Inet},{unix_socket, undefined}]), {_Pid, Port} = http_test_lib:dummy_server(ip_comm, Inet, [{content_cb, ?MODULE}]), Port; server_start(sim_https, SslConfig) -> Inet = inet_version(), - ok = httpc:set_options([{ipfamily, Inet}]), + ok = httpc:set_options([{ipfamily, Inet},{unix_socket, undefined}]), {_Pid, Port} = http_test_lib:dummy_server(ssl, Inet, [{ssl, SslConfig}, {content_cb, ?MODULE}]), Port; +server_start(http_unix_socket, Config) -> + Inet = local, + Socket = proplists:get_value(unix_socket, Config), + ok = httpc:set_options([{ipfamily, Inet},{unix_socket, Socket}]), + {_Pid, Port} = http_test_lib:dummy_server(unix_socket, Inet, [{content_cb, ?MODULE}, + {unix_socket, Socket}]), + Port; + server_start(_, HttpdConfig) -> {ok, Pid} = inets:start(httpd, HttpdConfig), Serv = inets:services_info(), + ok = httpc:set_options([{ipfamily, inet_version()},{unix_socket, undefined}]), {value, {_, _, Info}} = lists:keysearch(Pid, 2, Serv), proplists:get_value(port, Info). @@ -1385,14 +1457,31 @@ server_config(http, Config) -> {mime_type, "text/plain"}, {script_alias, {"/cgi-bin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}} ]; - +server_config(http_internal, Config) -> + ServerRoot = proplists:get_value(server_root, Config), + [{port, 0}, + {server_name,"httpc_test"}, + {server_root, ServerRoot}, + {document_root, proplists:get_value(doc_root, Config)}, + {bind_address, any}, + {ipfamily, inet_version()}, + {mime_type, "text/plain"}, + {erl_script_alias, {"", [httpc_SUITE]}} + ]; server_config(https, Config) -> [{socket_type, {essl, ssl_config(Config)}} | server_config(http, Config)]; server_config(sim_https, Config) -> ssl_config(Config); +server_config(http_unix_socket, _Config) -> + Socket = ?UNIX_SOCKET, + [{unix_socket, Socket}]; + server_config(_, _) -> []. +esi_post(Sid, _Env, _Input) -> + mod_esi:deliver(Sid, ["OK"]). + start_apps(https) -> inets_test_lib:start_apps([crypto, public_key, ssl]); start_apps(sim_https) -> @@ -2131,6 +2220,19 @@ handle_uri(_,"/delay_close.html",_,_,Socket,_) -> handle_uri("HEAD",_,_,_,_,_) -> "HTTP/1.1 200 ok\r\n" ++ "Content-Length:0\r\n\r\n"; +handle_uri("PUT","/v1/kv/foo",_,_,_,_) -> + "HTTP/1.1 200 OK\r\n" ++ + "Date: Tue, 20 Feb 2018 14:39:08 GMT\r\n" ++ + "Content-Length: 5\r\n\r\n" ++ + "Content-Type: application/json\r\n\r\n" ++ + "true\n"; +handle_uri("GET","/v1/kv/foo",_,_,_,_) -> + "HTTP/1.1 200 OK\r\n" ++ + "Date: Tue, 20 Feb 2018 14:39:08 GMT\r\n" ++ + "Content-Length: 24\r\n" ++ + "Content-Type: application/json\r\n\r\n" ++ + "[{\"Value\": \"aGVsbG8=\"}]\n"; + handle_uri(_,_,_,_,_,DefaultResponse) -> DefaultResponse. diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 05cf4f6cc3..1fad9afe33 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 6.4.5 +INETS_VSN = 6.5 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/kernel/doc/src/code.xml b/lib/kernel/doc/src/code.xml index c94f612c01..bd95819636 100644 --- a/lib/kernel/doc/src/code.xml +++ b/lib/kernel/doc/src/code.xml @@ -34,26 +34,28 @@ <p>This module contains the interface to the Erlang <em>code server</em>, which deals with the loading of compiled code into a running Erlang runtime system.</p> - <p>The runtime system can be started in <em>embedded</em> or - <em>interactive</em> mode. Which one is decided by command-line + <p>The runtime system can be started in <em>interactive</em> or + <em>embedded</em> mode. Which one is decided by the command-line flag <c>-mode</c>:</p> <pre> % <input>erl -mode interactive</input></pre> <p>The modes are as follows:</p> <list type="bulleted"> <item> - <p>In embedded mode, all code is loaded during system startup - according to the boot script. (Code can also be loaded later - by explicitly ordering the code server to do so).</p> - </item> - <item> <p>In interactive mode, which is default, only some code is loaded - during system startup, basically the modules needed by the runtime + during system startup, basically the modules needed by the runtime system. Other code is dynamically loaded when first referenced. When a call to a function in a certain module is made, and the module is not loaded, the code server searches for and tries to load the module.</p> </item> + <item> + <p>In embedded mode, modules are not auto loaded. Trying to use + a module that has not been loaded results in an error. This mode is + recommended when the boot script loads all modules, as it is + typically done in OTP releases. (Code can still be loaded later + by explicitly ordering the code server to do so).</p> + </item> </list> <p>To prevent accidentally reloading of modules affecting the Erlang runtime system, directories <c>kernel</c>, <c>stdlib</c>, diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 8477b0e148..1b72769ce3 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -981,8 +981,7 @@ f.txt: {person, "kalle", 25}. </item> <tag><c>eisdir</c></tag> <item> - <p>The named file is not a regular file. It can be a - directory, a FIFO, or a device.</p> + <p>The named file is a directory.</p> </item> <tag><c>enotdir</c></tag> <item> diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index d7f224c38e..65fe9b9c07 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,26 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 5.4.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Add <c>os:cmd/2</c> that takes an options map as the + second argument.</p> + <p> + Add <c>max_size</c> as an option to <c>os:cmd/2</c> that + control the maximum size of the result that + <c>os:cmd/2</c> will return.</p> + <p> + Own Id: OTP-14823</p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 5.4.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/doc/src/os.xml b/lib/kernel/doc/src/os.xml index c27182ff0b..ef416ed233 100644 --- a/lib/kernel/doc/src/os.xml +++ b/lib/kernel/doc/src/os.xml @@ -103,7 +103,7 @@ </desc> </datatype> <datatype> - <name name="command_input"/> + <name name="os_command"/> <desc> <p>All characters needs to be valid characters on the specific OS using @@ -116,19 +116,31 @@ </p> </desc> </datatype> + <datatype> + <name name="os_command_opts"/> + <desc> + <p>Options for <seealso marker="#cmd/2"><c>os:cmd/2</c></seealso></p> + <taglist> + <tag><c>max_size</c></tag> + <item> + <p>The maximum size of the data returned by the <c>os:cmd</c> call. + See the <seealso marker="#cmd/2"><c>os:cmd/2</c></seealso> + documentation for more details.</p> + </item> + </taglist> + </desc> + </datatype> </datatypes> - + <funcs> <func> <name name="cmd" arity="1"/> + <name name="cmd" arity="2"/> <fsummary>Execute a command in a shell of the target OS.</fsummary> <desc> <p>Executes <c><anno>Command</anno></c> in a command shell of the - target OS, - captures the standard output of the command, and returns this - result as a string. This function is a replacement of - the previous function <c>unix:cmd/1</c>; they are equivalent on a - Unix platform.</p> + target OS, captures the standard output of the command, + and returns this result as a string.</p> <warning><p>Previous implementation used to allow all characters as long as they were integer values greater than or equal to zero. This sometimes lead to unwanted results since null characters @@ -142,6 +154,21 @@ DirOut = os:cmd("dir"), % on Win32 platform</code> called from another program (for example, <c>os:cmd/1</c>) can differ, compared with the standard output of the command when called directly from an OS command shell.</p> + <p><c>os:cmd/2</c> was added in kernel-5.5 (OTP-20.2.1). It makes it + possible to pass an options map as the second argument in order to + control the behaviour of <c>os:cmd</c>. The possible options are: + </p> + <taglist> + <tag><c>max_size</c></tag> + <item> + <p>The maximum size of the data returned by the <c>os:cmd</c> call. + This option is a safety feature that should be used when the command + executed can return a very large, possibly infinite, result.</p> + <code type="none"> +> os:cmd("cat /dev/zero", #{ max_size => 20 }). +[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]</code> + </item> + </taglist> </desc> </func> diff --git a/lib/kernel/doc/src/rpc.xml b/lib/kernel/doc/src/rpc.xml index adec2d9520..fab616e630 100644 --- a/lib/kernel/doc/src/rpc.xml +++ b/lib/kernel/doc/src/rpc.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2016</year> + <year>1996</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -217,7 +217,7 @@ <list type="bulleted"> <item>A list of the nodes that do not exist</item> <item>A list of the nodes where the server does not exist</item> - <item>A list of the nodes where the server terminatd before sending + <item>A list of the nodes where the server terminated before sending any reply.</item> </list> </desc> @@ -268,8 +268,9 @@ on the specified nodes and collects the answers. It returns <c>{<anno>ResL</anno>, <anno>BadNodes</anno>}</c>, where <c><anno>BadNodes</anno></c> is a list - of the nodes that terminated or timed out during computation, - and <c><anno>ResL</anno></c> is a list of the return values. + of the nodes that do not exist, + and <c><anno>ResL</anno></c> is a list of the return values, + or <c>{badrpc, <anno>Reason</anno>}</c> for failing calls. <c><anno>Timeout</anno></c> is a time (integer) in milliseconds, or <c>infinity</c>.</p> <p>The following example is useful when new object code is to @@ -347,7 +348,7 @@ <func> <name name="pmap" arity="3"/> - <fsummary>Parallell evaluation of mapping a function over a + <fsummary>Parallel evaluation of mapping a function over a list.</fsummary> <desc> <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 418b0c50e1..f5a890cb95 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -340,8 +340,7 @@ handle_call(all_loaded, _From, S) -> {reply,all_loaded(Db),S}; handle_call({get_object_code,Mod}, _From, St) when is_atom(Mod) -> - Path = St#state.path, - case mod_to_bin(Path, Mod) of + case get_object_code(St, Mod) of {_,Bin,FName} -> {reply,{Mod,Bin,FName},St}; Error -> {reply,Error,St} end; @@ -1182,19 +1181,28 @@ load_file(Mod, From, St0) -> end, handle_pending_on_load(Action, Mod, From, St0). -load_file_1(Mod, From, #state{path=Path}=St) -> - case mod_to_bin(Path, Mod) of +load_file_1(Mod, From, St) -> + case get_object_code(St, Mod) of error -> {reply,{error,nofile},St}; {Mod,Binary,File} -> try_load_module_1(File, Mod, Binary, From, St) end. -mod_to_bin([Dir|Tail], Mod) -> - File = filename:append(Dir, atom_to_list(Mod) ++ objfile_extension()), +get_object_code(#state{path=Path}, Mod) when is_atom(Mod) -> + ModStr = atom_to_list(Mod), + case erl_prim_loader:is_basename(ModStr) of + true -> + mod_to_bin(Path, Mod, ModStr ++ objfile_extension()); + false -> + error + end. + +mod_to_bin([Dir|Tail], Mod, ModFile) -> + File = filename:append(Dir, ModFile), case erl_prim_loader:get_file(File) of error -> - mod_to_bin(Tail, Mod); + mod_to_bin(Tail, Mod, ModFile); {ok,Bin,_} -> case filename:pathtype(File) of absolute -> @@ -1203,10 +1211,9 @@ mod_to_bin([Dir|Tail], Mod) -> {Mod,Bin,absname(File)} end end; -mod_to_bin([], Mod) -> +mod_to_bin([], Mod, ModFile) -> %% At last, try also erl_prim_loader's own method - File = to_list(Mod) ++ objfile_extension(), - case erl_prim_loader:get_file(File) of + case erl_prim_loader:get_file(ModFile) of error -> error; % No more alternatives ! {ok,Bin,FName} -> diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index fb9f7fd7eb..5ea0ca991f 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -539,8 +539,8 @@ do_setnode(#hs_data{other_node = Node, socket = Socket, "no table space left for node ~w ** ~n", [Node]), ?shutdown(Node); - error:Other -> - exit({Other, erlang:get_stacktrace()}) + error:Other:Stacktrace -> + exit({Other, Stacktrace}) end; _ -> error_msg("** Distribution connection error, " diff --git a/lib/kernel/src/error_handler.erl b/lib/kernel/src/error_handler.erl index 59ca8e690d..a9582c6225 100644 --- a/lib/kernel/src/error_handler.erl +++ b/lib/kernel/src/error_handler.erl @@ -106,8 +106,8 @@ crash(M, F, A) -> crash(Tuple) -> try erlang:error(undef) catch - error:undef -> - Stk = [Tuple|tl(erlang:get_stacktrace())], + error:undef:Stacktrace -> + Stk = [Tuple|tl(Stacktrace)], erlang:raise(error, undef, Stk) end. diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index d05199897f..c2df1ee288 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -1382,8 +1382,8 @@ eval_stream2({ok,Form,EndLine}, Fd, H, Last, E, Bs0) -> try erl_eval:exprs(Form, Bs0) of {value,V,Bs} -> eval_stream(Fd, H, EndLine, {V}, E, Bs) - catch Class:Reason -> - Error = {EndLine,?MODULE,{Class,Reason,erlang:get_stacktrace()}}, + catch Class:Reason:StackTrace -> + Error = {EndLine,?MODULE,{Class,Reason,StackTrace}}, eval_stream(Fd, H, EndLine, Last, [Error|E], Bs0) end; eval_stream2({error,What,EndLine}, Fd, H, Last, E, Bs) -> diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index f8199fcf71..fd06f0f7d8 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -275,6 +275,7 @@ needs_trampolines(Architecture) -> arm -> true; powerpc -> true; ppc64 -> true; + amd64 -> true; _ -> false end. diff --git a/lib/kernel/src/os.erl b/lib/kernel/src/os.erl index fbc046c8f9..77c883f57f 100644 --- a/lib/kernel/src/os.erl +++ b/lib/kernel/src/os.erl @@ -21,11 +21,11 @@ %% Provides a common operating system interface. --export([type/0, version/0, cmd/1, find_executable/1, find_executable/2]). +-export([type/0, version/0, cmd/1, cmd/2, find_executable/1, find_executable/2]). -include("file.hrl"). --export_type([env_var_name/0, env_var_value/0, env_var_name_value/0, command_input/0]). +-export_type([env_var_name/0, env_var_value/0, env_var_name_value/0]). -export([getenv/0, getenv/1, getenv/2, putenv/2, unsetenv/1]). @@ -35,14 +35,17 @@ perf_counter/1, set_env_var/2, set_signal/2, system_time/0, system_time/1, timestamp/0, unset_env_var/1]). +-type os_command() :: atom() | io_lib:chars(). +-type os_command_opts() :: #{ max_size => non_neg_integer() | infinity }. + +-export_type([os_command/0, os_command_opts/0]). + -type env_var_name() :: nonempty_string(). -type env_var_value() :: string(). -type env_var_name_value() :: nonempty_string(). --type command_input() :: atom() | io_lib:chars(). - -spec list_env_vars() -> [{env_var_name(), env_var_value()}]. list_env_vars() -> erlang:nif_error(undef). @@ -260,14 +263,20 @@ extensions() -> %% Executes the given command in the default shell for the operating system. -spec cmd(Command) -> string() when - Command :: os:command_input(). + Command :: os_command(). cmd(Cmd) -> + cmd(Cmd, #{ }). + +-spec cmd(Command, Options) -> string() when + Command :: os_command(), + Options :: os_command_opts(). +cmd(Cmd, Opts) -> {SpawnCmd, SpawnOpts, SpawnInput, Eot} = mk_cmd(os:type(), validate(Cmd)), Port = open_port({spawn, SpawnCmd}, [binary, stderr_to_stdout, stream, in, hide | SpawnOpts]), MonRef = erlang:monitor(port, Port), true = port_command(Port, SpawnInput), - Bytes = get_data(Port, MonRef, Eot, []), + Bytes = get_data(Port, MonRef, Eot, [], 0, maps:get(max_size, Opts, infinity)), demonitor(MonRef, [flush]), String = unicode:characters_to_list(Bytes), if %% Convert to unicode list if possible otherwise return bytes @@ -332,12 +341,13 @@ validate2([List|Rest]) when is_list(List) -> validate2(List), validate2(Rest). -get_data(Port, MonRef, Eot, Sofar) -> +get_data(Port, MonRef, Eot, Sofar, Size, Max) -> receive {Port, {data, Bytes}} -> - case eot(Bytes, Eot) of + case eot(Bytes, Eot, Size, Max) of more -> - get_data(Port, MonRef, Eot, [Sofar,Bytes]); + get_data(Port, MonRef, Eot, [Sofar, Bytes], + Size + byte_size(Bytes), Max); Last -> catch port_close(Port), flush_until_down(Port, MonRef), @@ -348,13 +358,16 @@ get_data(Port, MonRef, Eot, Sofar) -> iolist_to_binary(Sofar) end. -eot(_Bs, <<>>) -> +eot(Bs, <<>>, Size, Max) when Size + byte_size(Bs) < Max -> more; -eot(Bs, Eot) -> +eot(Bs, <<>>, Size, Max) -> + binary:part(Bs, {0, Max - Size}); +eot(Bs, Eot, Size, Max) -> case binary:match(Bs, Eot) of - nomatch -> more; - {Pos, _} -> - binary:part(Bs,{0, Pos}) + {Pos, _} when Size + Pos < Max -> + binary:part(Bs,{0, Pos}); + _ -> + eot(Bs, <<>>, Size, Max) end. %% When port_close returns we know that all the diff --git a/lib/kernel/src/rpc.erl b/lib/kernel/src/rpc.erl index 0e0b7dffa3..b04aa9030b 100644 --- a/lib/kernel/src/rpc.erl +++ b/lib/kernel/src/rpc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -498,7 +498,7 @@ start_monitor(Node, Name) -> Module :: module(), Function :: atom(), Args :: [term()], - ResL :: [term()], + ResL :: [Res :: term() | {'badrpc', Reason :: term()}], BadNodes :: [node()]. multicall(M, F, A) -> @@ -509,14 +509,14 @@ multicall(M, F, A) -> Module :: module(), Function :: atom(), Args :: [term()], - ResL :: [term()], + ResL :: [Res :: term() | {'badrpc', Reason :: term()}], BadNodes :: [node()]; (Module, Function, Args, Timeout) -> {ResL, BadNodes} when Module :: module(), Function :: atom(), Args :: [term()], Timeout :: timeout(), - ResL :: [term()], + ResL :: [Res :: term() | {'badrpc', Reason :: term()}], BadNodes :: [node()]. multicall(Nodes, M, F, A) when is_list(Nodes) -> @@ -531,7 +531,7 @@ multicall(M, F, A, Timeout) -> Function :: atom(), Args :: [term()], Timeout :: timeout(), - ResL :: [term()], + ResL :: [Res :: term() | {'badrpc', Reason :: term()}], BadNodes :: [node()]. multicall(Nodes, M, F, A, infinity) diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 569753155f..902196def2 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -931,37 +931,34 @@ purge_stacktrace(Config) when is_list(Config) -> code:purge(code_b_test), try code_b_test:call(fun(b) -> ok end, a) catch - error:function_clause -> + error:function_clause:Stacktrace -> code:load_file(code_b_test), - case erlang:get_stacktrace() of + case Stacktrace of [{?MODULE,_,[a],_}, {code_b_test,call,2,_}, {?MODULE,purge_stacktrace,1,_}|_] -> - false = code:purge(code_b_test), - [] = erlang:get_stacktrace() + false = code:purge(code_b_test) end end, try code_b_test:call(nofun, 2) catch - error:function_clause -> + error:function_clause:Stacktrace2 -> code:load_file(code_b_test), - case erlang:get_stacktrace() of + case Stacktrace2 of [{code_b_test,call,[nofun,2],_}, {?MODULE,purge_stacktrace,1,_}|_] -> - false = code:purge(code_b_test), - [] = erlang:get_stacktrace() + false = code:purge(code_b_test) end end, Args = [erlang,error,[badarg]], try code_b_test:call(erlang, error, [badarg,Args]) catch - error:badarg -> + error:badarg:Stacktrace3 -> code:load_file(code_b_test), - case erlang:get_stacktrace() of + case Stacktrace3 of [{code_b_test,call,Args,_}, {?MODULE,purge_stacktrace,1,_}|_] -> - false = code:purge(code_b_test), - [] = erlang:get_stacktrace() + false = code:purge(code_b_test) end end, ok. diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index bbfaa9d147..f6791adf86 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -95,7 +95,11 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - +init_per_testcase(TC, Config) when TC == hostnames; + TC == nodenames -> + file:make_dir("hostnames_nodedir"), + file:write_file("hostnames_nodedir/ignore_core_files",""), + Config; init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Config. @@ -251,7 +255,7 @@ test_node(Name, Illigal) -> end, net_kernel:monitor_nodes(true), BinCommand = unicode:characters_to_binary(Command, utf8), - Prt = open_port({spawn, BinCommand}, [stream]), + Prt = open_port({spawn, BinCommand}, [stream,{cd,"hostnames_nodedir"}]), Node = list_to_atom(Name), receive {nodeup, Node} -> @@ -459,9 +463,9 @@ run_remote_test([FuncStr, TestNodeStr | Args]) -> 1 end catch - C:E -> + C:E:S -> io:format("Node ~p got EXCEPTION ~p:~p\nat ~p\n", - [node(), C, E, erlang:get_stacktrace()]), + [node(), C, E, S]), 2 end, io:format("Node ~p doing halt(~p).\n",[node(), Status]), diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 0cb8087a76..9a77454432 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -399,11 +399,11 @@ read_write_0(Str, {Func, ReadFun}, Options) -> io:format("~p:~p: ~p ERROR: ~ts vs~n ~w~n - ~p~n", [?MODULE, Line, Func, Str, ReadBytes, Options]), exit({error, ?LINE}); - error:What -> + error:What:Stacktrace -> io:format("~p:??: ~p ERROR: ~p from~n ~w~n ~p~n", [?MODULE, Func, What, Str, Options]), - io:format("\t~p~n", [erlang:get_stacktrace()]), + io:format("\t~p~n", [Stacktrace]), exit({error, ?LINE}) end. diff --git a/lib/kernel/test/gen_sctp_SUITE.erl b/lib/kernel/test/gen_sctp_SUITE.erl index 620ab235a0..9dde00652c 100644 --- a/lib/kernel/test/gen_sctp_SUITE.erl +++ b/lib/kernel/test/gen_sctp_SUITE.erl @@ -1038,8 +1038,7 @@ do_from_other_process(Fun) -> Result -> Parent ! {Ref,Result} catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> Parent ! {Ref,Class,Reason,Stacktrace} end end), @@ -1617,8 +1616,7 @@ s_start(Socket, Timeout, Parent) -> try s_loop(Socket, Timeout, Parent, Handler, gb_trees:empty()) catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> io:format(?MODULE_STRING":socket exception ~w:~w at~n" "~p.~n", [Class,Reason,Stacktrace]), erlang:raise(Class, Reason, Stacktrace) diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index 12d22519ce..0fe44e8a88 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -605,9 +605,9 @@ ok({ok,V}) -> V; ok(NotOk) -> try throw(not_ok) catch - Thrown -> + throw:Thrown:Stacktrace -> erlang:raise( - error, {Thrown, NotOk}, tl(erlang:get_stacktrace())) + error, {Thrown, NotOk}, tl(Stacktrace)) end. get_localaddr() -> diff --git a/lib/kernel/test/gen_udp_SUITE.erl b/lib/kernel/test/gen_udp_SUITE.erl index 96e495505a..6a50239c2a 100644 --- a/lib/kernel/test/gen_udp_SUITE.erl +++ b/lib/kernel/test/gen_udp_SUITE.erl @@ -755,9 +755,9 @@ ok({ok,V}) -> V; ok(NotOk) -> try throw(not_ok) catch - Thrown -> + throw:Thrown:Stacktrace -> erlang:raise( - error, {Thrown, NotOk}, tl(erlang:get_stacktrace())) + error, {Thrown, NotOk}, tl(Stacktrace)) end. diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index 77ac743a6e..2e5f8c7d2c 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -1104,11 +1104,9 @@ ifaddrs([{If,Opts}|IOs]) -> #ifopts{flags=F} = Ifopts = check_ifopts(Opts, #ifopts{name=If}), case F of {flags,Flags} -> - case lists:member(up, Flags) of - true -> - Ifopts#ifopts.addrs; - false -> - [] + case lists:member(running, Flags) of + true -> Ifopts#ifopts.addrs; + false -> [] end ++ ifaddrs(IOs); undefined -> ifaddrs(IOs) diff --git a/lib/kernel/test/inet_res_SUITE.erl b/lib/kernel/test/inet_res_SUITE.erl index 6691ad9c06..2a5b8d0044 100644 --- a/lib/kernel/test/inet_res_SUITE.erl +++ b/lib/kernel/test/inet_res_SUITE.erl @@ -217,10 +217,10 @@ proxy_start(TC, {NS,P}) -> spawn_link( fun () -> try proxy_start(TC, NS, P, Parent, Tag) - catch C:X -> + catch C:X:Stacktrace -> io:format( "~w: ~w:~p ~p~n", - [self(),C,X,erlang:get_stacktrace()]) + [self(),C,X,Stacktrace]) end end), receive {started,Tag,Port} -> diff --git a/lib/kernel/test/os_SUITE.erl b/lib/kernel/test/os_SUITE.erl index 8056321448..591fbb2125 100644 --- a/lib/kernel/test/os_SUITE.erl +++ b/lib/kernel/test/os_SUITE.erl @@ -26,7 +26,8 @@ null_in_command/1, space_in_name/1, bad_command/1, find_executable/1, unix_comment_in_command/1, deep_list_command/1, large_output_command/1, background_command/0, background_command/1, - message_leak/1, close_stdin/0, close_stdin/1, perf_counter_api/1]). + message_leak/1, close_stdin/0, close_stdin/1, max_size_command/1, + perf_counter_api/1]). -include_lib("common_test/include/ct.hrl"). @@ -39,7 +40,7 @@ all() -> space_in_name, bad_command, find_executable, unix_comment_in_command, deep_list_command, large_output_command, background_command, message_leak, - close_stdin, perf_counter_api]. + close_stdin, max_size_command, perf_counter_api]. groups() -> []. @@ -322,6 +323,19 @@ close_stdin(Config) -> "-1" = os:cmd(Fds). +max_size_command(_Config) -> + + Res20 = os:cmd("cat /dev/zero", #{ max_size => 20 }), + 20 = length(Res20), + + Res0 = os:cmd("cat /dev/zero", #{ max_size => 0 }), + 0 = length(Res0), + + Res32768 = os:cmd("cat /dev/zero", #{ max_size => 32768 }), + 32768 = length(Res32768), + + ResHello = string:trim(os:cmd("echo hello", #{ max_size => 20 })), + 5 = length(ResHello). %% Test that the os:perf_counter api works as expected perf_counter_api(_Config) -> diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 106bda01ca..91261e1d55 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 5.4.1 +KERNEL_VSN = 5.4.2 diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 95f7d4afc1..6d87544bd2 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -177,8 +177,8 @@ %% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia.hrl b/lib/mnesia/src/mnesia.hrl index da7e662288..e36222d098 100644 --- a/lib/mnesia/src/mnesia.hrl +++ b/lib/mnesia/src/mnesia.hrl @@ -47,6 +47,10 @@ -define(catch_val(Var), (try ?ets_lookup_element(mnesia_gvar, Var, 2) catch error:_ -> {'EXIT', {badarg, []}} end)). +-define(catch_val_and_stack(Var), + (try ?ets_lookup_element(mnesia_gvar, Var, 2) + catch error:_:_Stacktrace -> {'EXIT', _Stacktrace} end)). + %% It's important that counter is first, since we compare tid's -record(tid, diff --git a/lib/mnesia/src/mnesia_bup.erl b/lib/mnesia/src/mnesia_bup.erl index 34f16f178b..9dda340333 100644 --- a/lib/mnesia/src/mnesia_bup.erl +++ b/lib/mnesia/src/mnesia_bup.erl @@ -90,9 +90,9 @@ iterate(Mod, Fun, Opaque, Acc) -> catch throw:Err -> close_read(R2), Err; - _:Reason -> + _:Reason:Stacktrace -> close_read(R2), - {error, {Reason, erlang:get_stacktrace()}} + {error, {Reason, Stacktrace}} end catch throw:{error,_} = Err -> Err @@ -198,9 +198,9 @@ do_read_schema_section(R) -> try {R3, RawSchema} = safe_apply(R2, read, [R2#restore.bup_data]), do_read_schema_section(R3, verify_header(RawSchema), []) - catch T:E -> + catch T:E:S -> close_read(R2), - erlang:raise(T,E,erlang:get_stacktrace()) + erlang:raise(T,E,S) end. do_read_schema_section(R, {ok, B, C, []}, Acc) -> diff --git a/lib/mnesia/src/mnesia_checkpoint.erl b/lib/mnesia/src/mnesia_checkpoint.erl index 8112378ffd..8290237908 100644 --- a/lib/mnesia/src/mnesia_checkpoint.erl +++ b/lib/mnesia/src/mnesia_checkpoint.erl @@ -1269,9 +1269,9 @@ system_code_change(Cp, _Module, _OldVsn, _Extra) -> %%%%%%%%%%%%%%%%%%%%%%%%%% +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); - _VaLuE_ -> _VaLuE_ + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); + Value -> Value end. - diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 77013489b3..77e67a59db 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -185,9 +185,10 @@ max_loaders() -> worker_res }). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_dumper.erl b/lib/mnesia/src/mnesia_dumper.erl index f0ed7aef4a..31bcc1451f 100644 --- a/lib/mnesia/src/mnesia_dumper.erl +++ b/lib/mnesia/src/mnesia_dumper.erl @@ -191,8 +191,7 @@ do_perform_dump(Cont, InPlace, InitBy, Regulator, OldVersion) -> try insert_recs(Recs, InPlace, InitBy, Regulator, OldVersion) of Version -> do_perform_dump(C2, InPlace, InitBy, Regulator, Version) - catch _:R when R =/= fatal -> - ST = erlang:get_stacktrace(), + catch _:R:ST when R =/= fatal -> Reason = {"Transaction log dump error: ~tp~n", [{R, ST}]}, close_files(InPlace, {error, Reason}, InitBy), exit(Reason) @@ -325,8 +324,7 @@ perform_update(Tid, SchemaOps, _DumperMode, _UseDir) -> ?eval_debug_fun({?MODULE, post_dump}, [InitBy]), close_files(InPlace, ok, InitBy), ok - catch _:Reason when Reason =/= fatal -> - ST = erlang:get_stacktrace(), + catch _:Reason:ST when Reason =/= fatal -> Error = {error, {"Schema update error", {Reason, ST}}}, close_files(InPlace, Error, InitBy), fatal("Schema update error ~tp ~tp", [{Reason,ST}, SchemaOps]) @@ -1471,8 +1469,9 @@ regulate(RegulatorPid) -> {regulated, RegulatorPid} -> ok end. +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_frag.erl b/lib/mnesia/src/mnesia_frag.erl index c39f30e140..63fb4981fe 100644 --- a/lib/mnesia/src/mnesia_frag.erl +++ b/lib/mnesia/src/mnesia_frag.erl @@ -1157,9 +1157,10 @@ remove_node(Node, Cs) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Helpers +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_lib.erl b/lib/mnesia/src/mnesia_lib.erl index 53fdd76de8..56f6926df0 100644 --- a/lib/mnesia/src/mnesia_lib.erl +++ b/lib/mnesia/src/mnesia_lib.erl @@ -116,7 +116,7 @@ lock_table/1, mkcore/1, not_active_here/1, - other_val/1, + other_val/2, overload_read/0, overload_read/1, overload_set/2, @@ -435,8 +435,8 @@ validate_record(Tab, Obj) -> %% val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> other_val(Var, Stacktrace); _VaLuE_ -> _VaLuE_ end. @@ -446,9 +446,9 @@ set(Var, Val) -> unset(Var) -> ?ets_delete(mnesia_gvar, Var). -other_val(Var) -> +other_val(Var, Stacktrace) -> case other_val_1(Var) of - error -> pr_other(Var); + error -> pr_other(Var, Stacktrace); Val -> Val end. @@ -460,8 +460,8 @@ other_val_1(Var) -> _ -> error end. --spec pr_other(_) -> no_return(). -pr_other(Var) -> +-spec pr_other(_, _) -> no_return(). +pr_other(Var, Stacktrace) -> Why = case is_running() of no -> {node_not_running, node()}; @@ -469,7 +469,7 @@ pr_other(Var) -> end, verbose("~p (~tp) val(mnesia_gvar, ~tw) -> ~p ~tp ~n", [self(), process_info(self(), registered_name), - Var, Why, erlang:get_stacktrace()]), + Var, Why, Stacktrace]), mnesia:abort(Why). %% Some functions for list valued variables diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl index 4c6336cb73..bbceb9ba66 100644 --- a/lib/mnesia/src/mnesia_loader.erl +++ b/lib/mnesia/src/mnesia_loader.erl @@ -34,9 +34,10 @@ -include("mnesia.hrl"). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. @@ -535,7 +536,7 @@ init_table(Tab, _, Fun, _DetsInfo,_) -> try true = ets:init_table(Tab, Fun), ok - catch _:Else -> {Else, erlang:get_stacktrace()} + catch _:Else:Stacktrace -> {Else, Stacktrace} end. @@ -777,9 +778,9 @@ do_send_table(Pid, Tab, Storage, RemoteS) -> throw:receiver_died -> cleanup_tab_copier(Pid, Storage, Tab), ok; - error:Reason -> %% Prepare failed + error:Reason:Stacktrace -> %% Prepare failed cleanup_tab_copier(Pid, Storage, Tab), - {error, {tab_copier, Tab, {Reason, erlang:get_stacktrace()}}} + {error, {tab_copier, Tab, {Reason, Stacktrace}}} after unlink(whereis(mnesia_tm)) end. diff --git a/lib/mnesia/src/mnesia_locker.erl b/lib/mnesia/src/mnesia_locker.erl index 073b48abc0..710bf38cb5 100644 --- a/lib/mnesia/src/mnesia_locker.erl +++ b/lib/mnesia/src/mnesia_locker.erl @@ -97,10 +97,11 @@ init(Parent) -> end, loop(#state{supervisor = Parent}). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); - _VaLuE_ -> _VaLuE_ + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); + Value -> Value end. reply(From, R) -> diff --git a/lib/mnesia/src/mnesia_recover.erl b/lib/mnesia/src/mnesia_recover.erl index d792070332..475bf3c327 100644 --- a/lib/mnesia/src/mnesia_recover.erl +++ b/lib/mnesia/src/mnesia_recover.erl @@ -177,10 +177,10 @@ disconnect(Node) -> log_decision(D) -> cast({log_decision, D}). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _Reason} -> - mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_schema.erl b/lib/mnesia/src/mnesia_schema.erl index 71952af31c..de6c3fb4ee 100644 --- a/lib/mnesia/src/mnesia_schema.erl +++ b/lib/mnesia/src/mnesia_schema.erl @@ -181,9 +181,10 @@ exit_on_error({error, Reason}) -> exit_on_error(GoodRes) -> GoodRes. +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); Value -> Value end. @@ -2695,10 +2696,10 @@ prepare_op(_Tid, {op, transform, Fun, TabDef}, _WaitFor) -> Objs -> mnesia_lib:db_fixtable(Storage, Tab, false), {true, Objs, mandatory} - catch _:Reason -> + catch _:Reason:Stacktrace -> mnesia_lib:db_fixtable(Storage, Tab, false), mnesia_lib:important("Transform function failed: '~tp' in '~tp'", - [Reason, erlang:get_stacktrace()]), + [Reason, Stacktrace]), exit({"Bad transform function", Tab, Fun, node(), Reason}) end end; diff --git a/lib/mnesia/src/mnesia_tm.erl b/lib/mnesia/src/mnesia_tm.erl index ebf580d09e..eaebdf6d02 100644 --- a/lib/mnesia/src/mnesia_tm.erl +++ b/lib/mnesia/src/mnesia_tm.erl @@ -121,10 +121,11 @@ init(Parent) -> proc_lib:init_ack(Parent, {ok, self()}), doit_loop(#state{supervisor = Parent}). +%% Local function in order to avoid external function call val(Var) -> - case ?catch_val(Var) of - {'EXIT', _} -> mnesia_lib:other_val(Var); - _VaLuE_ -> _VaLuE_ + case ?catch_val_and_stack(Var) of + {'EXIT', Stacktrace} -> mnesia_lib:other_val(Var, Stacktrace); + Value -> Value end. reply({From,Ref}, R) -> @@ -597,9 +598,9 @@ recover_coordinator(Tid, Etabs) -> false -> %% When killed before store havn't been copied to ok %% to the new nested trans store. end - catch _:Reason -> + catch _:Reason:Stacktrace -> dbg_out("Recovery of coordinator ~p failed: ~tp~n", - [Tid, {Reason, erlang:get_stacktrace()}]), + [Tid, {Reason, Stacktrace}]), Protocol = asym_trans, tell_outcome(Tid, Protocol, node(), CheckNodes, TellNodes) end, @@ -825,8 +826,7 @@ execute_transaction(Fun, Args, Factor, Retries, Type) -> catch throw:Value -> %% User called throw Reason = {aborted, {throw, Value}}, return_abort(Fun, Args, Reason); - error:Reason -> - ST = erlang:get_stacktrace(), + error:Reason:ST -> check_exit(Fun, Args, Factor, Retries, {Reason,ST}, Type); _:Reason -> check_exit(Fun, Args, Factor, Retries, Reason, Type) @@ -1796,14 +1796,13 @@ do_update(Tid, Storage, [Op | Ops], OldRes) -> try do_update_op(Tid, Storage, Op) of ok -> do_update(Tid, Storage, Ops, OldRes); NewRes -> do_update(Tid, Storage, Ops, NewRes) - catch _:Reason -> + catch _:Reason:ST -> %% This may only happen when we recently have %% deleted our local replica, changed storage_type %% or transformed table %% BUGBUG: Updates may be lost if storage_type is changed. %% Determine actual storage type and try again. %% BUGBUG: Updates may be lost if table is transformed. - ST = erlang:get_stacktrace(), verbose("do_update in ~w failed: ~tp -> {'EXIT', ~tp}~n", [Tid, Op, {Reason, ST}]), do_update(Tid, Storage, Ops, OldRes) @@ -1914,11 +1913,10 @@ commit_clear([H|R], Tid, Storage, Tab, K, Obj) do_snmp(_, []) -> ok; do_snmp(Tid, [Head|Tail]) -> try mnesia_snmp_hook:update(Head) - catch _:Reason -> + catch _:Reason:ST -> %% This should only happen when we recently have %% deleted our local replica or recently deattached %% the snmp table - ST = erlang:get_stacktrace(), verbose("do_snmp in ~w failed: ~tp -> {'EXIT', ~tp}~n", [Tid, Head, {Reason, ST}]) end, diff --git a/lib/mnesia/test/mnesia_test_lib.erl b/lib/mnesia/test/mnesia_test_lib.erl index 78dbe7ffde..db0e7beac6 100644 --- a/lib/mnesia/test/mnesia_test_lib.erl +++ b/lib/mnesia/test/mnesia_test_lib.erl @@ -470,9 +470,9 @@ get_suite(Mod, {group, Suite}) -> {_, _, TCList} = lists:keyfind(Suite, 1, Groups), TCList catch - _:Reason -> + _:Reason:Stacktrace -> io:format("Not implemented ~p ~p (~p ~p)~n", - [Mod,Suite,Reason, erlang:get_stacktrace()]), + [Mod,Suite,Reason,Stacktrace]), 'NYI' end; get_suite(Mod, all) -> diff --git a/lib/mnesia/test/mnesia_test_lib.hrl b/lib/mnesia/test/mnesia_test_lib.hrl index ba7eb10ea2..90a3c686b0 100644 --- a/lib/mnesia/test/mnesia_test_lib.hrl +++ b/lib/mnesia/test/mnesia_test_lib.hrl @@ -55,25 +55,25 @@ ?error("Not Matching Actual result was:~n ~p~n",[_AR_0]), {fail,_AR_0} catch - exit:{aborted, _ER_1} when + exit:{aborted, _ER_1}:Stacktrace when element(1, _ER_1) =:= node_not_running; element(1, _ER_1) =:= bad_commit; element(1, _ER_1) =:= cyclic -> %% Need to re-raise these to restart transaction - erlang:raise(exit, {aborted, _ER_1}, erlang:get_stacktrace()); - exit:_AR_1 -> + erlang:raise(exit, {aborted, _ER_1}, Stacktrace); + exit:_AR_1:Stacktrace -> case fun(_AR_EXIT_) -> {'EXIT', _AR_EXIT_} end(_AR_1) of _AR_2 = ExpectedRes -> ?verbose("ok, ~n Result as expected:~p~n",[_AR_2]), {success,_AR_2}; _AR_2 -> ?error("Not Matching Actual result was:~n ~p~n ~p~n", - [_AR_2, erlang:get_stacktrace()]), + [_AR_2, Stacktrace]), {fail,_AR_2} end; - _T1_:_AR_1 -> + _T1_:_AR_1:Stacktrace -> ?error("Not Matching Actual result was:~n ~p~n ~p~n", - [{_T1_,_AR_1}, erlang:get_stacktrace()]), + [{_T1_,_AR_1}, Stacktrace]), {fail,{_T1_,_AR_1}} end end()). diff --git a/lib/observer/src/cdv_port_cb.erl b/lib/observer/src/cdv_port_cb.erl index b5cbe8132d..6bb8f07a74 100644 --- a/lib/observer/src/cdv_port_cb.erl +++ b/lib/observer/src/cdv_port_cb.erl @@ -34,7 +34,8 @@ -define(COL_CONN, ?COL_ID+1). -define(COL_NAME, ?COL_CONN+1). -define(COL_CTRL, ?COL_NAME+1). --define(COL_SLOT, ?COL_CTRL+1). +-define(COL_QUEUE, ?COL_CTRL+1). +-define(COL_SLOT, ?COL_QUEUE+1). @@ -44,6 +45,7 @@ col_to_elem(?COL_ID) -> #port.id; col_to_elem(?COL_CONN) -> #port.connected; col_to_elem(?COL_NAME) -> #port.name; col_to_elem(?COL_CTRL) -> #port.controls; +col_to_elem(?COL_QUEUE) -> #port.queue; col_to_elem(?COL_SLOT) -> #port.slot. col_spec() -> @@ -51,6 +53,7 @@ col_spec() -> {"Connected", ?wxLIST_FORMAT_LEFT, 120}, {"Name", ?wxLIST_FORMAT_LEFT, 150}, {"Controls", ?wxLIST_FORMAT_LEFT, 200}, + {"Queue", ?wxLIST_FORMAT_RIGHT, 100}, {"Slot", ?wxLIST_FORMAT_RIGHT, 50}]. get_info(_) -> @@ -96,9 +99,17 @@ format(D) -> info_fields() -> [{"Overview", [{"Name", name}, + {"State", state}, + {"Task Flags", task_flags}, {"Connected", {click,connected}}, {"Slot", slot}, - {"Controls", controls}]}, + {"Controls", controls}, + {"Input bytes", input}, + {"Output bytes", output}, + {"Queue bytes", queue}, + {"Port data", port_data}]}, {scroll_boxes, [{"Links",1,{click,links}}, - {"Monitors",1,{click,monitors}}]}]. + {"Monitors",1,{click,monitors}}, + {"Suspended",1,{click,suspended}} + ]}]. diff --git a/lib/observer/src/cdv_proc_cb.erl b/lib/observer/src/cdv_proc_cb.erl index f10650bbb7..0ea23dd7cb 100644 --- a/lib/observer/src/cdv_proc_cb.erl +++ b/lib/observer/src/cdv_proc_cb.erl @@ -149,6 +149,10 @@ info_fields() -> {"Old Heap", old_heap}, {"Heap Unused", heap_unused}, {"Old Heap Unused", old_heap_unused}, + {"Binary vheap", bin_vheap}, + {"Old Binary vheap", old_bin_vheap}, + {"Binary vheap unused", bin_vheap_unused}, + {"Old Binary vheap unused", old_bin_vheap_unused}, {"Number of Heap Fragements", num_heap_frag}, {"Heap Fragment Data",heap_frag_data}, {"New Heap Start", new_heap_start}, diff --git a/lib/observer/src/cdv_sched_cb.erl b/lib/observer/src/cdv_sched_cb.erl index 192aaf31a7..d2696a276f 100644 --- a/lib/observer/src/cdv_sched_cb.erl +++ b/lib/observer/src/cdv_sched_cb.erl @@ -31,7 +31,8 @@ %% Columns -define(COL_ID, 0). --define(COL_PROC, ?COL_ID+1). +-define(COL_TYPE, ?COL_ID+1). +-define(COL_PROC, ?COL_TYPE+1). -define(COL_PORT, ?COL_PROC+1). -define(COL_RQL, ?COL_PORT+1). -define(COL_PQL, ?COL_RQL+1). @@ -39,6 +40,7 @@ %% Callbacks for cdv_virtual_list_wx col_to_elem(id) -> col_to_elem(?COL_ID); col_to_elem(?COL_ID) -> #sched.name; +col_to_elem(?COL_TYPE) -> #sched.type; col_to_elem(?COL_PROC) -> #sched.process; col_to_elem(?COL_PORT) -> #sched.port; col_to_elem(?COL_RQL) -> #sched.run_q; @@ -46,6 +48,7 @@ col_to_elem(?COL_PQL) -> #sched.port_q. col_spec() -> [{"Id", ?wxLIST_FORMAT_RIGHT, 50}, + {"Type", ?wxLIST_FORMAT_CENTER, 100}, {"Current Process", ?wxLIST_FORMAT_CENTER, 130}, {"Current Port", ?wxLIST_FORMAT_CENTER, 130}, {"Run Queue Length", ?wxLIST_FORMAT_RIGHT, 180}, @@ -73,7 +76,8 @@ detail_pages() -> [{"Scheduler Information", fun init_gen_page/2}]. init_gen_page(Parent, Info0) -> - Fields = info_fields(), + Type = proplists:get_value(type, Info0), + Fields = info_fields(Type), Details = proplists:get_value(details, Info0), Info = if is_map(Details) -> Info0 ++ maps:to_list(Details); true -> Info0 @@ -81,15 +85,16 @@ init_gen_page(Parent, Info0) -> cdv_info_wx:start_link(Parent,{Fields,Info,[]}). %%% Internal -info_fields() -> +info_fields(Type) -> [{"Scheduler Overview", [{"Id", id}, + {"Type", type}, {"Current Process",process}, {"Current Port", port}, {"Sleep Info Flags", sleep_info}, {"Sleep Aux Work", sleep_aux} ]}, - {"Run Queues", + {run_queues_header(Type), [{"Flags", runq_flags}, {"Priority Max Length", runq_max}, {"Priority High Length", runq_high}, @@ -116,3 +121,8 @@ info_fields() -> {" ", {currp_stack, 11}} ]} ]. + +run_queues_header(normal) -> + "Run Queues"; +run_queues_header(DirtyX) -> + "Run Queues (common for all '" ++ atom_to_list(DirtyX) ++ "' schedulers)". diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index bba97624a7..d0c14db486 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -116,6 +116,10 @@ -define(allocator,allocator). -define(atoms,atoms). -define(binary,binary). +-define(dirty_cpu_scheduler,dirty_cpu_scheduler). +-define(dirty_cpu_run_queue,dirty_cpu_run_queue). +-define(dirty_io_scheduler,dirty_io_scheduler). +-define(dirty_io_run_queue,dirty_io_run_queue). -define(ende,ende). -define(erl_crash_dump,erl_crash_dump). -define(ets,ets). @@ -1222,6 +1226,18 @@ all_procinfo(Fd,Fun,Proc,WS,LineHead) -> "OldHeap unused" -> Bytes = list_to_integer(bytes(Fd))*WS, get_procinfo(Fd,Fun,Proc#proc{old_heap_unused=Bytes},WS); + "BinVHeap" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{bin_vheap=Bytes},WS); + "OldBinVHeap" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{old_bin_vheap=Bytes},WS); + "BinVHeap unused" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{bin_vheap_unused=Bytes},WS); + "OldBinVHeap unused" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{old_bin_vheap_unused=Bytes},WS); "New heap start" -> get_procinfo(Fd,Fun,Proc#proc{new_heap_start=bytes(Fd)},WS); "New heap top" -> @@ -1632,6 +1648,10 @@ port_to_tuple("#Port<"++Port) -> get_portinfo(Fd,Port) -> case line_head(Fd) of + "State" -> + get_portinfo(Fd,Port#port{state=bytes(Fd)}); + "Task Flags" -> + get_portinfo(Fd,Port#port{task_flags=bytes(Fd)}); "Slot" -> %% stored as integer so we can sort on it get_portinfo(Fd,Port#port{slot=list_to_integer(bytes(Fd))}); @@ -1656,6 +1676,10 @@ get_portinfo(Fd,Port) -> {Pid,Pid++" ("++Ref++")"} end || Mon <- Monitors0], get_portinfo(Fd,Port#port{monitors=Monitors}); + "Suspended" -> + Pids = split_pid_list_no_space(bytes(Fd)), + Suspended = [{Pid,Pid} || Pid <- Pids], + get_portinfo(Fd,Port#port{suspended=Suspended}); "Port controls linked-in driver" -> Str = lists:flatten(["Linked in driver: " | string(Fd)]), get_portinfo(Fd,Port#port{controls=Str}); @@ -1671,6 +1695,15 @@ get_portinfo(Fd,Port) -> "Port is UNIX fd not opened by emulator" -> Str = lists:flatten(["UNIX fd not opened by emulator: "| string(Fd)]), get_portinfo(Fd,Port#port{controls=Str}); + "Input" -> + get_portinfo(Fd,Port#port{input=list_to_integer(bytes(Fd))}); + "Output" -> + get_portinfo(Fd,Port#port{output=list_to_integer(bytes(Fd))}); + "Queue" -> + get_portinfo(Fd,Port#port{queue=list_to_integer(bytes(Fd))}); + "Port Data" -> + get_portinfo(Fd,Port#port{port_data=string(Fd)}); + "=" ++ _next_tag -> Port; Other -> @@ -2503,73 +2536,142 @@ get_indextableinfo1(Fd,IndexTable) -> %%----------------------------------------------------------------- %% Page with scheduler table information schedulers(File) -> - case lookup_index(?scheduler) of - [] -> - []; - Schedulers -> - Fd = open(File), - R = lists:map(fun({Name,Start}) -> - get_schedulerinfo(Fd,Name,Start) - end, - Schedulers), - close(Fd), - R - end. + Fd = open(File), -get_schedulerinfo(Fd,Name,Start) -> + Schds0 = case lookup_index(?scheduler) of + [] -> + []; + Normals -> + [{Normals, #sched{type=normal}}] + end, + Schds1 = case lookup_index(?dirty_cpu_scheduler) of + [] -> + Schds0; + DirtyCpus -> + [{DirtyCpus, get_dirty_runqueue(Fd, ?dirty_cpu_run_queue)} + | Schds0] + end, + Schds2 = case lookup_index(?dirty_io_scheduler) of + [] -> + Schds1; + DirtyIos -> + [{DirtyIos, get_dirty_runqueue(Fd, ?dirty_io_run_queue)} + | Schds1] + end, + + R = schedulers1(Fd, Schds2, []), + close(Fd), + R. + +schedulers1(_Fd, [], Acc) -> + Acc; +schedulers1(Fd, [{Scheds,Sched0} | Tail], Acc0) -> + Acc1 = lists:foldl(fun({Name,Start}, AccIn) -> + [get_schedulerinfo(Fd,Name,Start,Sched0) | AccIn] + end, + Acc0, + Scheds), + schedulers1(Fd, Tail, Acc1). + +get_schedulerinfo(Fd,Name,Start,Sched0) -> pos_bof(Fd,Start), - get_schedulerinfo1(Fd,#sched{name=Name}). + get_schedulerinfo1(Fd,Sched0#sched{name=list_to_integer(Name)}). -get_schedulerinfo1(Fd,Sched=#sched{details=Ds}) -> +sched_type(?dirty_cpu_run_queue) -> dirty_cpu; +sched_type(?dirty_io_run_queue) -> dirty_io. + +get_schedulerinfo1(Fd, Sched) -> + case get_schedulerinfo2(Fd, Sched) of + {more, Sched2} -> + get_schedulerinfo1(Fd, Sched2); + {done, Sched2} -> + Sched2 + end. + +get_schedulerinfo2(Fd, Sched=#sched{details=Ds}) -> case line_head(Fd) of "Current Process" -> - get_schedulerinfo1(Fd,Sched#sched{process=bytes(Fd, "None")}); + {more, Sched#sched{process=bytes(Fd, "None")}}; "Current Port" -> - get_schedulerinfo1(Fd,Sched#sched{port=bytes(Fd, "None")}); + {more, Sched#sched{port=bytes(Fd, "None")}}; + + "Scheduler Sleep Info Flags" -> + {more, Sched#sched{details=Ds#{sleep_info=>bytes(Fd, "None")}}}; + "Scheduler Sleep Info Aux Work" -> + {more, Sched#sched{details=Ds#{sleep_aux=>bytes(Fd, "None")}}}; + + "Current Process State" -> + {more, Sched#sched{details=Ds#{currp_state=>bytes(Fd)}}}; + "Current Process Internal State" -> + {more, Sched#sched{details=Ds#{currp_int_state=>bytes(Fd)}}}; + "Current Process Program counter" -> + {more, Sched#sched{details=Ds#{currp_prg_cnt=>string(Fd)}}}; + "Current Process CP" -> + {more, Sched#sched{details=Ds#{currp_cp=>string(Fd)}}}; + "Current Process Limited Stack Trace" -> + %% If there shall be last in scheduler information block + {done, Sched#sched{details=get_limited_stack(Fd, 0, Ds)}}; + + "=" ++ _next_tag -> + {done, Sched}; + + Other -> + case Sched#sched.type of + normal -> + get_runqueue_info2(Fd, Other, Sched); + _ -> + unexpected(Fd,Other,"dirty scheduler information"), + {done, Sched} + end + end. + +get_dirty_runqueue(Fd, Tag) -> + case lookup_index(Tag) of + [{_, Start}] -> + pos_bof(Fd,Start), + get_runqueue_info1(Fd,#sched{type=sched_type(Tag)}); + [] -> + #sched{} + end. + +get_runqueue_info1(Fd, Sched) -> + case get_runqueue_info2(Fd, line_head(Fd), Sched) of + {more, Sched2} -> + get_runqueue_info1(Fd, Sched2); + {done, Sched2} -> + Sched2 + end. + +get_runqueue_info2(Fd, LineHead, Sched=#sched{details=Ds}) -> + case LineHead of "Run Queue Max Length" -> RQMax = list_to_integer(bytes(Fd)), RQ = RQMax + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}}}; "Run Queue High Length" -> RQHigh = list_to_integer(bytes(Fd)), RQ = RQHigh + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}}}; "Run Queue Normal Length" -> RQNorm = list_to_integer(bytes(Fd)), RQ = RQNorm + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}}}; "Run Queue Low Length" -> RQLow = list_to_integer(bytes(Fd)), RQ = RQLow + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}}}; "Run Queue Port Length" -> RQ = list_to_integer(bytes(Fd)), - get_schedulerinfo1(Fd,Sched#sched{port_q=RQ}); - - "Scheduler Sleep Info Flags" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_info=>bytes(Fd, "None")}}); - "Scheduler Sleep Info Aux Work" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_aux=>bytes(Fd, "None")}}); + {more, Sched#sched{port_q=RQ}}; "Run Queue Flags" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{runq_flags=>bytes(Fd, "None")}}); + {more, Sched#sched{details=Ds#{runq_flags=>bytes(Fd, "None")}}}; - "Current Process State" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_state=>bytes(Fd)}}); - "Current Process Internal State" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_int_state=>bytes(Fd)}}); - "Current Process Program counter" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_prg_cnt=>string(Fd)}}); - "Current Process CP" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_cp=>string(Fd)}}); - "Current Process Limited Stack Trace" -> - %% If there shall be last in scheduler information block - Sched#sched{details=get_limited_stack(Fd, 0, Ds)}; "=" ++ _next_tag -> - Sched; + {done, Sched}; Other -> unexpected(Fd,Other,"scheduler information"), - Sched + {done, Sched} end. get_limited_stack(Fd, N, Ds) -> @@ -3000,6 +3102,10 @@ tag_to_atom("allocated_areas") -> ?allocated_areas; tag_to_atom("allocator") -> ?allocator; tag_to_atom("atoms") -> ?atoms; tag_to_atom("binary") -> ?binary; +tag_to_atom("dirty_cpu_scheduler") -> ?dirty_cpu_scheduler; +tag_to_atom("dirty_cpu_run_queue") -> ?dirty_cpu_run_queue; +tag_to_atom("dirty_io_scheduler") -> ?dirty_io_scheduler; +tag_to_atom("dirty_io_run_queue") -> ?dirty_io_run_queue; tag_to_atom("end") -> ?ende; tag_to_atom("erl_crash_dump") -> ?erl_crash_dump; tag_to_atom("ets") -> ?ets; diff --git a/lib/observer/src/crashdump_viewer.hrl b/lib/observer/src/crashdump_viewer.hrl index 6a93a089fd..252e19379d 100644 --- a/lib/observer/src/crashdump_viewer.hrl +++ b/lib/observer/src/crashdump_viewer.hrl @@ -80,6 +80,10 @@ old_heap, heap_unused, old_heap_unused, + bin_vheap, + old_bin_vheap, + bin_vheap_unused, + old_bin_vheap_unused, new_heap_start, new_heap_top, stack_top, @@ -95,19 +99,27 @@ -record(port, {id, + state, + task_flags=0, slot, connected, links, name, monitors, - controls}). + suspended, + controls, + input, + output, + queue, + port_data}). -record(sched, {name, + type, process, port, run_q=0, - port_q=0, + port_q, details=#{} }). diff --git a/lib/observer/src/observer_alloc_wx.erl b/lib/observer/src/observer_alloc_wx.erl index 7f4b3dd484..e0f319b6cc 100644 --- a/lib/observer/src/observer_alloc_wx.erl +++ b/lib/observer/src/observer_alloc_wx.erl @@ -79,8 +79,8 @@ init([Notebook, Parent, Config]) -> max = #{} } } - catch _:Err -> - io:format("~p crashed ~tp: ~tp~n",[?MODULE, Err, erlang:get_stacktrace()]), + catch _:Err:Stacktrace -> + io:format("~p crashed ~tp: ~tp~n",[?MODULE, Err, Stacktrace]), {stop, Err} end. diff --git a/lib/observer/src/observer_perf_wx.erl b/lib/observer/src/observer_perf_wx.erl index 5adfadb16e..eddb9327f3 100644 --- a/lib/observer/src/observer_perf_wx.erl +++ b/lib/observer/src/observer_perf_wx.erl @@ -86,8 +86,8 @@ init([Notebook, Parent, Config]) -> secs=maps:get(secs, Config, ?DISP_SECONDS)} }, {Panel, State0} - catch _:Err -> - io:format("~p crashed ~tp: ~tp~n",[?MODULE, Err, erlang:get_stacktrace()]), + catch _:Err:Stacktrace -> + io:format("~p crashed ~tp: ~tp~n",[?MODULE, Err, Stacktrace]), {stop, Err} end. diff --git a/lib/observer/src/observer_traceoptions_wx.erl b/lib/observer/src/observer_traceoptions_wx.erl index fbcf6d7fe9..ff8c40ce77 100644 --- a/lib/observer/src/observer_traceoptions_wx.erl +++ b/lib/observer/src/observer_traceoptions_wx.erl @@ -536,7 +536,7 @@ ms_from_string(Str) -> {error, List} -> throw([[Error, $\n] || {_, Error} <- List]) end catch error:_Reason -> - %% io:format("Bad term: ~ts~n ~tp in ~tp~n", [Str, _Reason, erlang:get_stacktrace()]), + %% io:format("Bad term: ~ts~n ~tp in ~tp~n", [Str, _Reason, Stacktrace]), throw("Invalid term") end. diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl index 32773a779e..41ca3f3ce9 100644 --- a/lib/observer/test/crashdump_viewer_SUITE.erl +++ b/lib/observer/test/crashdump_viewer_SUITE.erl @@ -820,10 +820,10 @@ dump_with_size_limit_reached(DataDir,Rel,DumpName,Max) -> "-env ERL_CRASH_DUMP_BYTES " ++ integer_to_list(Bytes)), {ok,#file_info{size=Size}} = file:read_file_info(CD), - if Size < Bytes -> + if Size =< Bytes -> %% This means that the dump was actually smaller than the %% randomly selected truncation size, so we'll just do it - %% again with a smaller numer + %% again with a smaller number ok = file:delete(CD), dump_with_size_limit_reached(DataDir,Rel,DumpName,Size-3); true -> diff --git a/lib/parsetools/include/yeccpre.hrl b/lib/parsetools/include/yeccpre.hrl index 91d6cd49a6..53e8124aaa 100644 --- a/lib/parsetools/include/yeccpre.hrl +++ b/lib/parsetools/include/yeccpre.hrl @@ -56,8 +56,7 @@ return_error(Line, Message) -> yeccpars0(Tokens, Tzr, State, States, Vstack) -> try yeccpars1(Tokens, Tzr, State, States, Vstack) catch - error: Error -> - Stacktrace = erlang:get_stacktrace(), + error: Error: Stacktrace -> try yecc_error_type(Error, Stacktrace) of Desc -> erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, diff --git a/lib/parsetools/src/yeccparser.erl b/lib/parsetools/src/yeccparser.erl index 6f6f66d56c..45d91a83a3 100644 --- a/lib/parsetools/src/yeccparser.erl +++ b/lib/parsetools/src/yeccparser.erl @@ -72,8 +72,7 @@ return_error(Line, Message) -> yeccpars0(Tokens, Tzr, State, States, Vstack) -> try yeccpars1(Tokens, Tzr, State, States, Vstack) catch - error: Error -> - Stacktrace = erlang:get_stacktrace(), + error: Error: Stacktrace -> try yecc_error_type(Error, Stacktrace) of Desc -> erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, @@ -167,21 +166,20 @@ yecctoken_location(Token) -> end. -compile({nowarn_unused_function, yecctoken2string/1}). -yecctoken2string({atom, _, A}) -> io_lib:write(A); +yecctoken2string({atom, _, A}) -> io_lib:write_atom(A); yecctoken2string({integer,_,N}) -> io_lib:write(N); yecctoken2string({float,_,F}) -> io_lib:write(F); yecctoken2string({char,_,C}) -> io_lib:write_char(C); yecctoken2string({var,_,V}) -> io_lib:format("~s", [V]); yecctoken2string({string,_,S}) -> io_lib:write_string(S); yecctoken2string({reserved_symbol, _, A}) -> io_lib:write(A); -yecctoken2string({_Cat, _, Val}) -> io_lib:format("~p",[Val]); +yecctoken2string({_Cat, _, Val}) -> io_lib:format("~tp", [Val]); yecctoken2string({dot, _}) -> "'.'"; -yecctoken2string({'$end', _}) -> - []; +yecctoken2string({'$end', _}) -> []; yecctoken2string({Other, _}) when is_atom(Other) -> - io_lib:write(Other); + io_lib:write_atom(Other); yecctoken2string(Other) -> - io_lib:write(Other). + io_lib:format("~tp", [Other]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl index a7166b91ed..2d3b15326b 100644 --- a/lib/parsetools/test/yecc_SUITE.erl +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -1674,8 +1674,7 @@ format_error(Message) -> yeccpars0(Tokens, MFA) -> try yeccpars1(Tokens, MFA, 0, [], []) catch - error: Error -> - Stacktrace = erlang:get_stacktrace(), + error: Error : Stacktrace -> try yecc_error_type(Error, Stacktrace) of {syntax_error, Token} -> yeccerror(Token); diff --git a/lib/reltool/src/reltool_app_win.erl b/lib/reltool/src/reltool_app_win.erl index 663144861f..335b40fa74 100644 --- a/lib/reltool/src/reltool_app_win.erl +++ b/lib/reltool/src/reltool_app_win.erl @@ -102,8 +102,8 @@ init(Parent, WxEnv, Xref, C, AppName) -> try do_init(Parent, WxEnv, Xref, C, AppName) catch - error:Reason -> - exit({Reason, erlang:get_stacktrace()}) + error:Reason:Stacktrace -> + exit({Reason, Stacktrace}) end. do_init(Parent, WxEnv, Xref, C, AppName) -> diff --git a/lib/reltool/src/reltool_mod_win.erl b/lib/reltool/src/reltool_mod_win.erl index dcd802a5de..8272d864c2 100644 --- a/lib/reltool/src/reltool_mod_win.erl +++ b/lib/reltool/src/reltool_mod_win.erl @@ -107,8 +107,8 @@ init(Parent, WxEnv, Xref, RelPid, C, ModName) -> try do_init(Parent, WxEnv, Xref, RelPid, C, ModName) catch - error:Reason -> - exit({Reason, erlang:get_stacktrace()}) + error:Reason:Stacktrace -> + exit({Reason, Stacktrace}) end. do_init(Parent, WxEnv, Xref, RelPid, C, ModName) -> diff --git a/lib/reltool/src/reltool_server.erl b/lib/reltool/src/reltool_server.erl index 853191c696..66ee1a3742 100644 --- a/lib/reltool/src/reltool_server.erl +++ b/lib/reltool/src/reltool_server.erl @@ -135,8 +135,8 @@ init([{parent,Parent}|_] = Options) -> catch throw:{error,Reason} -> proc_lib:init_ack(Parent,{error,Reason}); - error:Reason -> - exit({Reason, erlang:get_stacktrace()}) + error:Reason:Stacktrace -> + exit({Reason, Stacktrace}) end. do_init(Options) -> diff --git a/lib/reltool/src/reltool_sys_win.erl b/lib/reltool/src/reltool_sys_win.erl index 92df270752..d87b1dc91f 100644 --- a/lib/reltool/src/reltool_sys_win.erl +++ b/lib/reltool/src/reltool_sys_win.erl @@ -135,9 +135,9 @@ init(Options) -> try do_init(Options) catch - error:Reason -> - io:format("~tp: ~tp~n",[Reason, erlang:get_stacktrace()]), - exit({Reason, erlang:get_stacktrace()}) + error:Reason:Stacktrace -> + io:format("~tp: ~tp~n",[Reason, Stacktrace]), + exit({Reason, Stacktrace}) end. do_init([{safe_config, Safe}, {parent, Parent} | Options]) -> diff --git a/lib/runtime_tools/doc/src/Makefile b/lib/runtime_tools/doc/src/Makefile index ec19a4ce59..a9b0056a93 100644 --- a/lib/runtime_tools/doc/src/Makefile +++ b/lib/runtime_tools/doc/src/Makefile @@ -41,7 +41,13 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # Target Specs # ---------------------------------------------------- XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = dbg.xml dyntrace.xml erts_alloc_config.xml system_information.xml msacc.xml +XML_REF3_FILES = \ + dbg.xml \ + dyntrace.xml \ + erts_alloc_config.xml \ + system_information.xml \ + msacc.xml \ + scheduler.xml XML_REF6_FILES = runtime_tools_app.xml XML_PART_FILES = part.xml diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 93e3e26fda..74300ba3fc 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,23 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.12.4</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + New family of <c>erts_alloc</c> strategies: Age Order + First Fit. Similar to "address order", but instead the + oldest possible carrier is always chosen for allocation.</p> + <p> + Own Id: OTP-14917 Aux Id: ERIERL-88 </p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.12.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/runtime_tools/doc/src/ref_man.xml b/lib/runtime_tools/doc/src/ref_man.xml index d2fb7a29af..eb3a6f0f5c 100644 --- a/lib/runtime_tools/doc/src/ref_man.xml +++ b/lib/runtime_tools/doc/src/ref_man.xml @@ -37,6 +37,7 @@ <xi:include href="dyntrace.xml"/> <xi:include href="erts_alloc_config.xml"/> <xi:include href="msacc.xml"/> + <xi:include href="scheduler.xml"/> <xi:include href="system_information.xml"/> </application> diff --git a/lib/runtime_tools/doc/src/scheduler.xml b/lib/runtime_tools/doc/src/scheduler.xml new file mode 100644 index 0000000000..dd8bf73bae --- /dev/null +++ b/lib/runtime_tools/doc/src/scheduler.xml @@ -0,0 +1,135 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title></title> + <prepared></prepared> + <responsible></responsible> + <docno>1</docno> + <approved></approved> + <checked></checked> + <date></date> + <rev></rev> + <file>scheduler.xml</file> + </header> + <module>scheduler</module> + <modulesummary>Measure scheduler utilization</modulesummary> + <description> + <p>This module contains utility functions for easier measurement and + calculation of scheduler utilization, otherwise obtained from calling the + more primitive <seealso marker="erts:erlang#statistics_scheduler_wall_time"> + <c>statistics(scheduler_wall_time)</c></seealso>.</p> + <p>The simplest usage is to call <seealso marker="#utilization-1"> + <c>scheduler:utilization(Seconds)</c></seealso>.</p> + </description> + + <datatypes> + <datatype> + <name name="sched_sample"/> + </datatype> + <datatype> + <name name="sched_type"/> + </datatype> + <datatype> + <name name="sched_id"/> + </datatype> + <datatype> + <name name="sched_util_result"/> + <desc> + <p>A list of tuples containing results for individual schedulers + as well as aggregated averages. <c>Util</c> is the scheduler utilization + as a floating point value between 0.0 and 1.0. <c>Percent</c> is the + same utilization as a more human readable string expressed in percent.</p> + <taglist> + <tag><c>{normal, SchedulerId, Util, Percent}</c></tag> + <item>Scheduler utilization of a normal scheduler with number + <c>SchedulerId</c>.</item> + <tag><c>{cpu, SchedulerId, Util, Percent}</c></tag> + <item>Scheduler utilization of a dirty-cpu scheduler with number + <c>SchedulerId</c>.</item> + <tag><c>{io, SchedulerId, Util, Percent}</c></tag> + <item>Scheduler utilization of a dirty-io scheduler with number + <c>SchedulerId</c>. This tuple will only exist if both samples were + taken with <seealso marker="#sample_all-0"><c>sample_all/0</c></seealso>.</item> + <tag><c>{total, Util, Percent}</c></tag> + <item>Total utilization of all normal and dirty-cpu schedulers.</item> + <tag><c>{weighted, Util, Percent}</c></tag> + <item>Total utilization of all normal and dirty-cpu schedulers, + weighted against maximum amount of available CPU time.</item> + </taglist> + </desc> + </datatype> + </datatypes> + + <funcs> + + <func> + <name name="sample" arity="0"/> + <fsummary>Get scheduler utilization sample.</fsummary> + <desc> + <p>Return a scheduler utilization sample for normal and dirty-cpu + schedulers.</p> + </desc> + </func> + + <func> + <name name="sample_all" arity="0"/> + <fsummary>Get scheduler utilization sample.</fsummary> + <desc> + <p>Return a scheduler utilization sample for all schedulers, + including dirty-io schedulers.</p> + </desc> + </func> + + <func> + <name name="utilization" arity="1" clause_i="1"/> + <fsummary>Measure scheduler utilizations during a period of time.</fsummary> + <desc> + <p>Measure utilization for normal and dirty-cpu schedulers during + <c><anno>Seconds</anno></c> seconds, and then return the result.</p> + </desc> + </func> + + <func> + <name name="utilization" arity="1" clause_i="2"/> + <fsummary>Measure scheduler utilizations since sample.</fsummary> + <desc> + <p>Calculate scheduler utilizations for the time interval from when + <c><anno>Sample</anno></c> was taken and "now". The same as calling + <c>scheduler:utilization(Sample, scheduler:sample_all())</c>.</p> + </desc> + </func> + + <func> + <name name="utilization" arity="2"/> + <fsummary>Measure scheduler utilizations between two samples.</fsummary> + <desc> + <p>Calculates scheduler utilizations for the time interval between + the two samples obtained from calling + <seealso marker="#sample-0"><c>sample/0</c></seealso> or + <seealso marker="#sample_all-0"><c>sample_all/0</c></seealso>.</p> + </desc> + </func> + + </funcs> + </erlref> diff --git a/lib/runtime_tools/doc/src/specs.xml b/lib/runtime_tools/doc/src/specs.xml index 978bd39e55..33fe7fa370 100644 --- a/lib/runtime_tools/doc/src/specs.xml +++ b/lib/runtime_tools/doc/src/specs.xml @@ -2,4 +2,5 @@ <specs xmlns:xi="http://www.w3.org/2001/XInclude"> <xi:include href="../specs/specs_system_information.xml"/> <xi:include href="../specs/specs_msacc.xml"/> + <xi:include href="../specs/specs_scheduler.xml"/> </specs> diff --git a/lib/runtime_tools/src/Makefile b/lib/runtime_tools/src/Makefile index 5a99c6e240..6faa9c2e35 100644 --- a/lib/runtime_tools/src/Makefile +++ b/lib/runtime_tools/src/Makefile @@ -45,6 +45,7 @@ MODULES= \ system_information \ observer_backend \ ttb_autostart\ + scheduler\ msacc HRL_FILES= ../include/observer_backend.hrl diff --git a/lib/runtime_tools/src/erts_alloc_config.erl b/lib/runtime_tools/src/erts_alloc_config.erl index 342363d08d..6c1a945484 100644 --- a/lib/runtime_tools/src/erts_alloc_config.erl +++ b/lib/runtime_tools/src/erts_alloc_config.erl @@ -265,7 +265,13 @@ strategy_str(aoff) -> strategy_str(aoffcbf) -> "Address order first fit carrier best fit"; strategy_str(aoffcaobf) -> - "Address order first fit carrier adress order best fit". + "Address order first fit carrier adress order best fit"; +strategy_str(ageffcaoff) -> + "Age order first fit carrier address order first fit"; +strategy_str(ageffcbf) -> + "Age order first fit carrier best fit"; +strategy_str(ageffcaobf) -> + "Age order first fit carrier adress order best fit". default_acul(A, S) -> case carrier_migration_support(S) of diff --git a/lib/runtime_tools/src/observer_backend.erl b/lib/runtime_tools/src/observer_backend.erl index a1edde8516..7ec3b38930 100644 --- a/lib/runtime_tools/src/observer_backend.erl +++ b/lib/runtime_tools/src/observer_backend.erl @@ -279,7 +279,7 @@ get_table_list(mnesia, Opts) -> end, [Tab|Acc] catch _:_What -> - %% io:format("Skipped ~p: ~p ~p ~n",[Id, _What, erlang:get_stacktrace()]), + %% io:format("Skipped ~p: ~p ~p ~n",[Id, _What, Stacktrace]), Acc end end, diff --git a/lib/runtime_tools/src/runtime_tools.app.src b/lib/runtime_tools/src/runtime_tools.app.src index 449532e5c4..09a9b447c2 100644 --- a/lib/runtime_tools/src/runtime_tools.app.src +++ b/lib/runtime_tools/src/runtime_tools.app.src @@ -23,6 +23,7 @@ {modules, [appmon_info, dbg,observer_backend,runtime_tools, runtime_tools_sup,erts_alloc_config, ttb_autostart,dyntrace,system_information, + scheduler, msacc]}, {registered, [runtime_tools_sup]}, {applications, [kernel, stdlib]}, diff --git a/lib/runtime_tools/src/scheduler.erl b/lib/runtime_tools/src/scheduler.erl new file mode 100644 index 0000000000..c896b671ac --- /dev/null +++ b/lib/runtime_tools/src/scheduler.erl @@ -0,0 +1,152 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% @doc Utility functions for easier measurement of scheduler utilization +%% using erlang:statistics(scheduler_wall_time). + +-module(scheduler). + +-export([sample/0, + sample_all/0, + utilization/1, + utilization/2]). + +-export_type([sched_sample/0]). + + +-opaque sched_sample() :: + {scheduler_wall_time | scheduler_wall_time_all, + [{sched_type(), sched_id(), ActiveTime::integer(), TotalTime::integer()}]}. + +-type sched_type() :: normal | cpu | io. + +-type sched_id() :: integer(). + +-spec sample() -> sched_sample(). +sample() -> + sample(scheduler_wall_time). + +-spec sample_all() -> sched_sample(). +sample_all() -> + sample(scheduler_wall_time_all). + +sample(Stats) -> + case erlang:statistics(Stats) of + undefined -> + erlang:system_flag(scheduler_wall_time, true), + sample(Stats); + + List -> + Sorted = lists:sort(List), + Tagged = lists:map(fun({I, A, T}) -> {sched_tag(I), I, A, T} end, + Sorted), + {Stats, Tagged} + end. + +-type sched_util_result() :: + [{sched_type(), sched_id(), float(), string()} | + {total, float(), string()} | + {weighted, float(), string()}]. + +-spec utilization(Seconds) -> sched_util_result() when + Seconds :: pos_integer(); + (Sample) -> sched_util_result() when + Sample :: sched_sample(). +utilization(Seconds) when is_integer(Seconds), Seconds > 0 -> + OldFlag = erlang:system_flag(scheduler_wall_time, true), + T0 = sample(), + receive after Seconds*1000 -> ok end, + T1 = sample(), + case OldFlag of + false -> + erlang:system_flag(scheduler_wall_time, OldFlag); + true -> + ok + end, + utilization(T0,T1); + +utilization({Stats, _}=T0) when Stats =:= scheduler_wall_time; + Stats =:= scheduler_wall_time_all -> + utilization(T0, sample(Stats)). + +-spec utilization(Sample1, Sample2) -> sched_util_result() when + Sample1 :: sched_sample(), + Sample2 :: sched_sample(). +utilization({Stats, Ts0}, {Stats, Ts1}) -> + Diffs = lists:map(fun({{Tag, I, A0, T0}, {Tag, I, A1, T1}}) -> + {Tag, I, (A1 - A0), (T1 - T0)} + end, + lists:zip(Ts0,Ts1)), + + {Lst0, {A, T, N}} = lists:foldl(fun({Tag, I, Adiff, Tdiff}, {Lst, Acc}) -> + R = safe_div(Adiff, Tdiff), + {[{Tag, I, R, percent(R)} | Lst], + acc(Tag, Adiff, Tdiff, Acc)} + end, + {[], {0, 0, 0}}, + Diffs), + + Total = safe_div(A, T), + Lst1 = lists:reverse(Lst0), + Lst2 = case erlang:system_info(logical_processors_available) of + unknown -> Lst1; + LPA -> + Weighted = Total * (N / LPA), + [{weighted, Weighted, percent(Weighted)} | Lst1] + end, + [{total, Total, percent(Total)} | Lst2]; + +utilization({scheduler_wall_time, _}=T0, + {scheduler_wall_time_all, Ts1}) -> + utilization(T0, {scheduler_wall_time, remove_io(Ts1)}); + +utilization({scheduler_wall_time_all, Ts0}, + {scheduler_wall_time, _}=T1) -> + utilization({scheduler_wall_time, remove_io(Ts0)}, T1). + +%% Do not include dirty-io in totals +acc(io, _, _, Acc) -> + Acc; +acc(Tag, Adiff, Tdiff, {Asum, Tsum, N}) when Tag =:= normal; Tag =:= cpu -> + {Adiff+Asum, Tdiff+Tsum, N+1}. + + +remove_io(Ts) -> + lists:filter(fun({io,_,_,_}) -> false; + (_) -> true end, + Ts). + +safe_div(A, B) -> + if B == 0.0 -> 0.0; + true -> A / B + end. + +sched_tag(Nr) -> + Normal = erlang:system_info(schedulers), + Cpu = Normal + erlang:system_info(dirty_cpu_schedulers), + case Nr of + _ when Nr =< Normal -> normal; + _ when Nr =< Cpu -> cpu; + _ -> io + end. + + +percent(F) -> + float_to_list(F*100, [{decimals,1}]) ++ [$%]. diff --git a/lib/runtime_tools/src/system_information.erl b/lib/runtime_tools/src/system_information.erl index 119d7cc3d4..136ee55b54 100644 --- a/lib/runtime_tools/src/system_information.erl +++ b/lib/runtime_tools/src/system_information.erl @@ -488,10 +488,10 @@ to_fd(Fd) -> ok = file:write(Fd, io_lib:format(Format, Args)) end, - EmitChunk("{system_information_version, ~p}.~n" + EmitChunk("{system_information_version, ~w}.~n" "{system_information,[" - "{init_arguments,~p}," - "{code_paths,~p},", + "{init_arguments,~w}," + "{code_paths,~w},", [?REPORT_FILE_VSN, init:get_arguments(), code:get_path()]), @@ -499,12 +499,12 @@ to_fd(Fd) -> emit_code_info(EmitChunk), EmitChunk( "," %% Note the leading comma! - "{system_info,~p}," - "{erts_compile_info,~p}," - "{beam_dynamic_libraries,~p}," - "{environment_erts,~p}," - "{environment,~p}," - "{sanity_check,~p}" + "{system_info,~w}," + "{erts_compile_info,~w}," + "{beam_dynamic_libraries,~w}," + "{environment_erts,~w}," + "{environment,~w}," + "{sanity_check,~w}" "]}.~n", [erlang_system_info(), erlang:system_info(compile_info), @@ -533,11 +533,11 @@ emit_application_info(EmitChunk, Path) -> Description = proplists:get_value(description, Info, []), Version = proplists:get_value(vsn, Info, []), - EmitChunk("{application, {~p,[" - "{description,~p}," - "{vsn,~p}," - "{path,~p}," - "{runtime_dependencies,~p},", + EmitChunk("{application, {~w,[" + "{description,~w}," + "{vsn,~w}," + "{path,~w}," + "{runtime_dependencies,~w},", [App, Description, Version, Path, RtDeps]), emit_module_info_from_path(EmitChunk, Path), EmitChunk("]}}", []) @@ -545,7 +545,7 @@ emit_application_info(EmitChunk, Path) -> emit_code_path_info(EmitChunk, Path) -> EmitChunk("{code, [" - "{path, ~p},", [Path]), + "{path, ~w},", [Path]), emit_module_info_from_path(EmitChunk, Path), EmitChunk("]}", []). @@ -573,11 +573,11 @@ emit_module_info(EmitChunk, Beam) -> _ -> true end, - EmitChunk("{~p,[" - "{loaded,~p}," - "{native,~p}," - "{compiler,~p}," - "{md5,~p}" + EmitChunk("{~w,[" + "{loaded,~w}," + "{native,~w}," + "{compiler,~w}," + "{md5,~w}" "]}", [Mod, Loaded, Native, CompilerVersion, hexstring(Md5)]). diff --git a/lib/runtime_tools/test/Makefile b/lib/runtime_tools/test/Makefile index 61377ea09e..de37b2570d 100644 --- a/lib/runtime_tools/test/Makefile +++ b/lib/runtime_tools/test/Makefile @@ -9,6 +9,7 @@ MODULES = \ system_information_SUITE \ dbg_SUITE \ erts_alloc_config_SUITE \ + scheduler_SUITE \ msacc_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/runtime_tools/test/scheduler_SUITE.erl b/lib/runtime_tools/test/scheduler_SUITE.erl new file mode 100644 index 0000000000..1c80253371 --- /dev/null +++ b/lib/runtime_tools/test/scheduler_SUITE.erl @@ -0,0 +1,104 @@ +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(scheduler_SUITE). + +-export([suite/0, all/0]). + +%% Test cases +-export([basic/1]). + +all() -> [basic]. + + +suite() -> [{ct_hooks,[ts_install_cth]}]. + + +basic(_Config) -> + S1 = scheduler:sample(), + S2 = scheduler:sample_all(), + + check(scheduler:utilization(1)), + + check(scheduler:utilization(S1)), + check(scheduler:utilization(S2)), + check(scheduler:utilization(S1, scheduler:sample())), + check(scheduler:utilization(S2, scheduler:sample())), + + S3 = scheduler:sample_all(), + U13 = scheduler:utilization(S1, S3), + U13 = scheduler:utilization(S1, remove_io(S3)), + check(U13), + + U23all = scheduler:utilization(S2, S3), + check(U23all), + U23 = scheduler:utilization(S2, remove_io(S3)), + U23 = scheduler:utilization(remove_io(S2), S3), + U23 = remove_io(U23all), + check(U23), + + ok. + + +check([{total, Tf, Ts} | List]=U) -> + io:format("\nU = ~p\n", [U]), + check_values(Tf, Ts, true), + + SchdList = case hd(List) of + {weighted, Wf, Ws} -> + check_values(Wf, Ws, false), + tl(List); + _ -> + unknown = erlang:system_info(logical_processors_available), + List + end, + + lists:foreach(fun({Type, Id, F, S}) when ((Type =:= normal) or (Type =:= cpu) or (Type =:= io)), + is_integer(Id) -> + check_values(F, S, true) + end, + SchdList), + ok. + +check_values(F, S, Max100) -> + true = is_float(F), + true = F >= 0.0, + + $% = lists:last(S), + Sf = list_to_float(lists:droplast(S)), + true = Sf >= 0.0, + true = case Max100 of + true -> + true = F =< 1.0, + true = Sf =< 100.0; + false -> + true + end, + MaxDiff = 0.055555555555555555, %% change to 0.05 when float_to_list/2 is fixed + true = abs(F*100 - Sf) =< MaxDiff, + ok. + + +remove_io({scheduler_wall_time_all,Lst}) -> + {scheduler_wall_time, remove_io(Lst)}; +remove_io(Lst) -> + lists:filter(fun({io,_,_,_}) -> false; + (_) -> true end, + Lst). diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index 872bd5db1d..0dc6a48570 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.12.3 +RUNTIME_TOOLS_VSN = 1.12.4 diff --git a/lib/sasl/doc/src/release_handler.xml b/lib/sasl/doc/src/release_handler.xml index 8f073807fb..975188f489 100644 --- a/lib/sasl/doc/src/release_handler.xml +++ b/lib/sasl/doc/src/release_handler.xml @@ -61,6 +61,7 @@ <list type="bulleted"> <item>A release upgrade file, <c>relup</c></item> <item>A system configuration file, <c>sys.config</c></item> + <item>A system configuration source file, <c>sys.config.src</c></item> </list> <p>The <c>relup</c> file contains instructions for how to upgrade to, or downgrade from, this version of the release.</p> @@ -819,4 +820,3 @@ release_handler:set_unpacked(RelFile, [{myapp,"1.0","/home/user"},...]). <seealso marker="systools"><c>systools(3)</c></seealso></p> </section> </erlref> - diff --git a/lib/sasl/doc/src/systools.xml b/lib/sasl/doc/src/systools.xml index e7c3c499da..4842c732b1 100644 --- a/lib/sasl/doc/src/systools.xml +++ b/lib/sasl/doc/src/systools.xml @@ -349,10 +349,11 @@ myapp-1/ebin/myapp.app the release version as specified in <c>Name.rel</c>.</p> <p><c>releases/RelVsn</c> contains the boot script <c>Name.boot</c> renamed to <c>start.boot</c> and, if found, - the files <c>relup</c> and <c>sys.config</c>. These files + the files <c>relup</c> and <c>sys.config</c> or <c>sys.config.src</c>. These files are searched for in the same directory as <c>Name.rel</c>, in the current working directory, and in any directories - specified using option <c>path</c>.</p> + specified using option <c>path</c>. In the case of <c>sys.config</c> + it is not included if <c>sys.config.src</c> is found.</p> <p>If the release package is to contain a new Erlang runtime system, the <c>bin</c> directory of the specified runtime system <c>{erts,Dir}</c> is copied to <c>erts-ErtsVsn/bin</c>.</p> @@ -397,4 +398,3 @@ myapp-1/ebin/myapp.app <seealso marker="script"><c>script(4)</c></seealso></p> </section> </erlref> - diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 9d960b7361..fa3182cc08 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -310,6 +310,7 @@ add_apply_upgrade(Script,Args) -> %% RelVsn/start.boot %% relup %% sys.config +%% sys.config.src %% erts-EVsn[/bin] %%----------------------------------------------------------------- @@ -1552,6 +1553,7 @@ create_kernel_procs(Appls) -> %% RelVsn/start.boot %% relup %% sys.config +%% sys.config.src %% erts-EVsn[/bin] %% %% The VariableN.tar.gz files can also be stored as own files not @@ -1707,14 +1709,18 @@ add_system_files(Tar, RelName, Release, Path1) -> add_to_tar(Tar, Relup, filename:join(RelVsnDir, "relup")) end, - case lookup_file("sys.config", Path) of - false -> - ignore; - Sys -> - check_sys_config(Sys), - add_to_tar(Tar, Sys, filename:join(RelVsnDir, "sys.config")) + case lookup_file("sys.config.src", Path) of + false -> + case lookup_file("sys.config", Path) of + false -> + ignore; + Sys -> + check_sys_config(Sys), + add_to_tar(Tar, Sys, filename:join(RelVsnDir, "sys.config")) + end; + SysSrc -> + add_to_tar(Tar, SysSrc, filename:join(RelVsnDir, "sys.config.src")) end, - ok. lookup_file(Name, [Dir|Path]) -> diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl index 824820c214..edd2efdf05 100644 --- a/lib/sasl/test/release_handler_SUITE.erl +++ b/lib/sasl/test/release_handler_SUITE.erl @@ -2580,15 +2580,15 @@ check_gg_info(Node,OtherAlive,OtherDead,Synced,N) -> GGI = rpc:call(Node, global_group, info, []), GI = rpc:call(Node, global, info,[]), try do_check_gg_info(OtherAlive,OtherDead,Synced,GGI,GI) - catch _:E when N==0 -> + catch _:E:Stacktrace when N==0 -> ?t:format("~nERROR: check_gg_info failed for ~p:~n~p~n" "when GGI was: ~p~nand GI was: ~p~n", - [Node,{E,erlang:get_stacktrace()},GGI,GI]), + [Node,{E,Stacktrace},GGI,GI]), ?t:fail("check_gg_info failed"); - _:E -> + _:E:Stacktrace -> ?t:format("~nWARNING: check_gg_info failed for ~p:~n~p~n" "when GGI was: ~p~nand GI was: ~p~n", - [Node,{E,erlang:get_stacktrace()},GGI,GI]), + [Node,{E,Stacktrace},GGI,GI]), timer:sleep(1000), check_gg_info(Node,OtherAlive,OtherDead,Synced,N-1) end. diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 07748d975f..c8b2f31120 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -67,7 +67,7 @@ groups() -> otp_3065_circular_dependenies, included_and_used_sort_script]}, {tar, [], [tar_options, normal_tar, no_mod_vsn_tar, system_files_tar, - invalid_system_files_tar, variable_tar, + system_src_file_tar, invalid_system_files_tar, variable_tar, src_tests_tar, var_tar, exref_tar, link_tar, no_sasl_tar, otp_9507_path_ebin]}, {relup, [], @@ -945,12 +945,47 @@ system_files_tar(Config) -> ok. + system_files_tar(cleanup,Config) -> Dir = ?privdir, file:delete(filename:join(Dir,"sys.config")), file:delete(filename:join(Dir,"relup")), ok. +%% make_tar: Check that sys.config.src and not sys.config is included +system_src_file_tar(Config) -> + {ok, OldDir} = file:get_cwd(), + + {LatestDir, LatestName} = create_script(latest,Config), + + DataDir = filename:absname(?copydir), + LibDir = fname([DataDir, d_normal, lib]), + P = [fname([LibDir, 'db-2.1', ebin]), + fname([LibDir, 'fe-3.1', ebin])], + + ok = file:set_cwd(LatestDir), + + %% Add dummy sys.config and sys.config.src + ok = file:write_file("sys.config.src","[${SOMETHING}].\n"), + ok = file:write_file("sys.config","[].\n"), + + {ok, _, _} = systools:make_script(LatestName, [silent, {path, P}]), + ok = systools:make_tar(LatestName, [{path, P}]), + ok = check_tar(fname(["releases","LATEST","sys.config.src"]), LatestName), + {error, _} = check_tar(fname(["releases","LATEST","sys.config"]), LatestName), + {ok, _, _} = systools:make_tar(LatestName, [{path, P}, silent]), + ok = check_tar(fname(["releases","LATEST","sys.config.src"]), LatestName), + {error, _} = check_tar(fname(["releases","LATEST","sys.config"]), LatestName), + + ok = file:set_cwd(OldDir), + + ok. + +system_src_file_tar(cleanup,Config) -> + Dir = ?privdir, + file:delete(filename:join(Dir,"sys.config")), + file:delete(filename:join(Dir,"sys.config.src")), + ok. %% make_tar: Check that make_tar fails if relup or sys.config exist %% but do not have valid content diff --git a/lib/snmp/doc/src/snmp_impl_example_agent.xml b/lib/snmp/doc/src/snmp_impl_example_agent.xml index a86006a0a7..e576fa51f3 100644 --- a/lib/snmp/doc/src/snmp_impl_example_agent.xml +++ b/lib/snmp/doc/src/snmp_impl_example_agent.xml @@ -47,6 +47,7 @@ EX1-MIB DEFINITIONS ::= BEGIN IMPORTS + experimental FROM RFC1155-SMI RowStatus FROM STANDARD-MIB DisplayString FROM RFC1213-MIB OBJECT-TYPE FROM RFC-1212 @@ -81,7 +82,7 @@ EX1-MIB DEFINITIONS ::= BEGIN FriendsEntry ::= SEQUENCE { - fIndex + fIndex INTEGER, fName DisplayString, @@ -105,6 +106,7 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "Name of friend" ::= { friendsEntry 2 } + fAddress OBJECT-TYPE SYNTAX DisplayString (SIZE (0..255)) ACCESS read-write @@ -112,6 +114,7 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "Address of friend" ::= { friendsEntry 3 } + fStatus OBJECT-TYPE SYNTAX RowStatus ACCESS read-write @@ -119,12 +122,13 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "The status of this conceptual row." ::= { friendsEntry 4 } + fTrap TRAP-TYPE ENTERPRISE example1 VARIABLES { myName, fIndex } DESCRIPTION - "This trap is sent when something happens to - the friend specified by fIndex." + "This trap is sent when something happens to + the friend specified by fIndex." ::= 1 END </code> diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 3a2f55a487..f5cb3ec254 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,41 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.6.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Adjusted supervisor timeouts</p> + <p> + Own Id: OTP-14907</p> + </item> + <item> + <p> + Remove ERROR messages for slow process exits</p> + <p> + Own Id: OTP-14930</p> + </item> + </list> + </section> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add option <c>save_accepted_host</c> to + <c>ssh:connection</c>. This option, if set to false, + inhibits saving host keys to e.g the file + <c>known_hosts</c>.</p> + <p> + Own Id: OTP-14935</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.6.4</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -60,7 +95,6 @@ </list> </section> - <section><title>Improvements and New Features</title> <list> <item> @@ -90,7 +124,6 @@ </section> <section><title>Ssh 4.6.2</title> - <section><title>Fixed Bugs and Malfunctions</title> <list> <item> @@ -370,6 +403,40 @@ </section> + +<section><title>Ssh 4.4.2.2</title> + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Default exec is disabled when a user-defined shell is + enabled.</p> + <p> + Own Id: OTP-14881</p> + </item> + </list> + </section> +</section> + + +<section><title>Ssh 4.4.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Trailing white space was removed at end of the + hello-string. This caused interoperability problems with + some other ssh-implementations (e.g OpenSSH 7.3p1 on + Solaris 11)</p> + <p> + Own Id: OTP-14763 Aux Id: ERIERL-74 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.4.2</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -740,6 +807,93 @@ </section> +<section><title>Ssh 4.2.2.5</title> + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Default exec is disabled when a user-defined shell is + enabled.</p> + <p> + Own Id: OTP-14881</p> + </item> + </list> + </section> +</section> + + +<section><title>Ssh 4.2.2.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Trailing white space was removed at end of the + hello-string. This caused interoperability problems with + some other ssh-implementations (e.g OpenSSH 7.3p1 on + Solaris 11)</p> + <p> + Own Id: OTP-14763 Aux Id: ERIERL-74 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 4.2.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The key exchange algorithm + diffie-hellman-group-exchange-sha* has a server-option + <c>{dh_gex_limits,{Min,Max}}</c>. There was a hostkey + signature validation error on the client side if the + option was used and the <c>Min</c> or the <c>Max</c> + differed from the corresponding values obtained from the + client.</p> + <p> + This bug is now corrected.</p> + <p> + Own Id: OTP-14166</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Key exchange algorithms + diffie-hellman-group-exchange-sha* optimized, up to a + factor of 11 for the slowest ( = biggest and safest) one.</p> + <p> + Own Id: OTP-14169 Aux Id: seq-13261 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 4.2.2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Upgrade of an established client connection could crash + because the ssh client supervisors children had wrong + type. This is fixed now.</p> + <p> + Own Id: OTP-13782 Aux Id: seq13158 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.2.2.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 337f4094cc..acf94ff6af 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -227,6 +227,18 @@ </item> </list> </item> + + <tag><c><![CDATA[{save_accepted_host, boolean()}]]></c></tag> + <item> + <p>If <c>true</c>, the client saves an accepted host key to avoid the + accept question the next time the same host is connected. If the option + <c>key_cb</c> is not present, the key is saved in the file "known_hosts". + </p> + <p>If <c>false</c>, the key is not saved and the key will still be unknown + at the next access of the same host. + </p> + </item> + <tag><c><![CDATA[{user_interaction, boolean()}]]></c></tag> <item> <p>If <c>false</c>, disables the client to connect to the server diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index ed7fbf9cf3..129426a6d5 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -464,11 +464,16 @@ <v>FileInfo = record()</v> </type> <desc> - <p>Returns a <c><![CDATA[file_info]]></c> record from the file specified by + <p>Returns a <c><![CDATA[file_info]]></c> record from the file system object specified by <c><![CDATA[Name]]></c> or <c><![CDATA[Handle]]></c>. See <seealso marker="kernel:file#read_file_info-2">file:read_file_info/2</seealso> for information about the record. </p> + <p> + Depending on the underlying OS:es links might be followed and info on the final file, directory + etc is returned. See <seealso marker="#read_link_info-2">ssh_sftp::read_link_info/2</seealso> + on how to get information on links instead. + </p> </desc> </func> diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index 032d87bdad..25d537c624 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -184,7 +184,6 @@ channel_info(ConnectionRef, ChannelId, Options) -> daemon(Port) -> daemon(Port, []). - daemon(Socket, UserOptions) when is_port(Socket) -> try #{} = Options = ssh_options:handle_options(server, UserOptions), @@ -267,8 +266,6 @@ daemon(Host0, Port0, UserOptions0) when 0 =< Port0, Port0 =< 65535, daemon(_, _, _) -> {error, badarg}. - - %%-------------------------------------------------------------------- -spec daemon_info(daemon_ref()) -> ok_error( [{atom(), term()}] ). diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 3dee1c5521..4711f54fb5 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -35,6 +35,8 @@ -define(DEFAULT_TRANSPORT, {tcp, gen_tcp, tcp_closed} ). +-define(DEFAULT_SHELL, {shell, start, []} ). + -define(MAX_RND_PADDING_LEN, 15). -define(SUPPORTED_AUTH_METHODS, "publickey,keyboard-interactive,password"). diff --git a/lib/ssh/src/ssh_acceptor_sup.erl b/lib/ssh/src/ssh_acceptor_sup.erl index a24664793b..fc564a359b 100644 --- a/lib/ssh/src/ssh_acceptor_sup.erl +++ b/lib/ssh/src/ssh_acceptor_sup.erl @@ -86,10 +86,7 @@ child_spec(Address, Port, Profile, Options) -> Timeout = ?GET_INTERNAL_OPT(timeout, Options, ?DEFAULT_TIMEOUT), #{id => id(Address, Port, Profile), start => {ssh_acceptor, start_link, [Port, Address, Options, Timeout]}, - restart => transient, - shutdown => 5500, %brutal_kill, - type => worker, - modules => [ssh_acceptor] + restart => transient % because a crashed listener could be replaced by a new one }. id(Address, Port, Profile) -> diff --git a/lib/ssh/src/ssh_channel_sup.erl b/lib/ssh/src/ssh_channel_sup.erl index 6b01dc334d..8444533fd1 100644 --- a/lib/ssh/src/ssh_channel_sup.erl +++ b/lib/ssh/src/ssh_channel_sup.erl @@ -26,7 +26,7 @@ -behaviour(supervisor). --export([start_link/1, start_child/2]). +-export([start_link/1, start_child/5]). %% Supervisor callback -export([init/1]). @@ -37,7 +37,14 @@ start_link(Args) -> supervisor:start_link(?MODULE, [Args]). -start_child(Sup, ChildSpec) -> +start_child(Sup, Callback, Id, Args, Exec) -> + ChildSpec = + #{id => make_ref(), + start => {ssh_channel, start_link, [self(), Id, Callback, Args, Exec]}, + restart => temporary, + type => worker, + modules => [ssh_channel] + }, supervisor:start_child(Sup, ChildSpec). %%%========================================================================= diff --git a/lib/ssh/src/ssh_cli.erl b/lib/ssh/src/ssh_cli.erl index 62854346b0..958c342f5f 100644 --- a/lib/ssh/src/ssh_cli.erl +++ b/lib/ssh/src/ssh_cli.erl @@ -127,7 +127,8 @@ handle_ssh_msg({ssh_cm, ConnectionHandler, cm = ConnectionHandler}}; handle_ssh_msg({ssh_cm, ConnectionHandler, - {exec, ChannelId, WantReply, Cmd}}, #state{exec=undefined} = State) -> + {exec, ChannelId, WantReply, Cmd}}, #state{exec=undefined, + shell=?DEFAULT_SHELL} = State) -> {Reply, Status} = exec(Cmd), write_chars(ConnectionHandler, ChannelId, io_lib:format("~p\n", [Reply])), @@ -136,6 +137,15 @@ handle_ssh_msg({ssh_cm, ConnectionHandler, ssh_connection:exit_status(ConnectionHandler, ChannelId, Status), ssh_connection:send_eof(ConnectionHandler, ChannelId), {stop, ChannelId, State#state{channel = ChannelId, cm = ConnectionHandler}}; + +handle_ssh_msg({ssh_cm, ConnectionHandler, + {exec, ChannelId, WantReply, _Cmd}}, #state{exec = undefined} = State) -> + write_chars(ConnectionHandler, ChannelId, 1, "Prohibited.\n"), + ssh_connection:reply_request(ConnectionHandler, WantReply, success, ChannelId), + ssh_connection:exit_status(ConnectionHandler, ChannelId, 255), + ssh_connection:send_eof(ConnectionHandler, ChannelId), + {stop, ChannelId, State#state{channel = ChannelId, cm = ConnectionHandler}}; + handle_ssh_msg({ssh_cm, ConnectionHandler, {exec, ChannelId, WantReply, Cmd}}, State) -> NewState = start_shell(ConnectionHandler, Cmd, State), @@ -453,11 +463,14 @@ move_cursor(From, To, #ssh_pty{width=Width, term=Type}) -> %% %%% make sure that there is data to send %% %%% before calling ssh_connection:send write_chars(ConnectionHandler, ChannelId, Chars) -> + write_chars(ConnectionHandler, ChannelId, ?SSH_EXTENDED_DATA_DEFAULT, Chars). + +write_chars(ConnectionHandler, ChannelId, Type, Chars) -> case has_chars(Chars) of false -> ok; true -> ssh_connection:send(ConnectionHandler, ChannelId, - ?SSH_EXTENDED_DATA_DEFAULT, + Type, Chars) end. diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index 7e9ee78fd2..946ae2967b 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -812,22 +812,20 @@ start_channel(Cb, Id, Args, SubSysSup, Opts) -> start_channel(Cb, Id, Args, SubSysSup, undefined, Opts). start_channel(Cb, Id, Args, SubSysSup, Exec, Opts) -> - ChildSpec = child_spec(Cb, Id, Args, Exec), ChannelSup = ssh_subsystem_sup:channel_supervisor(SubSysSup), - assert_limit_num_channels_not_exceeded(ChannelSup, Opts), - ssh_channel_sup:start_child(ChannelSup, ChildSpec). + case max_num_channels_not_exceeded(ChannelSup, Opts) of + true -> + ssh_channel_sup:start_child(ChannelSup, Cb, Id, Args, Exec); + false -> + throw(max_num_channels_exceeded) + end. -assert_limit_num_channels_not_exceeded(ChannelSup, Opts) -> +max_num_channels_not_exceeded(ChannelSup, Opts) -> MaxNumChannels = ?GET_OPT(max_channels, Opts), NumChannels = length([x || {_,_,worker,[ssh_channel]} <- supervisor:which_children(ChannelSup)]), - if - %% Note that NumChannels is BEFORE starting a new one - NumChannels < MaxNumChannels -> - ok; - true -> - throw(max_num_channels_exceeded) - end. + %% Note that NumChannels is BEFORE starting a new one + NumChannels < MaxNumChannels. %%-------------------------------------------------------------------- %%% Internal functions @@ -874,14 +872,6 @@ check_subsystem(SsName, Options) -> Value end. -child_spec(Callback, Id, Args, Exec) -> - Name = make_ref(), - StartFunc = {ssh_channel, start_link, [self(), Id, Callback, Args, Exec]}, - Restart = temporary, - Shutdown = 3600, - Type = worker, - {Name, StartFunc, Restart, Shutdown, Type, [ssh_channel]}. - start_cli(#connection{cli_spec = no_cli}, _) -> {error, cli_disabled}; start_cli(#connection{options = Options, diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 0ca960ef96..e11d3adee4 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -1174,17 +1174,25 @@ handle_event({call,_}, _, StateName, _) when not ?CONNECTED(StateName) -> handle_event({call,From}, {request, ChannelPid, ChannelId, Type, Data, Timeout}, StateName, D0) when ?CONNECTED(StateName) -> - D = handle_request(ChannelPid, ChannelId, Type, Data, true, From, D0), - %% Note reply to channel will happen later when reply is recived from peer on the socket - start_channel_request_timer(ChannelId, From, Timeout), - {keep_state, cache_request_idle_timer_check(D)}; + case handle_request(ChannelPid, ChannelId, Type, Data, true, From, D0) of + {error,Error} -> + {keep_state, D0, {reply,From,{error,Error}}}; + D -> + %% Note reply to channel will happen later when reply is recived from peer on the socket + start_channel_request_timer(ChannelId, From, Timeout), + {keep_state, cache_request_idle_timer_check(D)} + end; handle_event({call,From}, {request, ChannelId, Type, Data, Timeout}, StateName, D0) when ?CONNECTED(StateName) -> - D = handle_request(ChannelId, Type, Data, true, From, D0), - %% Note reply to channel will happen later when reply is recived from peer on the socket - start_channel_request_timer(ChannelId, From, Timeout), - {keep_state, cache_request_idle_timer_check(D)}; + case handle_request(ChannelId, Type, Data, true, From, D0) of + {error,Error} -> + {keep_state, D0, {reply,From,{error,Error}}}; + D -> + %% Note reply to channel will happen later when reply is recived from peer on the socket + start_channel_request_timer(ChannelId, From, Timeout), + {keep_state, cache_request_idle_timer_check(D)} + end; handle_event({call,From}, {data, ChannelId, Type, Data, Timeout}, StateName, D0) when ?CONNECTED(StateName) -> @@ -1460,13 +1468,12 @@ terminate(shutdown, StateName, State0) -> State0), finalize_termination(StateName, State); -%% terminate({shutdown,Msg}, StateName, State0) when is_record(Msg,ssh_msg_disconnect)-> -%% State = send_msg(Msg, State0), -%% finalize_termination(StateName, Msg, State); - terminate({shutdown,_R}, StateName, State) -> finalize_termination(StateName, State); +terminate(kill, StateName, State) -> + finalize_termination(StateName, State); + terminate(Reason, StateName, State0) -> %% Others, e.g undef, {badmatch,_} log_error(Reason), @@ -1774,21 +1781,31 @@ is_usable_user_pubkey(A, Ssh) -> %%%---------------------------------------------------------------- handle_request(ChannelPid, ChannelId, Type, Data, WantReply, From, D) -> case ssh_channel:cache_lookup(cache(D), ChannelId) of - #channel{remote_id = Id} = Channel -> + #channel{remote_id = Id, + sent_close = false} = Channel -> update_sys(cache(D), Channel, Type, ChannelPid), send_msg(ssh_connection:channel_request_msg(Id, Type, WantReply, Data), add_request(WantReply, ChannelId, From, D)); - undefined -> - D + + _ when WantReply==true -> + {error,closed}; + + _ -> + D end. handle_request(ChannelId, Type, Data, WantReply, From, D) -> case ssh_channel:cache_lookup(cache(D), ChannelId) of - #channel{remote_id = Id} -> + #channel{remote_id = Id, + sent_close = false} -> send_msg(ssh_connection:channel_request_msg(Id, Type, WantReply, Data), add_request(WantReply, ChannelId, From, D)); - undefined -> - D + + _ when WantReply==true -> + {error,closed}; + + _ -> + D end. %%%---------------------------------------------------------------- diff --git a/lib/ssh/src/ssh_connection_sup.erl b/lib/ssh/src/ssh_connection_sup.erl index 60ee8b7c73..2e8450090a 100644 --- a/lib/ssh/src/ssh_connection_sup.erl +++ b/lib/ssh/src/ssh_connection_sup.erl @@ -52,10 +52,7 @@ init(_) -> }, ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. start => {ssh_connection_handler, start_link, []}, - restart => temporary, - shutdown => 4000, - type => worker, - modules => [ssh_connection_handler] + restart => temporary % because there is no way to restart a crashed connection } ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index 68c99743ee..1e10f72956 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -268,7 +268,7 @@ default(server) -> }, {shell, def} => - #{default => {shell, start, []}, + #{default => ?DEFAULT_SHELL, chk => fun({M,F,A}) -> is_atom(M) andalso is_atom(F) andalso is_list(A); (V) -> check_function1(V) orelse check_function2(V) end, @@ -439,6 +439,12 @@ default(client) -> class => user_options }, + {save_accepted_host, def} => + #{default => true, + chk => fun erlang:is_boolean/1, + class => user_options + }, + {pref_public_key_algs, def} => #{default => ssh_transport:default_algorithms(public_key), chk => fun check_pref_public_key_algs/1, diff --git a/lib/ssh/src/ssh_subsystem_sup.erl b/lib/ssh/src/ssh_subsystem_sup.erl index 8db051095c..77da240a66 100644 --- a/lib/ssh/src/ssh_subsystem_sup.erl +++ b/lib/ssh/src/ssh_subsystem_sup.erl @@ -74,18 +74,14 @@ ssh_connection_child_spec(Role, Address, Port, _Profile, Options) -> #{id => id(Role, ssh_connection_sup, Address, Port), start => {ssh_connection_sup, start_link, [Options]}, restart => temporary, - shutdown => 5000, - type => supervisor, - modules => [ssh_connection_sup] + type => supervisor }. ssh_channel_child_spec(Role, Address, Port, _Profile, Options) -> #{id => id(Role, ssh_channel_sup, Address, Port), start => {ssh_channel_sup, start_link, [Options]}, restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_channel_sup] + type => supervisor }. id(Role, Sup, Address, Port) -> diff --git a/lib/ssh/src/ssh_sup.erl b/lib/ssh/src/ssh_sup.erl index eaec7a54e4..8183016ba5 100644 --- a/lib/ssh/src/ssh_sup.erl +++ b/lib/ssh/src/ssh_sup.erl @@ -36,15 +36,14 @@ init(_) -> intensity => 10, period => 3600 }, - ChildSpecs = [#{id => Module, - start => {Module, start_link, []}, - restart => permanent, - shutdown => 4000, %brutal_kill, - type => supervisor, - modules => [Module] + ChildSpecs = [#{id => sshd_sup, + start => {sshd_sup, start_link, []}, + type => supervisor + }, + #{id => sshc_sup, + start => {sshc_sup, start_link, []}, + type => supervisor } - || Module <- [sshd_sup, - sshc_sup] ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/ssh_system_sup.erl b/lib/ssh/src/ssh_system_sup.erl index e70abf59c2..17f990c5d8 100644 --- a/lib/ssh/src/ssh_system_sup.erl +++ b/lib/ssh/src/ssh_system_sup.erl @@ -63,9 +63,7 @@ init([Address, Port, Profile, Options]) -> [#{id => id(ssh_acceptor_sup, Address, Port, Profile), start => {ssh_acceptor_sup, start_link, [Address, Port, Profile, Options]}, restart => transient, - shutdown => infinity, - type => supervisor, - modules => [ssh_acceptor_sup] + type => supervisor }]; _ -> [] @@ -124,9 +122,8 @@ start_subsystem(SystemSup, Role, Address, Port, Profile, Options) -> #{id => make_ref(), start => {ssh_subsystem_sup, start_link, [Role, Address, Port, Profile, Options]}, restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_subsystem_sup]}, + type => supervisor + }, supervisor:start_child(SystemSup, SubsystemSpec). stop_subsystem(SystemSup, SubSys) -> diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index ad9efc4755..975053d301 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -889,10 +889,13 @@ known_host_key(#ssh{opts = Opts, key_cb = {KeyCb,KeyCbOpts}, peer = {PeerName,_} {_,true} -> ok; {_,false} -> + DoAdd = ?GET_OPT(save_accepted_host, Opts), case accepted_host(Ssh, PeerName, Public, Opts) of - true -> + true when DoAdd == true -> {_,R} = add_host_key(KeyCb, PeerName, Public, [{key_cb_private,KeyCbOpts}|UserOpts]), R; + true when DoAdd == false -> + ok; false -> {error, rejected_by_user}; {error,E} -> diff --git a/lib/ssh/src/sshc_sup.erl b/lib/ssh/src/sshc_sup.erl index 133b2c6450..fd4d8a3c07 100644 --- a/lib/ssh/src/sshc_sup.erl +++ b/lib/ssh/src/sshc_sup.erl @@ -60,10 +60,7 @@ init(_) -> }, ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. start => {ssh_connection_handler, start_link, []}, - restart => temporary, - shutdown => 4000, - type => worker, - modules => [ssh_connection_handler] + restart => temporary % because there is no way to restart a crashed connection } ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/sshd_sup.erl b/lib/ssh/src/sshd_sup.erl index c23e65d955..779a861a54 100644 --- a/lib/ssh/src/sshd_sup.erl +++ b/lib/ssh/src/sshd_sup.erl @@ -90,10 +90,8 @@ init(_) -> child_spec(Address, Port, Profile, Options) -> #{id => id(Address, Port, Profile), start => {ssh_system_sup, start_link, [Address, Port, Profile, Options]}, - restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_system_sup] + restart => temporary, + type => supervisor }. id(Address, Port, Profile) -> diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 202b0afe57..365f25fabb 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -60,7 +60,7 @@ login_bad_pwd_no_retry5/1, misc_ssh_options/1, openssh_zlib_basic_test/1, - packet_size_zero/1, + packet_size/1, pass_phrase/1, peername_sockname/1, send/1, @@ -111,7 +111,7 @@ all() -> double_close, daemon_opt_fd, multi_daemon_opt_fd, - packet_size_zero, + packet_size, ssh_info_print, {group, login_bad_pwd_no_retry}, shell_exit_status @@ -764,11 +764,11 @@ cli(Config) when is_list(Config) -> {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), ssh_connection:shell(ConnectionRef, ChannelId), - ok = ssh_connection:send(ConnectionRef, ChannelId, <<"q">>), + ssh_connection:send(ConnectionRef, ChannelId, <<"q">>), receive {ssh_cm, ConnectionRef, {data,0,0, <<"\r\nYou are accessing a dummy, type \"q\" to exit\r\n\n">>}} -> - ok = ssh_connection:send(ConnectionRef, ChannelId, <<"q">>) + ssh_connection:send(ConnectionRef, ChannelId, <<"q">>) after 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) end, @@ -1104,7 +1104,7 @@ multi_daemon_opt_fd(Config) -> end || {S,Pid,C} <- Tests]. %%-------------------------------------------------------------------- -packet_size_zero(Config) -> +packet_size(Config) -> SystemDir = proplists:get_value(data_dir, Config), PrivDir = proplists:get_value(priv_dir, Config), UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth @@ -1119,21 +1119,31 @@ packet_size_zero(Config) -> {user_interaction, false}, {user, "vego"}, {password, "morot"}]), - - {ok,Chan} = ssh_connection:session_channel(Conn, 1000, _MaxPacketSize=0, 60000), - ok = ssh_connection:shell(Conn, Chan), + lists:foreach( + fun(MaxPacketSize) -> + ct:log("Try max_packet_size=~p",[MaxPacketSize]), + {ok,Ch} = ssh_connection:session_channel(Conn, 1000, MaxPacketSize, 60000), + ok = ssh_connection:shell(Conn, Ch), + rec(Server, Conn, Ch, MaxPacketSize) + end, [0, 1, 10, 25]), ssh:close(Conn), - ssh:stop_daemon(Server), + ssh:stop_daemon(Server). +rec(Server, Conn, Ch, MaxSz) -> receive - {ssh_cm,Conn,{data,Chan,_Type,_Msg1}} = M -> - ct:log("Got ~p",[M]), - ct:fail(doesnt_obey_max_packet_size_0) - after 5000 -> - ok - end. - + {ssh_cm,Conn,{data,Ch,_,M}} when size(M) =< MaxSz -> + ct:log("~p: ~p",[MaxSz,M]), + rec(Server, Conn, Ch, MaxSz); + {ssh_cm,Conn,{data,Ch,_,_}} = M -> + ct:log("Max pkt size=~p. Got ~p",[MaxSz,M]), + ssh:close(Conn), + ssh:stop_daemon(Server), + ct:fail("Does not obey max_packet_size=~p",[MaxSz]) + after + 2000 -> ok + end. + %%-------------------------------------------------------------------- shell_no_unicode(Config) -> new_do_shell(proplists:get_value(io,Config), @@ -1491,7 +1501,7 @@ new_do_shell(IO, N, Ops=[{Order,Arg}|More]) -> ct:fail("*** Expected ~p, but got ~p",[string:strip(ExpStr),RecStr]) end after 30000 -> - ct:log("Meassage queue of ~p:~n~p", + ct:log("Message queue of ~p:~n~p", [self(), erlang:process_info(self(), messages)]), case Order of expect -> ct:fail("timeout, expected ~p",[string:strip(Arg)]); diff --git a/lib/ssh/test/ssh_compat_SUITE.erl b/lib/ssh/test/ssh_compat_SUITE.erl index 74ab5aca3a..f7eda1dc08 100644 --- a/lib/ssh/test/ssh_compat_SUITE.erl +++ b/lib/ssh/test/ssh_compat_SUITE.erl @@ -32,7 +32,8 @@ -compile(export_all). -define(USER,"sshtester"). --define(PWD, "foobar"). +-define(PASSWD, "foobar"). +-define(BAD_PASSWD, "NOT-"?PASSWD). -define(DOCKER_PFX, "ssh_compat_suite-ssh"). %%-------------------------------------------------------------------- @@ -44,25 +45,22 @@ suite() -> {timetrap,{seconds,40}}]. all() -> - [{group,G} || G <- vers()]. +%% [check_docker_present] ++ + [{group,G} || G <- ssh_image_versions()]. groups() -> - [{G, [], tests()} || G <- vers()]. - -tests() -> - [login_with_password_otp_is_client, - login_with_password_otp_is_server, - login_with_keyboard_interactive_otp_is_client, - login_with_keyboard_interactive_otp_is_server, - login_with_all_public_keys_otp_is_client, - login_with_all_public_keys_otp_is_server, - all_algorithms_otp_is_client, - all_algorithms_otp_is_server + [{otp_client, [], [login_otp_is_client, + all_algorithms_sftp_exec_reneg_otp_is_client, + send_recv_big_with_renegotiate_otp_is_client + ]}, + {otp_server, [], [login_otp_is_server, + all_algorithms_sftp_exec_reneg_otp_is_server + ]} | + [{G, [], [{group,otp_client}, {group,otp_server}]} || G <- ssh_image_versions()] ]. - -vers() -> +ssh_image_versions() -> try %% Find all useful containers in such a way that undefined command, too low %% priviliges, no containers and containers found give meaningful result: @@ -94,28 +92,60 @@ end_per_suite(Config) -> %%% os:cmd("docker rm $(docker ps -aq -f status=exited)"), %% Remove dangling images: %%% os:cmd("docker rmi $(docker images -f dangling=true -q)"), + catch ssh:stop(), Config. +init_per_group(otp_server, Config) -> + case proplists:get_value(common_remote_client_algs, Config) of + undefined -> + SSHver = proplists:get_value(ssh_version, Config, ""), + {skip,"No "++SSHver++ " client found in docker"}; + _ -> + Config + end; + +init_per_group(otp_client, Config) -> + Config; -init_per_group(G, Config) -> - case lists:member(G, vers()) of +init_per_group(G, Config0) -> + case lists:member(G, ssh_image_versions()) of true -> + %% This group is for one of the images + Vssh = atom_to_list(G), + Cmnt = io_lib:format("+++ ~s +++",[Vssh]), + ct:comment("~s",[Cmnt]), try start_docker(G) of {ok,ID} -> - ct:log("==> ~p",[G]), - [Vssh|VsslRest] = string:tokens(atom_to_list(G), "-"), - Vssl = lists:flatten(lists:join($-,VsslRest)), - ct:comment("+++ ~s + ~s +++",[Vssh,Vssl]), + ct:log("==> ~p started",[G]), %% Find the algorithms that both client and server supports: {IP,Port} = ip_port([{id,ID}]), - try common_algs([{id,ID}|Config], IP, Port) of - {ok, RemoteServerCommon, RemoteClientCommon} -> - [{ssh_version,Vssh},{ssl_version,Vssl}, - {id,ID}, - {common_server_algs,RemoteServerCommon}, - {common_client_algs,RemoteClientCommon} - |Config]; + ct:log("Try contact ~p:~p",[IP,Port]), + Config1 = [{id,ID}, + {ssh_version,Vssh} + | Config0], + try common_algs(Config1, IP, Port) of + {ok, ServerHello, RemoteServerCommon, ClientHello, RemoteClientCommon} -> + case chk_hellos([ServerHello,ClientHello], Cmnt) of + Cmnt -> + ok; + NewCmnt -> + ct:comment("~s",[NewCmnt]) + end, + AuthMethods = + %% This should be obtained by quering the peer, but that + %% is a bit hard. It is possible with ssh_protocol_SUITE + %% techniques, but it can wait. + case Vssh of + "dropbear" ++ _ -> + [password, publickey]; + _ -> + [password, 'keyboard-interactive', publickey] + end, + [{common_remote_server_algs,RemoteServerCommon}, + {common_remote_client_algs,RemoteClientCommon}, + {common_authmethods,AuthMethods} + |Config1]; Other -> ct:log("Error in init_per_group: ~p",[Other]), stop_docker(ID), @@ -138,188 +168,301 @@ init_per_group(G, Config) -> end; false -> - Config + Config0 end. -end_per_group(_, Config) -> - catch stop_docker(proplists:get_value(id,Config)), - Config. +end_per_group(G, Config) -> + case lists:member(G, ssh_image_versions()) of + true -> + catch stop_docker(proplists:get_value(id,Config)); + false -> + ok + end. %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -login_with_password_otp_is_client(Config) -> - {IP,Port} = ip_port(Config), - {ok,C} = ssh:connect(IP, Port, [{auth_methods,"password"}, - {user,?USER}, - {password,?PWD}, - {user_dir, new_dir(Config)}, - {silently_accept_hosts,true}, - {user_interaction,false} - ]), - ssh:close(C). - -%%-------------------------------------------------------------------- -login_with_password_otp_is_server(Config) -> - {Server, Host, HostPort} = - ssh_test_lib:daemon(0, - [{auth_methods,"password"}, - {system_dir, setup_local_hostdir('ssh-rsa',Config)}, - {user_dir, new_dir(Config)}, - {user_passwords, [{?USER,?PWD}]}, - {failfun, fun ssh_test_lib:failfun/2} - ]), - R = exec_from_docker(Config, Host, HostPort, - "'lists:concat([\"Answer=\",1+2]).\r\n'", - [<<"Answer=3">>], - ""), - ssh:stop_daemon(Server), - R. - -%%-------------------------------------------------------------------- -login_with_keyboard_interactive_otp_is_client(Config) -> - {DockerIP,DockerPort} = ip_port(Config), - {ok,C} = ssh:connect(DockerIP, DockerPort, - [{auth_methods,"keyboard-interactive"}, - {user,?USER}, - {password,?PWD}, - {user_dir, new_dir(Config)}, - {silently_accept_hosts,true}, - {user_interaction,false} - ]), - ssh:close(C). - -%%-------------------------------------------------------------------- -login_with_keyboard_interactive_otp_is_server(Config) -> - {Server, Host, HostPort} = - ssh_test_lib:daemon(0, - [{auth_methods,"keyboard-interactive"}, - {system_dir, setup_local_hostdir('ssh-rsa',Config)}, - {user_dir, new_dir(Config)}, - {user_passwords, [{?USER,?PWD}]}, - {failfun, fun ssh_test_lib:failfun/2} - ]), - R = exec_from_docker(Config, Host, HostPort, - "'lists:concat([\"Answer=\",1+3]).\r\n'", - [<<"Answer=4">>], - ""), - ssh:stop_daemon(Server), - R. +check_docker_present(_Config) -> + ct:log("This testcase is just to show in Monitor that we have a test host with docker installed",[]), + {fail, "Test is OK: just showing docker is available"}. %%-------------------------------------------------------------------- -login_with_all_public_keys_otp_is_client(Config) -> - CommonAlgs = [{public_key_from_host,A} - || {public_key,A} <- proplists:get_value(common_server_algs, Config)], - {DockerIP,DockerPort} = ip_port(Config), - chk_all_algos(CommonAlgs, Config, - fun(_Tag,Alg) -> - ssh:connect(DockerIP, DockerPort, - [{auth_methods, "publickey"}, - {user, ?USER}, - {user_dir, setup_remote_auth_keys_and_local_priv(Alg, Config)}, - {silently_accept_hosts,true}, - {user_interaction,false} - ]) +login_otp_is_client(Config) -> + {IP,Port} = ip_port(Config), + PublicKeyAlgs = [A || {public_key,A} <- proplists:get_value(common_remote_server_algs, Config)], + CommonAuths = + [{AuthMethod,Alg} || AuthMethod <- proplists:get_value(common_authmethods, Config), + Alg <- case AuthMethod of + publickey -> + PublicKeyAlgs; + _ -> + [' '] + end + ], + + chk_all_algos(?FUNCTION_NAME, CommonAuths, Config, + fun(AuthMethod,Alg) -> + {Opts,Dir} = + case AuthMethod of + publickey -> + {[], setup_remote_auth_keys_and_local_priv(Alg, Config)}; + _ -> + {[{password,?PASSWD}], new_dir(Config)} + end, + ssh:connect(IP, Port, [{auth_methods, atom_to_list(AuthMethod)}, + {user,?USER}, + {user_dir, Dir}, + {silently_accept_hosts,true}, + {user_interaction,false} + | Opts + ]) end). + %%-------------------------------------------------------------------- -login_with_all_public_keys_otp_is_server(Config) -> - CommonAlgs = [{public_key_to_host,A} - || {public_key,A} <- proplists:get_value(common_client_algs, Config)], - UserDir = new_dir(Config), - {Server, Host, HostPort} = - ssh_test_lib:daemon(0, - [{auth_methods, "publickey"}, - {system_dir, setup_local_hostdir('ssh-rsa',Config)}, - {user_dir, UserDir}, - {user_passwords, [{?USER,?PWD}]}, - {failfun, fun ssh_test_lib:failfun/2} - ]), - - R = chk_all_algos(CommonAlgs, Config, - fun(_Tag,Alg) -> - setup_remote_priv_and_local_auth_keys(Alg, clear_dir(UserDir), Config), - exec_from_docker(Config, Host, HostPort, - "'lists:concat([\"Answer=\",1+4]).\r\n'", - [<<"Answer=5">>], - "") - end), - ssh:stop_daemon(Server), - R. +login_otp_is_server(Config) -> + PublicKeyAlgs = [A || {public_key,A} <- proplists:get_value(common_remote_client_algs, Config)], + CommonAuths = + [{AuthMethod,Alg} || AuthMethod <- proplists:get_value(common_authmethods, Config), + Alg <- case AuthMethod of + publickey -> + PublicKeyAlgs; + _ -> + [' '] + end + ], + SysDir = setup_local_hostdir(hd(PublicKeyAlgs), Config), + chk_all_algos(?FUNCTION_NAME, CommonAuths, Config, + fun(AuthMethod,Alg) -> + {Opts,UsrDir} = + case AuthMethod of + publickey -> + {[{user_passwords, [{?USER,?BAD_PASSWD}]}], + setup_remote_priv_and_local_auth_keys(Alg, Config) + }; + _ -> + {[{user_passwords, [{?USER,?PASSWD}]}], + new_dir(Config) + } + end, + {Server, Host, HostPort} = + ssh_test_lib:daemon(0, + [{auth_methods, atom_to_list(AuthMethod)}, + {system_dir, SysDir}, + {user_dir, UsrDir}, + {failfun, fun ssh_test_lib:failfun/2} + | Opts + ]), + R = exec_from_docker(Config, Host, HostPort, + "'lists:concat([\"Answer=\",1+3]).\r\n'", + [<<"Answer=4">>], + ""), + ssh:stop_daemon(Server), + R + end). %%-------------------------------------------------------------------- -all_algorithms_otp_is_client(Config) -> - CommonAlgs = proplists:get_value(common_server_algs, Config), +all_algorithms_sftp_exec_reneg_otp_is_client(Config) -> + CommonAlgs = proplists:get_value(common_remote_server_algs, Config), {IP,Port} = ip_port(Config), - chk_all_algos(CommonAlgs, Config, + chk_all_algos(?FUNCTION_NAME, CommonAlgs, Config, fun(Tag, Alg) -> - ssh:connect(IP, Port, [{user,?USER}, - {password,?PWD}, - {auth_methods, "password"}, - {user_dir, new_dir(Config)}, - {preferred_algorithms, [{Tag,[Alg]}]}, - {silently_accept_hosts,true}, - {user_interaction,false} + ConnRes = + ssh:connect(IP, Port, + [{user,?USER}, + {password,?PASSWD}, + {auth_methods, "password"}, + {user_dir, new_dir(Config)}, + {preferred_algorithms, [{Tag,[Alg]}]}, + {silently_accept_hosts,true}, + {user_interaction,false} + ]) , + test_erl_client_reneg(ConnRes, % Seems that max 10 channels may be open in sshd + [{exec,1}, + {sftp,5}, + {no_subsyst,1}, + {setenv, 1}, + {sftp_async,1} ]) end). %%-------------------------------------------------------------------- -all_algorithms_otp_is_server(Config) -> - CommonAlgs = proplists:get_value(common_client_algs, Config), +all_algorithms_sftp_exec_reneg_otp_is_server(Config) -> + CommonAlgs = proplists:get_value(common_remote_client_algs, Config), UserDir = setup_remote_priv_and_local_auth_keys('ssh-rsa', Config), - chk_all_algos(CommonAlgs, Config, + chk_all_algos(?FUNCTION_NAME, CommonAlgs, Config, fun(Tag,Alg) -> HostKeyAlg = case Tag of public_key -> Alg; _ -> 'ssh-rsa' end, + SftpRootDir = new_dir(Config), + %% ct:log("Rootdir = ~p",[SftpRootDir]), {Server, Host, HostPort} = ssh_test_lib:daemon(0, [{preferred_algorithms, [{Tag,[Alg]}]}, {system_dir, setup_local_hostdir(HostKeyAlg, Config)}, {user_dir, UserDir}, - {user_passwords, [{?USER,?PWD}]}, - {failfun, fun ssh_test_lib:failfun/2} + {user_passwords, [{?USER,?PASSWD}]}, + {failfun, fun ssh_test_lib:failfun/2}, + {subsystems, + [ssh_sftpd:subsystem_spec([{cwd,SftpRootDir}, + {root,SftpRootDir}]), + {"echo_10",{ssh_echo_server,[10,[{dbg,true}]]}} + ]} ]), - R = exec_from_docker(Config, Host, HostPort, - "hi_there.\r\n", - [<<"hi_there">>], - ""), + R = do([fun() -> + exec_from_docker(Config, Host, HostPort, + "hi_there.\r\n", + [<<"hi_there">>], + "") + end, + fun() -> + sftp_tests_erl_server(Config, Host, HostPort, SftpRootDir, UserDir) + end + ]), ssh:stop_daemon(Server), R end). %%-------------------------------------------------------------------- +send_recv_big_with_renegotiate_otp_is_client(Config) -> + %% Connect to the remote openssh server: + {IP,Port} = ip_port(Config), + {ok,C} = ssh:connect(IP, Port, [{user,?USER}, + {password,?PASSWD}, + {user_dir, setup_remote_auth_keys_and_local_priv('ssh-rsa', Config)}, + {silently_accept_hosts,true}, + {user_interaction,false} + ]), + + %% Open a channel and exec the Linux 'cat' command at the openssh side. + %% This 'cat' will read stdin and write to stdout until an eof is read from stdin. + {ok, Ch1} = ssh_connection:session_channel(C, infinity), + success = ssh_connection:exec(C, Ch1, "cat", infinity), + + %% Build big binary + HalfSizeBytes = 100*1000*1000, + Data = << <<X:32>> || X <- lists:seq(1, HalfSizeBytes div 4)>>, + + %% Send the data. Must spawn a process to avoid deadlock. The client will block + %% until all is sent through the send window. But the server will stop receiveing + %% when the servers send-window towards the client is full. + %% Since the client can't receive before the server has received all but 655k from the client + %% ssh_connection:send/4 is blocking... + spawn_link( + fun() -> + ct:comment("Sending ~p Mbytes with renegotiation in the middle",[2*byte_size(Data)/1000000]), + %% ct:log("sending first ~p bytes",[byte_size(Data)]), + ok = ssh_connection:send(C, Ch1, Data, 10000), + %% ct:log("Init renegotiation test",[]), + Kex1 = renegotiate_test(init, C), + %% ct:log("sending next ~p bytes",[byte_size(Data)]), + ok = ssh_connection:send(C, Ch1, Data, 10000), + %% ct:log("Finnish renegotiation test",[]), + renegotiate_test(Kex1, C), + %% ct:log("sending eof",[]), + ok = ssh_connection:send_eof(C, Ch1) + %%, ct:log("READY, sent ~p bytes",[2*byte_size(Data)]) + end), + + {eof,ReceivedData} = + loop_until(fun({eof,_}) -> true; + (_ ) -> false + end, + fun(Acc) -> + %%ct:log("Get more ~p",[ ExpectedSize-byte_size(Acc) ]), + receive + {ssh_cm, C, {eof,Ch}} when Ch==Ch1 -> + %% ct:log("eof received",[]), + {eof,Acc}; + + {ssh_cm, C, {data,Ch,0,B}} when Ch==Ch1, + is_binary(B) -> + %% ct:log("(1) Received ~p bytes (total ~p), missing ~p bytes", + %% [byte_size(B), + %% byte_size(B)+byte_size(Acc), + %% 2*byte_size(Data)-(byte_size(B)+byte_size(Acc))]), + ssh_connection:adjust_window(C, Ch1, byte_size(B)), + <<Acc/binary, B/binary>> + end + end, + <<>>), + + ExpectedData = <<Data/binary, Data/binary>>, + case ReceivedData of + ExpectedData -> + %% ct:log("Correct data returned",[]), + %% receive close messages + loop_until(fun(Left) -> %% ct:log("Expect: ~p",[Left]), + Left == [] + end, + fun([Next|Rest]) -> + receive + {ssh_cm,C,Next} -> Rest + end + end, + [%% Already received: {eof, Ch1}, + {exit_status,Ch1,0}, + {closed,Ch1}] + ), + ok; + _ when is_binary(ReceivedData) -> + ct:fail("~p bytes echoed but ~p expected", [byte_size(ReceivedData), 2*byte_size(Data)]) + end. + +%%-------------------------------------------------------------------- %% Utilities --------------------------------------------------------- %%-------------------------------------------------------------------- -exec_from_docker(WhatEver, {0,0,0,0}, HostPort, Command, Expects, ExtraSshArg) -> - exec_from_docker(WhatEver, host_ip(), HostPort, Command, Expects, ExtraSshArg); +%%-------------------------------------------------------------------- +%% +%% A practical meta function +%% +loop_until(CondFun, DoFun, Acc) -> + case CondFun(Acc) of + true -> + Acc; + false -> + loop_until(CondFun, DoFun, DoFun(Acc)) + end. + +%%-------------------------------------------------------------------- +%% +%% Exec the Command in the docker. Add the arguments ExtraSshArg in the +%% ssh command. +%% +%% If Expects is returned, then return 'ok', else return {fail,Msg}. +%% exec_from_docker(Config, HostIP, HostPort, Command, Expects, ExtraSshArg) when is_binary(hd(Expects)), is_list(Config) -> {DockerIP,DockerPort} = ip_port(Config), {ok,C} = ssh:connect(DockerIP, DockerPort, [{user,?USER}, - {password,?PWD}, + {password,?PASSWD}, {user_dir, new_dir(Config)}, {silently_accept_hosts,true}, {user_interaction,false} ]), - R = exec_from_docker(C, HostIP, HostPort, Command, Expects, ExtraSshArg), + R = exec_from_docker(C, HostIP, HostPort, Command, Expects, ExtraSshArg, Config), ssh:close(C), - R; - -exec_from_docker(C, HostIP, HostPort, Command, Expects, ExtraSshArg) when is_binary(hd(Expects)) -> - SSH_from_docker = - lists:concat(["sshpass -p ",?PWD," ", - "/buildroot/ssh/bin/ssh -p ",HostPort," -o 'CheckHostIP=no' -o 'StrictHostKeyChecking=no' ", - ExtraSshArg," ", - inet_parse:ntoa(HostIP)," " - ]), - ExecCommand = SSH_from_docker ++ Command, - R = exec(C, ExecCommand), - case R of - {ok,{ExitStatus,Result}} when ExitStatus == 0 -> + R. + +exec_from_docker(C, DestIP, DestPort, Command, Expects, ExtraSshArg, Config) when is_binary(hd(Expects)) -> + ExecCommand = + lists:concat( + ["sshpass -p ",?PASSWD," " + | case proplists:get_value(ssh_version,Config) of + "dropbear" ++ _ -> + ["dbclient -y -y -p ",DestPort," ",ExtraSshArg," ",iptoa(DestIP)," "]; + + _ -> %% OpenSSH or compatible + ["/buildroot/ssh/bin/ssh -o 'CheckHostIP=no' -o 'StrictHostKeyChecking=no' ", + ExtraSshArg," -p ",DestPort," ",iptoa(DestIP)," "] + end]) ++ Command, + + case exec(C, ExecCommand) of + {ok,{ExitStatus,Result}} = R when ExitStatus == 0 -> case binary:match(Result, Expects) of nomatch -> ct:log("Result of~n ~s~nis~n ~p",[ExecCommand,R]), @@ -327,28 +470,26 @@ exec_from_docker(C, HostIP, HostPort, Command, Expects, ExtraSshArg) when is_bin _ -> ok end; - {ok,_} -> + {ok,_} = R -> ct:log("Result of~n ~s~nis~n ~p",[ExecCommand,R]), {fail, "Exit status =/= 0"}; - _ -> + R -> ct:log("Result of~n ~s~nis~n ~p",[ExecCommand,R]), {fail, "Couldn't login to host"} end. - - exec(C, Cmd) -> - ct:log("~s",[Cmd]), + %% ct:log("~s",[Cmd]), {ok,Ch} = ssh_connection:session_channel(C, 10000), success = ssh_connection:exec(C, Ch, Cmd, 10000), - exec_result(C, Ch). + result_of_exec(C, Ch). -exec_result(C, Ch) -> - exec_result(C, Ch, undefined, <<>>). +result_of_exec(C, Ch) -> + result_of_exec(C, Ch, undefined, <<>>). -exec_result(C, Ch, ExitStatus, Acc) -> +result_of_exec(C, Ch, ExitStatus, Acc) -> receive {ssh_cm,C,{closed,Ch}} -> %%ct:log("CHAN ~p got *closed*",[Ch]), @@ -356,29 +497,37 @@ exec_result(C, Ch, ExitStatus, Acc) -> {ssh_cm,C,{exit_status,Ch,ExStat}} when ExitStatus == undefined -> %%ct:log("CHAN ~p got *exit status ~p*",[Ch,ExStat]), - exec_result(C, Ch, ExStat, Acc); + result_of_exec(C, Ch, ExStat, Acc); {ssh_cm,C,{data,Ch,_,Data}=_X} when ExitStatus == undefined -> %%ct:log("CHAN ~p got ~p",[Ch,_X]), - exec_result(C, Ch, ExitStatus, <<Acc/binary, Data/binary>>); + result_of_exec(C, Ch, ExitStatus, <<Acc/binary, Data/binary>>); _Other -> %%ct:log("OTHER: ~p",[_Other]), - exec_result(C, Ch, ExitStatus, Acc) + result_of_exec(C, Ch, ExitStatus, Acc) after 5000 -> - %%ct:log("NO MORE, received so far:~n~s",[Acc]), + ct:log("NO MORE, received so far:~n~s",[Acc]), {error, timeout} end. -chk_all_algos(CommonAlgs, Config, DoTestFun) when is_function(DoTestFun,2) -> +%%-------------------------------------------------------------------- +%% +%% Loop through all {Tag,Alg} pairs in CommonAlgs, call DoTestFun(Tag,Alg) which +%% returns one of {ok,C}, ok, or Other. +%% +%% The chk_all_algos returns 'ok' or {fail,FaledAlgosList} +%% + +chk_all_algos(FunctionName, CommonAlgs, Config, DoTestFun) when is_function(DoTestFun,2) -> ct:comment("~p algorithms",[length(CommonAlgs)]), %% Check each algorithm Failed = lists:foldl( fun({Tag,Alg}, FailedAlgos) -> - ct:log("Try ~p",[Alg]), + %% ct:log("Try ~p",[Alg]), case DoTestFun(Tag,Alg) of {ok,C} -> ssh:close(C), @@ -387,10 +536,10 @@ chk_all_algos(CommonAlgs, Config, DoTestFun) when is_function(DoTestFun,2) -> FailedAlgos; Other -> ct:log("FAILED! ~p ~p: ~p",[Tag,Alg,Other]), - [Alg|FailedAlgos] + [{Alg,Other}|FailedAlgos] end end, [], CommonAlgs), - ct:pal("~s", [format_result_table_use_all_algos(Config, CommonAlgs, Failed)]), + ct:pal("~s", [format_result_table_use_all_algos(FunctionName, Config, CommonAlgs, Failed)]), case Failed of [] -> ok; @@ -398,6 +547,41 @@ chk_all_algos(CommonAlgs, Config, DoTestFun) when is_function(DoTestFun,2) -> {fail, Failed} end. + + +%%%---------------------------------------------------------------- +%%% +%%% Call all Funs as Fun() which returns 'ok', {ok,C} or Other. +%%% do/1 returns 'ok' or the first encountered value that is not +%%% successful. +%%% + +do(Funs) -> + do(Funs, 1). + +do([Fun|Funs], N) -> + case Fun() of + ok -> + %% ct:log("Fun ~p ok",[N]), + do(Funs, N-1); + {ok,C} -> + %% ct:log("Fun ~p {ok,C}",[N]), + ssh:close(C), + do(Funs, N-1); + Other -> + ct:log("Fun ~p FAILED:~n~p",[N, Other]), + Other + end; + +do([], _) -> + %% ct:log("All Funs ok",[]), + ok. + +%%-------------------------------------------------------------------- +%% +%% Functions to set up local and remote host's and user's keys and directories +%% + setup_local_hostdir(KeyAlg, Config) -> setup_local_hostdir(KeyAlg, new_dir(Config), Config). setup_local_hostdir(KeyAlg, HostDir, Config) -> @@ -428,7 +612,7 @@ setup_remote_auth_keys_and_local_priv(KeyAlg, IP, Port, UserDir, Config) -> ok = file:write_file(DstFile++".pub", Publ), %% Remote auth_methods with public key {ok,Ch,Cc} = ssh_sftp:start_channel(IP, Port, [{user, ?USER }, - {password, ?PWD }, + {password, ?PASSWD }, {auth_methods, "password"}, {silently_accept_hosts,true}, {user_interaction,false} @@ -460,7 +644,7 @@ setup_remote_priv_and_local_auth_keys(KeyAlg, IP, Port, UserDir, Config) -> ok = file:write_file(AuthKeyFile, Publ), %% Remote private and public key {ok,Ch,Cc} = ssh_sftp:start_channel(IP, Port, [{user, ?USER }, - {password, ?PWD }, + {password, ?PASSWD }, {auth_methods, "password"}, {silently_accept_hosts,true}, {user_interaction,false} @@ -485,6 +669,7 @@ priv_pub_keys(KeySubDir, Type, Config, KeyAlg) -> {ok, {Priv,Publ}}. +%%%---------------- The default filenames src_filename(user, 'ssh-rsa' ) -> "id_rsa"; src_filename(user, 'rsa-sha2-256' ) -> "id_rsa"; src_filename(user, 'rsa-sha2-512' ) -> "id_rsa"; @@ -516,7 +701,11 @@ dst_filename(host, 'ecdsa-sha2-nistp384') -> "ssh_host_ecdsa_key"; dst_filename(host, 'ecdsa-sha2-nistp521') -> "ssh_host_ecdsa_key". -format_result_table_use_all_algos(Config, CommonAlgs, Failed) -> +%%-------------------------------------------------------------------- +%% +%% Format the result table for chk_all_algos/4 +%% +format_result_table_use_all_algos(FunctionName, Config, CommonAlgs, Failed) -> %% Write a nice table with the result AlgHead = 'Algorithm', AlgWidth = lists:max([length(atom_to_list(A)) || {_,A} <- CommonAlgs]), @@ -529,23 +718,25 @@ format_result_table_use_all_algos(Config, CommonAlgs, Failed) -> end, {io_lib:format('~s ~*s ~s~n', [Tag, -AlgWidth, A, - case lists:member(A,Failed) of - true -> "<<<< FAIL <<<<"; - false-> "(ok)" + case proplists:get_value(A,Failed) of + undefined -> "(ok)"; + Err -> io_lib:format("<<<< FAIL <<<< ~p",[Err]) end]), T} end, undefined, CommonAlgs), Vssh = proplists:get_value(ssh_version,Config,""), - Vssl = proplists:get_value(ssl_version,Config,""), - io_lib:format("~nResults, Peer versions: ~s and ~s~n" + io_lib:format("~nResults of ~p, Peer version: ~s~n~n" "Tag ~*s Result~n" "=====~*..=s=======~n~s" - ,[Vssh,Vssl, - -AlgWidth,AlgHead, + ,[FunctionName, Vssh, + -AlgWidth, AlgHead, AlgWidth, "", ResultTable]). - +%%-------------------------------------------------------------------- +%% +%% Docker handling: start_docker/1 and stop_docker/1 +%% start_docker(Ver) -> Cmnd = lists:concat(["docker run -itd --rm -p 1234 ",?DOCKER_PFX,":",Ver]), Id0 = os:cmd(Cmnd), @@ -572,6 +763,10 @@ is_docker_sha(L) -> (_) -> false end, L). +%%-------------------------------------------------------------------- +%% +%% Misc docker info functions + ip_port(Config) -> {_Ver,{IP,Port},_} = proplists:get_value(id,Config), {IP,Port}. @@ -590,6 +785,23 @@ ip(Id) -> {ok,IP} = inet:parse_address(IPstr), IP. +%%-------------------------------------------------------------------- +%% +%% Normalize the host returned from ssh_test_lib + +iptoa({0,0,0,0}) -> inet_parse:ntoa(host_ip()); +iptoa(IP) -> inet_parse:ntoa(IP). + +host_ip() -> + {ok,Name} = inet:gethostname(), + {ok,#hostent{h_addr_list = [IP|_]}} = inet_res:gethostbyname(Name), + IP. + +%%-------------------------------------------------------------------- +%% +%% Create a new fresh directory or clear an existing one +%% + new_dir(Config) -> PrivDir = proplists:get_value(priv_dir, Config), SubDirName = integer_to_list(erlang:system_time()), @@ -626,20 +838,34 @@ delete_all_contents(Dir) -> end end, Fs). +%%-------------------------------------------------------------------- +%% +%% Find the intersection of algoritms for otp ssh and the docker ssh. +%% Returns {ok, ServerHello, Server, ClientHello, Client} where Server are the algorithms common +%% with the docker server and analogous for Client. +%% +%% Client may be undefined if no usable client is found. +%% +%% Both Server and Client are lists of {Tag,AlgName}. +%% + common_algs(Config, IP, Port) -> case remote_server_algs(IP, Port) of - {ok, {RemoteHelloBin, RemoteServerKexInit}} -> + {ok, {ServerHello, RemoteServerKexInit}} -> + RemoteServerAlgs = kexint_msg2default_algorithms(RemoteServerKexInit), + Server = find_common_algs(RemoteServerAlgs, + use_algorithms(ServerHello)), + ct:log("Remote server:~n~p~n~p",[ServerHello, RemoteServerAlgs]), case remote_client_algs(Config) of - {ok,{_Hello,RemoteClientKexInit}} -> - RemoteServerAlgs = kexint_msg2default_algorithms(RemoteServerKexInit), - Server = find_common_algs(RemoteServerAlgs, - use_algorithms(RemoteHelloBin)), + {ok,{ClientHello,RemoteClientKexInit}} -> RemoteClientAlgs = kexint_msg2default_algorithms(RemoteClientKexInit), Client = find_common_algs(RemoteClientAlgs, - use_algorithms(RemoteHelloBin)), - ct:log("Docker server algorithms:~n ~p~n~nDocker client algorithms:~n ~p", - [RemoteServerAlgs,RemoteClientAlgs]), - {ok, Server, Client}; + use_algorithms(ClientHello)), + ct:log("Remote client:~n~p~n~p",[ClientHello, RemoteClientAlgs]), + {ok, ServerHello, Server, ClientHello, Client}; + {error,_} =TO -> + ct:log("Remote client algs can't be found: ~p",[TO]), + {ok, ServerHello, Server, undefined, undefined}; Other -> Other end; @@ -648,6 +874,24 @@ common_algs(Config, IP, Port) -> end. +chk_hellos(Hs, Str) -> + lists:foldl( + fun(H, Acc) -> + try binary:split(H, <<"-">>, [global]) + of + %% [<<"SSH">>,<<"2.0">>|_] -> + %% Acc; + [<<"SSH">>,OldVer = <<"1.",_/binary>>|_] -> + io_lib:format("~s, Old SSH ver ~s",[Acc,OldVer]); + _ -> + Acc + catch + _:_ -> + Acc + end + end, Str, Hs). + + find_common_algs(Remote, Local) -> [{T,V} || {T,Vs} <- ssh_test_lib:extract_algos( ssh_test_lib:intersection(Remote, @@ -685,12 +929,18 @@ kexint_msg2default_algorithms(#ssh_msg_kexinit{kex_algorithms = Kex, {server2client,ssh_test_lib:to_atoms(CompS2C)}]}]. - +%%-------------------------------------------------------------------- +%% +%% Find the algorithms supported by the remote server +%% +%% Connect with tcp to the server, send a hello and read the returned +%% server hello and kexinit message. +%% remote_server_algs(IP, Port) -> case try_gen_tcp_connect(IP, Port, 5) of {ok,S} -> ok = gen_tcp:send(S, "SSH-2.0-CheckAlgs\r\n"), - receive_hello(S, <<>>); + receive_hello(S); {error,Error} -> {error,Error} end. @@ -709,6 +959,13 @@ try_gen_tcp_connect(_, _, _) -> {error, "No contact"}. +%%-------------------------------------------------------------------- +%% +%% Find the algorithms supported by the remote client +%% +%% Set up a fake ssh server and make the remote client connect to it. Use +%% hello message and the kexinit message. +%% remote_client_algs(Config) -> Parent = self(), Ref = make_ref(), @@ -719,7 +976,7 @@ remote_client_algs(Config) -> Parent ! {addr,Ref,IP,Port}, {ok,S} = gen_tcp:accept(Sl), ok = gen_tcp:send(S, "SSH-2.0-CheckAlgs\r\n"), - Parent ! {Ref,receive_hello(S, <<>>)} + Parent ! {Ref,receive_hello(S)} end), receive {addr,Ref,IP,Port} -> @@ -732,14 +989,28 @@ remote_client_algs(Config) -> receive {Ref, Result} -> Result - after 15000 -> - {error, timeout2} + after 5000 -> + {error, {timeout,2}} end - after 15000 -> - {error, timeout1} + after 5000 -> + {error, {timeout,1}} end. +%%% Receive a few packets from the remote server or client and find what is supported: + +receive_hello(S) -> + try + receive_hello(S, <<>>) + of + Result -> + Result + catch + Class:Error -> + ST = erlang:get_stacktrace(), + {error, {Class,Error,ST}} + end. + receive_hello(S, Ack) -> %% The Ack is to collect bytes until the full message is received @@ -747,20 +1018,19 @@ receive_hello(S, Ack) -> {tcp, S, Bin0} when is_binary(Bin0) -> case binary:split(<<Ack/binary, Bin0/binary>>, [<<"\r\n">>,<<"\r">>,<<"\n">>]) of [Hello = <<"SSH-2.0-",_/binary>>, NextPacket] -> - ct:log("Got 2.0 hello (~p), ~p bytes to next msg",[Hello,size(NextPacket)]), + %% ct:log("Got 2.0 hello (~p), ~p bytes to next msg",[Hello,size(NextPacket)]), {ok, {Hello, receive_kexinit(S, NextPacket)}}; [Hello = <<"SSH-1.99-",_/binary>>, NextPacket] -> - ct:comment("Old SSH ~s",["1.99"]), - ct:log("Got 1.99 hello (~p), ~p bytes to next msg",[Hello,size(NextPacket)]), + %% ct:log("Got 1.99 hello (~p), ~p bytes to next msg",[Hello,size(NextPacket)]), {ok, {Hello, receive_kexinit(S, NextPacket)}}; [Bin] when size(Bin) < 256 -> - ct:log("Got part of hello (~p chars):~n~s~n~s",[size(Bin),Bin, - [io_lib:format('~2.16.0b ',[C]) - || C <- binary_to_list(Bin0) - ] - ]), + %% ct:log("Got part of hello (~p chars):~n~s~n~s",[size(Bin),Bin, + %% [io_lib:format('~2.16.0b ',[C]) + %% || C <- binary_to_list(Bin0) + %% ] + %% ]), receive_hello(S, Bin0); _ -> @@ -804,11 +1074,326 @@ receive_kexinit(S, Ack) -> throw(timeout) end. +%%%---------------------------------------------------------------- +%%% Test of sftp from the OpenSSH client side +%%% +sftp_tests_erl_server(Config, ServerIP, ServerPort, ServerRootDir, UserDir) -> + try + Cmnds = prepare_local_directory(ServerRootDir), + call_sftp_in_docker(Config, ServerIP, ServerPort, Cmnds, UserDir), + check_local_directory(ServerRootDir) + catch + Class:Error -> + ST = erlang:get_stacktrace(), + {error, {Class,Error,ST}} + end. -host_ip() -> - {ok,Name} = inet:gethostname(), - {ok,#hostent{h_addr_list = [IP|_]}} = inet_res:gethostbyname(Name), - IP. +prepare_local_directory(ServerRootDir) -> + file:write_file(filename:join(ServerRootDir,"tst1"), + <<"Some test text">> + ), + ["get tst1", + "put tst1 tst2", + "put tst1 tst3", + "rename tst1 ex_tst1", + "rm tst3", + "mkdir mydir", + "cd mydir", + "put tst1 file_1", + "put tst1 unreadable_file", + "chmod 222 unreadable_file", + "exit"]. + +check_local_directory(ServerRootDir) -> + case lists:sort(ok(file:list_dir(ServerRootDir)) -- [".",".."]) of + ["ex_tst1","mydir","tst2"] -> + {ok,Expect} = file:read_file(filename:join(ServerRootDir,"ex_tst1")), + case file:read_file(filename:join(ServerRootDir,"tst2")) of + {ok,Expect} -> + case lists:sort(ok(file:list_dir(filename:join(ServerRootDir,"mydir"))) -- [".",".."]) of + ["file_1","unreadable_file"] -> + case file:read_file(filename:join([ServerRootDir,"mydir","file_1"])) of + {ok,Expect} -> + case file:read_file(filename:join([ServerRootDir,"mydir","unreadable_file"])) of + {error,_} -> + ok; + {ok,_} -> + {error, {could_read_unreadable,"mydir/unreadable_file"}} + end; + {ok,Other} -> + ct:log("file_1:~n~s~nExpected:~n~s",[Other,Expect]), + {error, {bad_contents_in_file,"mydir/file_1"}} + end; + Other -> + ct:log("Directory ~s~n~p",[filename:join(ServerRootDir,"mydir"),Other]), + {error,{bad_dir_contents,"mydir"}} + end; + {ok,Other} -> + ct:log("tst2:~n~s~nExpected:~n~s",[Other,Expect]), + {error, {bad_contents_in_file,"tst2"}} + end; + ["tst1"] -> + {error,{missing_file,"tst2"}}; + Other -> + ct:log("Directory ~s~n~p",[ServerRootDir,Other]), + {error,{bad_dir_contents,"/"}} + end. + +call_sftp_in_docker(Config, ServerIP, ServerPort, Cmnds, UserDir) -> + {DockerIP,DockerPort} = ip_port(Config), + {ok,C} = ssh:connect(DockerIP, DockerPort, + [{user,?USER}, + {password,?PASSWD}, + {user_dir, UserDir}, + {silently_accept_hosts,true}, + {user_interaction,false} + ]), + + %% Make commands for "expect" in the docker: + PreExpectCmnds = ["spawn /buildroot/ssh/bin/sftp -oPort="++integer_to_list(ServerPort)++ + " -oCheckHostIP=no -oStrictHostKeyChecking=no " ++ + iptoa(ServerIP)++"\n" + ], + PostExpectCmnds= [], + ExpectCmnds = + PreExpectCmnds ++ + ["expect \"sftp>\" {send \""++Cmnd++"\n\"}\n" || Cmnd <- Cmnds] ++ + PostExpectCmnds, + + %% Make an commands file in the docker + {ok,Ch} = ssh_sftp:start_channel(C, [{timeout,10000}]), + ok = ssh_sftp:write_file(Ch, "commands", erlang:iolist_to_binary(ExpectCmnds)), + ok = ssh_sftp:stop_channel(Ch), + + %% Call expect in the docker + {ok, Ch1} = ssh_connection:session_channel(C, infinity), + Kex1 = renegotiate_test(init, C), + success = ssh_connection:exec(C, Ch1, "expect commands", infinity), + + renegotiate_test(Kex1, C), + recv_log_msgs(C, Ch1), + + %% Done. + ssh:close(C). + +recv_log_msgs(C, Ch) -> + receive + {ssh_cm,C,{closed,Ch}} -> + %% ct:log("Channel closed ~p",[{closed,1}]), + ok; + {ssh_cm,C,{data,Ch,1,Msg}} -> + ct:log("*** ERROR from docker:~n~s",[Msg]), + recv_log_msgs(C, Ch); + {ssh_cm,C,_Msg} -> + %% ct:log("Got ~p",[_Msg]), + recv_log_msgs(C, Ch) + end. +%%%---------------------------------------------------------------- +%%%---------------------------------------------------------------- +%%% +%%% Tests from the Erlang client side +%%% +%%%---------------------------------------------------------------- +%%%---------------------------------------------------------------- +test_erl_client_reneg({ok,C}, Spec) -> + %% Start the test processes on the connection C: + Parent = self(), + Pids = [spawn( + fun() -> + Parent ! {self(), TestType, Id, one_test_erl_client(TestType,Id,C)} + end + ) + || {TestType,N} <- Spec, + Id <- lists:seq(1,N)], + + Kex1 = renegotiate_test(init, C), + + %% Collect the results: + case lists:filter( + fun(R) -> R=/=ok end, + [receive + {Pid,_TestType,_Id,ok} -> + %% ct:log("Test ~p:~p passed!", [_TestType,_Id]), + ok; + {Pid,TestType,Id,OtherResult} -> + ct:log("~p:~p ~p ~p~n~p",[?MODULE,?LINE,TestType,Id,OtherResult]), + {error,TestType,Id} + end || Pid <- Pids]) + of + [] -> + renegotiate_test(Kex1, C), + {ok,C}; + Other -> + renegotiate_test(Kex1, C), + Other + end; + +test_erl_client_reneg(Error, _) -> + Error. + + +one_test_erl_client(exec, Id, C) -> + {ok, Ch} = ssh_connection:session_channel(C, infinity), + success = ssh_connection:exec(C, Ch, "echo Hi there", 5000), + case loop_until(fun({eof,_}) -> true; + (_ ) -> false + end, + fun(Acc) -> + receive + {ssh_cm, C, {eof,Ch}} -> + {eof,Acc}; + {ssh_cm, C, {data,Ch,0,B}} when is_binary(B) -> + <<Acc/binary, B/binary>> + end + end, + <<>>) of + {eof,<<"Hi there\n">>} -> + ok; + Other -> + ct:pal("exec Got other ~p", [Other]), + {error, {exec,Id,bad_msg,Other,undefined}} + end; + +one_test_erl_client(no_subsyst, Id, C) -> + {ok, Ch} = ssh_connection:session_channel(C, infinity), + case ssh_connection:subsystem(C, Ch, "foo", infinity) of + failure -> + ok; + Other -> + ct:pal("no_subsyst Got other ~p", [Other]), + {error, {no_subsyst,Id,bad_ret,Other,undefined}} + end; + +one_test_erl_client(setenv, Id, C) -> + {ok, Ch} = ssh_connection:session_channel(C, infinity), + Var = "ENV_TEST", + Value = lists:concat(["env_test_",Id,"_",erlang:system_time()]), + Env = case ssh_connection:setenv(C, Ch, Var, Value, infinity) of + success -> binary_to_list(Value++"\n"); + failure -> <<"\n">> + end, + success = ssh_connection:exec(C, Ch, "echo $"++Var, 5000), + case loop_until(fun({eof,_}) -> true; + (_ ) -> false + end, + fun(Acc) -> + receive + {ssh_cm, C, {eof,Ch}} -> + {eof,Acc}; + {ssh_cm, C, {data,Ch,0,B}} when is_binary(B) -> + <<Acc/binary, B/binary>> + end + end, + <<>>) of + {eof,Env} -> + ok; + Other -> + ct:pal("setenv Got other ~p", [Other]), + {error, {setenv,Id,bad_msg,Other,undefined}} + end; + +one_test_erl_client(SFTP, Id, C) when SFTP==sftp ; SFTP==sftp_async -> + try + {ok,Ch} = ssh_sftp:start_channel(C, [{timeout,10000}]), + %% A new fresh name of a new file tree: + RootDir = lists:concat(["r_",Id,"_",erlang:system_time()]), + %% Check that it does not exist: + false = lists:member(RootDir, ok(ssh_sftp:list_dir(Ch, "."))), + %% Create it: + ok = ssh_sftp:make_dir(Ch, RootDir), + {ok, #file_info{type=directory, access=read_write}} = ssh_sftp:read_file_info(Ch, RootDir), + R = do_sftp_tests_erl_client(SFTP, C, Ch, Id, RootDir), + catch ssh_sftp:stop_channel(Ch), + R + catch + Class:Error -> + ST = erlang:get_stacktrace(), + {error, {SFTP,Id,Class,Error,ST}} + end. + + + +do_sftp_tests_erl_client(sftp_async, _C, Ch, _Id, RootDir) -> + FileName1 = "boring_name", + F1 = filename:join(RootDir, FileName1), + %% Open a new handle and start writing: + {ok,Handle1} = ssh_sftp:open(Ch, F1, [write,binary]), + {async,Aref1} = ssh_sftp:awrite(Ch, Handle1, <<0:250000/unsigned-unit:8>>), + wait_for_async_result(Aref1); + +do_sftp_tests_erl_client(sftp, _C, Ch, _Id, RootDir) -> + FileName0 = "f0", + F0 = filename:join(RootDir, FileName0), + + %% Create and write a file: + ok = ssh_sftp:write_file(Ch, + F0 = filename:join(RootDir, FileName0), + Data0 = mkbin(1234,240)), + {ok,Data0} = ssh_sftp:read_file(Ch, F0), + {ok, #file_info{type=regular, access=read_write, size=1234}} = ssh_sftp:read_file_info(Ch, F0), + + %% Re-write: + {ok,Handle0} = ssh_sftp:open(Ch, F0, [write,read,binary]), + ok = ssh_sftp:pwrite(Ch, Handle0, 16, Data0_1=mkbin(10,255)), + + <<B1:16/binary, _:10/binary, B2:(1234-26)/binary>> = Data0, + FileContents = <<B1:16/binary, Data0_1:10/binary, B2:(1234-26)/binary>>, + + <<_:1/binary, Part:25/binary, _/binary>> = FileContents, + {ok, Part} = ssh_sftp:pread(Ch, Handle0, 1, 25), + + %% Check: + {ok, FileContents} = ssh_sftp:pread(Ch, Handle0, 0, 1234), + ok = ssh_sftp:close(Ch, Handle0), + + %% Check in another way: + {ok, FileContents} = ssh_sftp:read_file(Ch, F0), + + %% Remove write access rights and check that it can't be written: + ok = ssh_sftp:write_file_info(Ch, F0, #file_info{mode=8#400}), %read}), + {ok, #file_info{type=regular, access=read}} = ssh_sftp:read_file_info(Ch, F0), + {error,permission_denied} = ssh_sftp:write_file(Ch, F0, mkbin(10,14)), + + %% Test deletion of file and dir: + [FileName0] = ok(ssh_sftp:list_dir(Ch, RootDir)) -- [".", ".."], + ok = ssh_sftp:delete(Ch, F0), + [] = ok(ssh_sftp:list_dir(Ch, RootDir)) -- [".", ".."], + ok = ssh_sftp:del_dir(Ch, RootDir), + false = lists:member(RootDir, ok(ssh_sftp:list_dir(Ch, "."))), + ok. + + +wait_for_async_result(Aref) -> + receive + {async_reply, Aref, Result} -> + Result + after + 60000 -> + timeout + end. + + +mkbin(Size, Byte) -> + list_to_binary(lists:duplicate(Size,Byte)). + +ok({ok,X}) -> X. + +%%%---------------------------------------------------------------- +renegotiate_test(init, ConnectionRef) -> + Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), + ssh_connection_handler:renegotiate(ConnectionRef), + %%ct:log("Renegotiate test initiated!",[]), + Kex1; + +renegotiate_test(Kex1, ConnectionRef) -> + case ssh_test_lib:get_kex_init(ConnectionRef) of + Kex1 -> + ct:log("Renegotiate test failed, Kex1 == Kex2!",[]), + error(renegotiate_failed); + _ -> + %% ct:log("Renegotiate test passed!",[]), + ok + end. diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh new file mode 100755 index 0000000000..85973081d0 --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh @@ -0,0 +1,28 @@ +#!/bin/sh + +# ./create-dropbear-ssh + +# This way of fetching the tar-file separate from the docker commands makes +# http-proxy handling way easier. The wget command handles the $https_proxy +# variable while the docker command must have /etc/docker/something changed +# and the docker server restarted. That is not possible without root access. + +# Make a Dockerfile. This method simplifies env variable handling considerably: +cat - > TempDockerFile <<EOF + + FROM ubuntubuildbase + + WORKDIR /buildroot + + RUN apt-get -y update + RUN apt-get -y upgrade + RUN apt-get -y install openssh-sftp-server +%% RUN echo 81 | apt-get -y install dropbear + +EOF + +# Build the image: +docker build -t ssh_compat_suite-ssh-dropbear -f ./TempDockerFile . + +# Cleaning +rm -fr ./TempDockerFile $TMP diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh-run b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh-run new file mode 100755 index 0000000000..d98c0cfaa3 --- /dev/null +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-dropbear-ssh-run @@ -0,0 +1,27 @@ +#!/bin/sh + +# ./create-dropbear-ssh-run + +VER=v2016.72 + +# This way of fetching the tar-file separate from the docker commands makes +# http-proxy handling way easier. The wget command handles the $https_proxy +# variable while the docker command must have /etc/docker/something changed +# and the docker server restarted. That is not possible without root access. + +# Make a Dockerfile. This method simplifies env variable handling considerably: +cat - > TempDockerFile <<EOF + + FROM ssh_compat_suite-ssh-dropbear-installed:${VER} + + WORKDIR /buildroot + + CMD dropbear -F -p 1234 + +EOF + +# Build the image: +docker build -t ssh_compat_suite-ssh:dropbear${VER} -f ./TempDockerFile . + +# Cleaning +rm -fr ./TempDockerFile $TMP diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssh-image b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssh-image index 983c57b18b..2e08408841 100755 --- a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssh-image +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssh-image @@ -47,7 +47,8 @@ cat - > TempDockerFile <<EOF RUN ./configure --without-pie \ --prefix=/buildroot/ssh \ --with-ssl-dir=/buildroot/ssl \ - --with-pam + --with-pam \ + LDFLAGS=-Wl,-R/buildroot/ssl/lib RUN make RUN make install RUN echo UsePAM yes >> /buildroot/ssh/etc/sshd_config diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssl-image b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssl-image index 66f8358b8a..4ab2a8bddc 100755 --- a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssl-image +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create-ssl-image @@ -23,6 +23,16 @@ case "$1" in ;; esac +case $1$2 in + openssl0.9.8[a-l]) + CONFIG_FLAGS=no-asm + ;; + *) + CONFIG_FLAGS= + ;; +esac + + # This way of fetching the tar-file separate from the docker commands makes # http-proxy handling way easier. The wget command handles the $https_proxy # variable while the docker command must have /etc/docker/something changed @@ -42,10 +52,10 @@ cat - > TempDockerFile <<EOF WORKDIR ${FAM}-${VER} - RUN ./config --prefix=/buildroot/ssl + RUN ./config --prefix=/buildroot/ssl ${CONFIG_FLAGS} RUN make - RUN make install + RUN make install_sw RUN echo Built ${FAM}-${VER} EOF diff --git a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all index 16b9c21d9f..0dcf8cb570 100755 --- a/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all +++ b/lib/ssh/test/ssh_compat_SUITE_data/build_scripts/create_all @@ -3,19 +3,21 @@ UBUNTU_VERSION=16.04 SSH_SSL_VERSIONS=(\ - openssh 4.4p1 openssl 0.9.8zh \ - openssh 4.5p1 openssl 0.9.8zh \ - openssh 5.0p1 openssl 0.9.8zh \ - openssh 6.2p2 openssl 0.9.8zh \ - openssh 6.3p1 openssl 0.9.8zh \ - \ - openssh 7.1p1 openssl 1.0.0t \ - \ - openssh 7.1p1 openssl 1.0.1p \ - \ - openssh 6.6p1 openssl 1.0.2n \ - openssh 7.1p1 openssl 1.0.2n \ - openssh 7.6p1 openssl 1.0.2n \ + openssh 4.4p1 openssl 0.9.8c \ + openssh 4.5p1 openssl 0.9.8m \ + openssh 5.0p1 openssl 0.9.8za \ + openssh 6.2p2 openssl 0.9.8c \ + openssh 6.3p1 openssl 0.9.8zh \ + \ + openssh 7.1p1 openssl 1.0.0a \ + \ + openssh 7.1p1 openssl 1.0.1p \ + \ + openssh 6.6p1 openssl 1.0.2n \ + openssh 7.1p1 openssl 1.0.2n \ + openssh 7.6p1 openssl 1.0.2n \ + \ + openssh 7.6p1 libressl 2.6.4 \ ) if [ "x$1" == "x-b" ] diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index ba4518cfe6..9587c0c251 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -45,6 +45,8 @@ all() -> {group, openssh}, small_interrupted_send, interrupted_send, + exec_erlang_term, + exec_erlang_term_non_default_shell, start_shell, start_shell_exec, start_shell_exec_fun, @@ -85,6 +87,7 @@ init_per_suite(Config) -> ?CHECK_CRYPTO(Config). end_per_suite(Config) -> + catch ssh:stop(), Config. %%-------------------------------------------------------------------- @@ -542,6 +545,79 @@ start_shell_exec(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- +exec_erlang_term(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = proplists:get_value(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"} + ]), + + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir}]), + + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + + success = ssh_connection:exec(ConnectionRef, ChannelId0, + "1+2.", infinity), + TestResult = + receive + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"3",_/binary>>}} = R -> + ct:log("Got expected ~p",[R]); + Other -> + ct:log("Got unexpected ~p",[Other]) + after 5000 -> + {fail,"Exec Timeout"} + end, + + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid), + TestResult. + +%%-------------------------------------------------------------------- +exec_erlang_term_non_default_shell(Config) when is_list(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = proplists:get_value(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {shell, fun(U, H) -> start_our_shell(U, H) end} + ]), + + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir} + ]), + + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + + success = ssh_connection:exec(ConnectionRef, ChannelId0, + "1+2.", infinity), + TestResult = + receive + {ssh_cm, ConnectionRef, {data, _ChannelId, 0, <<"3",_/binary>>}} = R -> + ct:log("Got unexpected ~p",[R]), + {fail,"Could exec erlang term although non-erlang shell"}; + Other -> + ct:log("Got expected ~p",[Other]) + after 5000 -> + {fail, "Exec Timeout"} + end, + + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid), + TestResult. + +%%-------------------------------------------------------------------- start_shell_exec_fun() -> [{doc, "start shell to exec command"}]. @@ -800,6 +876,8 @@ stop_listener(Config) when is_list(Config) -> ssh:stop_daemon(Pid0), ssh:stop_daemon(Pid1); Error -> + ssh:close(ConnectionRef0), + ssh:stop_daemon(Pid0), ct:fail({unexpected, Error}) end. @@ -819,11 +897,22 @@ start_subsystem_on_closed_channel(Config) -> {user_interaction, false}, {user_dir, UserDir}]), - {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), - ok = ssh_connection:close(ConnectionRef, ChannelId), + {ok, ChannelId1} = ssh_connection:session_channel(ConnectionRef, infinity), + ok = ssh_connection:close(ConnectionRef, ChannelId1), + {error, closed} = ssh_connection:ptty_alloc(ConnectionRef, ChannelId1, []), + {error, closed} = ssh_connection:subsystem(ConnectionRef, ChannelId1, "echo_n", 5000), + {error, closed} = ssh_connection:exec(ConnectionRef, ChannelId1, "testing1.\n", 5000), + {error, closed} = ssh_connection:send(ConnectionRef, ChannelId1, "exit().\n", 5000), - {error, closed} = ssh_connection:subsystem(ConnectionRef, ChannelId, "echo_n", infinity), + %% Test that there could be a gap between close and an operation (Bugfix OTP-14939): + {ok, ChannelId2} = ssh_connection:session_channel(ConnectionRef, infinity), + ok = ssh_connection:close(ConnectionRef, ChannelId2), + timer:sleep(2000), + {error, closed} = ssh_connection:ptty_alloc(ConnectionRef, ChannelId2, []), + {error, closed} = ssh_connection:subsystem(ConnectionRef, ChannelId2, "echo_n", 5000), + {error, closed} = ssh_connection:exec(ConnectionRef, ChannelId2, "testing1.\n", 5000), + {error, closed} = ssh_connection:send(ConnectionRef, ChannelId2, "exit().\n", 5000), ssh:close(ConnectionRef), ssh:stop_daemon(Pid). diff --git a/lib/ssh/test/ssh_engine_SUITE.erl b/lib/ssh/test/ssh_engine_SUITE.erl index daf93891e9..c131a70973 100644 --- a/lib/ssh/test/ssh_engine_SUITE.erl +++ b/lib/ssh/test/ssh_engine_SUITE.erl @@ -55,16 +55,22 @@ basic_tests() -> init_per_suite(Config) -> ssh:start(), ?CHECK_CRYPTO( - case load_engine() of - {ok,E} -> - [{engine,E}|Config]; - {error, notsup} -> - {skip, "Engine not supported on this OpenSSL version"}; - {error, bad_engine_id} -> - {skip, "Dynamic Engine not supported"}; - Other -> - ct:log("Engine load failed: ~p",[Other]), - {fail, "Engine load failed"} + case crypto:info_lib() of + [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}] -> + {skip, "Strange Engine stuff"}; + + _ -> + case load_engine() of + {ok,E} -> + [{engine,E}|Config]; + {error, notsup} -> + {skip, "Engine not supported on this OpenSSL version"}; + {error, bad_engine_id} -> + {skip, "Dynamic Engine not supported"}; + Other -> + ct:log("Engine load failed: ~p",[Other]), + {fail, "Engine load failed"} + end end ). diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 144ec7f8fd..12a85c40aa 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -70,7 +70,8 @@ hostkey_fingerprint_check_sha256/1, hostkey_fingerprint_check_sha384/1, hostkey_fingerprint_check_sha512/1, - hostkey_fingerprint_check_list/1 + hostkey_fingerprint_check_list/1, + save_accepted_host_option/1 ]). %%% Common test callbacks @@ -124,6 +125,7 @@ all() -> id_string_own_string_server, id_string_own_string_server_trail_space, id_string_random_server, + save_accepted_host_option, {group, hardening_tests} ]. @@ -206,32 +208,23 @@ end_per_group(_, Config) -> %%-------------------------------------------------------------------- init_per_testcase(_TestCase, Config) -> ssh:start(), - Config. - -end_per_testcase(TestCase, Config) when TestCase == server_password_option; - TestCase == server_userpassword_option; - TestCase == server_pwdfun_option; - TestCase == server_pwdfun_4_option -> + %% Create a clean user_dir UserDir = filename:join(proplists:get_value(priv_dir, Config), nopubkey), ssh_test_lib:del_dirs(UserDir), - end_per_testcase(Config); -end_per_testcase(_TestCase, Config) -> - end_per_testcase(Config). + file:make_dir(UserDir), + [{user_dir,UserDir}|Config]. -end_per_testcase(_Config) -> +end_per_testcase(_TestCase, Config) -> ssh:stop(), ok. %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -%%-------------------------------------------------------------------- %%% validate to server that uses the 'password' option server_password_option(Config) when is_list(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, UserDir}, @@ -262,12 +255,10 @@ server_password_option(Config) when is_list(Config) -> %%% validate to server that uses the 'password' option server_userpassword_option(Config) when is_list(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {user_passwords, [{"vego", "morot"}]}]), ConnectionRef = @@ -297,15 +288,13 @@ server_userpassword_option(Config) when is_list(Config) -> %%-------------------------------------------------------------------- %%% validate to server that uses the 'pwdfun' option server_pwdfun_option(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), CHKPWD = fun("foo",Pwd) -> Pwd=="bar"; (_,_) -> false end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {pwdfun,CHKPWD}]), ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, @@ -335,9 +324,7 @@ server_pwdfun_option(Config) -> %%-------------------------------------------------------------------- %%% validate to server that uses the 'pwdfun/4' option server_pwdfun_4_option(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), PWDFUN = fun("foo",Pwd,{_,_},undefined) -> Pwd=="bar"; ("fie",Pwd,{_,_},undefined) -> {Pwd=="bar",new_state}; @@ -345,7 +332,7 @@ server_pwdfun_4_option(Config) -> (_,_,_,_) -> false end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {pwdfun,PWDFUN}]), ConnectionRef1 = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, @@ -395,9 +382,7 @@ server_pwdfun_4_option(Config) -> %%-------------------------------------------------------------------- server_pwdfun_4_option_repeat(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), %% Test that the state works Parent = self(), @@ -406,7 +391,7 @@ server_pwdfun_4_option_repeat(Config) -> (_,P,_,S) -> Parent!{P,S}, {false,S+1} end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {auth_methods,"keyboard-interactive"}, {pwdfun,PWDFUN}]), @@ -490,9 +475,7 @@ user_dir_option(Config) -> %%-------------------------------------------------------------------- %%% validate client that uses the 'ssh_msg_debug_fun' option ssh_msg_debug_fun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, @@ -530,9 +513,7 @@ ssh_msg_debug_fun_option_client(Config) -> %%-------------------------------------------------------------------- connectfun_disconnectfun_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -576,9 +557,7 @@ connectfun_disconnectfun_server(Config) -> %%-------------------------------------------------------------------- connectfun_disconnectfun_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -607,9 +586,7 @@ connectfun_disconnectfun_client(Config) -> %%-------------------------------------------------------------------- %%% validate client that uses the 'ssh_msg_debug_fun' option ssh_msg_debug_fun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -651,9 +628,7 @@ ssh_msg_debug_fun_option_server(Config) -> %%-------------------------------------------------------------------- disconnectfun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -686,9 +661,7 @@ disconnectfun_option_server(Config) -> %%-------------------------------------------------------------------- disconnectfun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -720,9 +693,7 @@ disconnectfun_option_client(Config) -> %%-------------------------------------------------------------------- unexpectedfun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -763,9 +734,7 @@ unexpectedfun_option_server(Config) -> %%-------------------------------------------------------------------- unexpectedfun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -840,14 +809,9 @@ supported_hash(HashAlg) -> really_do_hostkey_fingerprint_check(Config, HashAlg) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDirServer = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDirServer), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), - UserDirClient = - ssh_test_lib:create_random_dir(Config), % Ensure no 'known_hosts' disturbs - %% All host key fingerprints. Trust that public_key has checked the ssh_hostkey_fingerprint %% function since that function is used by the ssh client... FPs0 = [case HashAlg of @@ -873,7 +837,7 @@ really_do_hostkey_fingerprint_check(Config, HashAlg) -> %% Start daemon with the public keys that we got fingerprints from {Pid, Host0, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDirServer}, + {user_dir, UserDir}, {password, "morot"}]), Host = ssh_test_lib:ntoa(Host0), FP_check_fun = fun(PeerName, FP) -> @@ -896,7 +860,8 @@ really_do_hostkey_fingerprint_check(Config, HashAlg) -> end}, {user, "foo"}, {password, "morot"}, - {user_dir, UserDirClient}, + {user_dir, UserDir}, + {save_accepted_host, false}, % Ensure no 'known_hosts' disturbs {user_interaction, false}]), ssh:stop_daemon(Pid). @@ -987,9 +952,7 @@ ms_passed(T0) -> %%-------------------------------------------------------------------- ssh_daemon_minimal_remote_max_packet_size_option(Config) -> SystemDir = proplists:get_value(data_dir, Config), - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), {Server, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {user_dir, UserDir}, @@ -1314,6 +1277,33 @@ try_to_connect(Connect, Host, Port, Pid, Tref, N) -> end. %%-------------------------------------------------------------------- +save_accepted_host_option(Config) -> + UserDir = proplists:get_value(user_dir, Config), + KnownHosts = filename:join(UserDir, "known_hosts"), + SysDir = proplists:get_value(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {user_passwords, [{"vego", "morot"}]} + ]), + {error,enoent} = file:read_file(KnownHosts), + + {ok,_C1} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}, + {save_accepted_host, false}, + {user_dir, UserDir}]), + {error,enoent} = file:read_file(KnownHosts), + + {ok,_C2} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}, + {user_dir, UserDir}]), + {ok,_} = file:read_file(KnownHosts), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_sup_SUITE.erl b/lib/ssh/test/ssh_sup_SUITE.erl index 3920a1c592..1df55834b1 100644 --- a/lib/ssh/test/ssh_sup_SUITE.erl +++ b/lib/ssh/test/ssh_sup_SUITE.erl @@ -42,7 +42,9 @@ suite() -> all() -> [default_tree, sshc_subtree, sshd_subtree, sshd_subtree_profile, - killed_acceptor_restarts]. + killed_acceptor_restarts, + shell_channel_tree + ]. groups() -> []. @@ -199,7 +201,7 @@ killed_acceptor_restarts(Config) -> Port2 = ssh_test_lib:daemon_port(DaemonPid2), true = (Port /= Port2), - ct:pal("~s",[lists:flatten(ssh_info:string())]), + ct:log("~s",[lists:flatten(ssh_info:string())]), {ok,[{AccPid,ListenAddr,Port}]} = acceptor_pid(DaemonPid), {ok,[{AccPid2,ListenAddr,Port2}]} = acceptor_pid(DaemonPid2), @@ -216,11 +218,14 @@ killed_acceptor_restarts(Config) -> %% Make acceptor restart: exit(AccPid, kill), + ?wait_match(undefined, process_info(AccPid)), %% Check it is a new acceptor: - {ok,[{AccPid1,ListenAddr,Port}]} = acceptor_pid(DaemonPid), - true = (AccPid /= AccPid1), - true = (AccPid2 /= AccPid1), + ?wait_match({ok,[{AccPid1,ListenAddr,Port}]}, AccPid1=/=AccPid, + acceptor_pid(DaemonPid), + AccPid1, + 500, 30), + AccPid1 =/= AccPid2, %% Connect second client and check it is alive: {ok,C2} = ssh:connect("localhost", Port, [{silently_accept_hosts, true}, @@ -230,21 +235,113 @@ killed_acceptor_restarts(Config) -> {user_dir, UserDir}]), [{client_version,_}] = ssh:connection_info(C2,[client_version]), - ct:pal("~s",[lists:flatten(ssh_info:string())]), + ct:log("~s",[lists:flatten(ssh_info:string())]), %% Check first client is still alive: [{client_version,_}] = ssh:connection_info(C1,[client_version]), ok = ssh:stop_daemon(DaemonPid2), - timer:sleep(15000), + ?wait_match(undefined, process_info(DaemonPid2), 1000, 30), [{client_version,_}] = ssh:connection_info(C1,[client_version]), [{client_version,_}] = ssh:connection_info(C2,[client_version]), ok = ssh:stop_daemon(DaemonPid), - timer:sleep(15000), + ?wait_match(undefined, process_info(DaemonPid), 1000, 30), {error,closed} = ssh:connection_info(C1,[client_version]), {error,closed} = ssh:connection_info(C2,[client_version]). + +%%------------------------------------------------------------------------- +shell_channel_tree(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = proplists:get_value(data_dir, Config), + TimeoutShell = + fun() -> + io:format("TimeoutShell started!~n",[]), + timer:sleep(5000), + ct:log("~p TIMEOUT!",[self()]) + end, + {Daemon, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {shell, fun(_User) -> + spawn(TimeoutShell) + end + } + ]), + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir}]), + + [ChannelSup|_] = Sups0 = chk_empty_con_daemon(Daemon), + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + ok = ssh_connection:shell(ConnectionRef,ChannelId0), + + ?wait_match([{_, GroupPid,worker,[ssh_channel]}], + supervisor:which_children(ChannelSup), + [GroupPid]), + {links,GroupLinks} = erlang:process_info(GroupPid, links), + [ShellPid] = GroupLinks--[ChannelSup], + ct:log("GroupPid = ~p, ShellPid = ~p",[GroupPid,ShellPid]), + + receive + {ssh_cm,ConnectionRef, {data, ChannelId0, 0, <<"TimeoutShell started!\r\n">>}} -> + receive + %%---- wait for the subsystem to terminate + {ssh_cm,ConnectionRef,{closed,ChannelId0}} -> + ct:log("Subsystem terminated",[]), + case {chk_empty_con_daemon(Daemon), + process_info(GroupPid), + process_info(ShellPid)} of + {Sups0, undefined, undefined} -> + %% SUCCESS + ssh:stop_daemon(Daemon); + {Sups0, _, undefined} -> + ssh:stop_daemon(Daemon), + ct:fail("Group proc lives!"); + {Sups0, undefined, _} -> + ssh:stop_daemon(Daemon), + ct:fail("Shell proc lives!"); + _ -> + ssh:stop_daemon(Daemon), + ct:fail("Sup tree changed!") + end + after 10000 -> + ssh:close(ConnectionRef), + ssh:stop_daemon(Daemon), + ct:fail("CLI Timeout") + end + after 10000 -> + ssh:close(ConnectionRef), + ssh:stop_daemon(Daemon), + ct:fail("CLI Timeout") + end. + + +chk_empty_con_daemon(Daemon) -> + ?wait_match([{_,SubSysSup, supervisor,[ssh_subsystem_sup]}, + {{ssh_acceptor_sup,_,_,_}, AccSup, supervisor,[ssh_acceptor_sup]}], + supervisor:which_children(Daemon), + [SubSysSup,AccSup]), + ?wait_match([{{server,ssh_connection_sup, _,_}, + ConnectionSup, supervisor, + [ssh_connection_sup]}, + {{server,ssh_channel_sup,_ ,_}, + ChannelSup,supervisor, + [ssh_channel_sup]}], + supervisor:which_children(SubSysSup), + [ConnectionSup,ChannelSup]), + ?wait_match([{{ssh_acceptor_sup,_,_,_},_,worker,[ssh_acceptor]}], + supervisor:which_children(AccSup)), + ?wait_match([{_, _, worker,[ssh_connection_handler]}], + supervisor:which_children(ConnectionSup)), + ?wait_match([], supervisor:which_children(ChannelSup)), + [ChannelSup, ConnectionSup, SubSysSup, AccSup]. + %%------------------------------------------------------------------------- %% Help functions %%------------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index 2d7bf75847..57ae2dbac2 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -56,7 +56,9 @@ daemon(Host, Port, Options) -> ct:log("~p:~p Calling ssh:daemon(~p, ~p, ~p)",[?MODULE,?LINE,Host,Port,Options]), case ssh:daemon(Host, Port, Options) of {ok, Pid} -> - {ok,L} = ssh:daemon_info(Pid), + R = ssh:daemon_info(Pid), + ct:log("~p:~p ssh:daemon_info(~p) ->~n ~p",[?MODULE,?LINE,Pid,R]), + {ok,L} = R, ListenPort = proplists:get_value(port, L), ListenIP = proplists:get_value(ip, L), {Pid, ListenIP, ListenPort}; @@ -199,15 +201,17 @@ init_io_server(TestCase) -> loop_io_server(TestCase, Buff0) -> receive - {input, TestCase, Line} -> + {input, TestCase, Line} = _INP -> + %%ct:log("io_server ~p:~p ~p got ~p",[?MODULE,?LINE,self(),_INP]), loop_io_server(TestCase, Buff0 ++ [Line]); - {io_request, From, ReplyAs, Request} -> + {io_request, From, ReplyAs, Request} = _REQ-> + %%ct:log("io_server ~p:~p ~p got ~p",[?MODULE,?LINE,self(),_REQ]), {ok, Reply, Buff} = io_request(Request, TestCase, From, ReplyAs, Buff0), io_reply(From, ReplyAs, Reply), loop_io_server(TestCase, Buff); {'EXIT',_, _} = _Exit -> -%% ct:log("ssh_test_lib:loop_io_server/2 got ~p",[_Exit]), + ct:log("ssh_test_lib:loop_io_server/2 got ~p",[_Exit]), ok after 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) diff --git a/lib/ssh/test/ssh_test_lib.hrl b/lib/ssh/test/ssh_test_lib.hrl index eaf856e6e8..4b6579bd71 100644 --- a/lib/ssh/test/ssh_test_lib.hrl +++ b/lib/ssh/test/ssh_test_lib.hrl @@ -16,12 +16,12 @@ %%------------------------------------------------------------------------- %% Help macro %%------------------------------------------------------------------------- --define(wait_match(Pattern, FunctionCall, Bind, Timeout, Ntries), +-define(wait_match(Pattern, Guard, FunctionCall, Bind, Timeout, Ntries), Bind = (fun() -> F = fun(N, F1) -> case FunctionCall of - Pattern -> Bind; + Pattern when Guard -> Bind; _ when N>0 -> ct:pal("Must sleep ~p ms at ~p:~p",[Timeout,?MODULE,?LINE]), timer:sleep(Timeout), @@ -34,6 +34,9 @@ end)() ). +-define(wait_match(Pattern, FunctionCall, Bind, Timeout, Ntries), + ?wait_match(Pattern, true, FunctionCall, Bind, Timeout, Ntries)). + -define(wait_match(Pattern, FunctionCall, Timeout, Ntries), ?wait_match(Pattern, FunctionCall, ok, Timeout, Ntries)). -define(wait_match(Pattern, FunctionCall, Bind), ?wait_match(Pattern, FunctionCall, Bind, 500, 10) ). diff --git a/lib/ssh/test/ssh_to_openssh_SUITE.erl b/lib/ssh/test/ssh_to_openssh_SUITE.erl index b20764ce47..9df404d7ed 100644 --- a/lib/ssh/test/ssh_to_openssh_SUITE.erl +++ b/lib/ssh/test/ssh_to_openssh_SUITE.erl @@ -48,19 +48,9 @@ all() -> end. groups() -> - [{erlang_client, [], [erlang_shell_client_openssh_server, - erlang_client_openssh_server_exec_compressed, - erlang_client_openssh_server_setenv, - erlang_client_openssh_server_publickey_dsa, - erlang_client_openssh_server_publickey_rsa, - erlang_client_openssh_server_password, - erlang_client_openssh_server_kexs, - erlang_client_openssh_server_nonexistent_subsystem, - erlang_client_openssh_server_renegotiate + [{erlang_client, [], [erlang_shell_client_openssh_server ]}, - {erlang_server, [], [erlang_server_openssh_client_public_key_dsa, - erlang_server_openssh_client_public_key_rsa, - erlang_server_openssh_client_renegotiate + {erlang_server, [], [erlang_server_openssh_client_renegotiate ]} ]. @@ -100,15 +90,6 @@ end_per_group(_, Config) -> Config. -init_per_testcase(erlang_server_openssh_client_public_key_dsa, Config) -> - chk_key(sshc, 'ssh-dss', ".ssh/id_dsa", Config); -init_per_testcase(erlang_server_openssh_client_public_key_rsa, Config) -> - chk_key(sshc, 'ssh-rsa', ".ssh/id_rsa", Config); -init_per_testcase(erlang_client_openssh_server_publickey_dsa, Config) -> - chk_key(sshd, 'ssh-dss', ".ssh/id_dsa", Config); -init_per_testcase(erlang_client_openssh_server_publickey_rsa, Config) -> - chk_key(sshd, 'ssh-rsa', ".ssh/id_rsa", Config); - init_per_testcase(erlang_server_openssh_client_renegotiate, Config) -> case os:type() of {unix,_} -> ssh:start(), Config; @@ -122,27 +103,6 @@ end_per_testcase(_TestCase, _Config) -> ssh:stop(), ok. - -chk_key(Pgm, Name, File, Config) -> - case ssh_test_lib:openssh_supports(Pgm, public_key, Name) of - false -> - {skip,lists:concat(["openssh client does not support ",Name])}; - true -> - {ok,[[Home]]} = init:get_argument(home), - KeyFile = filename:join(Home, File), - case file:read_file(KeyFile) of - {ok, Pem} -> - case public_key:pem_decode(Pem) of - [{_,_, not_encrypted}] -> - init_per_testcase('__default__',Config); - _ -> - {skip, {error, "Has pass phrase can not be used by automated test case"}} - end; - _ -> - {skip, lists:concat(["no ~/",File])} - end - end. - %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- @@ -160,219 +120,6 @@ erlang_shell_client_openssh_server(Config) when is_list(Config) -> receive_logout(), receive_normal_exit(Shell). -%-------------------------------------------------------------------- -erlang_client_openssh_server_exec() -> - [{doc, "Test api function ssh_connection:exec"}]. - -erlang_client_openssh_server_exec(Config) when is_list(Config) -> - ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user_interaction, false}]), - {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), - success = ssh_connection:exec(ConnectionRef, ChannelId0, - "echo testing", infinity), - Data0 = {ssh_cm, ConnectionRef, {data, ChannelId0, 0, <<"testing\n">>}}, - case ssh_test_lib:receive_exec_result(Data0) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId0); - {unexpected_msg,{ssh_cm, ConnectionRef, {exit_status, ChannelId0, 0}} - = ExitStatus0} -> - ct:log("0: Collected data ~p", [ExitStatus0]), - ssh_test_lib:receive_exec_result(Data0, - ConnectionRef, ChannelId0); - Other0 -> - ct:fail(Other0) - end, - - {ok, ChannelId1} = ssh_connection:session_channel(ConnectionRef, infinity), - success = ssh_connection:exec(ConnectionRef, ChannelId1, - "echo testing1", infinity), - Data1 = {ssh_cm, ConnectionRef, {data, ChannelId1, 0, <<"testing1\n">>}}, - case ssh_test_lib:receive_exec_result(Data1) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId1); - {unexpected_msg,{ssh_cm, ConnectionRef, {exit_status, ChannelId1, 0}} - = ExitStatus1} -> - ct:log("0: Collected data ~p", [ExitStatus1]), - ssh_test_lib:receive_exec_result(Data1, - ConnectionRef, ChannelId1); - Other1 -> - ct:fail(Other1) - end. - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_exec_compressed() -> - [{doc, "Test that compression option works"}]. - -erlang_client_openssh_server_exec_compressed(Config) when is_list(Config) -> - CompressAlgs = [zlib, '[email protected]',none], - case ssh_test_lib:ssh_supports(CompressAlgs, compression) of - {false,L} -> - {skip, io_lib:format("~p compression is not supported",[L])}; - - true -> - ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user_interaction, false}, - {preferred_algorithms, - [{compression,CompressAlgs}]}]), - {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), - success = ssh_connection:exec(ConnectionRef, ChannelId, - "echo testing", infinity), - Data = {ssh_cm, ConnectionRef, {data, ChannelId, 0, <<"testing\n">>}}, - case ssh_test_lib:receive_exec_result(Data) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId); - {unexpected_msg,{ssh_cm, ConnectionRef, - {exit_status, ChannelId, 0}} = ExitStatus} -> - ct:log("0: Collected data ~p", [ExitStatus]), - ssh_test_lib:receive_exec_result(Data, ConnectionRef, ChannelId); - Other -> - ct:fail(Other) - end - end. - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_kexs() -> - [{doc, "Test that we can connect with different KEXs."}]. - -erlang_client_openssh_server_kexs(Config) when is_list(Config) -> - KexAlgos = try proplists:get_value(kex, proplists:get_value(common_algs,Config)) - catch _:_ -> [] - end, - comment(KexAlgos), - case KexAlgos of - [] -> {skip, "No common kex algorithms"}; - _ -> - Success = - lists:foldl( - fun(Kex, Acc) -> - ConnectionRef = - ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user_interaction, false}, - {preferred_algorithms, - [{kex,[Kex]}]}]), - - {ok, ChannelId} = - ssh_connection:session_channel(ConnectionRef, infinity), - success = - ssh_connection:exec(ConnectionRef, ChannelId, - "echo testing", infinity), - - ExpectedData = {ssh_cm, ConnectionRef, {data, ChannelId, 0, <<"testing\n">>}}, - case ssh_test_lib:receive_exec_result(ExpectedData) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId), - Acc; - {unexpected_msg,{ssh_cm, ConnectionRef, - {exit_status, ChannelId, 0}} = ExitStatus} -> - ct:log("0: Collected data ~p", [ExitStatus]), - ssh_test_lib:receive_exec_result(ExpectedData, ConnectionRef, ChannelId), - Acc; - Other -> - ct:log("~p failed: ~p",[Kex,Other]), - false - end - end, true, KexAlgos), - case Success of - true -> - ok; - false -> - {fail, "Kex failed for one or more algos"} - end - end. - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_setenv() -> - [{doc, "Test api function ssh_connection:setenv"}]. - -erlang_client_openssh_server_setenv(Config) when is_list(Config) -> - ConnectionRef = - ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user_interaction, false}]), - {ok, ChannelId} = - ssh_connection:session_channel(ConnectionRef, infinity), - Env = case ssh_connection:setenv(ConnectionRef, ChannelId, - "ENV_TEST", "testing_setenv", - infinity) of - success -> - <<"tesing_setenv\n">>; - failure -> - <<"\n">> - end, - success = ssh_connection:exec(ConnectionRef, ChannelId, - "echo $ENV_TEST", infinity), - Data = {ssh_cm, ConnectionRef, {data, ChannelId, 0, Env}}, - case ssh_test_lib:receive_exec_result(Data) of - expected -> - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId); - {unexpected_msg,{ssh_cm, ConnectionRef, - {data,0,1, UnxpectedData}}} -> - %% Some os may return things as - %% ENV_TEST: Undefined variable.\n" - ct:log("UnxpectedData: ~p", [UnxpectedData]), - ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId); - {unexpected_msg,{ssh_cm, ConnectionRef, {exit_status, ChannelId, 0}} - = ExitStatus} -> - ct:log("0: Collected data ~p", [ExitStatus]), - ssh_test_lib:receive_exec_result(Data, - ConnectionRef, ChannelId); - Other -> - ct:fail(Other) - end. - -%%-------------------------------------------------------------------- - -%% setenv not meaningfull on erlang ssh daemon! - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_publickey_rsa(Config) -> - erlang_client_openssh_server_publickey_X(Config, 'ssh-rsa'). - -erlang_client_openssh_server_publickey_dsa(Config) -> - erlang_client_openssh_server_publickey_X(Config, 'ssh-dss'). - - -erlang_client_openssh_server_publickey_X(_Config, Alg) -> - ConnectionRef = - ssh_test_lib:connect(?SSH_DEFAULT_PORT, - [{pref_public_key_algs, [Alg]}, - {user_interaction, false}, - {auth_methods, "publickey"}, - silently_accept_hosts]), - {ok, Channel} = - ssh_connection:session_channel(ConnectionRef, infinity), - ok = ssh_connection:close(ConnectionRef, Channel), - ok = ssh:close(ConnectionRef). - -%%-------------------------------------------------------------------- -erlang_server_openssh_client_public_key_dsa() -> - [{timetrap, {seconds,(?TIMEOUT div 1000)+10}}]. -erlang_server_openssh_client_public_key_dsa(Config) when is_list(Config) -> - erlang_server_openssh_client_public_key_X(Config, 'ssh-dss'). - -erlang_server_openssh_client_public_key_rsa() -> - [{timetrap, {seconds,(?TIMEOUT div 1000)+10}}]. -erlang_server_openssh_client_public_key_rsa(Config) when is_list(Config) -> - erlang_server_openssh_client_public_key_X(Config, 'ssh-rsa'). - - -erlang_server_openssh_client_public_key_X(Config, Alg) -> - SystemDir = proplists:get_value(data_dir, Config), - PrivDir = proplists:get_value(priv_dir, Config), - KnownHosts = filename:join(PrivDir, "known_hosts"), - {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {preferred_algorithms,[{public_key, [Alg]}]}, - {auth_methods, "publickey"}, - {failfun, fun ssh_test_lib:failfun/2}]), - ct:sleep(500), - - Cmd = ssh_test_lib:open_sshc_cmd(Host, Port, - [" -o UserKnownHostsFile=", KnownHosts, - " -o StrictHostKeyChecking=no"], - "1+1."), - OpenSsh = ssh_test_lib:open_port({spawn, Cmd}), - ssh_test_lib:rcv_expected({data,<<"2\n">>}, OpenSsh, ?TIMEOUT), - ssh:stop_daemon(Pid). - %%-------------------------------------------------------------------- %% Test that the Erlang/OTP server can renegotiate with openSSH erlang_server_openssh_client_renegotiate(Config) -> @@ -430,108 +177,6 @@ erlang_server_openssh_client_renegotiate(Config) -> end. %%-------------------------------------------------------------------- -erlang_client_openssh_server_renegotiate(_Config) -> - process_flag(trap_exit, true), - IO = ssh_test_lib:start_io_server(), - Ref = make_ref(), - Parent = self(), - - Shell = - spawn_link( - fun() -> - Host = ssh_test_lib:hostname(), - Options = [{user_interaction, false}, - {silently_accept_hosts,true}], - group_leader(IO, self()), - {ok, ConnRef} = ssh:connect(Host, ?SSH_DEFAULT_PORT, Options), - ct:log("Parent = ~p, IO = ~p, Shell = ~p, ConnRef = ~p~n",[Parent, IO, self(), ConnRef]), - case ssh_connection:session_channel(ConnRef, infinity) of - {ok,ChannelId} -> - success = ssh_connection:ptty_alloc(ConnRef, ChannelId, []), - Args = [{channel_cb, ssh_shell}, - {init_args,[ConnRef, ChannelId]}, - {cm, ConnRef}, {channel_id, ChannelId}], - {ok, State} = ssh_channel:init([Args]), - Parent ! {ok, Ref, ConnRef}, - ssh_channel:enter_loop(State); - Error -> - Parent ! {error, Ref, Error} - end, - receive - nothing -> ok - end - end), - - receive - {error, Ref, Error} -> - ct:fail("Error=~p",[Error]); - {ok, Ref, ConnectionRef} -> - IO ! {input, self(), "echo Hej1\n"}, - receive_data("Hej1", ConnectionRef), - Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), - ssh_connection_handler:renegotiate(ConnectionRef), - IO ! {input, self(), "echo Hej2\n"}, - receive_data("Hej2", ConnectionRef), - Kex2 = ssh_test_lib:get_kex_init(ConnectionRef), - IO ! {input, self(), "exit\n"}, - receive_logout(), - receive_normal_exit(Shell), - true = (Kex1 =/= Kex2) - end. - -%%-------------------------------------------------------------------- -erlang_client_openssh_server_password() -> - [{doc, "Test client password option"}]. -erlang_client_openssh_server_password(Config) when is_list(Config) -> - %% to make sure we don't public-key-auth - UserDir = proplists:get_value(data_dir, Config), - {error, Reason0} = - ssh:connect(any, ?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, - {user, "foo"}, - {password, "morot"}, - {user_interaction, false}, - {user_dir, UserDir}]), - - ct:log("Test of user foo that does not exist. " - "Error msg: ~p~n", [Reason0]), - - User = string:strip(os:cmd("whoami"), right, $\n), - - case length(string:tokens(User, " ")) of - 1 -> - {error, Reason1} = - ssh:connect(any, ?SSH_DEFAULT_PORT, - [{silently_accept_hosts, true}, - {user, User}, - {password, "foo"}, - {user_interaction, false}, - {user_dir, UserDir}]), - ct:log("Test of wrong Pasword. " - "Error msg: ~p~n", [Reason1]); - _ -> - ct:log("Whoami failed reason: ~n", []) - end. - -%%-------------------------------------------------------------------- - -erlang_client_openssh_server_nonexistent_subsystem() -> - [{doc, "Test client password option"}]. -erlang_client_openssh_server_nonexistent_subsystem(Config) when is_list(Config) -> - - ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, - [{user_interaction, false}, - silently_accept_hosts]), - - {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), - - failure = ssh_connection:subsystem(ConnectionRef, ChannelId, "foo", infinity). - -%%-------------------------------------------------------------------- -% -%% Not possible to send password with openssh without user interaction -%% -%%-------------------------------------------------------------------- -%%-------------------------------------------------------------------- %%% Internal functions ----------------------------------------------- %%-------------------------------------------------------------------- receive_data(Data, Conn) -> diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 004db6e3a2..480e955ec4 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.6.4 - +SSH_VSN = 4.6.5 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 79176f5edf..bdf8711b2f 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -307,6 +307,21 @@ </section> </section> +<section><title>SSL 8.1.3.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix alert handling so that unexpected messages are logged + and alerted correctly</p> + <p> + Own Id: OTP-14929</p> + </item> + </list> + </section> +</section> + <section><title>SSL 8.1.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 80b639155b..4f72114ae9 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -141,23 +141,25 @@ <tag><c>sslsocket() =</c></tag> <item><p>opaque()</p></item> - <tag><marker id="type-protocol"/><c> protocol_versions() =</c></tag> + <tag><marker id="type-protocol"/><c> protocol_version() =</c></tag> <item><p><c> ssl_tls_protocol() | dtls_protocol() </c></p></item> - <tag><marker id="type-protocol"/><c> ssl_tls_protocol() =</c></tag> <item><p><c>sslv3 | tlsv1 | 'tlsv1.1' | 'tlsv1.2'</c></p></item> <tag><marker id="type-protocol"/><c> dtls_protocol() =</c></tag> <item><p><c>'dtlsv1' | 'dtlsv1.2'</c></p></item> <tag><c>ciphers() =</c></tag> - <item><p><c>= [ciphersuite()] | string()</c></p> - <p>According to old API.</p></item> + <item><p><c>= [ciphersuite()]</c></p> + <p>Tuples and string formats accepted by versions + before ssl-8.2.4 will be converted for backwards compatibility</p></item> <tag><c>ciphersuite() =</c></tag> - - <item><p><c>{key_exchange(), cipher(), MAC::hash()} | - {key_exchange(), cipher(), MAC::hash(), PRF::hash()}</c></p></item> + <item><p><c> + #{key_exchange := key_exchange(), + cipher := cipher(), + mac := MAC::hash() | aead, + prf := PRF::hash() | default_prf} </c></p></item> <tag><c>key_exchange()=</c></tag> <item><p><c>rsa | dhe_dss | dhe_rsa | dh_anon | psk | dhe_psk @@ -174,6 +176,12 @@ <tag><c>prf_random() =</c></tag> <item><p><c>client_random | server_random</c></p></item> + <tag><c>cipher_filters() =</c></tag> + <item><p><c> [{key_exchange | cipher | mac | prf, algo_filter()}])</c></p></item> + + <tag><c>algo_filter() =</c></tag> + <item><p>fun(key_exchange() | cipher() | hash() | aead | default_prf) -> true | false </p></item> + <tag><c>srp_param_type() =</c></tag> <item><p><c>srp_1024 | srp_1536 | srp_2048 | srp_3072 | srp_4096 | srp_6144 | srp_8192</c></p></item> @@ -465,7 +473,8 @@ marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_valid marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_validation/3</seealso> with the selected CA as trusted anchor and the rest of the chain.</p></item> - <tag><c>{versions, [protocol_versions()]}</c></tag> + <tag><c>{versions, [protocol_version()]}</c></tag> + <item><p>TLS protocol versions supported by started clients and servers. This option overrides the application environment option <c>protocol_version</c> and <c>dtls_protocol_version</c>. If the environment option is not set, it defaults @@ -838,23 +847,55 @@ fun(srp, Username :: string(), UserState :: term()) -> </section> <funcs> + + <func> + <name>append_cipher_suites(Deferred, Suites) -> ciphers() </name> + <fsummary></fsummary> + <type> + <v>Deferred = ciphers() | cipher_filters() </v> + <v>Suites = ciphers() </v> + </type> + <desc><p>Make <c>Deferred</c> suites become the least preferred + suites, that is put them at the end of the cipher suite list + <c>Suites</c> after removing them from <c>Suites</c> if + present. <c>Deferred</c> may be a list of cipher suits or a + list of filters in which case the filters are use on <c>Suites</c> to + extract the Deferred cipher list.</p> + </desc> + </func> + <func> <name>cipher_suites() -></name> - <name>cipher_suites(Type) -> ciphers()</name> - <name>cipher_suites(Type, protocol_version()) -> ciphers()</name> + <name>cipher_suites(Type) -> old_ciphers()</name> <fsummary>Returns a list of supported cipher suites.</fsummary> <type> <v>Type = erlang | openssl | all</v> </type> - <desc><p>Returns a list of supported cipher suites. + <desc> + <p>Returns a list of supported cipher suites. + This function will become deprecated in OTP 21, and replaced + by <seealso marker="#cipher_suites-2">ssl:cipher-suites/2</seealso> <c>cipher_suites()</c> is equivalent to <c>cipher_suites(erlang).</c> Type <c>openssl</c> is provided for backwards compatibility with the old SSL, which used OpenSSL. <c>cipher_suites(all)</c> returns all available cipher suites. The cipher suites not present in <c>cipher_suites(erlang)</c> but included in <c>cipher_suites(all)</c> are not used unless explicitly configured - by the user. If the version option is not specified, the highest supported - TLS version will be used to determine the supported cipher suites</p> + by the user.</p> + </desc> + </func> + + <func> + <name>cipher_suites(Supported, Version) -> ciphers()</name> + <fsummary>Returns a list of all default or + all supported cipher suites.</fsummary> + <type> + <v> Supported = default | all | anonymous </v> + <v> Version = protocol_version() </v> + </type> + <desc><p>Returns all default or all supported (except anonymous), + or all anonymous cipher suites for a + TLS version</p> </desc> </func> @@ -1019,6 +1060,21 @@ fun(srp, Username :: string(), UserState :: term()) -> </desc> </func> + <func> + <name>filter_cipher_suites(Suites, Filters) -> ciphers()</name> + <fsummary></fsummary> + <type> + <v> Suites = ciphers()</v> + <v> Filters = cipher_filters()</v> + </type> + <desc><p>Removes cipher suites if any of the filter functions + returns false for any part of the cipher suite. This function + also calls default filter functions to make sure the cipher + suites are supported by crypto. If no filter function is supplied for some + part the default behaviour is fun(Algorithm) -> true.</p> + </desc> + </func> + <func> <name>format_error(Reason) -> string()</name> <fsummary>Returns an error string.</fsummary> @@ -1116,6 +1172,22 @@ fun(srp, Username :: string(), UserState :: term()) -> <p>Returns the address and port number of the peer.</p> </desc> </func> + + <func> + <name>prepend_cipher_suites(Preferred, Suites) -> ciphers()</name> + <fsummary></fsummary> + <type> + <v>Preferred = ciphers() | cipher_filters() </v> + <v>Suites = ciphers() </v> + </type> + <desc><p>Make <c>Preferred</c> suites become the most preferred + suites that is put them at the head of the cipher suite list + <c>Suites</c> after removing them from <c>Suites</c> if + present. <c>Preferred</c> may be a list of cipher suits or a + list of filters in which case the filters are use on <c>Suites</c> to + extract the preferred cipher list. </p> + </desc> + </func> <func> <name>prf(Socket, Secret, Label, Seed, WantedLength) -> {ok, binary()} | {error, reason()}</name> diff --git a/lib/ssl/doc/src/using_ssl.xml b/lib/ssl/doc/src/using_ssl.xml index c369c3c133..3ef33df719 100644 --- a/lib/ssl/doc/src/using_ssl.xml +++ b/lib/ssl/doc/src/using_ssl.xml @@ -153,7 +153,51 @@ ok</code> </section> </section> - <section> + <section> + <title>Customizing cipher suits</title> + + <p>Fetch default cipher suite list for an TLS/DTLS version. Change default + to all to get all possible cipher suites.</p> + <code type="erl">1> Default = ssl:cipher_suites(default, 'tlsv1.2'). + [#{cipher => aes_256_gcm,key_exchange => ecdhe_ecdsa, + mac => aead,prf => sha384}, ....] +</code> + + <p>In OTP 20 it is desirable to remove all cipher suites + that uses rsa kexchange (removed from default in 21) </p> + <code type="erl">2> NoRSA = + ssl:filter_cipher_suites(Default, + [{key_exchange, fun(rsa) -> false; + (_) -> true end}]). + [...] + </code> + + <p> Pick just a few suites </p> + <code type="erl"> 3> Suites = + ssl:filter_cipher_suites(Default, + [{key_exchange, fun(ecdh_ecdsa) -> true; + (_) -> false end}, + {cipher, fun(aes_128_cbc) ->true; + (_) ->false end}]). + [#{cipher => aes_128_cbc,key_exchange => ecdh_ecdsa, + mac => sha256,prf => sha256}, + #{cipher => aes_128_cbc,key_exchange => ecdh_ecdsa,mac => sha, + prf => default_prf}] + </code> + + <p> Make some particular suites the most preferred, or least + preferred by changing prepend to append.</p> + <code type="erl"> 4>ssl:prepend_cipher_suites(Suites, Default). + [#{cipher => aes_128_cbc,key_exchange => ecdh_ecdsa, + mac => sha256,prf => sha256}, + #{cipher => aes_128_cbc,key_exchange => ecdh_ecdsa,mac => sha, + prf => default_prf}, + #{cipher => aes_256_cbc,key_exchange => ecdhe_ecdsa, + mac => sha384,prf => sha384}, ...] + </code> + </section> + + <section> <title>Using an Engine Stored Key</title> <p>Erlang ssl application is able to use private keys provided diff --git a/lib/ssl/src/dtls_v1.erl b/lib/ssl/src/dtls_v1.erl index 51ee8ec047..0f6344b6f7 100644 --- a/lib/ssl/src/dtls_v1.erl +++ b/lib/ssl/src/dtls_v1.erl @@ -21,7 +21,7 @@ -include("ssl_cipher.hrl"). --export([suites/1, all_suites/1, hmac_hash/3, ecc_curves/1, +-export([suites/1, all_suites/1, anonymous_suites/1,hmac_hash/3, ecc_curves/1, corresponding_tls_version/1, corresponding_dtls_version/1, cookie_secret/0, cookie_timeout/0]). @@ -40,6 +40,12 @@ all_suites(Version) -> end, ssl_cipher:all_suites(corresponding_tls_version(Version))). +anonymous_suites(Version) -> + lists:filter(fun(Cipher) -> + is_acceptable_cipher(ssl_cipher:suite_definition(Cipher)) + end, + ssl_cipher:anonymous_suites(corresponding_tls_version(Version))). + hmac_hash(MacAlg, MacSecret, Value) -> tls_v1:hmac_hash(MacAlg, MacSecret, Value). diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index a298012f26..0b035d31be 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -39,7 +39,9 @@ ]). %% SSL/TLS protocol handling --export([cipher_suites/0, cipher_suites/1, cipher_suites/2, eccs/0, eccs/1, versions/0, +-export([cipher_suites/0, cipher_suites/1, cipher_suites/2, filter_cipher_suites/2, + prepend_cipher_suites/2, append_cipher_suites/2, + eccs/0, eccs/1, versions/0, format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, connection_information/1, connection_information/2]). %% Misc @@ -379,35 +381,91 @@ negotiated_protocol(#sslsocket{pid = Pid}) -> cipher_suites() -> cipher_suites(erlang). %%-------------------------------------------------------------------- --spec cipher_suites(erlang | openssl | all) -> [ssl_cipher:old_erl_cipher_suite() | string()]. +-spec cipher_suites(erlang | openssl | all) -> + [ssl_cipher:old_erl_cipher_suite() | string()]. %% Description: Returns all supported cipher suites. %%-------------------------------------------------------------------- cipher_suites(erlang) -> - Version = tls_record:highest_protocol_version([]), - cipher_suites(erlang, Version); + [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(default)]; + cipher_suites(openssl) -> - Version = tls_record:highest_protocol_version([]), - cipher_suites(openssl, Version); + [ssl_cipher:openssl_suite_name(Suite) || + Suite <- available_suites(default)]; + cipher_suites(all) -> - Version = tls_record:highest_protocol_version([]), - cipher_suites(all, Version). + [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(all)]. %%-------------------------------------------------------------------- --spec cipher_suites(erlang | openssl | all, tls_record:tls_version() | - dtls_record:dtls_version()) -> [ssl_cipher:old_erl_cipher_suite() | string()]. -%% Description: Returns all supported cipher suites. +-spec cipher_suites(default | all | anonymous, tls_record:tls_version() | dtls_record:dtls_version() | + tls_record:tls_atom_version() | dtls_record:dtls_atom_version()) -> + [ssl_cipher:erl_cipher_suite()]. +%% Description: Returns all default and all supported cipher suites for a +%% TLS/DTLS version %%-------------------------------------------------------------------- -cipher_suites(Type, Version) when Version == 'dtlsv1'; - Version == 'dtlsv1.2' -> - cipher_suites(Type, dtls_record:protocol_version(Version)); -cipher_suites(Type, Version) when is_atom(Version) -> - cipher_suites(Type, tls_record:protocol_version(Version)); -cipher_suites(erlang, Version) -> - [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(default, Version)]; -cipher_suites(openssl, Version) -> - [ssl_cipher:openssl_suite_name(Suite) || Suite <- available_suites(default, Version)]; -cipher_suites(all, Version) -> - [ssl_cipher:erl_suite_definition(Suite) || Suite <- available_suites(all, Version)]. +cipher_suites(Base, Version) when Version == 'tlsv1.2'; + Version == 'tlsv1.1'; + Version == tlsv1; + Version == sslv3 -> + cipher_suites(Base, tls_record:protocol_version(Version)); +cipher_suites(Base, Version) when Version == 'dtlsv1.2'; + Version == 'dtlsv1'-> + cipher_suites(Base, dtls_record:protocol_version(Version)); +cipher_suites(Base, Version) -> + [ssl_cipher:suite_definition(Suite) || Suite <- supported_suites(Base, Version)]. + +%%-------------------------------------------------------------------- +-spec filter_cipher_suites([ssl_cipher:erl_cipher_suite()], + [{key_exchange | cipher | mac | prf, fun()}] | []) -> + [ssl_cipher:erl_cipher_suite()]. +%% Description: Removes cipher suites if any of the filter functions returns false +%% for any part of the cipher suite. This function also calls default filter functions +%% to make sure the cipher suite are supported by crypto. +%%-------------------------------------------------------------------- +filter_cipher_suites(Suites, Filters0) -> + #{key_exchange_filters := KexF, + cipher_filters := CipherF, + mac_filters := MacF, + prf_filters := PrfF} + = ssl_cipher:crypto_support_filters(), + Filters = #{key_exchange_filters => add_filter(proplists:get_value(key_exchange, Filters0), KexF), + cipher_filters => add_filter(proplists:get_value(cipher, Filters0), CipherF), + mac_filters => add_filter(proplists:get_value(mac, Filters0), MacF), + prf_filters => add_filter(proplists:get_value(prf, Filters0), PrfF)}, + ssl_cipher:filter_suites(Suites, Filters). +%%-------------------------------------------------------------------- +-spec prepend_cipher_suites([ssl_cipher:erl_cipher_suite()] | + [{key_exchange | cipher | mac | prf, fun()}], + [ssl_cipher:erl_cipher_suite()]) -> + [ssl_cipher:erl_cipher_suite()]. +%% Description: Make <Preferred> suites become the most prefered +%% suites that is put them at the head of the cipher suite list +%% and remove them from <Suites> if present. <Preferred> may be a +%% list of cipher suits or a list of filters in which case the +%% filters are use on Suites to extract the the preferred +%% cipher list. +%% -------------------------------------------------------------------- +prepend_cipher_suites([First | _] = Preferred, Suites0) when is_map(First) -> + Suites = Preferred ++ (Suites0 -- Preferred), + Suites; +prepend_cipher_suites(Filters, Suites) -> + Preferred = filter_cipher_suites(Suites, Filters), + Preferred ++ (Suites -- Preferred). +%%-------------------------------------------------------------------- +-spec append_cipher_suites(Deferred :: [ssl_cipher:erl_cipher_suite()] | + [{key_exchange | cipher | mac | prf, fun()}], + [ssl_cipher:erl_cipher_suite()]) -> + [ssl_cipher:erl_cipher_suite()]. +%% Description: Make <Deferred> suites suites become the +%% least prefered suites that is put them at the end of the cipher suite list +%% and removed them from <Suites> if present. +%% +%%-------------------------------------------------------------------- +append_cipher_suites([First | _] = Deferred, Suites0) when is_map(First)-> + Suites = (Suites0 -- Deferred) ++ Deferred, + Suites; +append_cipher_suites(Filters, Suites) -> + Deferred = filter_cipher_suites(Suites, Filters), + (Suites -- Deferred) ++ Deferred. %%-------------------------------------------------------------------- -spec eccs() -> tls_v1:curves(). @@ -661,14 +719,21 @@ tls_version({254, _} = Version) -> %%%-------------------------------------------------------------- %%% Internal functions %%%-------------------------------------------------------------------- - %% Possible filters out suites not supported by crypto -available_suites(default, Version) -> +available_suites(default) -> + Version = tls_record:highest_protocol_version([]), ssl_cipher:filter_suites(ssl_cipher:suites(Version)); - -available_suites(all, Version) -> +available_suites(all) -> + Version = tls_record:highest_protocol_version([]), ssl_cipher:filter_suites(ssl_cipher:all_suites(Version)). +supported_suites(default, Version) -> + ssl_cipher:suites(Version); +supported_suites(all, Version) -> + ssl_cipher:all_suites(Version); +supported_suites(anonymous, Version) -> + ssl_cipher:anonymous_suites(Version). + do_listen(Port, #config{transport_info = {Transport, _, _, _}} = Config, tls_connection) -> tls_socket:listen(Transport, Port, Config); @@ -1178,17 +1243,21 @@ handle_cipher_option(Value, Version) when is_list(Value) -> binary_cipher_suites(Version, []) -> %% Defaults to all supported suites that does %% not require explicit configuration - ssl_cipher:filter_suites(ssl_cipher:suites(tls_version(Version))); + default_binary_suites(Version); +binary_cipher_suites(Version, [Map|_] = Ciphers0) when is_map(Map) -> + Ciphers = [ssl_cipher:suite(C) || C <- Ciphers0], + binary_cipher_suites(Version, Ciphers); binary_cipher_suites(Version, [Tuple|_] = Ciphers0) when is_tuple(Tuple) -> Ciphers = [ssl_cipher:suite(tuple_to_map(C)) || C <- Ciphers0], binary_cipher_suites(Version, Ciphers); binary_cipher_suites(Version, [Cipher0 | _] = Ciphers0) when is_binary(Cipher0) -> - All = ssl_cipher:all_suites(tls_version(Version)), + All = ssl_cipher:all_suites(Version) ++ + ssl_cipher:anonymous_suites(Version), case [Cipher || Cipher <- Ciphers0, lists:member(Cipher, All)] of [] -> %% Defaults to all supported suites that does %% not require explicit configuration - ssl_cipher:filter_suites(ssl_cipher:suites(tls_version(Version))); + default_binary_suites(Version); Ciphers -> Ciphers end; @@ -1201,6 +1270,9 @@ binary_cipher_suites(Version, Ciphers0) -> Ciphers = [ssl_cipher:openssl_suite(C) || C <- string:lexemes(Ciphers0, ":")], binary_cipher_suites(Version, Ciphers). +default_binary_suites(Version) -> + ssl_cipher:filter_suites(ssl_cipher:suites(Version)). + tuple_to_map({Kex, Cipher, Mac}) -> #{key_exchange => Kex, cipher => Cipher, @@ -1209,9 +1281,19 @@ tuple_to_map({Kex, Cipher, Mac}) -> tuple_to_map({Kex, Cipher, Mac, Prf}) -> #{key_exchange => Kex, cipher => Cipher, - mac => Mac, + mac => tuple_to_map_mac(Cipher, Mac), prf => Prf}. +%% Backwards compatible +tuple_to_map_mac(aes_128_gcm, _) -> + aead; +tuple_to_map_mac(aes_256_gcm, _) -> + aead; +tuple_to_map_mac(chacha20_poly1305, _) -> + aead; +tuple_to_map_mac(_, MAC) -> + MAC. + handle_eccs_option(Value, Version) when is_list(Value) -> {_Major, Minor} = tls_version(Version), try tls_v1:ecc_curves(Minor, Value) of @@ -1490,3 +1572,8 @@ reject_alpn_next_prot_options([Opt| AlpnNextOpts], Opts) -> false -> reject_alpn_next_prot_options(AlpnNextOpts, Opts) end. + +add_filter(undefined, Filters) -> + Filters; +add_filter(Filter, Filters) -> + [Filter | Filters]. diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 7c5cff3665..a83ce42455 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -36,9 +36,11 @@ -export([security_parameters/2, security_parameters/3, suite_definition/1, erl_suite_definition/1, cipher_init/3, decipher/6, cipher/5, decipher_aead/6, cipher_aead/6, - suite/1, suites/1, all_suites/1, - ec_keyed_suites/0, chacha_suites/1, anonymous_suites/1, psk_suites/1, srp_suites/0, - rc4_suites/1, des_suites/1, rsa_suites/1, openssl_suite/1, openssl_suite_name/1, filter/2, filter_suites/1, + suite/1, suites/1, all_suites/1, crypto_support_filters/0, + ec_keyed_suites/0, chacha_suites/1, anonymous_suites/1, psk_suites/1, psk_suites_anon/1, + srp_suites/0, srp_suites_anon/0, + rc4_suites/1, des_suites/1, rsa_suites/1, openssl_suite/1, openssl_suite_name/1, + filter/2, filter_suites/1, filter_suites/2, hash_algorithm/1, sign_algorithm/1, is_acceptable_hash/2, is_fallback/1, random_bytes/1, calc_mac_hash/4, is_stream_ciphersuite/1]). @@ -53,7 +55,7 @@ -type key_algo() :: null | rsa | dhe_rsa | dhe_dss | ecdhe_ecdsa| ecdh_ecdsa | ecdh_rsa| srp_rsa| srp_dss | psk | dhe_psk | rsa_psk | dh_anon | ecdh_anon | srp_anon. -type erl_cipher_suite() :: #{key_exchange := key_algo(), cipher := cipher(), - mac := hash(), + mac := hash() | aead, prf := hash() | default_prf %% Old cipher suites, version dependent }. -type old_erl_cipher_suite() :: {key_algo(), cipher(), hash()} % Pre TLS 1.2 @@ -94,7 +96,7 @@ security_parameters(Version, CipherSuite, SecParams) -> expanded_key_material_length = expanded_key_material(Cipher), key_material_length = key_material(Cipher), iv_size = iv_size(Cipher), - mac_algorithm = hash_algorithm(Hash), + mac_algorithm = mac_algorithm(Hash), prf_algorithm = prf_algorithm(PrfHashAlg, Version), hash_size = hash_size(Hash)}. @@ -321,12 +323,12 @@ suites({_, Minor}) -> all_suites({3, _} = Version) -> suites(Version) ++ chacha_suites(Version) - ++ anonymous_suites(Version) ++ psk_suites(Version) ++ srp_suites() ++ rc4_suites(Version) ++ des_suites(Version) ++ rsa_suites(Version); + all_suites(Version) -> dtls_v1:all_suites(Version). %%-------------------------------------------------------------------- @@ -350,12 +352,12 @@ chacha_suites(_) -> %% if explicitly set by user. Intended only for testing. %%-------------------------------------------------------------------- anonymous_suites({3, N}) -> - anonymous_suites(N); + srp_suites_anon() ++ anonymous_suites(N); anonymous_suites({254, _} = Version) -> - anonymous_suites(dtls_v1:corresponding_tls_version(Version)) - -- [?TLS_DH_anon_WITH_RC4_128_MD5]; + dtls_v1:anonymous_suites(Version); anonymous_suites(N) when N >= 3 -> + psk_suites_anon(N) ++ [?TLS_DH_anon_WITH_AES_128_GCM_SHA256, ?TLS_DH_anon_WITH_AES_256_GCM_SHA384, ?TLS_DH_anon_WITH_AES_128_CBC_SHA256, @@ -364,20 +366,20 @@ anonymous_suites(N) ?TLS_ECDH_anon_WITH_AES_256_CBC_SHA, ?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA, ?TLS_DH_anon_WITH_RC4_128_MD5]; - -anonymous_suites(2) -> +anonymous_suites(2 = N) -> + psk_suites_anon(N) ++ [?TLS_ECDH_anon_WITH_AES_128_CBC_SHA, ?TLS_ECDH_anon_WITH_AES_256_CBC_SHA, ?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA, ?TLS_DH_anon_WITH_DES_CBC_SHA, ?TLS_DH_anon_WITH_RC4_128_MD5]; - anonymous_suites(N) when N == 0; N == 1 -> - [?TLS_DH_anon_WITH_RC4_128_MD5, - ?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA, - ?TLS_DH_anon_WITH_DES_CBC_SHA - ]. + psk_suites_anon(N) ++ + [?TLS_DH_anon_WITH_RC4_128_MD5, + ?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA, + ?TLS_DH_anon_WITH_DES_CBC_SHA + ]. %%-------------------------------------------------------------------- -spec psk_suites(ssl_record:ssl_version() | integer()) -> [cipher_suite()]. @@ -390,41 +392,52 @@ psk_suites({3, N}) -> psk_suites(N) when N >= 3 -> [ - ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384, - ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, + ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, + ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 + ] ++ psk_suites(0); +psk_suites(_) -> + [?TLS_RSA_PSK_WITH_AES_256_CBC_SHA, + ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA, + ?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, + ?TLS_RSA_PSK_WITH_RC4_128_SHA]. + +%%-------------------------------------------------------------------- +-spec psk_suites_anon(ssl_record:ssl_version() | integer()) -> [cipher_suite()]. +%% +%% Description: Returns a list of the anonymous PSK cipher suites, only supported +%% if explicitly set by user. +%%-------------------------------------------------------------------- +psk_suites_anon({3, N}) -> + psk_suites_anon(N); +psk_suites_anon(N) + when N >= 3 -> + [ + ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_PSK_WITH_AES_256_GCM_SHA384, ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, - ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, ?TLS_PSK_WITH_AES_256_CBC_SHA384, ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256, - ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256, ?TLS_PSK_WITH_AES_128_GCM_SHA256, ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, - ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256, ?TLS_PSK_WITH_AES_128_CBC_SHA256 - ] ++ psk_suites(0); -psk_suites(_) -> - [?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA, - ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, - ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA, + ] ++ psk_suites_anon(0); +psk_suites_anon(_) -> + [?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, ?TLS_PSK_WITH_AES_256_CBC_SHA, ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA, - ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA, ?TLS_PSK_WITH_AES_128_CBC_SHA, ?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA, - ?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_ECDHE_PSK_WITH_RC4_128_SHA, ?TLS_DHE_PSK_WITH_RC4_128_SHA, - ?TLS_RSA_PSK_WITH_RC4_128_SHA, ?TLS_PSK_WITH_RC4_128_SHA]. - %%-------------------------------------------------------------------- -spec srp_suites() -> [cipher_suite()]. %% @@ -432,15 +445,24 @@ psk_suites(_) -> %% if explicitly set by user. %%-------------------------------------------------------------------- srp_suites() -> - [?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA, - ?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA, + [?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA, ?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA, - ?TLS_SRP_SHA_WITH_AES_128_CBC_SHA, ?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA, ?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA, - ?TLS_SRP_SHA_WITH_AES_256_CBC_SHA, ?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA, ?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA]. + +%%-------------------------------------------------------------------- +-spec srp_suites_anon() -> [cipher_suite()]. +%% +%% Description: Returns a list of the SRP anonymous cipher suites, only supported +%% if explicitly set by user. +%%-------------------------------------------------------------------- +srp_suites_anon() -> + [?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA, + ?TLS_SRP_SHA_WITH_AES_128_CBC_SHA, + ?TLS_SRP_SHA_WITH_AES_256_CBC_SHA]. + %%-------------------------------------------------------------------- -spec rc4_suites(Version::ssl_record:ssl_version() | integer()) -> [cipher_suite()]. %% @@ -750,32 +772,32 @@ suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA) -> suite_definition(?TLS_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => psk, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => psk, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_psk, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_psk, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => rsa_psk, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => rsa_psk, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => psk, @@ -1115,42 +1137,42 @@ suite_definition(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384) -> suite_definition(?TLS_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DH_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DH_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_dss, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_dss, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_dss, @@ -1160,74 +1182,74 @@ suite_definition(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> suite_definition(?TLS_DH_DSS_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_dss, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_DH_anon_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_anon, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DH_anon_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_anon, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; %% RFC 5289 ECC AES-GCM Cipher Suites suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdh_ecdsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdh_ecdsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdhe_rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; suite_definition(?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdh_rsa, cipher => aes_128_gcm, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdh_rsa, cipher => aes_256_gcm, - mac => null, + mac => aead, prf => sha384}; %% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites suite_definition(?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => chacha20_poly1305, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => ecdhe_ecdsa, cipher => chacha20_poly1305, - mac => null, + mac => aead, prf => sha256}; suite_definition(?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => dhe_rsa, cipher => chacha20_poly1305, - mac => null, + mac => aead, prf => sha256}. %%-------------------------------------------------------------------- @@ -1428,32 +1450,32 @@ suite(#{key_exchange := rsa_psk, %%% TLS 1.2 PSK Cipher Suites RFC 5487 suite(#{key_exchange := psk, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_PSK_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := psk, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_PSK_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dhe_psk, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dhe_psk, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := rsa_psk, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := rsa_psk, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := psk, @@ -1739,119 +1761,119 @@ suite(#{key_exchange := ecdh_rsa, %% RFC 5288 AES-GCM Cipher Suites suite(#{key_exchange := rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_RSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dhe_rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dhe_rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dh_rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DH_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dh_rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DH_RSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dhe_dss, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dhe_dss, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dh_dss, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DH_DSS_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dh_dss, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DH_DSS_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := dh_anon, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DH_anon_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := dh_anon, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_DH_anon_WITH_AES_256_GCM_SHA384; %% RFC 5289 ECC AES-GCM Cipher Suites suite(#{key_exchange := ecdhe_ecdsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := ecdhe_ecdsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := ecdh_ecdsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := ecdh_ecdsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := ecdhe_rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := ecdhe_rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384; suite(#{key_exchange := ecdh_rsa, cipher := aes_128_gcm, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256; suite(#{key_exchange := ecdh_rsa, cipher := aes_256_gcm, - mac := null, + mac := aead, prf := sha384}) -> ?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384; %% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites suite(#{key_exchange := ecdhe_rsa, cipher := chacha20_poly1305, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256; suite(#{key_exchange := ecdhe_ecdsa, cipher := chacha20_poly1305, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256; suite(#{key_exchange := dhe_rsa, cipher := chacha20_poly1305, - mac := null, + mac := aead, prf := sha256}) -> ?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256. @@ -2011,9 +2033,9 @@ openssl_suite("ECDH-RSA-AES256-GCM-SHA384") -> ?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384. %%-------------------------------------------------------------------- --spec openssl_suite_name(cipher_suite()) -> openssl_cipher_suite(). +-spec openssl_suite_name(cipher_suite()) -> openssl_cipher_suite() | erl_cipher_suite(). %% -%% Description: Return openssl cipher suite name. +%% Description: Return openssl cipher suite name if possible %%------------------------------------------------------------------- openssl_suite_name(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA) -> "DHE-RSA-AES256-SHA"; @@ -2223,38 +2245,74 @@ filter(DerCert, Ciphers) -> {_, ecdsa} -> Ciphers1 -- rsa_signed_suites() end. - %%-------------------------------------------------------------------- --spec filter_suites([cipher_suite()]) -> [cipher_suite()]. +-spec filter_suites([erl_cipher_suite()] | [cipher_suite()], map()) -> + [erl_cipher_suite()] | [cipher_suite()]. +%% +%% Description: Filter suites using supplied filter funs +%%------------------------------------------------------------------- +filter_suites(Suites, Filters) -> + ApplyFilters = fun(Suite) -> + filter_suite(Suite, Filters) + end, + lists:filter(ApplyFilters, Suites). + +filter_suite(#{key_exchange := KeyExchange, + cipher := Cipher, + mac := Hash, + prf := Prf}, + #{key_exchange_filters := KeyFilters, + cipher_filters := CipherFilters, + mac_filters := HashFilters, + prf_filters := PrfFilters}) -> + all_filters(KeyExchange, KeyFilters) andalso + all_filters(Cipher, CipherFilters) andalso + all_filters(Hash, HashFilters) andalso + all_filters(Prf, PrfFilters); +filter_suite(Suite, Filters) -> + filter_suite(suite_definition(Suite), Filters). + +%%-------------------------------------------------------------------- +-spec filter_suites([erl_cipher_suite()] | [cipher_suite()]) -> + [erl_cipher_suite()] | [cipher_suite()]. %% %% Description: Filter suites for algorithms supported by crypto. %%------------------------------------------------------------------- -filter_suites(Suites = [Value|_]) when is_map(Value) -> - Algos = crypto:supports(), - Hashs = proplists:get_value(hashs, Algos), - lists:filter(fun(#{key_exchange := KeyExchange, - cipher := Cipher, - mac := Hash, - prf := Prf}) -> - is_acceptable_keyexchange(KeyExchange, proplists:get_value(public_keys, Algos)) andalso - is_acceptable_cipher(Cipher, proplists:get_value(ciphers, Algos)) andalso - is_acceptable_hash(Hash, Hashs) andalso - is_acceptable_prf(Prf, Hashs) - end, Suites); - filter_suites(Suites) -> + Filters = crypto_support_filters(), + filter_suites(Suites, Filters). + +all_filters(_, []) -> + true; +all_filters(Value, [Filter| Rest]) -> + case Filter(Value) of + true -> + all_filters(Value, Rest); + false -> + false + end. +crypto_support_filters() -> Algos = crypto:supports(), Hashs = proplists:get_value(hashs, Algos), - lists:filter(fun(Suite) -> - #{key_exchange := KeyExchange, - cipher := Cipher, - mac := Hash, - prf := Prf} = suite_definition(Suite), - is_acceptable_keyexchange(KeyExchange, proplists:get_value(public_keys, Algos)) andalso - is_acceptable_cipher(Cipher, proplists:get_value(ciphers, Algos)) andalso - is_acceptable_hash(Hash, Hashs) andalso - is_acceptable_prf(Prf, Hashs) - end, Suites). + #{key_exchange_filters => + [fun(KeyExchange) -> + is_acceptable_keyexchange(KeyExchange, + proplists:get_value(public_keys, Algos)) + end], + cipher_filters => + [fun(Cipher) -> + is_acceptable_cipher(Cipher, + proplists:get_value(ciphers, Algos)) + end], + mac_filters => + [fun(Hash) -> + is_acceptable_hash(Hash, Hashs) + end], + prf_filters => + [fun(Prf) -> + is_acceptable_prf(Prf, + proplists:get_value(hashs, Algos)) + end]}. is_acceptable_keyexchange(KeyExchange, _Algos) when KeyExchange == psk; KeyExchange == null -> @@ -2473,6 +2531,11 @@ prf_algorithm(default_prf, {3, _}) -> prf_algorithm(Algo, _) -> hash_algorithm(Algo). +mac_algorithm(aead) -> + aead; +mac_algorithm(Algo) -> + hash_algorithm(Algo). + hash_algorithm(null) -> ?NULL; hash_algorithm(md5) -> ?MD5; hash_algorithm(sha) -> ?SHA; %% Only sha always refers to "SHA-1" @@ -2503,6 +2566,10 @@ sign_algorithm(Other) when is_integer(Other) andalso ((Other >= 224) and (Other hash_size(null) -> 0; +%% The AEAD MAC hash size is not used in the context +%% of calculating the master secret. See RFC 5246 Section 6.2.3.3. +hash_size(aead) -> + 0; hash_size(md5) -> 16; hash_size(sha) -> diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index d046145dff..2031735a71 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -1148,8 +1148,8 @@ handle_common_event(internal, #change_cipher_spec{type = <<1>>}, StateName, StateName, State); handle_common_event(_Type, Msg, StateName, #state{negotiated_version = Version} = State, _) -> - Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE), - handle_own_alert(Alert, Version, {StateName, Msg}, State). + Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE, {unexpected_msg, Msg}), + handle_own_alert(Alert, Version, StateName, State). handle_call({application_data, _Data}, _, _, _, _) -> %% In renegotiation priorities handshake, send data when handshake is finished diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 0c55af9174..09160e2f9c 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -774,9 +774,8 @@ decode_suites('3_bytes', Dec) -> %%==================================================================== available_suites(UserSuites, Version) -> - lists:filtermap(fun(Suite) -> - lists:member(Suite, ssl_cipher:all_suites(Version)) - end, UserSuites). + VersionSuites = ssl_cipher:all_suites(Version) ++ ssl_cipher:anonymous_suites(Version), + lists:filtermap(fun(Suite) -> lists:member(Suite, VersionSuites) end, UserSuites). available_suites(ServerCert, UserSuites, Version, undefined, Curve) -> ssl_cipher:filter(ServerCert, available_suites(UserSuites, Version)) @@ -1056,7 +1055,9 @@ select_curve(undefined, _, _) -> %% %% Description: Handles signature_algorithms hello extension (server) %%-------------------------------------------------------------------- -select_hashsign(_, undefined, _, _, _Version) -> +select_hashsign(_, _, KeyExAlgo, _, _Version) when KeyExAlgo == dh_anon; + KeyExAlgo == ecdh_anon; + KeyExAlgo == srp_anon -> {null, anon}; %% The signature_algorithms extension was introduced with TLS 1.2. Ignore it if we have %% negotiated a lower version. diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index f13bd53a7c..2781203557 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -163,7 +163,8 @@ api_tests() -> server_name_indication_option, accept_pool, prf, - socket_options + socket_options, + cipher_suites ]. api_tests_tls() -> @@ -207,7 +208,7 @@ tls_cipher_tests() -> rc4_ecdsa_cipher_suites]. cipher_tests() -> - [cipher_suites, + [old_cipher_suites, cipher_suites_mix, ciphers_rsa_signed_certs, ciphers_rsa_signed_certs_openssl_names, @@ -704,8 +705,6 @@ secret_connection_info(Config) when is_list(Config) -> ct:log("Testcase ~p, Client ~p Server ~p ~n", [self(), Client, Server]), - - Version = ssl_test_lib:protocol_version(Config), ssl_test_lib:check_result(Server, true, Client, true), @@ -1130,11 +1129,16 @@ fallback(Config) when is_list(Config) -> %%-------------------------------------------------------------------- cipher_format() -> - [{doc, "Test that cipher conversion from tuples to binarys works"}]. + [{doc, "Test that cipher conversion from maps | tuples | stings to binarys works"}]. cipher_format(Config) when is_list(Config) -> - {ok, Socket} = ssl:listen(0, [{ciphers, ssl:cipher_suites()}]), - ssl:close(Socket). - + {ok, Socket0} = ssl:listen(0, [{ciphers, ssl:cipher_suites(default, 'tlsv1.2')}]), + ssl:close(Socket0), + %% Legacy + {ok, Socket1} = ssl:listen(0, [{ciphers, ssl:cipher_suites()}]), + ssl:close(Socket1), + {ok, Socket2} = ssl:listen(0, [{ciphers, ssl:cipher_suites(openssl)}]), + ssl:close(Socket2). + %%-------------------------------------------------------------------- peername() -> @@ -1285,20 +1289,76 @@ sockname_result(S) -> ssl:sockname(S). %%-------------------------------------------------------------------- + cipher_suites() -> - [{doc,"Test API function cipher_suites/0"}]. + [{doc,"Test API function cipher_suites/2, filter_cipher_suites/2" + " and prepend|append_cipher_suites/2"}]. cipher_suites(Config) when is_list(Config) -> - MandatoryCipherSuiteTLS1_0TLS1_1 = {rsa,'3des_ede_cbc',sha}, - MandatoryCipherSuiteTLS1_0TLS1_2 = {rsa,'aes_128_cbc',sha} , - [_|_] = Suites = ssl:cipher_suites(), - AllSuites = ssl:cipher_suites(all), - %% The mandantory suites will no longer be supported by default - %% due to security reasons - true = lists:member(MandatoryCipherSuiteTLS1_0TLS1_1, AllSuites), - true = lists:member(MandatoryCipherSuiteTLS1_0TLS1_2, AllSuites), + MandatoryCipherSuiteTLS1_0TLS1_1 = #{key_exchange => rsa, + cipher => '3des_ede_cbc', + mac => sha, + prf => default_prf}, + MandatoryCipherSuiteTLS1_0TLS1_2 = #{key_exchange =>rsa, + cipher => 'aes_128_cbc', + mac => sha, + prf => default_prf}, + Version = ssl_test_lib:protocol_version(Config), + All = [_|_] = ssl:cipher_suites(all, Version), + Default = [_|_] = ssl:cipher_suites(default, Version), + Anonymous = [_|_] = ssl:cipher_suites(anonymous, Version), + true = length(Default) < length(All), + Filters = [{key_exchange, + fun(dhe_rsa) -> + true; + (_) -> + false + end + }, + {cipher, + fun(aes_256_cbc) -> + true; + (_) -> + false + end + }, + {mac, + fun(sha) -> + true; + (_) -> + false + end + } + ], + Cipher = #{cipher => aes_256_cbc, + key_exchange => dhe_rsa, + mac => sha, + prf => default_prf}, + [Cipher] = ssl:filter_cipher_suites(All, Filters), + [Cipher | Rest0] = ssl:prepend_cipher_suites([Cipher], Default), + [Cipher | Rest0] = ssl:prepend_cipher_suites(Filters, Default), + true = lists:member(Cipher, Default), + false = lists:member(Cipher, Rest0), + [Cipher | Rest1] = lists:reverse(ssl:append_cipher_suites([Cipher], Default)), + [Cipher | Rest1] = lists:reverse(ssl:append_cipher_suites(Filters, Default)), + true = lists:member(Cipher, Default), + false = lists:member(Cipher, Rest1), + [] = lists:dropwhile(fun(X) -> not lists:member(X, Default) end, Anonymous), + [] = lists:dropwhile(fun(X) -> not lists:member(X, All) end, Anonymous), + true = lists:member(MandatoryCipherSuiteTLS1_0TLS1_1, All), + true = lists:member(MandatoryCipherSuiteTLS1_0TLS1_2, All). + +%%-------------------------------------------------------------------- + +old_cipher_suites() -> + [{doc,"Test API function cipher_suites/0"}]. + +old_cipher_suites(Config) when is_list(Config) -> + MandatoryCipherSuite = {rsa, '3des_ede_cbc', sha}, + [_|_] = Suites = ssl:cipher_suites(), Suites = ssl:cipher_suites(erlang), - [_|_] =ssl:cipher_suites(openssl). + [_|_] = ssl:cipher_suites(openssl), + true = lists:member(MandatoryCipherSuite, ssl:cipher_suites(all)). %%-------------------------------------------------------------------- cipher_suites_mix() -> @@ -3800,9 +3860,23 @@ rizzo() -> vunrable to Rizzo/Dungon attack"}]. rizzo(Config) when is_list(Config) -> - Ciphers = [X || X ={_,Y,_} <- ssl:cipher_suites(), Y =/= rc4_128], Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, NVersion), + [{key_exchange, + fun(Alg) when Alg == ecdh_rsa; Alg == ecdhe_rsa-> + true; + (_) -> + false + end}, + {cipher, + fun(rc4_128) -> + false; + (_) -> + true + end}]), + run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_rizzo, []}). %%-------------------------------------------------------------------- @@ -3814,8 +3888,13 @@ no_rizzo_rc4(Config) when is_list(Config) -> Version = proplists:get_value(name, Prop), NVersion = ssl_test_lib:protocol_version(Config, tuple), %% Test uses RSA certs - Ciphers = ssl_test_lib:rc4_suites(NVersion) -- [{ecdhe_ecdsa,rc4_128,sha}, - {ecdh_ecdsa,rc4_128,sha}], + Ciphers = ssl:filter_cipher_suites(ssl_test_lib:rc4_suites(NVersion), + [{key_exchange, + fun(Alg) when Alg == ecdh_rsa; Alg == ecdhe_rsa-> + true; + (_) -> + false + end}]), run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_no_rizzo, []}). @@ -3826,10 +3905,21 @@ rizzo_one_n_minus_one(Config) when is_list(Config) -> Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), NVersion = ssl_test_lib:protocol_version(Config, tuple), - AllSuites = ssl_test_lib:available_suites(NVersion), - Ciphers = [X || X ={_,Y,_} <- AllSuites, Y =/= rc4_128], + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(all, NVersion), + [{key_exchange, + fun(Alg) when Alg == ecdh_rsa; Alg == ecdhe_rsa-> + true; + (_) -> + false + end}, + {cipher, + fun(rc4_128) -> + false; + (_) -> + true + end}]), run_send_recv_rizzo(Ciphers, Config, Version, - {?MODULE, send_recv_result_active_rizzo, []}). + {?MODULE, send_recv_result_active_rizzo, []}). rizzo_zero_n() -> [{doc,"Test that the 0/n-split mitigation of Rizzo/Dungon attack can be explicitly selected"}]. @@ -3838,8 +3928,13 @@ rizzo_zero_n(Config) when is_list(Config) -> Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), NVersion = ssl_test_lib:protocol_version(Config, tuple), - AllSuites = ssl_test_lib:available_suites(NVersion), - Ciphers = [X || X ={_,Y,_} <- AllSuites, Y =/= rc4_128], + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(default, NVersion), + [{cipher, + fun(rc4_128) -> + false; + (_) -> + true + end}]), run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_no_rizzo, []}). @@ -3847,9 +3942,16 @@ rizzo_disabled() -> [{doc,"Test that the mitigation of Rizzo/Dungon attack can be explicitly disabled"}]. rizzo_disabled(Config) when is_list(Config) -> - Ciphers = [X || X ={_,Y,_} <- ssl:cipher_suites(), Y =/= rc4_128], Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(default, NVersion), + [{cipher, + fun(rc4_128) -> + false; + (_) -> + true + end}]), run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_no_rizzo, []}). @@ -4624,19 +4726,21 @@ rizzo_test(Cipher, Config, Version, Mfa) -> [{Cipher, Error}] end. -client_server_opts({KeyAlgo,_,_}, Config) +client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == rsa orelse KeyAlgo == dhe_rsa orelse - KeyAlgo == ecdhe_rsa -> + KeyAlgo == ecdhe_rsa orelse + KeyAlgo == rsa_psk orelse + KeyAlgo == srp_rsa -> {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_opts, Config)}; -client_server_opts({KeyAlgo,_,_}, Config) when KeyAlgo == dss orelse KeyAlgo == dhe_dss -> +client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == dss orelse KeyAlgo == dhe_dss -> {ssl_test_lib:ssl_options(client_dsa_opts, Config), ssl_test_lib:ssl_options(server_dsa_opts, Config)}; -client_server_opts({KeyAlgo,_,_}, Config) when KeyAlgo == ecdh_ecdsa orelse KeyAlgo == ecdhe_ecdsa -> +client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == ecdh_ecdsa orelse KeyAlgo == ecdhe_ecdsa -> {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_ecdsa_opts, Config)}; -client_server_opts({KeyAlgo,_,_}, Config) when KeyAlgo == ecdh_rsa -> +client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == ecdh_rsa -> {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)}. diff --git a/lib/ssl/test/ssl_engine_SUITE.erl b/lib/ssl/test/ssl_engine_SUITE.erl index bc221d35fd..71891356e8 100644 --- a/lib/ssl/test/ssl_engine_SUITE.erl +++ b/lib/ssl/test/ssl_engine_SUITE.erl @@ -39,23 +39,28 @@ init_per_suite(Config) -> catch crypto:stop(), try crypto:start() of ok -> - ssl_test_lib:clean_start(), - case crypto:get_test_engine() of - {ok, EngineName} -> - try crypto:engine_load(<<"dynamic">>, - [{<<"SO_PATH">>, EngineName}, - <<"LOAD">>], - []) of - {ok, Engine} -> - [{engine, Engine} |Config]; - {error, Reason} -> - ct:pal("Reason ~p", [Reason]), - {skip, "No dynamic engine support"} - catch error:notsup -> - {skip, "No engine support in OpenSSL"} - end; - {error, notexist} -> - {skip, "Test engine not found"} + case crypto:info_lib() of + [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}] -> + {skip, "Problem with engine on OpenSSL 1.0.1s-freebsd"}; + _ -> + ssl_test_lib:clean_start(), + case crypto:get_test_engine() of + {ok, EngineName} -> + try crypto:engine_load(<<"dynamic">>, + [{<<"SO_PATH">>, EngineName}, + <<"LOAD">>], + []) of + {ok, Engine} -> + [{engine, Engine} |Config]; + {error, Reason} -> + ct:pal("Reason ~p", [Reason]), + {skip, "No dynamic engine support"} + catch error:notsup -> + {skip, "No engine support in OpenSSL"} + end; + {error, notexist} -> + {skip, "Test engine not found"} + end end catch _:_ -> {skip, "Crypto did not start"} diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 94d10b2f9b..26ef311615 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -1024,59 +1024,50 @@ string_regex_filter(Str, Search) when is_list(Str) -> string_regex_filter(_Str, _Search) -> false. -anonymous_suites({3,_ } = Version) -> - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:anonymous_suites(Version))]; -anonymous_suites(DTLSVersion) -> - Version = dtls_v1:corresponding_tls_version(DTLSVersion), - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:anonymous_suites(Version)), - not ssl_cipher:is_stream_ciphersuite(tuple_to_map(ssl_cipher:erl_suite_definition(S)))]. - -psk_suites({3,_ } = Version) -> - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:psk_suites(Version))]; -psk_suites(DTLSVersion) -> - Version = dtls_v1:corresponding_tls_version(DTLSVersion), - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:psk_suites(Version)), - not ssl_cipher:is_stream_ciphersuite(tuple_to_map(ssl_cipher:erl_suite_definition(S)))]. - -psk_anon_suites({3,_ } = Version) -> - [Suite || Suite <- psk_suites(Version), is_psk_anon_suite(Suite)]; -psk_anon_suites(DTLSVersion) -> - Version = dtls_v1:corresponding_tls_version(DTLSVersion), - [Suite || Suite <- psk_suites(Version), is_psk_anon_suite(Suite), - not ssl_cipher:is_stream_ciphersuite(tuple_to_map(Suite))]. +anonymous_suites(Version) -> + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:anonymous_suites(Version)],[]). +psk_suites(Version) -> + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:psk_suites(Version)], []). + +psk_anon_suites(Version) -> + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:psk_suites_anon(Version)], + [{key_exchange, + fun(psk) -> + true; + (psk_dhe) -> + true; + (_) -> + false + end}]). + srp_suites() -> - [ssl_cipher:erl_suite_definition(Suite) || - Suite <- - ssl_cipher:filter_suites([tuple_to_map(S) || - S <- [{srp_anon,'3des_ede_cbc', sha}, - {srp_rsa, '3des_ede_cbc', sha}, - {srp_anon, aes_128_cbc, sha}, - {srp_rsa, aes_128_cbc, sha}, - {srp_anon, aes_256_cbc, sha}, - {srp_rsa, aes_256_cbc, sha}]])]. + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:srp_suites()], + [{key_exchange, + fun(srp_rsa) -> + true; + (_) -> + false + end}]). srp_anon_suites() -> - [ssl_cipher:erl_suite_definition(Suite) || - Suite <- - ssl_cipher:filter_suites([tuple_to_map(S) || - S <-[{srp_anon, '3des_ede_cbc', sha}, - {srp_anon, aes_128_cbc, sha}, - {srp_anon, aes_256_cbc, sha}]])]. + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:srp_suites_anon()], + []). srp_dss_suites() -> - [ssl_cipher:erl_suite_definition(Suite) || - Suite <- - ssl_cipher:filter_suites([tuple_to_map(S) || - S <- [{srp_dss, '3des_ede_cbc', sha}, - {srp_dss, aes_128_cbc, sha}, - {srp_dss, aes_256_cbc, sha}]])]. - + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <- ssl_cipher:srp_suites()], + [{key_exchange, + fun(srp_dss) -> + true; + (_) -> + false + end}]). chacha_suites(Version) -> [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:chacha_suites(Version))]. + rc4_suites(Version) -> - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:rc4_suites(Version))]. + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <-ssl_cipher:rc4_suites(Version)], []). des_suites(Version) -> - [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:des_suites(Version))]. + ssl:filter_cipher_suites([ssl_cipher:suite_definition(S) || S <-ssl_cipher:des_suites(Version)], []). tuple_to_map({Kex, Cipher, Mac}) -> #{key_exchange => Kex, @@ -1313,6 +1304,32 @@ cipher_restriction(Config0) -> Config0 end. +openssl_dsa_support() -> + case os:cmd("openssl version") of + "LibreSSL 2.6.1" ++ _ -> + true; + "LibreSSL 2.6.2" ++ _ -> + true; + "LibreSSL 2.6" ++ _ -> + false; + "LibreSSL 2.4" ++ _ -> + true; + "LibreSSL 2.3" ++ _ -> + true; + "LibreSSL 2.2" ++ _ -> + true; + "LibreSSL 2.1" ++ _ -> + true; + "LibreSSL 2.0" ++ _ -> + true; + "LibreSSL" ++ _ -> + false; + "OpenSSL 1.0.1" ++ Rest -> + hd(Rest) >= s; + _ -> + true + end. + check_sane_openssl_version(Version) -> case supports_ssl_tls_version(Version) of true -> @@ -1391,7 +1408,9 @@ filter_suites(Ciphers0, AtomVersion) -> Supported0 = ssl_cipher:suites(Version) ++ ssl_cipher:anonymous_suites(Version) ++ ssl_cipher:psk_suites(Version) + ++ ssl_cipher:psk_suites_anon(Version) ++ ssl_cipher:srp_suites() + ++ ssl_cipher:srp_suites_anon() ++ ssl_cipher:rc4_suites(Version), Supported1 = ssl_cipher:filter_suites(Supported0), Supported2 = [ssl_cipher:erl_suite_definition(S) || S <- Supported1], diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index 33cdc325f4..dcdea6beb5 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -70,6 +70,9 @@ all_versions_tests() -> erlang_server_openssl_client, erlang_client_openssl_server_dsa_cert, erlang_server_openssl_client_dsa_cert, + erlang_client_openssl_server_anon, + erlang_server_openssl_client_anon, + erlang_server_openssl_client_anon_with_cert, erlang_server_openssl_client_reuse_session, erlang_client_openssl_server_renegotiate, erlang_client_openssl_server_nowrap_seqnum, @@ -89,6 +92,9 @@ dtls_all_versions_tests() -> erlang_server_openssl_client, erlang_client_openssl_server_dsa_cert, erlang_server_openssl_client_dsa_cert, + erlang_client_openssl_server_anon, + erlang_server_openssl_client_anon, + erlang_server_openssl_client_anon_with_cert, erlang_server_openssl_client_reuse_session, erlang_client_openssl_server_renegotiate, erlang_client_openssl_server_nowrap_seqnum, @@ -143,10 +149,15 @@ init_per_suite(Config0) -> try crypto:start() of ok -> ssl_test_lib:clean_start(), - - Config1 = ssl_test_lib:make_rsa_cert(Config0), - Config2 = ssl_test_lib:make_dsa_cert(Config1), - ssl_test_lib:cipher_restriction(Config2) + Config = + case ssl_test_lib:openssl_dsa_support() of + true -> + Config1 = ssl_test_lib:make_rsa_cert(Config0), + ssl_test_lib:make_dsa_cert(Config1); + false -> + ssl_test_lib:make_rsa_cert(Config0) + end, + ssl_test_lib:cipher_restriction(Config) catch _:_ -> {skip, "Crypto did not start"} end @@ -199,15 +210,27 @@ init_per_testcase(expired_session, Config) -> ssl:start(), Config; -init_per_testcase(TestCase, Config) when TestCase == ciphers_rsa_signed_certs; - TestCase == ciphers_dsa_signed_certs -> - ct:timetrap({seconds, 90}), - special_init(TestCase, Config); - +init_per_testcase(TestCase, Config) when + TestCase == ciphers_dsa_signed_certs; + TestCase == erlang_client_openssl_server_dsa_cert; + TestCase == erlang_server_openssl_client_dsa_cert; + TestCase == erlang_client_openssl_server_dsa_cert; + TestCase == erlang_server_openssl_client_dsa_cert -> + case ssl_test_lib:openssl_dsa_support() of + true -> + special_init(TestCase, Config); + false -> + {skip, "DSA not supported by OpenSSL"} + end; init_per_testcase(TestCase, Config) -> ct:timetrap({seconds, 35}), special_init(TestCase, Config). +special_init(TestCase, Config) when + TestCase == ciphers_rsa_signed_certs; + TestCase == ciphers_dsa_signed_certs-> + ct:timetrap({seconds, 90}), + Config; special_init(TestCase, Config) when TestCase == erlang_client_openssl_server_renegotiate; TestCase == erlang_client_openssl_server_nowrap_seqnum; @@ -533,7 +556,121 @@ erlang_server_openssl_client_dsa_cert(Config) when is_list(Config) -> ssl_test_lib:close_port(OpenSslPort), process_flag(trap_exit, false). -%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +erlang_client_openssl_server_anon() -> + [{doc,"Test erlang client with openssl server, anonymous"}]. +erlang_client_openssl_server_anon(Config) when is_list(Config) -> + process_flag(trap_exit, true), + %% OpenSSL expects a certificate and key, even if the cipher spec + %% is restructed to aNULL, so we use 'server_rsa_opts' here + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_anon_opts, Config), + VersionTuple = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl_test_lib:anonymous_suites(VersionTuple), + + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + Data = "From openssl to erlang", + + Port = ssl_test_lib:inet_port(node()), + CertFile = proplists:get_value(certfile, ServerOpts), + KeyFile = proplists:get_value(keyfile, ServerOpts), + Version = ssl_test_lib:protocol_version(Config), + Exe = "openssl", + Args = ["s_server", "-accept", integer_to_list(Port), + ssl_test_lib:version_flag(Version), + "-cert", CertFile, "-key", KeyFile, + "-cipher", "aNULL", "-msg"], + + OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), + + ssl_test_lib:wait_for_openssl_server(Port, proplists:get_value(protocol, Config)), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + erlang_ssl_receive, [Data]}}, + {options, [{ciphers, Ciphers} | ClientOpts]}]), + + true = port_command(OpensslPort, Data), + + ssl_test_lib:check_result(Client, ok), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close_port(OpensslPort), + ssl_test_lib:close(Client), + process_flag(trap_exit, false), + ok. +%%-------------------------------------------------------------------- +erlang_server_openssl_client_anon() -> + [{doc,"Test erlang server with openssl client, anonymous"}]. +erlang_server_openssl_client_anon(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ServerOpts = ssl_test_lib:ssl_options(server_anon_opts, Config), + VersionTuple = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl_test_lib:anonymous_suites(VersionTuple), + + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Data = "From openssl to erlang", + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, erlang_ssl_receive, [Data]}}, + {options, [{ciphers, Ciphers} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Version = ssl_test_lib:protocol_version(Config), + Exe = "openssl", + Args = ["s_client", "-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), + ssl_test_lib:version_flag(Version), + "-cipher", "aNULL", "-msg"], + + OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), + true = port_command(OpenSslPort, Data), + + ssl_test_lib:check_result(Server, ok), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close(Server), + ssl_test_lib:close_port(OpenSslPort), + process_flag(trap_exit, false). + + %%-------------------------------------------------------------------- + erlang_server_openssl_client_anon_with_cert() -> + [{doc,"Test erlang server with openssl client, anonymous (with cert)"}]. + erlang_server_openssl_client_anon_with_cert(Config) when is_list(Config) -> + process_flag(trap_exit, true), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + VersionTuple = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl_test_lib:anonymous_suites(VersionTuple), + + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Data = "From openssl to erlang", + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, erlang_ssl_receive, [Data]}}, + {options, [{ciphers, Ciphers} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Version = ssl_test_lib:protocol_version(Config), + Exe = "openssl", + Args = ["s_client", "-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), + ssl_test_lib:version_flag(Version), + "-cipher", "aNULL", "-msg"], + + OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), + true = port_command(OpenSslPort, Data), + + ssl_test_lib:check_result(Server, ok), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close(Server), + ssl_test_lib:close_port(OpenSslPort), + process_flag(trap_exit, false). + +%%-------------------------------------------------------------------- erlang_server_openssl_client_reuse_session() -> [{doc, "Test erlang server with openssl client that reconnects with the" diff --git a/lib/stdlib/doc/src/assert_hrl.xml b/lib/stdlib/doc/src/assert_hrl.xml index ea23cca2ee..33f29f38da 100644 --- a/lib/stdlib/doc/src/assert_hrl.xml +++ b/lib/stdlib/doc/src/assert_hrl.xml @@ -93,7 +93,7 @@ erlc -DNOASSERT=true *.erl</code> <taglist> <tag><c>assert(BoolExpr)</c></tag> <item></item> - <tag><c>URKAassert(BoolExpr, Comment)</c></tag> + <tag><c>assert(BoolExpr, Comment)</c></tag> <item> <p>Tests that <c>BoolExpr</c> completes normally returning <c>true</c>.</p> diff --git a/lib/stdlib/doc/src/erl_tar.xml b/lib/stdlib/doc/src/erl_tar.xml index 337028568a..14c543ee2b 100644 --- a/lib/stdlib/doc/src/erl_tar.xml +++ b/lib/stdlib/doc/src/erl_tar.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2003</year><year>2017</year> + <year>2003</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -136,6 +136,9 @@ <v>Filename = filename()|{NameInArchive,FilenameOrBin}</v> <v>Options = [Option]</v> <v>Option = dereference|verbose|{chunks,ChunkSize}</v> + <v>|{atime,non_neg_integer()}|{mtime,non_neg_integer()}</v> + <v>|{ctime,non_neg_integer()}|{uid,non_neg_integer()}</v> + <v>|{gid,non_neg_integer()}</v> <v>ChunkSize = positive_integer()</v> <v>RetValue = ok|{error,{Filename,Reason}}</v> <v>Reason = term()</v> @@ -167,6 +170,42 @@ <seealso marker="ssh:ssh_sftp#open_tar/3"> <c>ssh_sftp:open_tar/3</c></seealso>.</p> </item> + <tag><c>{atime,non_neg_integer()}</c></tag> + <item> + <p>Sets the last time, as + <seealso marker="erts:time_correction#POSIX_Time"> + POSIX time</seealso>, when the file was read. See also + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> + <tag><c>{mtime,non_neg_integer()}</c></tag> + <item> + <p>Sets the last time, as + <seealso marker="erts:time_correction#POSIX_Time"> + POSIX time</seealso>, when the file was written. See also + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> + <tag><c>{ctime,non_neg_integer()}</c></tag> + <item> + <p>Sets the time, as + <seealso marker="erts:time_correction#POSIX_Time"> + POSIX time</seealso>, when the file was created. See also + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> + <tag><c>{uid,non_neg_integer()}</c></tag> + <item> + <p>Sets the file owner. + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> + <tag><c>{gid,non_neg_integer()}</c></tag> + <item> + <p>Sets the group that the file owner belongs to. + <seealso marker="kernel:file#read_file_info/1"> + <c>file:read_file_info/1</c></seealso>.</p> + </item> </taglist> </desc> </func> @@ -378,7 +417,7 @@ <v>Reason = term()</v> </type> <desc> - <p>Cconverts an error reason term to a human-readable error message + <p>Converts an error reason term to a human-readable error message string.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index a0ec22c515..305376a425 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -487,6 +487,11 @@ Error: fun containing local Erlang function calls <p>The pid of the heir of the table, or <c>none</c> if no heir is set.</p> </item> + <tag><c>{id,</c><seealso marker="#type-tid"> + <c>tid()</c></seealso><c>}</c></tag> + <item> + <p>The table identifier.</p> + </item> <tag><c>{keypos, integer() >= 1}</c></tag> <item> <p>The key position.</p> @@ -963,11 +968,11 @@ ets:is_compiled_ms(Broken).</code> <func> <name name="match_spec_run" arity="2"/> <fsummary>Perform matching, using a compiled match specification on a - list of tuples.</fsummary> + list of terms.</fsummary> <desc> <p>Executes the matching specified in a compiled <seealso marker="#match_spec">match specification</seealso> on a list - of tuples. Term <c><anno>CompiledMatchSpec</anno></c> is to be + of terms. Term <c><anno>CompiledMatchSpec</anno></c> is to be the result of a call to <seealso marker="#match_spec_compile/1"> <c>match_spec_compile/1</c></seealso> and is hence the internal representation of the match specification one wants to use.</p> @@ -985,7 +990,7 @@ Table = ets:new... MatchSpec = ... % The following call... ets:match_spec_run(ets:tab2list(Table), -ets:match_spec_compile(MatchSpec)), + ets:match_spec_compile(MatchSpec)), % ...gives the same result as the more common (and more efficient) ets:select(Table, MatchSpec),</code> <note> @@ -1074,10 +1079,13 @@ ets:select(Table, MatchSpec),</code> </item> <tag><c>named_table</c></tag> <item> - <p>If this option is present, name <c><anno>Name</anno></c> is - associated with the table identifier. The name can then - be used instead of the table identifier in subsequent - operations.</p> + <p>If this option is present, the table is registered under its + <c><anno>Name</anno></c> which can then be used instead of the + table identifier in subsequent operations.</p> + <p>The function will also return the <c><anno>Name</anno></c> + instead of the table identifier. To get the table identifier of a + named table, use + <seealso marker="#whereis/1"><c>whereis/1</c></seealso>.</p> </item> <tag><c>{keypos,<anno>Pos</anno>}</c></tag> <item> @@ -2037,6 +2045,21 @@ true</pre> </list> </desc> </func> + + <func> + <name name="whereis" arity="1"/> + <fsummary>Retrieves the tid() of a named table.</fsummary> + <desc> + <p>This function returns the + <seealso marker="#type-tid"><c>tid()</c></seealso> of the named table + identified by <c><anno>TableName</anno></c>, or <c>undefined</c> if + no such table exists. The <c>tid()</c> can be used in place of the + table name in all operations, which is slightly faster since the name + does not have to be resolved on each call.</p> + <p>If the table is deleted, the <c>tid()</c> will be invalid even if + another named table is created with the same name.</p> + </desc> + </func> </funcs> </erlref> diff --git a/lib/stdlib/doc/src/timer.xml b/lib/stdlib/doc/src/timer.xml index fcaccdb2cb..350847bf7d 100644 --- a/lib/stdlib/doc/src/timer.xml +++ b/lib/stdlib/doc/src/timer.xml @@ -270,7 +270,7 @@ <item> <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, <anno>Arguments</anno>)</c> and measures the elapsed real time as - reported by <seealso marker="os:timestamp/0"> + reported by <seealso marker="kernel:os#timestamp/0"> <c>os:timestamp/0</c></seealso>.</p> <p>Returns <c>{<anno>Time</anno>, <anno>Value</anno>}</c>, where <c><anno>Time</anno></c> is the elapsed real time in diff --git a/lib/stdlib/doc/src/uri_string.xml b/lib/stdlib/doc/src/uri_string.xml index 21f470e763..88d4600611 100644 --- a/lib/stdlib/doc/src/uri_string.xml +++ b/lib/stdlib/doc/src/uri_string.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2017</year><year>2017</year> + <year>2017</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -24,7 +24,7 @@ <title>uri_string</title> <prepared>Péter Dimitrov</prepared> <docno>1</docno> - <date>2017-10-24</date> + <date>2018-02-07</date> <rev>A</rev> </header> <module>uri_string</module> @@ -32,7 +32,11 @@ <description> <p>This module contains functions for parsing and handling URIs (<url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>) and - form-urlencoded query strings (<url href="https://www.w3.org/TR/html5/forms.html">HTML5</url>). + form-urlencoded query strings (<url href="https://www.w3.org/TR/html52/">HTML 5.2</url>). + </p> + <p> + Parsing and serializing non-UTF-8 form-urlencoded query strings are also supported + (<url href="https://www.w3.org/TR/html50/">HTML 5.0</url>). </p> <p>A URI is an identifier consisting of a sequence of characters matching the syntax rule named <em>URI</em> in <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>. @@ -70,7 +74,8 @@ <seealso marker="#transcode/2"><c>transcode/2</c></seealso> </item> <item>Transforming URIs into a normalized form<br></br> - <seealso marker="#normalize/1"><c>normalize/1</c></seealso> + <seealso marker="#normalize/1"><c>normalize/1</c></seealso><br></br> + <seealso marker="#normalize/2"><c>normalize/2</c></seealso> </item> <item>Composing form-urlencoded query strings from a list of key-value pairs<br></br> <seealso marker="#compose_query/1"><c>compose_query/1</c></seealso><br></br> @@ -151,8 +156,10 @@ <p>Composes a form-urlencoded <c><anno>QueryString</anno></c> based on a <c><anno>QueryList</anno></c>, a list of non-percent-encoded key-value pairs. Form-urlencoding is defined in section - 4.10.22.6 of the <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - specification. + 4.10.21.6 of the <url href="https://www.w3.org/TR/html52/">HTML 5.2</url> + specification and in section 4.10.22.6 of the + <url href="https://www.w3.org/TR/html50/">HTML 5.0</url> specification for + non-UTF-8 encodings. </p> <p>See also the opposite operation <seealso marker="#dissect_query/1"> <c>dissect_query/1</c></seealso>. @@ -209,12 +216,11 @@ <p>Dissects an urlencoded <c><anno>QueryString</anno></c> and returns a <c><anno>QueryList</anno></c>, a list of non-percent-encoded key-value pairs. Form-urlencoding is defined in section - 4.10.22.6 of the <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - specification. + 4.10.21.6 of the <url href="https://www.w3.org/TR/html52/">HTML 5.2</url> + specification and in section 4.10.22.6 of the + <url href="https://www.w3.org/TR/html50/">HTML 5.0</url> specification for + non-UTF-8 encodings. </p> - <p>It is not as strict for its input as the decoding algorithm defined by - <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - and accepts all unicode characters.</p> <p>See also the opposite operation <seealso marker="#compose_query/1"> <c>compose_query/1</c></seealso>. </p> @@ -233,7 +239,7 @@ <name name="normalize" arity="1"/> <fsummary>Syntax-based normalization.</fsummary> <desc> - <p>Transforms <c><anno>URIString</anno></c> into a normalized form + <p>Transforms an <c><anno>URI</anno></c> into a normalized form using Syntax-Based Normalization as defined by <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>.</p> <p>This function implements case normalization, percent-encoding @@ -247,6 +253,33 @@ <![CDATA[<<"mid/6">>]]> 3> uri_string:normalize("http://localhost:80"). "https://localhost/" +4> <input>uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g",</input> +4> host => "localhost-örebro"}). +"http://localhost-%C3%B6rebro/a/g" + </pre> + </desc> + </func> + + <func> + <name name="normalize" arity="2"/> + <fsummary>Syntax-based normalization.</fsummary> + <desc> + <p>Same as <c>normalize/1</c> but with an additional + <c><anno>Options</anno></c> parameter, that controls if the normalized URI + shall be returned as an uri_map(). + There is one supported option: <c>return_map</c>. + </p> + <p><em>Example:</em></p> + <pre> +1> <input>uri_string:normalize("/a/b/c/./../../g", [return_map]).</input> +#{path => "/a/g"} +2> <![CDATA[uri_string:normalize(<<"mid/content=5/../6">>, [return_map]).]]> +<![CDATA[#{path => <<"mid/6">>}]]> +3> uri_string:normalize("http://localhost:80", [return_map]). +#{scheme => "http",path => "/",host => "localhost"} +4> <input>uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g",</input> +4> host => "localhost-örebro"}, [return_map]). +#{scheme => "http",path => "/a/g",host => "localhost-örebro"} </pre> </desc> </func> diff --git a/lib/stdlib/include/assert.hrl b/lib/stdlib/include/assert.hrl index 2fbaeba0b2..2ec89e7d8a 100644 --- a/lib/stdlib/include/assert.hrl +++ b/lib/stdlib/include/assert.hrl @@ -309,7 +309,7 @@ {unexpected_success, __V}]}) catch Class:Term -> ok; - __C:__T -> + __C:__T:__S -> erlang:error({assertException, [{module, ?MODULE}, {line, ?LINE}, @@ -318,8 +318,7 @@ "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace()}}]}) + {__C, __T, __S}}]}) end end)()) end). @@ -338,7 +337,7 @@ {unexpected_success, __V}]}) catch Class:Term -> ok; - __C:__T -> + __C:__T:__S -> erlang:error({assertException, [{module, ?MODULE}, {line, ?LINE}, @@ -348,8 +347,7 @@ "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace()}}]}) + {__C, __T, __S}}]}) end end)()) end). @@ -378,7 +376,7 @@ try (Expr) of _ -> ok catch - __C:__T -> + __C:__T:__S -> case __C of Class -> case __T of @@ -391,9 +389,7 @@ "{ "++(??Class)++" , " ++(??Term)++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace() - }}]}); + {__C, __T, __S}}]}); _ -> ok end; _ -> ok @@ -407,7 +403,7 @@ try (Expr) of _ -> ok catch - __C:__T -> + __C:__T:__S -> case __C of Class -> case __T of @@ -421,9 +417,7 @@ "{ "++(??Class)++" , " ++(??Term)++" , [...] }"}, {unexpected_exception, - {__C, __T, - erlang:get_stacktrace() - }}]}); + {__C, __T, __S}}]}); _ -> ok end; _ -> ok diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 06c15fceda..24349c74e8 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -148,7 +148,8 @@ chunks(File, Chunks, Options) -> try read_chunk_data(File, Chunks, Options) catch Error -> Error end. --spec all_chunks(beam()) -> {'ok', 'beam_lib', [{chunkid(), dataB()}]}. +-spec all_chunks(beam()) -> + {'ok', 'beam_lib', [{chunkid(), dataB()}]} | {'error', 'beam_lib', info_rsn()}. all_chunks(File) -> read_all_chunks(File). diff --git a/lib/stdlib/src/binary.erl b/lib/stdlib/src/binary.erl index 6a64133b45..7d0e42489e 100644 --- a/lib/stdlib/src/binary.erl +++ b/lib/stdlib/src/binary.erl @@ -47,23 +47,39 @@ at(_, _) -> -spec bin_to_list(Subject) -> [byte()] when Subject :: binary(). -bin_to_list(_) -> - erlang:nif_error(undef). +bin_to_list(Subject) -> + binary_to_list(Subject). -spec bin_to_list(Subject, PosLen) -> [byte()] when Subject :: binary(), PosLen :: part(). -bin_to_list(_, _) -> - erlang:nif_error(undef). +bin_to_list(Subject, {Pos, Len}) -> + bin_to_list(Subject, Pos, Len); +bin_to_list(_Subject, _BadArg) -> + erlang:error(badarg). -spec bin_to_list(Subject, Pos, Len) -> [byte()] when Subject :: binary(), Pos :: non_neg_integer(), Len :: integer(). -bin_to_list(_, _, _) -> - erlang:nif_error(undef). +bin_to_list(Subject, Pos, Len) when not is_binary(Subject); + not is_integer(Pos); + not is_integer(Len) -> + %% binary_to_list/3 allows bitstrings as long as the slice fits, and we + %% want to badarg when Pos/Len aren't integers instead of raising badarith + %% when adjusting args for binary_to_list/3. + erlang:error(badarg); +bin_to_list(Subject, Pos, 0) when Pos >= 0, Pos =< byte_size(Subject) -> + %% binary_to_list/3 doesn't handle this case. + []; +bin_to_list(_Subject, _Pos, 0) -> + erlang:error(badarg); +bin_to_list(Subject, Pos, Len) when Len < 0 -> + bin_to_list(Subject, Pos + Len, -Len); +bin_to_list(Subject, Pos, Len) when Len > 0 -> + binary_to_list(Subject, Pos + 1, Pos + Len). -spec compile_pattern(Pattern) -> cp() when Pattern :: binary() | [binary()]. diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index 9a447af5b7..3597e61c26 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -1034,8 +1034,8 @@ appcall(App, M, F, Args) -> try apply(M, F, Args) catch - error:undef -> - case erlang:get_stacktrace() of + error:undef:S -> + case S of [{M,F,Args,_}|_] -> Arity = length(Args), io:format("Call to ~w:~w/~w in application ~w failed.\n", diff --git a/lib/stdlib/src/dets.erl b/lib/stdlib/src/dets.erl index 4e3fe0e5c1..e1a36abc70 100644 --- a/lib/stdlib/src/dets.erl +++ b/lib/stdlib/src/dets.erl @@ -1288,8 +1288,8 @@ init(Parent, Server) -> catch exit:normal -> exit(normal); - _:Bad -> - bug_found(no_name, Op, Bad, From), + _:Bad:Stacktrace -> + bug_found(no_name, Op, Bad, Stacktrace, From), exit(Bad) % give up end end. @@ -1371,8 +1371,8 @@ do_apply_op(Op, From, Head, N) -> catch exit:normal -> exit(normal); - _:Bad -> - bug_found(Head#head.name, Op, Bad, From), + _:Bad:Stacktrace -> + bug_found(Head#head.name, Op, Bad, Stacktrace, From), open_file_loop(Head, N) end. @@ -1581,7 +1581,7 @@ apply_op(Op, From, Head, N) -> ok end. -bug_found(Name, Op, Bad, From) -> +bug_found(Name, Op, Bad, Stacktrace, From) -> case dets_utils:debug_mode() of true -> %% If stream_op/5 found more requests, this is not @@ -1590,7 +1590,7 @@ bug_found(Name, Op, Bad, From) -> ("** dets: Bug was found when accessing table ~tw,~n" "** dets: operation was ~tp and reply was ~tw.~n" "** dets: Stacktrace: ~tw~n", - [Name, Op, Bad, erlang:get_stacktrace()]); + [Name, Op, Bad, Stacktrace]); false -> error_logger:format ("** dets: Bug was found when accessing table ~tw~n", diff --git a/lib/stdlib/src/dets_utils.erl b/lib/stdlib/src/dets_utils.erl index 17f55ebdc2..4c8ea9e82b 100644 --- a/lib/stdlib/src/dets_utils.erl +++ b/lib/stdlib/src/dets_utils.erl @@ -377,7 +377,8 @@ corrupt_reason(Head, Reason0) -> no_disk_map -> Reason0; DM -> - ST = erlang:get_stacktrace(), + {current_stacktrace, ST} = + erlang:process_info(self(), current_stacktrace), PD = get(), {Reason0, ST, PD, DM} end, diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index 00e6a10d8a..77cc88eb08 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1197,21 +1197,21 @@ skip_else(_Else, From, St, Sis) -> %% macro_expansion(Tokens, Anno) %% Extract the macro parameters and the expansion from a macro definition. -macro_pars([{')',_Lp}, {',',Ld}|Ex], Args) -> - {ok, {lists:reverse(Args), macro_expansion(Ex, Ld)}}; -macro_pars([{var,_,Name}, {')',_Lp}, {',',Ld}|Ex], Args) -> +macro_pars([{')',_Lp}, {',',_Ld}=Comma|Ex], Args) -> + {ok, {lists:reverse(Args), macro_expansion(Ex, Comma)}}; +macro_pars([{var,_,Name}, {')',_Lp}, {',',_Ld}=Comma|Ex], Args) -> false = lists:member(Name, Args), %Prolog is nice - {ok, {lists:reverse([Name|Args]), macro_expansion(Ex, Ld)}}; + {ok, {lists:reverse([Name|Args]), macro_expansion(Ex, Comma)}}; macro_pars([{var,_L,Name}, {',',_}|Ts], Args) -> false = lists:member(Name, Args), macro_pars(Ts, [Name|Args]). -macro_expansion([{')',_Lp},{dot,_Ld}], _Anno0) -> []; -macro_expansion([{dot,_}=Dot], _Anno0) -> +macro_expansion([{')',_Lp},{dot,_Ld}], _T0) -> []; +macro_expansion([{dot,_}=Dot], _T0) -> throw({error,loc(Dot),missing_parenthesis}); -macro_expansion([T|Ts], _Anno0) -> +macro_expansion([T|Ts], _T0) -> [T|macro_expansion(Ts, T)]; -macro_expansion([], Anno0) -> throw({error,loc(Anno0),premature_end}). +macro_expansion([], T0) -> throw({error,loc(T0),premature_end}). %% expand_macros(Tokens, St) %% expand_macro(Tokens, MacroToken, RestTokens) diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl index 5ee584d612..d8b8f466b1 100644 --- a/lib/stdlib/src/erl_tar.erl +++ b/lib/stdlib/src/erl_tar.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -457,26 +457,61 @@ add(Reader, NameOrBin, NameInArchive, Options) do_add(#reader{access=write}=Reader, Name, NameInArchive, Options) when is_list(NameInArchive), is_list(Options) -> - RF = fun(F) -> file:read_link_info(F, [{time, posix}]) end, + RF = apply_file_info_opts_fun(Options, read_link_info), Opts = #add_opts{read_info=RF}, - add1(Reader, Name, NameInArchive, add_opts(Options, Opts)); + add1(Reader, Name, NameInArchive, add_opts(Options, Options, Opts)); do_add(#reader{access=read},_,_,_) -> {error, eacces}; do_add(Reader,_,_,_) -> {error, {badarg, Reader}}. -add_opts([dereference|T], Opts) -> - RF = fun(F) -> file:read_file_info(F, [{time, posix}]) end, - add_opts(T, Opts#add_opts{read_info=RF}); -add_opts([verbose|T], Opts) -> - add_opts(T, Opts#add_opts{verbose=true}); -add_opts([{chunks,N}|T], Opts) -> - add_opts(T, Opts#add_opts{chunk_size=N}); -add_opts([_|T], Opts) -> - add_opts(T, Opts); -add_opts([], Opts) -> +add_opts([dereference|T], AllOptions, Opts) -> + RF = apply_file_info_opts_fun(AllOptions, read_file_info), + add_opts(T, AllOptions, Opts#add_opts{read_info=RF}); +add_opts([verbose|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{verbose=true}); +add_opts([{chunks,N}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{chunk_size=N}); +add_opts([{atime,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{atime=Value}); +add_opts([{mtime,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{mtime=Value}); +add_opts([{ctime,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{ctime=Value}); +add_opts([{uid,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{uid=Value}); +add_opts([{gid,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{gid=Value}); +add_opts([_|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts); +add_opts([], _AllOptions, Opts) -> Opts. +apply_file_info_opts(Opts, {ok, FileInfo}) -> + {ok, do_apply_file_info_opts(Opts, FileInfo)}; +apply_file_info_opts(_Opts, Other) -> + Other. + +do_apply_file_info_opts([{atime,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{atime=Value}); +do_apply_file_info_opts([{mtime,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{mtime=Value}); +do_apply_file_info_opts([{ctime,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{ctime=Value}); +do_apply_file_info_opts([{uid,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{uid=Value}); +do_apply_file_info_opts([{gid,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{gid=Value}); +do_apply_file_info_opts([_|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo); +do_apply_file_info_opts([], FileInfo) -> + FileInfo. + +apply_file_info_opts_fun(Options, InfoFunction) -> + fun(F) -> + apply_file_info_opts(Options, file:InfoFunction(F, [{time, posix}])) + end. + add1(#reader{}=Reader, Name, NameInArchive, #add_opts{read_info=ReadInfo}=Opts) when is_list(Name) -> Res = case ReadInfo(Name) of @@ -515,9 +550,11 @@ add1(Reader, Bin, NameInArchive, Opts) when is_binary(Bin) -> name = NameInArchive, size = byte_size(Bin), typeflag = ?TYPE_REGULAR, - atime = Now, - mtime = Now, - ctime = Now, + atime = add_opts_time(Opts#add_opts.atime, Now), + mtime = add_opts_time(Opts#add_opts.mtime, Now), + ctime = add_opts_time(Opts#add_opts.ctime, Now), + uid = Opts#add_opts.uid, + gid = Opts#add_opts.gid, mode = 8#100644}, {ok, Reader2} = add_header(Reader, Header, Opts), Padding = skip_padding(byte_size(Bin)), @@ -527,6 +564,9 @@ add1(Reader, Bin, NameInArchive, Opts) when is_binary(Bin) -> {error, Reason} -> {error, {NameInArchive, Reason}} end. +add_opts_time(undefined, Now) -> Now; +add_opts_time(Time, _Now) -> Time. + add_directory(Reader, DirName, NameInArchive, Info, Opts) -> case file:list_dir(DirName) of {ok, []} -> @@ -1650,8 +1690,12 @@ write_file(Name, Bin) -> case file:write_file(Name, Bin) of ok -> ok; {error,enoent} -> - ok = make_dirs(Name, file), - write_file(Name, Bin); + case make_dirs(Name, file) of + ok -> + write_file(Name, Bin); + {error,Reason} -> + throw({error, Reason}) + end; {error,Reason} -> throw({error, Reason}) end. diff --git a/lib/stdlib/src/erl_tar.hrl b/lib/stdlib/src/erl_tar.hrl index cff0c2f500..5d6cecbb66 100644 --- a/lib/stdlib/src/erl_tar.hrl +++ b/lib/stdlib/src/erl_tar.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% Copyright Ericsson AB 1997-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,7 +21,12 @@ -record(add_opts, { read_info, %% Fun to use for read file/link info. chunk_size = 0, %% For file reading when sending to sftp. 0=do not chunk - verbose = false}). %% Verbose on/off. + verbose = false, %% Verbose on/off. + atime = undefined, + mtime = undefined, + ctime = undefined, + uid = 0, + gid = 0}). -type add_opts() :: #add_opts{}. %% Options used when reading a tar archive. @@ -36,7 +41,12 @@ -type add_opt() :: dereference | verbose | - {chunks, pos_integer()}. + {chunks, pos_integer()} | + {atime, non_neg_integer()} | + {mtime, non_neg_integer()} | + {ctime, non_neg_integer()} | + {uid, non_neg_integer()} | + {gid, non_neg_integer()}. -type extract_opt() :: {cwd, string()} | {files, [string()]} | diff --git a/lib/stdlib/src/escript.erl b/lib/stdlib/src/escript.erl index 132f8efbbe..beea9927d2 100644 --- a/lib/stdlib/src/escript.erl +++ b/lib/stdlib/src/escript.erl @@ -283,8 +283,7 @@ start(EscriptOptions) -> throw:Str -> io:format("escript: ~ts\n", [Str]), my_halt(127); - _:Reason -> - Stk = erlang:get_stacktrace(), + _:Reason:Stk -> io:format("escript: Internal error: ~tp\n", [Reason]), io:format("~tp\n", [Stk]), my_halt(127) @@ -759,8 +758,8 @@ run(Module, Args) -> Module:main(Args), my_halt(0) catch - Class:Reason -> - fatal(format_exception(Class, Reason)) + Class:Reason:StackTrace -> + fatal(format_exception(Class, Reason, StackTrace)) end. -spec interpret(_, _, _, _) -> no_return(). @@ -793,8 +792,8 @@ interpret(Forms, HasRecs, File, Args) -> end}), my_halt(0) catch - Class:Reason -> - fatal(format_exception(Class, Reason)) + Class:Reason:StackTrace -> + fatal(format_exception(Class, Reason, StackTrace)) end. report_errors(Errors) -> @@ -873,7 +872,7 @@ eval_exprs([E|Es], Bs0, Lf, Ef, RBs) -> {value,_V,Bs} = erl_eval:expr(E, Bs0, Lf, Ef, RBs1), eval_exprs(Es, Bs, Lf, Ef, RBs). -format_exception(Class, Reason) -> +format_exception(Class, Reason, StackTrace) -> Enc = encoding(), P = case Enc of latin1 -> "P"; @@ -882,7 +881,6 @@ format_exception(Class, Reason) -> PF = fun(Term, I) -> io_lib:format("~." ++ integer_to_list(I) ++ P, [Term, 50]) end, - StackTrace = erlang:get_stacktrace(), StackFun = fun(M, _F, _A) -> (M =:= erl_eval) or (M =:= ?MODULE) end, lib:format_exception(1, Class, Reason, StackTrace, StackFun, PF, Enc). @@ -916,8 +914,8 @@ hidden_apply(App, M, F, Args) -> try apply(fun() -> M end(), F, Args) catch - error:undef -> - case erlang:get_stacktrace() of + error:undef:StackTrace -> + case StackTrace of [{M,F,Args,_} | _] -> Arity = length(Args), Text = io_lib:format("Call to ~w:~w/~w in application ~w failed.\n", diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index b6548626f3..6a559f0be5 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -73,7 +73,8 @@ select_count/2, select_delete/2, select_replace/2, select_reverse/1, select_reverse/2, select_reverse/3, setopts/2, slot/2, take/2, - update_counter/3, update_counter/4, update_element/3]). + update_counter/3, update_counter/4, update_element/3, + whereis/1]). %% internal exports -export([internal_request_all/0]). @@ -145,6 +146,7 @@ give_away(_, _, _) -> InfoList :: [InfoTuple], InfoTuple :: {compressed, boolean()} | {heir, pid() | none} + | {id, tid()} | {keypos, pos_integer()} | {memory, non_neg_integer()} | {name, atom()} @@ -162,7 +164,7 @@ info(_) -> -spec info(Tab, Item) -> Value | undefined when Tab :: tab(), - Item :: compressed | fixed | heir | keypos | memory + Item :: compressed | fixed | heir | id | keypos | memory | name | named_table | node | owner | protection | safe_fixed | safe_fixed_monotonic_time | size | stats | type | write_concurrency | read_concurrency, @@ -277,7 +279,7 @@ match_spec_compile(_) -> erlang:nif_error(undef). -spec match_spec_run_r(List, CompiledMatchSpec, list()) -> list() when - List :: [tuple()], + List :: [term()], CompiledMatchSpec :: comp_match_spec(). match_spec_run_r(_, _, _) -> @@ -512,12 +514,17 @@ update_counter(_, _, _, _) -> update_element(_, _, _) -> erlang:nif_error(undef). +-spec whereis(TableName) -> tid() | undefined when + TableName :: atom(). +whereis(_) -> + erlang:nif_error(undef). + %%% End of BIFs -opaque comp_match_spec() :: reference(). -spec match_spec_run(List, CompiledMatchSpec) -> list() when - List :: [tuple()], + List :: [term()], CompiledMatchSpec :: comp_match_spec(). match_spec_run(List, CompiledMS) -> @@ -882,10 +889,10 @@ tab2file(Tab, File, Options) -> _ = disk_log:close(Name), _ = file:delete(File), exit(ExReason); - error:ErReason -> + error:ErReason:StackTrace -> _ = disk_log:close(Name), _ = file:delete(File), - erlang:raise(error,ErReason,erlang:get_stacktrace()) + erlang:raise(error,ErReason,StackTrace) end catch throw:TReason2 -> @@ -1060,9 +1067,9 @@ file2tab(File, Opts) -> exit:ExReason -> ets:delete(Tab), exit(ExReason); - error:ErReason -> + error:ErReason:StackTrace -> ets:delete(Tab), - erlang:raise(error,ErReason,erlang:get_stacktrace()) + erlang:raise(error,ErReason,StackTrace) end after _ = disk_log:close(Name) diff --git a/lib/stdlib/src/file_sorter.erl b/lib/stdlib/src/file_sorter.erl index 3aeaff8dc4..7f74e71136 100644 --- a/lib/stdlib/src/file_sorter.erl +++ b/lib/stdlib/src/file_sorter.erl @@ -1314,9 +1314,9 @@ infun(W) -> {cont, W#w{in = NFun}, Objs}; Error -> error(Error, W1) - catch Class:Reason -> + catch Class:Reason:Stacktrace -> cleanup(W1), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end. outfun(A, #w{inout_value = Val} = W) when Val =/= no_value -> @@ -1336,9 +1336,9 @@ outfun(A, W) -> W#w{out = NF}; Error -> error(Error, W1) - catch Class:Reason -> + catch Class:Reason:Stacktrace -> cleanup(W1), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end. is_keypos(Keypos) when is_integer(Keypos), Keypos > 0 -> diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index ac172325b5..77a46419f6 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -109,7 +109,7 @@ -define( STACKTRACE(), - try throw(ok) catch _ -> erlang:get_stacktrace() end). + element(2, erlang:process_info(self(), current_stacktrace))). %%%========================================================================= %%% API @@ -369,7 +369,7 @@ init_it(Mod, Args) -> {ok, Mod:init(Args)} catch throw:R -> {ok, R}; - Class:R -> {'EXIT', Class, R, erlang:get_stacktrace()} + Class:R:S -> {'EXIT', Class, R, S} end. %%%======================================================================== @@ -634,7 +634,7 @@ try_dispatch(Mod, Func, Msg, State) -> catch throw:R -> {ok, R}; - error:undef = R when Func == handle_info -> + error:undef = R:Stacktrace when Func == handle_info -> case erlang:function_exported(Mod, handle_info, 2) of false -> error_logger:warning_msg("** Undefined handle_info in ~p~n" @@ -642,10 +642,10 @@ try_dispatch(Mod, Func, Msg, State) -> [Mod, Msg]), {ok, {noreply, State}}; true -> - {'EXIT', error, R, erlang:get_stacktrace()} + {'EXIT', error, R, Stacktrace} end; - Class:R -> - {'EXIT', Class, R, erlang:get_stacktrace()} + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end. try_handle_call(Mod, Msg, From, State) -> @@ -654,8 +654,8 @@ try_handle_call(Mod, Msg, From, State) -> catch throw:R -> {ok, R}; - Class:R -> - {'EXIT', Class, R, erlang:get_stacktrace()} + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end. try_terminate(Mod, Reason, State) -> @@ -666,8 +666,8 @@ try_terminate(Mod, Reason, State) -> catch throw:R -> {ok, R}; - Class:R -> - {'EXIT', Class, R, erlang:get_stacktrace()} + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end; false -> {ok, ok} diff --git a/lib/stdlib/src/gen_statem.erl b/lib/stdlib/src/gen_statem.erl index 1a7736fc7e..f95b2ea9cd 100644 --- a/lib/stdlib/src/gen_statem.erl +++ b/lib/stdlib/src/gen_statem.erl @@ -365,7 +365,7 @@ event_type(Type) -> -define( STACKTRACE(), - try throw(ok) catch _ -> erlang:get_stacktrace() end). + element(2, erlang:process_info(self(), current_stacktrace))). -define(not_sys_debug, []). %% @@ -590,11 +590,11 @@ call_dirty(ServerRef, Request, Timeout, T) -> {ok,Reply} -> Reply catch - Class:Reason -> + Class:Reason:Stacktrace -> erlang:raise( Class, {Reason,{?MODULE,call,[ServerRef,Request,Timeout]}}, - erlang:get_stacktrace()) + Stacktrace) end. call_clean(ServerRef, Request, Timeout, T) -> @@ -608,9 +608,8 @@ call_clean(ServerRef, Request, Timeout, T) -> ServerRef, '$gen_call', Request, T) of Result -> {Ref,Result} - catch Class:Reason -> - {Ref,Class,Reason, - erlang:get_stacktrace()} + catch Class:Reason:Stacktrace -> + {Ref,Class,Reason,Stacktrace} end end), Mref = monitor(process, Pid), @@ -697,8 +696,7 @@ init_it(Starter, Parent, ServerRef, Module, Args, Opts) -> catch Result -> init_result(Starter, Parent, ServerRef, Module, Result, Opts); - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> Name = gen:get_proc_name(ServerRef), gen:unregister_name(ServerRef), proc_lib:init_ack(Starter, {error,Reason}), @@ -1584,8 +1582,8 @@ call_callback_mode(#state{module = Module} = S) -> catch CallbackMode -> callback_mode_result(S, CallbackMode); - Class:Reason -> - [Class,Reason,erlang:get_stacktrace()] + Class:Reason:Stacktrace -> + [Class,Reason,Stacktrace] end. callback_mode_result(S, CallbackMode) -> @@ -1638,8 +1636,8 @@ call_state_function( catch Result -> {Result,S}; - Class:Reason -> - [Class,Reason,erlang:get_stacktrace()] + Class:Reason:Stacktrace -> + [Class,Reason,Stacktrace] end. @@ -1827,8 +1825,7 @@ terminate( _ -> ok catch _ -> ok; - C:R -> - ST = erlang:get_stacktrace(), + C:R:ST -> error_info( C, R, ST, S, Q, format_status(terminate, get(), S)), diff --git a/lib/stdlib/src/proc_lib.erl b/lib/stdlib/src/proc_lib.erl index 8e10cbe93b..1991585c13 100644 --- a/lib/stdlib/src/proc_lib.erl +++ b/lib/stdlib/src/proc_lib.erl @@ -231,8 +231,8 @@ init_p(Parent, Ancestors, Fun) when is_function(Fun) -> try Fun() catch - Class:Reason -> - exit_p(Class, Reason, erlang:get_stacktrace()) + Class:Reason:Stacktrace -> + exit_p(Class, Reason, Stacktrace) end. -spec init_p(pid(), [pid()], atom(), atom(), [term()]) -> term(). @@ -246,8 +246,8 @@ init_p_do_apply(M, F, A) -> try apply(M, F, A) catch - Class:Reason -> - exit_p(Class, Reason, erlang:get_stacktrace()) + Class:Reason:Stacktrace -> + exit_p(Class, Reason, Stacktrace) end. -spec wake_up(atom(), atom(), [term()]) -> term(). @@ -256,8 +256,8 @@ wake_up(M, F, A) when is_atom(M), is_atom(F), is_list(A) -> try apply(M, F, A) catch - Class:Reason -> - exit_p(Class, Reason, erlang:get_stacktrace()) + Class:Reason:Stacktrace -> + exit_p(Class, Reason, Stacktrace) end. exit_p(Class, Reason, Stacktrace) -> diff --git a/lib/stdlib/src/qlc.erl b/lib/stdlib/src/qlc.erl index f11f9d0a0b..3a66f6930b 100644 --- a/lib/stdlib/src/qlc.erl +++ b/lib/stdlib/src/qlc.erl @@ -301,11 +301,11 @@ eval(QH, Options) -> post_funs(Post) end end - catch Term -> - case erlang:get_stacktrace() of + catch throw:Term:Stacktrace -> + case Stacktrace of [?THROWN_ERROR | _] -> Term; - Stacktrace -> + _ -> erlang:raise(throw, Term, Stacktrace) end end @@ -359,11 +359,11 @@ fold(Fun, Acc0, QH, Options) -> post_funs(Post) end end - catch Term -> - case erlang:get_stacktrace() of + catch throw:Term:Stacktrace -> + case Stacktrace of [?THROWN_ERROR | _] -> Term; - Stacktrace -> + _ -> erlang:raise(throw, Term, Stacktrace) end end @@ -457,11 +457,11 @@ info(QH, Options) -> debug -> % Not documented. Intended for testing only. Info end - catch Term -> - case erlang:get_stacktrace() of + catch throw:Term:Stacktrace -> + case Stacktrace of [?THROWN_ERROR | _] -> Term; - Stacktrace -> + _ -> erlang:raise(throw, Term, Stacktrace) end end @@ -1056,9 +1056,9 @@ cursor_process(H, GUnique, GCache, TmpDir, SpawnOptions, MaxList, TmpUsage) -> Prep = prepare_qlc(H, not_a_list, GUnique, GCache, TmpDir, MaxList, TmpUsage), setup_qlc(Prep, Setup) - catch Class:Reason -> - Parent ! {self(), {caught, Class, Reason, - erlang:get_stacktrace()}}, + catch Class:Reason:Stacktrace -> + Parent ! {self(), + {caught, Class, Reason, Stacktrace}}, exit(normal) end, Parent ! {self(), ok}, @@ -1075,8 +1075,8 @@ parent_fun(Pid, Parent) -> {TPid, {parent_fun, Fun}} -> V = try {value, Fun()} - catch Class:Reason -> - {parent_fun_caught, Class, Reason, erlang:get_stacktrace()} + catch Class:Reason:Stacktrace -> + {parent_fun_caught, Class, Reason, Stacktrace} end, TPid ! {Parent, V}, parent_fun(Pid, Parent); @@ -1101,9 +1101,9 @@ reply(Parent, MonRef, Post, Cont) -> throw_error(Cont) end catch - Class:Reason -> + Class:Reason:Stacktrace -> post_funs(Post), - Message = {caught, Class, Reason, erlang:get_stacktrace()}, + Message = {caught, Class, Reason, Stacktrace}, Parent ! {self(), Message}, exit(normal) end, @@ -1392,9 +1392,8 @@ next_loop(Pid, L, N) when N =/= 0 -> {caught, throw, Error, [?THROWN_ERROR | _]} -> Error; {caught, Class, Reason, Stacktrace} -> - CurrentStacktrace = try erlang:error(foo) - catch error:_ -> erlang:get_stacktrace() - end, + {current_stacktrace, CurrentStacktrace} = + erlang:process_info(self(), current_stacktrace), erlang:raise(Class, Reason, Stacktrace ++ CurrentStacktrace); error -> erlang:error({qlc_cursor_pid_no_longer_exists, Pid}) @@ -2627,9 +2626,9 @@ table_handle(#qlc_table{trav_fun = TraverseFun, trav_MS = TravMS, Parent =:= self() -> try ParentFun() - catch Class:Reason -> + catch Class:Reason:Stacktrace -> post_funs(Post), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end; true -> case monitor_request(Parent, {parent_fun, ParentFun}) of @@ -3033,9 +3032,9 @@ file_sort_handle(H, Kp, SortOptions, TmpDir, Compressed, Post, LocalPost) -> {terms, BTerms} -> try {[binary_to_term(B) || B <- BTerms], Post, LocalPost} - catch Class:Reason -> + catch Class:Reason:Stacktrace -> post_funs(Post), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end end. @@ -3045,9 +3044,9 @@ do_sort(In, Out, Sort, SortOptions, Post) -> {error, Reason} -> throw_reason(Reason); Reply -> Reply end - catch Class:Term -> + catch Class:Term:Stacktrace -> post_funs(Post), - erlang:raise(Class, Term, erlang:get_stacktrace()) + erlang:raise(Class, Term, Stacktrace) end. do_sort(In, Out, sort, SortOptions) -> @@ -3797,9 +3796,9 @@ call(undefined, _Arg, Default, _Post) -> call(Fun, Arg, _Default, Post) -> try Fun(Arg) - catch Class:Reason -> + catch Class:Reason:Stacktrace -> post_funs(Post), - erlang:raise(Class, Reason, erlang:get_stacktrace()) + erlang:raise(Class, Reason, Stacktrace) end. grd(undefined, _Arg) -> diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index ad4984b64c..e4153e7899 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -645,8 +645,7 @@ eval_exprs(Es, Shell, Bs0, RT, Lf, Ef, W) -> catch exit:normal -> exit(normal); - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> M = {self(),Class,{Reason,Stacktrace}}, case do_catch(Class, Reason) of true -> @@ -807,8 +806,8 @@ restrict_handlers(RShMod, Shell, RT) -> -define(BAD_RETURN(M, F, V), try erlang:error(reason) - catch _:_ -> erlang:raise(exit, {restricted_shell_bad_return,V}, - [{M,F,3} | erlang:get_stacktrace()]) + catch _:_:S -> erlang:raise(exit, {restricted_shell_bad_return,V}, + [{M,F,3} | S]) end). local_allowed(F, As, RShMod, Bs, Shell, RT) when is_atom(F) -> diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index e01bb7d85e..4e89819e41 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -420,10 +420,12 @@ to_number(_, Number, Rest, _, Tail) -> %% Return the remaining string with prefix removed or else nomatch -spec prefix(String::unicode:chardata(), Prefix::unicode:chardata()) -> 'nomatch' | unicode:chardata(). -prefix(Str, []) -> Str; prefix(Str, Prefix0) -> - Prefix = unicode:characters_to_list(Prefix0), - case prefix_1(Str, Prefix) of + Result = case unicode:characters_to_list(Prefix0) of + [] -> Str; + Prefix -> prefix_1(Str, Prefix) + end, + case Result of [] when is_binary(Str) -> <<>>; Res -> Res end. diff --git a/lib/stdlib/src/uri_string.erl b/lib/stdlib/src/uri_string.erl index a84679c595..28d36ea229 100644 --- a/lib/stdlib/src/uri_string.erl +++ b/lib/stdlib/src/uri_string.erl @@ -227,7 +227,7 @@ %% External API %%------------------------------------------------------------------------- -export([compose_query/1, compose_query/2, - dissect_query/1, normalize/1, parse/1, + dissect_query/1, normalize/1, normalize/2, parse/1, recompose/1, transcode/2]). -export_type([error/0, uri_map/0, uri_string/0]). @@ -292,18 +292,36 @@ %%------------------------------------------------------------------------- %% Normalize URIs %%------------------------------------------------------------------------- --spec normalize(URIString) -> NormalizedURI when - URIString :: uri_string(), - NormalizedURI :: uri_string(). -normalize(URIString) -> - %% Percent-encoding normalization and case normalization for - %% percent-encoded triplets are achieved by running parse and - %% recompose on the input URI string. - recompose( - normalize_path_segment( - normalize_scheme_based( - normalize_case( - parse(URIString))))). +-spec normalize(URI) -> NormalizedURI when + URI :: uri_string() | uri_map(), + NormalizedURI :: uri_string() + | error(). +normalize(URIMap) -> + normalize(URIMap, []). + + +-spec normalize(URI, Options) -> NormalizedURI when + URI :: uri_string() | uri_map(), + Options :: [return_map], + NormalizedURI :: uri_string() | uri_map(). +normalize(URIMap, []) when is_map(URIMap) -> + recompose(normalize_map(URIMap)); +normalize(URIMap, [return_map]) when is_map(URIMap) -> + normalize_map(URIMap); +normalize(URIString, []) -> + case parse(URIString) of + Value when is_map(Value) -> + recompose(normalize_map(Value)); + Error -> + Error + end; +normalize(URIString, [return_map]) -> + case parse(URIString) of + Value when is_map(Value) -> + normalize_map(Value); + Error -> + Error + end. %%------------------------------------------------------------------------- @@ -385,7 +403,8 @@ transcode(URIString, Options) when is_list(URIString) -> %%------------------------------------------------------------------------- %% Functions for working with the query part of a URI as a list %% of key/value pairs. -%% HTML5 - 4.10.22.6 URL-encoded form data +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - non UTF-8 %%------------------------------------------------------------------------- %%------------------------------------------------------------------------- @@ -393,7 +412,7 @@ transcode(URIString, Options) when is_list(URIString) -> %% (application/x-www-form-urlencoded encoding algorithm) %%------------------------------------------------------------------------- -spec compose_query(QueryList) -> QueryString when - QueryList :: [{uri_string(), uri_string()}], + QueryList :: [{unicode:chardata(), unicode:chardata()}], QueryString :: uri_string() | error(). compose_query(List) -> @@ -401,7 +420,7 @@ compose_query(List) -> -spec compose_query(QueryList, Options) -> QueryString when - QueryList :: [{uri_string(), uri_string()}], + QueryList :: [{unicode:chardata(), unicode:chardata()}], Options :: [{encoding, atom()}], QueryString :: uri_string() | error(). @@ -432,7 +451,7 @@ compose_query([], _Options, IsList, Acc) -> %%------------------------------------------------------------------------- -spec dissect_query(QueryString) -> QueryList when QueryString :: uri_string(), - QueryList :: [{uri_string(), uri_string()}] + QueryList :: [{unicode:chardata(), unicode:chardata()}] | error(). dissect_query(<<>>) -> []; @@ -1755,7 +1774,8 @@ get_separator(_L) -> <<"&">>. -%% HTML5 - 4.10.22.6 URL-encoded form data - encoding +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - encoding (non UTF-8) form_urlencode(Cs, [{encoding, latin1}]) when is_list(Cs) -> B = convert_to_binary(Cs, utf8, utf8), html5_byte_encode(base10_encode(B)); @@ -1850,7 +1870,8 @@ dissect_query_value(<<>>, IsList, Acc, Key, Value) -> lists:reverse([{K,V}|Acc]). -%% Form-urldecode input based on RFC 1866 [8.2.1] +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - decoding (non UTF-8) form_urldecode(true, B) -> Result = base10_decode(form_urldecode(B, <<>>)), convert_to_list(Result, utf8); @@ -1903,6 +1924,12 @@ base10_decode_unicode(<<H,_/binary>>, _, _) -> %% Helper functions for normalize %%------------------------------------------------------------------------- +normalize_map(URIMap) -> + normalize_path_segment( + normalize_scheme_based( + normalize_case(URIMap))). + + %% 6.2.2.1. Case Normalization normalize_case(#{scheme := Scheme, host := Host} = Map) -> Map#{scheme => to_lower(Scheme), diff --git a/lib/stdlib/src/zip.erl b/lib/stdlib/src/zip.erl index 81f927f399..39be2abff6 100644 --- a/lib/stdlib/src/zip.erl +++ b/lib/stdlib/src/zip.erl @@ -457,8 +457,7 @@ do_zip(F, Files, Options) -> Out3 = Output({close, F}, Out2), {ok, Out3} catch - C:R -> - Stk = erlang:get_stacktrace(), + C:R:Stk -> zlib:close(Z), Output({close, F}, Out0), erlang:raise(C, R, Stk) diff --git a/lib/stdlib/test/array_SUITE.erl b/lib/stdlib/test/array_SUITE.erl index 5836f275ba..956582c4fd 100644 --- a/lib/stdlib/test/array_SUITE.erl +++ b/lib/stdlib/test/array_SUITE.erl @@ -141,10 +141,10 @@ t(What) -> io:format("Test ~p ~n",[T]), try ?MODULE:T([]) - catch _E:_R -> + catch _E:_R:_S -> Line = get(test_server_loc), io:format("Failed ~p:~p ~p ~p~n ~p~n", - [T,Line,_E,_R, erlang:get_stacktrace()]) + [T,Line,_E,_R,_S]) end end, What). @@ -161,8 +161,8 @@ extract_tests() -> end, [Call(Test) || Test <- Tests], io:format("Tests ~p~n", [Tests]) - catch _:Err -> - io:format("Error: ~p ~p~n", [Err, erlang:get_stacktrace()]) + catch _:Err:Stacktrace -> + io:format("Error: ~p ~p~n", [Err, Stacktrace]) end, file:close(In), file:close(Out). diff --git a/lib/stdlib/test/error_logger_h_SUITE.erl b/lib/stdlib/test/error_logger_h_SUITE.erl index 1f2a9fda0b..9dc04f27a1 100644 --- a/lib/stdlib/test/error_logger_h_SUITE.erl +++ b/lib/stdlib/test/error_logger_h_SUITE.erl @@ -257,8 +257,7 @@ match_output([Item|T], Lines0, AtNode, Depth) -> Lines -> match_output(T, Lines, AtNode, Depth) catch - C:E -> - Stk = erlang:get_stacktrace(), + C:E:Stk -> io:format("ITEM: ~p", [Item]), io:format("LINES: ~p", [Lines0]), erlang:raise(C, E, Stk) diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 07c8b60cbd..8b651f4b43 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -78,6 +78,7 @@ -export([ets_all/1]). -export([massive_ets_all/1]). -export([take/1]). +-export([whereis_table/1]). -export([init_per_testcase/2, end_per_testcase/2]). %% Convenience for manual testing @@ -137,7 +138,8 @@ all() -> otp_9423, ets_all, massive_ets_all, - take]. + take, + whereis_table]. groups() -> [{new, [], @@ -4099,6 +4101,7 @@ info_do(Opts) -> {value, {keypos, 2}} = lists:keysearch(keypos, 1, Res), {value, {protection, protected}} = lists:keysearch(protection, 1, Res), + {value, {id, Tab}} = lists:keysearch(id, 1, Res), true = ets:delete(Tab), undefined = ets:info(non_existing_table_xxyy), undefined = ets:info(non_existing_table_xxyy,type), @@ -5892,6 +5895,36 @@ take(Config) when is_list(Config) -> ets:delete(T3), ok. +whereis_table(Config) when is_list(Config) -> + %% Do we return 'undefined' when the named table doesn't exist? + undefined = ets:whereis(whereis_test), + + %% Does the tid() refer to the same table as the name? + whereis_test = ets:new(whereis_test, [named_table]), + Tid = ets:whereis(whereis_test), + + ets:insert(whereis_test, [{hello}, {there}]), + + [[{hello}],[{there}]] = ets:match(whereis_test, '$1'), + [[{hello}],[{there}]] = ets:match(Tid, '$1'), + + true = ets:delete_all_objects(Tid), + + [] = ets:match(whereis_test, '$1'), + [] = ets:match(Tid, '$1'), + + %% Does the name disappear when deleted through the tid()? + true = ets:delete(Tid), + undefined = ets:info(whereis_test), + {'EXIT',{badarg, _}} = (catch ets:match(whereis_test, '$1')), + + %% Is the old tid() broken when the table is re-created with the same + %% name? + whereis_test = ets:new(whereis_test, [named_table]), + [] = ets:match(whereis_test, '$1'), + {'EXIT',{badarg, _}} = (catch ets:match(Tid, '$1')), + + ok. %% %% Utility functions: @@ -6023,17 +6056,23 @@ etsmem() -> end}, {Mem,AllTabs}. -verify_etsmem({MemInfo,AllTabs}) -> + +verify_etsmem(MI) -> wait_for_test_procs(), + verify_etsmem(MI, 1). + +verify_etsmem({MemInfo,AllTabs}, Try) -> case etsmem() of {MemInfo,_} -> io:format("Ets mem info: ~p", [MemInfo]), - case MemInfo of - {ErlMem,EtsAlloc} when ErlMem == notsup; EtsAlloc == undefined -> + case {MemInfo, Try} of + {{ErlMem,EtsAlloc},_} when ErlMem == notsup; EtsAlloc == undefined -> %% Use 'erl +Mea max' to do more complete memory leak testing. {comment,"Incomplete or no mem leak testing"}; - _ -> - ok + {_, 1} -> + ok; + _ -> + {comment, "Transient memory discrepancy"} end; {MemInfo2, AllTabs2} -> @@ -6041,7 +6080,15 @@ verify_etsmem({MemInfo,AllTabs}) -> io:format("Actual: ~p", [MemInfo2]), io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]), io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), - ct:fail("Failed memory check") + case Try < 2 of + true -> + io:format("\nThis discrepancy could be caused by an " + "inconsistent memory \"snapshot\"" + "\nTry again...\n", []), + verify_etsmem({MemInfo, AllTabs}, Try+1); + false -> + ct:fail("Failed memory check") + end end. diff --git a/lib/stdlib/test/gen_statem_SUITE.erl b/lib/stdlib/test/gen_statem_SUITE.erl index c747db475a..7c8a386116 100644 --- a/lib/stdlib/test/gen_statem_SUITE.erl +++ b/lib/stdlib/test/gen_statem_SUITE.erl @@ -2040,9 +2040,9 @@ handle_event(Type, Event, State, Data) -> Result -> wrap_result(Result) catch - throw:Result -> + throw:Result:Stacktrace -> erlang:raise( - throw, wrap_result(Result), erlang:get_stacktrace()) + throw, wrap_result(Result), Stacktrace) end. unwrap_state([State]) -> diff --git a/lib/stdlib/test/proc_lib_SUITE.erl b/lib/stdlib/test/proc_lib_SUITE.erl index 7686889360..fbdcb518b2 100644 --- a/lib/stdlib/test/proc_lib_SUITE.erl +++ b/lib/stdlib/test/proc_lib_SUITE.erl @@ -446,8 +446,8 @@ init_dont_hang(Config) when is_list(Config) -> StartLinkRes = proc_lib:start(?MODULE, init_dont_hang_init, [self()], 1000), StartLinkRes = proc_lib:start(?MODULE, init_dont_hang_init, [self()], 1000, []), ok - catch _:Error -> - io:format("Error ~p /= ~p ~n",[erlang:get_stacktrace(), StartLinkRes]), + catch _:Error:Stacktrace -> + io:format("Error ~p /= ~p ~n",[Stacktrace, StartLinkRes]), exit(Error) end. diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 3d3241b33d..d753d929f5 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2017. All Rights Reserved. +%% Copyright Ericsson AB 2000-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -52,7 +52,9 @@ all() -> [seed, interval_int, interval_float, api_eq, reference, - {group, basic_stats}, uniform_real_conv, + {group, basic_stats}, + {group, distr_stats}, + uniform_real_conv, plugin, measure, {group, reference_jump} ]. @@ -60,8 +62,9 @@ all() -> groups() -> [{basic_stats, [parallel], [basic_stats_uniform_1, basic_stats_uniform_2, - basic_stats_standard_normal, - stats_standard_normal_box_muller, + basic_stats_standard_normal]}, + {distr_stats, [parallel], + [stats_standard_normal_box_muller, stats_standard_normal_box_muller_2, stats_standard_normal]}, {reference_jump, [parallel], @@ -70,6 +73,9 @@ groups() -> group(basic_stats) -> %% valgrind needs a lot of time [{timetrap,{minutes,10}}]; +group(distr_stats) -> + %% valgrind needs a lot of time + [{timetrap,{minutes,10}}]; group(reference_jump) -> %% valgrind needs a lot of time [{timetrap,{minutes,10}}]. @@ -82,9 +88,9 @@ test() -> try ok = ?MODULE:Test([]), io:format("~p: ok~n", [Test]) - catch _:Reason -> + catch _:Reason:Stacktrace -> io:format("Failed: ~p: ~p ~p~n", - [Test, Reason, erlang:get_stacktrace()]) + [Test, Reason, Stacktrace]) end end, Tests). @@ -98,8 +104,8 @@ seed(Config) when is_list(Config) -> Algs = algs(), Test = fun(Alg) -> try seed_1(Alg) - catch _:Reason -> - ct:fail({Alg, Reason, erlang:get_stacktrace()}) + catch _:Reason:Stacktrace -> + ct:fail({Alg, Reason, Stacktrace}) end end, [Test(Alg) || Alg <- Algs], @@ -437,7 +443,7 @@ stats_standard_normal_box_muller(Config) when is_list(Config) -> {Z, [S]} end, State = [rand:seed(exrop)], - stats_standard_normal(NormalS, State) + stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. @@ -462,7 +468,7 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) -> {Z, [S]} end, State = [rand:seed(exrop)], - stats_standard_normal(NormalS, State) + stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. @@ -472,21 +478,21 @@ stats_standard_normal(Config) when is_list(Config) -> try math:erfc(1.0) of _ -> stats_standard_normal( - fun rand:normal_s/1, rand:seed_s(exrop)) + fun rand:normal_s/1, rand:seed_s(exrop), 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. %% -stats_standard_normal(Fun, S) -> +stats_standard_normal(Fun, S, Retries) -> %%% %%% ct config: -%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.2}]}]}. +%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.0}]}]}. %%% Seconds = ct:get_config({?MODULE, ?FUNCTION_NAME, seconds}, 8), StdDevs = ct:get_config( {?MODULE, ?FUNCTION_NAME, std_devs}, - 4.2), % probability erfc(4.2/sqrt(2)) (1/37465) to fail a bucket + 4.0), % probability erfc(4.0/sqrt(2)) (1/15787) to fail a bucket %%% ct:timetrap({seconds, Seconds + 120}), %% Buckets is chosen to get a range where the the probability to land @@ -505,11 +511,11 @@ stats_standard_normal(Fun, S) -> P0 = math:erf(1 / W), Rounds = TargetHits * ceil(1.0 / P0), Histogram = array:new({default, 0}), - StopTime = erlang:monotonic_time(second) + Seconds, ct:pal( "Running standard normal test against ~w std devs for ~w seconds...", [StdDevs, Seconds]), - {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds} = + StopTime = erlang:monotonic_time(second) + Seconds, + {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds, NewS} = stats_standard_normal( InvDelta, Buckets, Histogram, Histogram, 0.0, Fun, S, Rounds, StopTime, Rounds, 0), @@ -522,16 +528,33 @@ stats_standard_normal(Fun, S) -> "Total rounds: ~w, tolerance: 1/~.2f..1/~.2f, " "outlier: ~.2f, probability 1/~.2f.", [TotalRounds, Precision, TopPrecision, Outlier, InvOP]), - {TotalRounds, [], []} = - {TotalRounds, + case + {bucket_error, TotalRounds, check_histogram( W, TotalRounds, StdDevs, PositiveHistogram, Buckets), check_histogram( - W, TotalRounds, StdDevs, NegativeHistogram, Buckets)}, - %% If the probability for getting this Outlier is lower than 1/50, - %% then this is fishy! - true = (1/50 =< OutlierProbability), - {comment, {tp, TopPrecision, op, InvOP}}. + W, TotalRounds, StdDevs, NegativeHistogram, Buckets)} + of + {_, _, [], []} when InvOP < 100 -> + {comment, {tp, TopPrecision, op, InvOP}}; + {_, _, [], []} -> + %% If the probability for getting this Outlier is lower than + %% 1/100, then this is fishy! + stats_standard_normal( + Fun, NewS, Retries, {outlier_fishy, InvOP}); + BucketErrors -> + stats_standard_normal( + Fun, NewS, Retries, BucketErrors) + end. +%% +stats_standard_normal(Fun, S, Retries, Failure) -> + case Retries - 1 of + 0 -> + ct:fail(Failure); + NewRetries -> + ct:pal("Retry due to TC glitch: ~p", [Failure]), + stats_standard_normal(Fun, S, NewRetries) + end. %% stats_standard_normal( InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier, @@ -544,7 +567,7 @@ stats_standard_normal( Fun, S, Rounds, StopTime, Rounds, TotalRounds + Rounds); _ -> {PositiveHistogram, NegativeHistogram, - Outlier, TotalRounds + Rounds} + Outlier, TotalRounds + Rounds, S} end; stats_standard_normal( InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier, @@ -571,9 +594,6 @@ increment_bucket(Bucket, Array) -> array:set(Bucket, array:get(Bucket, Array) + 1, Array). check_histogram(W, Rounds, StdDevs, Histogram, Buckets) -> - %%PrevBucket = 512, - %%Bucket = PrevBucket - 1, - %%P = 0.5 * math:erfc(PrevBucket / W), TargetP = 0.5 * math:erfc(Buckets / W), P = 0.0, N = 0, @@ -592,7 +612,7 @@ check_histogram( P = 0.5 * math:erfc(Bucket / W), BucketP = P - PrevP, if - TargetP =< BucketP -> + BucketP < TargetP -> check_histogram( W, Rounds, StdDevs, Histogram, TargetP, Bucket - 1, PrevBucket, PrevP, N); @@ -604,7 +624,7 @@ check_histogram( UpperLimit = ceil(Exp + Threshold), if N < LowerLimit; UpperLimit < N -> - [#{bucket => {Bucket, PrevBucket}, n => N, exp => Exp, + [#{bucket => {Bucket, PrevBucket}, n => N, lower => LowerLimit, upper => UpperLimit} | check_histogram( W, Rounds, StdDevs, Histogram, TargetP, @@ -722,12 +742,12 @@ uniform_real_conv_check(M, E, Gen) -> [["16#",integer_to_list(G,16),$\s]||G<-Gen]]), ct:fail({neq, FF, F}) catch - Error:Reason -> + Error:Reason:Stacktrace -> ct:pal( "~w:~p ~s: ~s~n", [Error, Reason, rand:float2str(F), [["16#",integer_to_list(G,16),$\s]||G<-Gen]]), - ct:fail({Error, Reason, F, erlang:get_stacktrace()}) + ct:fail({Error, Reason, F, Stacktrace}) end. diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index 71f86e32e5..7b82647416 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -894,10 +894,13 @@ match_limit(Config) when is_list(Config) -> %% Test that we get sub-binaries if subject is a binary and we capture %% binaries. sub_binaries(Config) when is_list(Config) -> - Bin = list_to_binary(lists:seq(1,255)), - {match,[B,C]}=re:run(Bin,"(a)",[{capture,all,binary}]), - 255 = binary:referenced_byte_size(B), - 255 = binary:referenced_byte_size(C), - {match,[D]}=re:run(Bin,"(a)",[{capture,[1],binary}]), - 255 = binary:referenced_byte_size(D), + %% The GC can auto-convert tiny sub-binaries to heap binaries, so we + %% extract large sequences to make the test more stable. + Bin = << <<I>> || I <- lists:seq(1, 4096) >>, + {match,[B,C]}=re:run(Bin,"a(.+)$",[{capture,all,binary}]), + true = byte_size(B) =/= byte_size(C), + 4096 = binary:referenced_byte_size(B), + 4096 = binary:referenced_byte_size(C), + {match,[D]}=re:run(Bin,"a(.+)$",[{capture,[1],binary}]), + 4096 = binary:referenced_byte_size(D), ok. diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index d02a6eac0a..fdff2d24b8 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -486,6 +486,10 @@ to_float(_) -> prefix(_) -> ?TEST("", ["a"], nomatch), ?TEST("a", [""], "a"), + ?TEST("a", [[[]]], "a"), + ?TEST("a", [<<>>], "a"), + ?TEST("a", [[<<>>]], "a"), + ?TEST("a", [[[<<>>]]], "a"), ?TEST("b", ["a"], nomatch), ?TEST("a", ["a"], ""), ?TEST("å", ["a"], nomatch), @@ -877,9 +881,9 @@ test_1(Line, Func, Str, Args, Exp) -> catch error:Exp -> ok; - error:Reason -> + error:Reason:Stacktrace -> io:format("~p:~p: Crash ~p ~p~n", - [?MODULE,Line, Reason, erlang:get_stacktrace()]), + [?MODULE,Line, Reason, Stacktrace]), exit({error, Func}) end. @@ -944,10 +948,10 @@ check_types(Line, Func, [Str|_], Res) -> io:format("Failed: ~p ~p: ~p ~p~n",[Line, Func, T1, T2]), io:format(" ~p => ~p~n", [Str, Res]), error; - _:Reason -> - io:format("Crash: ~p in~n ~p~n",[Reason, erlang:get_stacktrace()]), + _:Reason:Stacktrace -> + io:format("Crash: ~p in~n ~p~n",[Reason, Stacktrace]), io:format("Failed: ~p ~p: ~p => ~p~n", [Line, Func, Str, Res]), - exit({Reason, erlang:get_stacktrace()}) + exit({Reason, Stacktrace}) end. check_types_1(T, T) -> diff --git a/lib/stdlib/test/tar_SUITE.erl b/lib/stdlib/test/tar_SUITE.erl index 4061008812..32a33283d1 100644 --- a/lib/stdlib/test/tar_SUITE.erl +++ b/lib/stdlib/test/tar_SUITE.erl @@ -28,7 +28,7 @@ extract_from_open_file/1, symlinks/1, open_add_close/1, cooked_compressed/1, memory/1,unicode/1,read_other_implementations/1, sparse/1, init/1, leading_slash/1, dotdot/1, - roundtrip_metadata/1]). + roundtrip_metadata/1, apply_file_info_opts/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). @@ -42,7 +42,8 @@ all() -> extract_filtered, symlinks, open_add_close, cooked_compressed, memory, unicode, read_other_implementations, - sparse,init,leading_slash,dotdot,roundtrip_metadata]. + sparse,init,leading_slash,dotdot,roundtrip_metadata, + apply_file_info_opts]. groups() -> []. @@ -989,6 +990,31 @@ do_roundtrip_metadata(Dir, File) -> ok end. +apply_file_info_opts(Config) when is_list(Config) -> + ok = file:set_cwd(proplists:get_value(priv_dir, Config)), + + ok = file:make_dir("empty_directory"), + ok = file:write_file("file", "contents"), + + Opts = [{atime, 0}, {mtime, 0}, {ctime, 0}, {uid, 0}, {gid, 0}], + TarFile = "reproducible.tar", + {ok, Tar} = erl_tar:open(TarFile, [write]), + ok = erl_tar:add(Tar, "file", Opts), + ok = erl_tar:add(Tar, "empty_directory", Opts), + ok = erl_tar:add(Tar, <<"contents">>, "memory_file", Opts), + erl_tar:close(Tar), + + ok = file:make_dir("extracted"), + erl_tar:extract(TarFile, [{cwd, "extracted"}]), + + {ok, #file_info{mtime=0}} = + file:read_file_info("extracted/empty_directory", [{time, posix}]), + {ok, #file_info{mtime=0}} = + file:read_file_info("extracted/file", [{time, posix}]), + {ok, #file_info{mtime=0}} = + file:read_file_info("extracted/memory_file", [{time, posix}]), + + ok. %% Delete the given list of files. delete_files([]) -> ok; diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl index 632d9ae6e6..40b1c260a5 100644 --- a/lib/stdlib/test/unicode_util_SUITE.erl +++ b/lib/stdlib/test/unicode_util_SUITE.erl @@ -136,10 +136,10 @@ verify_gc(Line0, N, Acc) -> io:format("Expected: ~p~n", [Res]), io:format("Got: ~w~n", [Other]), Acc+1; - Cl:R -> + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[N, Line, Str]), io:format("Expected: ~p~n", [Res]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end. gc_test_data([[247]|Rest], Str, [First|GCs]) -> @@ -175,29 +175,29 @@ verify_nfd(Data0, LineNo, _Acc) -> C3GC = fetch(C1, fun unicode_util:nfd/1), C3GC = fetch(C2, fun unicode_util:nfd/1), C3GC = fetch(C3, fun unicode_util:nfd/1) - catch _Cl:{badmatch, Other} = _R-> + catch _Cl:{badmatch, Other} = _R: Stacktrace -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C1, C1]), io:format("Expected: ~ts ~w~n", [C3GC, C3GC]), io:format("Got: ~ts ~w~n", [Other, Other]), - erlang:raise(_Cl,_R,erlang:get_stacktrace()); - Cl:R -> + erlang:raise(_Cl,_R,Stacktrace); + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C3]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end, C5GC = fetch(C5, fun unicode_util:gc/1), try C5GC = fetch(C4, fun unicode_util:nfd/1), C5GC = fetch(C5, fun unicode_util:nfd/1) - catch _Cl2:{badmatch, Other2} = _R2-> + catch _Cl2:{badmatch, Other2} = _R2:Stacktrace2 -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C1, C1]), io:format("Expected: ~ts ~w~n", [C5GC, C5GC]), io:format("Got: ~ts ~w~n", [Other2, Other2]), - erlang:raise(_Cl2,_R2,erlang:get_stacktrace()); - Cl2:R2 -> + erlang:raise(_Cl2,_R2,Stacktrace2); + Cl2:R2:Stacktrace2 -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C5]), - erlang:raise(Cl2,R2,erlang:get_stacktrace()) + erlang:raise(Cl2,R2,Stacktrace2) end, ok. @@ -218,29 +218,29 @@ verify_nfc(Data0, LineNo, _Acc) -> C2GC = fetch(C1, fun unicode_util:nfc/1), C2GC = fetch(C2, fun unicode_util:nfc/1), C2GC = fetch(C3, fun unicode_util:nfc/1) - catch _Cl:{badmatch, Other} = _R-> + catch _Cl:{badmatch, Other} = _R:Stacktrace -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C1, C1]), io:format("Expected: ~ts ~w~n", [C2GC, C2GC]), io:format("Got: ~ts ~w~n", [Other, Other]), - erlang:raise(_Cl,_R,erlang:get_stacktrace()); - Cl:R -> + erlang:raise(_Cl,_R,Stacktrace); + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C3]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end, C4GC = fetch(C4, fun unicode_util:gc/1), try C4GC = fetch(C4, fun unicode_util:nfc/1), C4GC = fetch(C5, fun unicode_util:nfc/1) - catch _Cl2:{badmatch, Other2} = _R2-> + catch _Cl2:{badmatch, Other2} = _R2:Stacktrace2 -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C1, C1]), io:format("Expected: ~ts ~w~n", [C4GC, C4GC]), io:format("Got: ~ts ~w~n", [Other2, Other2]), - erlang:raise(_Cl2,_R2,erlang:get_stacktrace()); - Cl2:R2 -> + erlang:raise(_Cl2,_R2,Stacktrace2); + Cl2:R2:Stacktrace2 -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C5]), - erlang:raise(Cl2,R2,erlang:get_stacktrace()) + erlang:raise(Cl2,R2,Stacktrace2) end, ok. @@ -263,15 +263,15 @@ verify_nfkd(Data0, LineNo, _Acc) -> C5GC = lists:flatten(fetch(C3, fun unicode_util:nfkd/1)), C5GC = lists:flatten(fetch(C4, fun unicode_util:nfkd/1)), C5GC = lists:flatten(fetch(C5, fun unicode_util:nfkd/1)) - catch _Cl:{badmatch, Other} = _R-> + catch _Cl:{badmatch, Other} = _R:Stacktrace -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C5, C5]), io:format("Expected: ~ts ~w~n", [C5GC, C5GC]), io:format("Got: ~ts ~w~n", [Other, Other]), - erlang:raise(_Cl,_R,erlang:get_stacktrace()); - Cl:R -> + erlang:raise(_Cl,_R,Stacktrace); + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C3]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end, ok. @@ -296,15 +296,15 @@ verify_nfkc(Data0, LineNo, _Acc) -> C4GC = lists:flatten(fetch(C4, fun unicode_util:nfkc/1)), C4GC = lists:flatten(fetch(C5, fun unicode_util:nfkc/1)) - catch _Cl:{badmatch, Other} = _R-> + catch _Cl:{badmatch, Other} = _R:Stacktrace -> io:format("Failed: ~p~nInput: ~ts~n\t=> ~w |~ts|~n",[LineNo, Data1, C4, C4]), io:format("Expected: ~ts ~w~n", [C4GC, C4GC]), io:format("Got: ~ts ~w~n", [Other, Other]), - erlang:raise(_Cl,_R,erlang:get_stacktrace()); - Cl:R -> + erlang:raise(_Cl,_R,Stacktrace); + Cl:R:Stacktrace -> io:format("~p: ~ts => |~tp|~n",[LineNo, Data1, C1]), io:format("Expected: ~p~n", [C3]), - erlang:raise(Cl,R,erlang:get_stacktrace()) + erlang:raise(Cl,R,Stacktrace) end, ok. diff --git a/lib/stdlib/test/uri_string_SUITE.erl b/lib/stdlib/test/uri_string_SUITE.erl index fef356355c..92f8bb3292 100644 --- a/lib/stdlib/test/uri_string_SUITE.erl +++ b/lib/stdlib/test/uri_string_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0, suite/0,groups/0, - normalize/1, + normalize/1, normalize_map/1, normalize_return_map/1, normalize_negative/1, parse_binary_fragment/1, parse_binary_host/1, parse_binary_host_ipv4/1, parse_binary_host_ipv6/1, parse_binary_path/1, parse_binary_pct_encoded_fragment/1, parse_binary_pct_encoded_query/1, @@ -68,6 +68,9 @@ suite() -> all() -> [ normalize, + normalize_map, + normalize_return_map, + normalize_negative, parse_binary_scheme, parse_binary_userinfo, parse_binary_pct_encoded_userinfo, @@ -912,6 +915,56 @@ normalize(_Config) -> <<"tftp://localhost">> = uri_string:normalize(<<"tftp://localhost:69">>). +normalize_map(_Config) -> + "/a/g" = uri_string:normalize(#{path => "/a/b/c/./../../g"}), + <<"mid/6">> = uri_string:normalize(#{path => <<"mid/content=5/../6">>}), + "http://localhost-%C3%B6rebro/a/g" = + uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g", + host => "localhost-örebro"}), + <<"http://localhost-%C3%B6rebro/a/g">> = + uri_string:normalize(#{scheme => <<"http">>,port => 80, + path => <<"/a/b/c/./../../g">>, + host => <<"localhost-örebro"/utf8>>}), + <<"https://localhost/">> = + uri_string:normalize(#{scheme => <<"https">>,port => 443,path => <<>>, + host => <<"localhost">>}), + <<"https://localhost:445/">> = + uri_string:normalize(#{scheme => <<"https">>,port => 445,path => <<>>, + host => <<"localhost">>}), + <<"ftp://localhost">> = + uri_string:normalize(#{scheme => <<"ftp">>,port => 21,path => <<>>, + host => <<"localhost">>}), + <<"ssh://localhost">> = + uri_string:normalize(#{scheme => <<"ssh">>,port => 22,path => <<>>, + host => <<"localhost">>}), + <<"sftp://localhost">> = + uri_string:normalize(#{scheme => <<"sftp">>,port => 22,path => <<>>, + host => <<"localhost">>}), + <<"tftp://localhost">> = + uri_string:normalize(#{scheme => <<"tftp">>,port => 69,path => <<>>, + host => <<"localhost">>}). + +normalize_return_map(_Config) -> + #{scheme := "http",path := "/a/g",host := "localhost-örebro"} = + uri_string:normalize("http://localhos%74-%c3%b6rebro:80/a/b/c/./../../g", + [return_map]), + #{scheme := <<"http">>,path := <<"/a/g">>, host := <<"localhost-örebro"/utf8>>} = + uri_string:normalize(<<"http://localhos%74-%c3%b6rebro:80/a/b/c/./../../g">>, + [return_map]), + #{scheme := <<"https">>,path := <<"/">>, host := <<"localhost">>} = + uri_string:normalize(#{scheme => <<"https">>,port => 443,path => <<>>, + host => <<"localhost">>}, [return_map]). + +normalize_negative(_Config) -> + {error,invalid_uri,":"} = + uri_string:normalize("http://local>host"), + {error,invalid_uri,":"} = + uri_string:normalize(<<"http://local>host">>), + {error,invalid_uri,":"} = + uri_string:normalize("http://[192.168.0.1]", [return_map]), + {error,invalid_uri,":"} = + uri_string:normalize(<<"http://[192.168.0.1]">>, [return_map]). + interop_query_utf8(_Config) -> Q = uri_string:compose_query([{"foo bar","1"}, {"合", "2"}]), Uri = uri_string:recompose(#{path => "/", query => Q}), diff --git a/lib/stdlib/test/zip_SUITE.erl b/lib/stdlib/test/zip_SUITE.erl index 1dfcda4ed0..e5ba629c55 100644 --- a/lib/stdlib/test/zip_SUITE.erl +++ b/lib/stdlib/test/zip_SUITE.erl @@ -909,8 +909,7 @@ do_fd_leak(Bad, N) -> ok -> do_fd_leak(Bad, N + 1) catch - C:R -> - Stk = erlang:get_stacktrace(), + C:R:Stk -> io:format("Bad error after ~p attempts\n", [N]), erlang:raise(C, R, Stk) end. diff --git a/lib/tools/doc/src/lcnt.xml b/lib/tools/doc/src/lcnt.xml index 5bdfc60448..0c24375b91 100644 --- a/lib/tools/doc/src/lcnt.xml +++ b/lib/tools/doc/src/lcnt.xml @@ -371,7 +371,7 @@ <v>Serial = integer()</v> </type> <desc> - <p>Creates a process id with creation 0. Example:</p> + <p>Creates a process id with creation 0.</p> </desc> </func> diff --git a/lib/tools/emacs/Makefile b/lib/tools/emacs/Makefile index 35c93ba4ed..ea4d6cb723 100644 --- a/lib/tools/emacs/Makefile +++ b/lib/tools/emacs/Makefile @@ -54,8 +54,6 @@ EL_FILES = $(EMACS_FILES:%=%.el) ELC_FILES = $(EMACS_FILES:%=%.elc) -TEST_FILES = test.erl.indented test.erl.orig - # ---------------------------------------------------- # Targets # ---------------------------------------------------- @@ -75,7 +73,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_spec: opt $(INSTALL_DIR) "$(RELSYSDIR)/emacs" - $(INSTALL_DATA) $(EL_FILES) $(README_FILES) $(TEST_FILES) \ + $(INSTALL_DATA) $(EL_FILES) $(README_FILES) \ "$(RELSYSDIR)/emacs" ifeq ($(DOCTYPE),pdf) @@ -89,19 +87,3 @@ release_docs_spec: docs $(INSTALL_DATA) $(MAN_FILES) "$(RELEASE_PATH)/man/man3" endif endif - -EMACS ?= emacs - -test_indentation: - @rm -f test.erl - @rm -f test_indent.el - @echo '(load "erlang-start")' >> test_indent.el - @echo '(find-file "test.erl.orig")' >> test_indent.el - @echo "(require 'cl) ; required with Emacs < 23 for ignore-errors" >> test_indent.el - @echo '(erlang-mode)' >> test_indent.el - @echo '(toggle-debug-on-error)' >> test_indent.el - @echo '(erlang-indent-current-buffer)' >> test_indent.el - @echo '(write-file "test.erl")' >> test_indent.el - $(EMACS) --batch -Q -L . -l test_indent.el - diff -u test.erl.indented test.erl - @echo "No differences between expected and actual indentation" diff --git a/lib/tools/emacs/erlang-skels.el b/lib/tools/emacs/erlang-skels.el index bdb3d9ad4a..534f50ab33 100644 --- a/lib/tools/emacs/erlang-skels.el +++ b/lib/tools/emacs/erlang-skels.el @@ -279,7 +279,8 @@ Please see the function `tempo-define-template'.") '((erlang-skel-include erlang-skel-large-header) "-behaviour(application)." n n "%% Application callbacks" n - "-export([start/2, stop/1])." n n + "-export([start/2, start_phase/3, stop/1, prep_stop/1," n> + "config_change/3])." n n (erlang-skel-double-separator-start 3) "%%% Application callbacks" n (erlang-skel-double-separator-end 3) n @@ -291,13 +292,14 @@ Please see the function `tempo-define-template'.") "%% application. If the application is structured according to the OTP" n "%% design principles as a supervision tree, this means starting the" n "%% top supervisor of the tree." n - "%%" n - "%% @spec start(StartType, StartArgs) -> {ok, Pid} |" n - "%% {ok, Pid, State} |" n - "%% {error, Reason}" n - "%% StartType = normal | {takeover, Node} | {failover, Node}" n - "%% StartArgs = term()" n (erlang-skel-separator-end 2) + "-spec start(StartType :: normal |" n> + "{takeover, Node :: node()} |" n> + "{failover, Node :: node()}," n> + "StartArgs :: term()) ->" n> + "{ok, Pid :: pid()} |" n> + "{ok, Pid :: pid(), State :: term()} |" n> + "{error, Reason :: term()}." n "start(_StartType, _StartArgs) ->" n> "case 'TopSupervisor':start_link() of" n> "{ok, Pid} ->" n> @@ -309,15 +311,52 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @private" n "%% @doc" n + "%% top supervisor of the tree." n + "%% Starts an application with included applications, when" n + "%% synchronization is needed between processes in the different" n + "%% applications during startup." + (erlang-skel-separator-end 2) + "-spec start_phase(Phase :: atom()," n> + "StartType :: normal |" n> + "{takeover, Node :: node()} |" n> + "{failover, Node :: node()}," n> + "PhaseArgs :: term()) -> ok | {error, Reason :: term()}." n + "start_phase(_Phase, _StartType, _PhaseArgs) ->" n> + "ok."n + n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n "%% This function is called whenever an application has stopped. It" n "%% is intended to be the opposite of Module:start/2 and should do" n "%% any necessary cleaning up. The return value is ignored." n - "%%" n - "%% @spec stop(State) -> void()" n (erlang-skel-separator-end 2) + "-spec stop(State :: term()) -> any()." n "stop(_State) ->" n> "ok." n n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n + "%% This function is called when an application is about to be stopped," n + "%% before shutting down the processes of the application." n + (erlang-skel-separator-end 2) + "-spec prep_stop(State :: term()) -> NewState :: term()." n + "prep_stop(State) ->" n> + "State." n + n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n + "%% This function is called by an application after a code replacement," n + "%% if the configuration parameters have changed." n + (erlang-skel-separator-end 2) + "-spec config_change(Changed :: [{Par :: atom(), Val :: term()}]," n> + "New :: [{Par :: atom(), Val :: term()}]," n> + "Removed :: [Par :: atom()]) -> ok." n + "config_change(_Changed, _New, _Removed) ->" n> + "ok." n + n (erlang-skel-double-separator-start 3) "%%% Internal functions" n (erlang-skel-double-separator-end 3) @@ -343,9 +382,12 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @doc" n "%% Starts the supervisor" n - "%%" n - "%% @spec start_link() -> {ok, Pid} | ignore | {error, Error}" n (erlang-skel-separator-end 2) + "-spec start_link() -> {ok, Pid :: pid()} |" n> + "{error, {already_started, Pid :: pid()}} |" n> + "{error, {shutdown, term()}} |" n> + "{error, term()} |" n> + "ignore." n "start_link() ->" n> "supervisor:start_link({local, ?SERVER}, ?MODULE, [])." n n @@ -359,11 +401,11 @@ Please see the function `tempo-define-template'.") "%% this function is called by the new process to find out about" n "%% restart strategy, maximum restart intensity, and child" n "%% specifications." n - "%%" n - "%% @spec init(Args) -> {ok, {SupFlags, [ChildSpec]}} |" n - "%% ignore |" n - "%% {error, Reason}" n (erlang-skel-separator-end 2) + "-spec init(Args :: term()) ->" n> + "{ok, {SupFlags :: supervisor:sup_flags()," n> + "[ChildSpec :: supervisor:child_spec()]}} |" n> + "ignore." n "init([]) ->" n "" n> "SupFlags = #{strategy => one_for_one," n> @@ -406,9 +448,11 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @doc" n "%% Starts the supervisor bridge" n - "%%" n - "%% @spec start_link() -> {ok, Pid} | ignore | {error, Error}" n (erlang-skel-separator-end 2) + "-spec start_link() -> {ok, Pid :: pid()} |" n> + "{error, {already_started, Pid :: pid()}} |" n> + "{error, term()} |" n> + "ignore." n "start_link() ->" n> "supervisor_bridge:start_link({local, ?SERVER}, ?MODULE, [])." n n @@ -422,11 +466,10 @@ Please see the function `tempo-define-template'.") "%% which calls Module:init/1 to start the subsystem. To ensure a" n "%% synchronized start-up procedure, this function does not return" n "%% until Module:init/1 has returned." n - "%%" n - "%% @spec init(Args) -> {ok, Pid, State} |" n - "%% ignore |" n - "%% {error, Reason}" n (erlang-skel-separator-end 2) + "-spec init(Args :: term()) -> {ok, Pid :: pid(), State :: term()} |" n> + "{error, Error :: term()} |" n> + "ignore." n "init([]) ->" n> "case 'AModule':start_link() of" n> "{ok, Pid} ->" n> @@ -442,10 +485,9 @@ Please see the function `tempo-define-template'.") "%% to terminate. It should be the opposite of Module:init/1 and stop" n "%% the subsystem and do any necessary cleaning up.The return value is" n "%% ignored." n - "%%" n - "%% @spec terminate(Reason, State) -> void()" n (erlang-skel-separator-end 2) - "terminate(Reason, State) ->" n> + "-spec terminate(Reason :: shutdown | term(), State :: term()) -> any()." n + "terminate(_Reason, _State) ->" n> "'AModule':stop()," n> "ok." n n @@ -464,9 +506,8 @@ Please see the function `tempo-define-template'.") "-export([start_link/0])." n n "%% gen_server callbacks" n - "-export([init/1, handle_call/3, handle_cast/2, " - "handle_info/2," n> - "terminate/2, code_change/3])." n n + "-export([init/1, handle_call/3, handle_cast/2, handle_info/2," n> + "terminate/2, code_change/3, format_status/2])." n n "-define(SERVER, ?MODULE)." n n @@ -478,9 +519,11 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @doc" n "%% Starts the server" n - "%%" n - "%% @spec start_link() -> {ok, Pid} | ignore | {error, Error}" n (erlang-skel-separator-end 2) + "-spec start_link() -> {ok, Pid :: pid()} |" n> + "{error, Error :: {already_started, pid()}} |" n> + "{error, Error :: term()} |" n> + "ignore." n "start_link() ->" n> "gen_server:start_link({local, ?SERVER}, ?MODULE, [], [])." n n @@ -492,12 +535,12 @@ Please see the function `tempo-define-template'.") "%% @private" n "%% @doc" n "%% Initializes the server" n - "%%" n - "%% @spec init(Args) -> {ok, State} |" n - "%% {ok, State, Timeout} |" n - "%% ignore |" n - "%% {stop, Reason}" n (erlang-skel-separator-end 2) + "-spec init(Args :: term()) -> {ok, State :: term()} |" n> + "{ok, State :: term(), Timeout :: timeout()} |" n> + "{ok, State :: term(), hibernate} |" n> + "{stop, Reason :: term()} |" n> + "ignore." n "init([]) ->" n> "process_flag(trap_exit, true)," n> "{ok, #state{}}." n @@ -506,15 +549,16 @@ Please see the function `tempo-define-template'.") "%% @private" n "%% @doc" n "%% Handling call messages" n - "%%" n - "%% @spec handle_call(Request, From, State) ->" n - "%% {reply, Reply, State} |" n - "%% {reply, Reply, State, Timeout} |" n - "%% {noreply, State} |" n - "%% {noreply, State, Timeout} |" n - "%% {stop, Reason, Reply, State} |" n - "%% {stop, Reason, State}" n (erlang-skel-separator-end 2) + "-spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) ->" n> + "{reply, Reply :: term(), NewState :: term()} |" n> + "{reply, Reply :: term(), NewState :: term(), Timeout :: timeout()} |" n> + "{reply, Reply :: term(), NewState :: term(), hibernate} |" n> + "{noreply, NewState :: term()} |" n> + "{noreply, NewState :: term(), Timeout :: timeout()} |" n> + "{noreply, NewState :: term(), hibernate} |" n> + "{stop, Reason :: term(), Reply :: term(), NewState :: term()} |" n> + "{stop, Reason :: term(), NewState :: term()}." n "handle_call(_Request, _From, State) ->" n> "Reply = ok," n> "{reply, Reply, State}." n @@ -523,23 +567,25 @@ Please see the function `tempo-define-template'.") "%% @private" n "%% @doc" n "%% Handling cast messages" n - "%%" n - "%% @spec handle_cast(Msg, State) -> {noreply, State} |" n - "%% {noreply, State, Timeout} |" n - "%% {stop, Reason, State}" n (erlang-skel-separator-end 2) - "handle_cast(_Msg, State) ->" n> + "-spec handle_cast(Request :: term(), State :: term()) ->" n> + "{noreply, NewState :: term()} |" n> + "{noreply, NewState :: term(), Timeout :: timeout()} |" n> + "{noreply, NewState :: term(), hibernate} |" n> + "{stop, Reason :: term(), NewState :: term()}." n + "handle_cast(_Request, State) ->" n> "{noreply, State}." n n (erlang-skel-separator-start 2) "%% @private" n "%% @doc" n "%% Handling all non call/cast messages" n - "%%" n - "%% @spec handle_info(Info, State) -> {noreply, State} |" n - "%% {noreply, State, Timeout} |" n - "%% {stop, Reason, State}" n (erlang-skel-separator-end 2) + "-spec handle_info(Info :: timeout() | term(), State :: term()) ->" n> + "{noreply, NewState :: term()} |" n> + "{noreply, NewState :: term(), Timeout :: timeout()} |" n> + "{noreply, NewState :: term(), hibernate} |" n> + "{stop, Reason :: normal | term(), NewState :: term()}." n "handle_info(_Info, State) ->" n> "{noreply, State}." n n @@ -550,9 +596,9 @@ Please see the function `tempo-define-template'.") "%% terminate. It should be the opposite of Module:init/1 and do any" n "%% necessary cleaning up. When it returns, the gen_server terminates" n "%% with Reason. The return value is ignored." n - "%%" n - "%% @spec terminate(Reason, State) -> void()" n (erlang-skel-separator-end 2) + "-spec terminate(Reason :: normal | shutdown | {shutdown, term()} | term()," n> + "State :: term()) -> any()." n "terminate(_Reason, _State) ->" n> "ok." n n @@ -560,12 +606,26 @@ Please see the function `tempo-define-template'.") "%% @private" n "%% @doc" n "%% Convert process state when code is changed" n - "%%" n - "%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}" n (erlang-skel-separator-end 2) + "-spec code_change(OldVsn :: term() | {down, term()}," n> + "State :: term()," n> + "Extra :: term()) -> {ok, NewState :: term()} |" n> + "{error, Reason :: term()}." n "code_change(_OldVsn, State, _Extra) ->" n> "{ok, State}." n n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n + "%% This function is called for changing the form and appearance" n + "%% of gen_server status when it is returned from sys:get_status/1,2" n + "%% or when it appears in termination error logs." n + (erlang-skel-separator-end 2) + "-spec format_status(Opt :: normal | terminate," n> + "Status :: list()) -> Status :: term()." n + "format_status(_Opt, Status) ->" n> + "Status." n + n (erlang-skel-double-separator-start 3) "%%% Internal functions" n (erlang-skel-double-separator-end 3) @@ -581,8 +641,8 @@ Please see the function `tempo-define-template'.") "-export([start_link/0, add_handler/0])." n n "%% gen_event callbacks" n - "-export([init/1, handle_event/2, handle_call/2, " n> - "handle_info/2, terminate/2, code_change/3])." n n + "-export([init/1, handle_event/2, handle_call/2, handle_info/2," n> + "terminate/2, code_change/3, format_status/2])." n n "-define(SERVER, ?MODULE)." n n @@ -594,18 +654,17 @@ Please see the function `tempo-define-template'.") (erlang-skel-separator-start 2) "%% @doc" n "%% Creates an event manager" n - "%%" n - "%% @spec start_link() -> {ok, Pid} | {error, Error}" n (erlang-skel-separator-end 2) + "-spec start_link() -> {ok, Pid :: pid()} |" n> + "{error, Error :: {already_started, pid()} | term()}." n "start_link() ->" n> "gen_event:start_link({local, ?SERVER})." n n (erlang-skel-separator-start 2) "%% @doc" n "%% Adds an event handler" n - "%%" n - "%% @spec add_handler() -> ok | {'EXIT', Reason} | term()" n (erlang-skel-separator-end 2) + "-spec add_handler() -> ok | {'EXIT', Reason :: term()} | term()." n "add_handler() ->" n> "gen_event:add_handler(?SERVER, ?MODULE, [])." n n @@ -617,9 +676,11 @@ Please see the function `tempo-define-template'.") "%% @doc" n "%% Whenever a new event handler is added to an event manager," n "%% this function is called to initialize the event handler." n - "%%" n - "%% @spec init(Args) -> {ok, State}" n (erlang-skel-separator-end 2) + "-spec init(Args :: term() | {Args :: term(), Term :: term()}) ->" n> + "{ok, State :: term()} |" n> + "{ok, State :: term(), hibernate} |" n> + "{error, Reason :: term()}." n "init([]) ->" n> "{ok, #state{}}." n n @@ -629,12 +690,13 @@ Please see the function `tempo-define-template'.") "%% Whenever an event manager receives an event sent using" n "%% gen_event:notify/2 or gen_event:sync_notify/2, this function is" n "%% called for each installed event handler to handle the event." n - "%%" n - "%% @spec handle_event(Event, State) ->" n - "%% {ok, State} |" n - "%% {swap_handler, Args1, State1, Mod2, Args2} |"n - "%% remove_handler" n (erlang-skel-separator-end 2) + "-spec handle_event(Event :: term(), State :: term()) ->" n> + "{ok, NewState :: term()} |" n> + "{ok, NewState :: term(), hibernate} |" n> + "remove_handler |" n> + "{swap_handler, Args1 :: term(), NewState :: term()," n> + "Handler2 :: atom() | {atom(), term()} , Args2 :: term()}." n> "handle_event(_Event, State) ->" n> "{ok, State}." n n @@ -644,12 +706,13 @@ Please see the function `tempo-define-template'.") "%% Whenever an event manager receives a request sent using" n "%% gen_event:call/3,4, this function is called for the specified" n "%% event handler to handle the request." n - "%%" n - "%% @spec handle_call(Request, State) ->" n - "%% {ok, Reply, State} |" n - "%% {swap_handler, Reply, Args1, State1, Mod2, Args2} |" n - "%% {remove_handler, Reply}" n (erlang-skel-separator-end 2) + "-spec handle_call(Request :: term(), State :: term()) ->" n> + "{ok, Reply :: term(), NewState :: term()} |" n> + "{ok, Reply :: term(), NewState :: term(), hibernate} |" n> + "{remove_handler, Reply :: term()} |" n> + "{swap_handler, Reply :: term(), Args1 :: term(), NewState :: term()," n> + "Handler2 :: atom() | {atom(), term()}, Args2 :: term()}." n "handle_call(_Request, State) ->" n> "Reply = ok," n> "{ok, Reply, State}." n @@ -660,12 +723,13 @@ Please see the function `tempo-define-template'.") "%% This function is called for each installed event handler when" n "%% an event manager receives any other message than an event or a" n "%% synchronous request (or a system message)." n - "%%" n - "%% @spec handle_info(Info, State) ->" n - "%% {ok, State} |" n - "%% {swap_handler, Args1, State1, Mod2, Args2} |" n - "%% remove_handler" n (erlang-skel-separator-end 2) + "-spec handle_info(Info :: term(), State :: term()) ->" n> + "{ok, NewState :: term()} |" n> + "{ok, NewState :: term(), hibernate} |" n> + "remove_handler |" n> + "{swap_handler, Args1 :: term(), NewState :: term()," n> + "Handler2 :: atom() | {atom(), term()}, Args2 :: term()}." n "handle_info(_Info, State) ->" n> "{ok, State}." n n @@ -675,22 +739,40 @@ Please see the function `tempo-define-template'.") "%% Whenever an event handler is deleted from an event manager, this" n "%% function is called. It should be the opposite of Module:init/1 and" n "%% do any necessary cleaning up." n - "%%" n - "%% @spec terminate(Reason, State) -> void()" n (erlang-skel-separator-end 2) - "terminate(_Reason, _State) ->" n> + "-spec terminate(Arg :: {stop, Reason :: term()} |" n> + "stop |" n> + "remove_handler |" n> + "{error, {'EXIT', Reason :: term()}} |" n> + "{error, Term :: term()} |" n> + "term()," n> + "State :: term()) -> any()." n + "terminate(_Arg, _State) ->" n> "ok." n n (erlang-skel-separator-start 2) "%% @private" n "%% @doc" n "%% Convert process state when code is changed" n - "%%" n - "%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}" n (erlang-skel-separator-end 2) + "-spec code_change(OldVsn :: term() | {down, term()}," n> + "State :: term()," n> + "Extra :: term()) -> {ok, NewState :: term()}." n "code_change(_OldVsn, State, _Extra) ->" n> "{ok, State}." n n + (erlang-skel-separator-start 2) + "%% @private" n + "%% @doc" n + "%% This function is called for changing the form and appearance" n + "%% of gen_event status when it is returned from sys:get_status/1,2" n + "%% or when it appears in termination error logs." n + (erlang-skel-separator-end 2) + "-spec format_status(Opt :: normal | terminate," n> + "Status :: list()) -> Status :: term()." n + "format_status(_Opt, Status) ->" n> + "Status." n + n (erlang-skel-double-separator-start 3) "%%% Internal functions" n (erlang-skel-double-separator-end 3) diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 6b93d63182..b3411c3ce7 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -4,7 +4,7 @@ ;; Author: Anders Lindgren ;; Keywords: erlang, languages, processes ;; Date: 2011-12-11 -;; Version: 2.8.0 +;; Version: 2.8.1 ;; Package-Requires: ((emacs "24.1")) ;; %CopyrightBegin% @@ -84,7 +84,7 @@ "The Erlang programming language." :group 'languages) -(defconst erlang-version "2.8.0" +(defconst erlang-version "2.8.1" "The version number of Erlang mode.") (defcustom erlang-root-dir nil @@ -2742,7 +2742,7 @@ Return nil if inside string, t if in a comment." (1+ (nth 2 stack-top))) ((= (char-syntax (following-char)) ?\)) (goto-char (nth 1 stack-top)) - (cond ((looking-at "[({]\\s *\\($\\|%\\)") + (cond ((erlang-record-or-function-args-p) ;; Line ends with parenthesis. (let ((previous (erlang-indent-find-preceding-expr)) (stack-pos (nth 2 stack-top))) @@ -2752,19 +2752,10 @@ Return nil if inside string, t if in a comment." (nth 2 stack-top)))) ((= (following-char) ?,) ;; a comma at the start of the line: line up with opening parenthesis. - (nth 2 stack-top)) + (min (nth 2 stack-top) + (erlang-indent-element stack-top indent-point token))) (t - (goto-char (nth 1 stack-top)) - (let ((base (cond ((looking-at "[({]\\s *\\($\\|%\\)") - ;; Line ends with parenthesis. - (erlang-indent-parenthesis (nth 2 stack-top))) - (t - ;; Indent to the same column as the first - ;; argument. - (goto-char (1+ (nth 1 stack-top))) - (skip-chars-forward " \t") - (current-column))))) - (erlang-indent-standard indent-point token base 't))))) + (erlang-indent-element stack-top indent-point token)))) ;; ((eq (car stack-top) '<<) ;; Element of binary (possible comprehension) expression, @@ -2773,13 +2764,11 @@ Return nil if inside string, t if in a comment." (+ 2 (nth 2 stack-top))) ((looking-at "\\(>>\\)[^_a-zA-Z0-9]") (nth 2 stack-top)) + ((= (following-char) ?,) + (min (+ (nth 2 stack-top) 1) + (- (erlang-indent-to-first-element stack-top 2) 1))) (t - (goto-char (nth 1 stack-top)) - ;; Indent to the same column as the first - ;; argument. - (goto-char (+ 2 (nth 1 stack-top))) - (skip-chars-forward " \t") - (current-column)))) + (erlang-indent-to-first-element stack-top 2)))) ((memq (car stack-top) '(icr fun spec)) ;; The default indentation is the column of the option @@ -2835,12 +2824,13 @@ Return nil if inside string, t if in a comment." (let ((base (erlang-indent-find-base stack indent-point off skip))) ;; Special cases (goto-char indent-point) - (cond ((looking-at "\\(end\\|after\\)\\($\\|[^_a-zA-Z0-9]\\)") + (cond ((looking-at "\\(;\\|end\\|after\\)\\($\\|[^_a-zA-Z0-9]\\)") (if (eq (car stack-top) '->) (erlang-pop stack)) - (if stack - (erlang-caddr (car stack)) - 0)) + (cond ((and stack (looking-at ";")) + (+ (erlang-caddr (car stack)) (- erlang-indent-level 2))) + (stack (erlang-caddr (car stack))) + (t off))) ((looking-at "catch\\b\\($\\|[^_a-zA-Z0-9]\\)") ;; Are we in a try (let ((start (if (eq (car stack-top) '->) @@ -2914,6 +2904,22 @@ Return nil if inside string, t if in a comment." (current-column))) start-alternativ)))))) ))) +(defun erlang-indent-to-first-element (stack-top extra) + ;; Indent to the same column as the first + ;; argument. extra should be 1 for lists tuples or 2 for binaries + (goto-char (+ (nth 1 stack-top) extra)) + (skip-chars-forward " \t") + (current-column)) + +(defun erlang-indent-element (stack-top indent-point token) + (goto-char (nth 1 stack-top)) + (let ((base (cond ((erlang-record-or-function-args-p) + ;; Line ends with parenthesis. + (erlang-indent-parenthesis (nth 2 stack-top))) + (t + (erlang-indent-to-first-element stack-top 1))))) + (erlang-indent-standard indent-point token base 't))) + (defun erlang-indent-standard (indent-point token base inside-parenthesis) "Standard indent when in blocks or tuple or arguments. Look at last thing to see in what state we are, move relative to the base." @@ -2939,6 +2945,9 @@ Return nil if inside string, t if in a comment." ;; Avoid treating comments a continued line. ((= (following-char) ?%) base) + ((and (= (following-char) ?,) inside-parenthesis) + ;; a comma at the start of the line line up with parenthesis + (- base 1)) ;; Continued line (e.g. line beginning ;; with an operator.) (t @@ -3028,11 +3037,21 @@ This assumes that the preceding expression is either simple (t col))) col)))) +(defun erlang-record-or-function-args-p () + (and (looking-at "[({]\\s *\\($\\|%\\)") + (or (eq (following-char) ?\( ) + (save-excursion + (ignore-errors (forward-sexp (- 1))) + (eq (preceding-char) ?#))))) + (defun erlang-indent-parenthesis (stack-position) (let ((previous (erlang-indent-find-preceding-expr))) - (if (> previous stack-position) - (+ stack-position erlang-argument-indent) - (+ previous erlang-argument-indent)))) + (cond ((eq previous stack-position) ;; tuple or map not a record + (1+ stack-position)) + ((> previous stack-position) + (+ stack-position erlang-argument-indent)) + (t + (+ previous erlang-argument-indent))))) (defun erlang-skip-blank (&optional lim) "Skip over whitespace and comments until limit reached." @@ -5166,7 +5185,6 @@ future, a new shell on an already running host will be started." ;; e.g. it does not assume that we are running an inferior ;; Erlang, there exists a lot of other possibilities. - (defvar erlang-shell-buffer-name "*erlang*" "The name of the Erlang link shell buffer.") @@ -5177,46 +5195,28 @@ Also see the description of `ielm-prompt-read-only'." :type 'boolean :package-version '(erlang . "2.8.0")) -(defvar erlang-shell-mode-map nil - "Keymap used by Erlang shells.") - - -(defvar erlang-shell-mode-hook nil - "User functions to run when an Erlang shell is started. - -This hook is used to change the behaviour of Erlang mode. It is -normally used by the user to personalise the programming environment. -When used in a site init file, it could be used to customise Erlang -mode for all users on the system. - -The function added to this hook is run every time a new Erlang -shell is started. +(defvar erlang-shell-mode-map + (let ((map (make-sparse-keymap))) + (define-key map "\M-\t" 'erlang-complete-tag) -See also `erlang-load-hook', a hook which is run once, when Erlang -mode is loaded, and `erlang-mode-hook' which is run every time a new -Erlang source file is loaded into Emacs.") + ;; Normally the other way around. + (define-key map "\C-a" 'comint-bol) + (define-key map "\C-c\C-a" 'beginning-of-line) + (define-key map "\C-d" nil) ; Was `comint-delchar-or-maybe-eof' + (define-key map "\M-\C-m" 'compile-goto-error) + map) + "Keymap used by Erlang shells.") (defvar erlang-input-ring-file-name "~/.erlang_history" "When non-nil, file name used to store Erlang shell history information.") - -(defun erlang-shell-mode () +(define-derived-mode erlang-shell-mode comint-mode "Erlang Shell" "Major mode for interacting with an Erlang shell. -We assume that we already are in Comint mode. - The following special commands are available: \\{erlang-shell-mode-map}" - (interactive) - (setq major-mode 'erlang-shell-mode) - (setq mode-name "Erlang Shell") (erlang-mode-variables) - (if erlang-shell-mode-map - nil - (setq erlang-shell-mode-map (copy-keymap comint-mode-map)) - (erlang-shell-mode-commands erlang-shell-mode-map)) - (use-local-map erlang-shell-mode-map) ;; Needed when compiling directly from the Erlang shell. (setq compilation-last-buffer (current-buffer)) (setq comint-prompt-regexp "^[^>=]*> *") @@ -5230,7 +5230,6 @@ The following special commands are available: 'inferior-erlang-strip-delete nil t) (add-hook 'comint-output-filter-functions 'inferior-erlang-strip-ctrl-m nil t) - (setq comint-input-ring-file-name erlang-input-ring-file-name) (comint-read-input-ring t) (make-local-variable 'kill-buffer-hook) @@ -5249,8 +5248,7 @@ The following special commands are available: (define-key map [menu-bar compilation] (cons "Errors" compilation-menu-map))) map)))) - (erlang-tags-init) - (run-hooks 'erlang-shell-mode-hook)) + (erlang-tags-init)) (defun erlang-mouse-2-command (event) @@ -5272,13 +5270,6 @@ Selects Comint or Compilation mode command as appropriate." (call-interactively (lookup-key compilation-mode-map "\C-m")) (call-interactively (lookup-key comint-mode-map "\C-m")))) -(defun erlang-shell-mode-commands (map) - (define-key map "\M-\t" 'erlang-complete-tag) - (define-key map "\C-a" 'comint-bol) ; Normally the other way around. - (define-key map "\C-c\C-a" 'beginning-of-line) - (define-key map "\C-d" nil) ; Was `comint-delchar-or-maybe-eof' - (define-key map "\M-\C-m" 'compile-goto-error)) - ;;; ;;; Inferior Erlang -- Run an Erlang shell as a subprocess. ;;; diff --git a/lib/tools/emacs/test.erl.indented b/lib/tools/emacs/test.erl.indented deleted file mode 100644 index 14a4eca7c3..0000000000 --- a/lib/tools/emacs/test.erl.indented +++ /dev/null @@ -1,784 +0,0 @@ -%% -*- Mode: erlang; indent-tabs-mode: nil -*- -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% - -%%%------------------------------------------------------------------- -%%% File : test.erl -%%% Author : Dan Gudmundsson <[email protected]> -%%% Description : Test emacs mode indention and font-locking -%%% this file is intentionally not indented. -%%% Copy the file and indent it and you should end up with test.erl.indented -%%% Created : 6 Oct 2009 by Dan Gudmundsson <[email protected]> -%%%------------------------------------------------------------------- - -%% Start off with syntax highlighting you have to verify this by looking here -%% and see that the code looks alright - --module(test). --compile(export_all). - -%% Used to cause an "Unbalanced parentheses" error. -foo(M) -> - M#{a :=<<"a">> - ,b:=1}. -foo() -> - #{a =><<"a">> - ,b=>1}. - -%% Module attributes should be highlighted - --export([t/1]). --record(record1, {a, - b, - c - }). --record(record2, { - a, - b - }). - --record(record3, {a = 8#42423 bor - 8#4234, - b = 8#5432 - bor 2#1010101 - c = 123 + - 234, - d}). - --record(record4, { - a = 8#42423 bor - 8#4234, - b = 8#5432 - bor 2#1010101 - c = 123 + - 234, - d}). - --record(record5, { a = 1 :: integer() - , b = foobar :: atom() - }). - --define(MACRO_1, macro). --define(MACRO_2(_), macro). - --spec t(integer()) -> any(). - --type ann() :: Var :: integer(). --type ann2() :: Var :: - 'return' - | 'return_white_spaces' - | 'return_comments' - | 'text' | ann(). --type paren() :: - (ann2()). --type t1() :: atom(). --type t2() :: [t1()]. --type t3(Atom) :: integer(Atom). --type t4() :: t3(foobar). --type t5() :: {t1(), t3(foo)}. --type t6() :: 1 | 2 | 3 | - 'foo' | 'bar'. --type t7() :: []. --type t71() :: [_]. --type t8() :: {any(),none(),pid(),port(), - reference(),float()}. --type t9() :: [1|2|3|foo|bar] | - list(a | b | c) | t71(). --type t10() :: {1|2|3|foo|t9()} | {}. --type t11() :: 1..2. --type t13() :: maybe_improper_list(integer(), t11()). --type t14() :: [erl_scan:foo() | - %% Should be highlighted - term() | - bool() | - byte() | - char() | - non_neg_integer() | nonempty_list() | - pos_integer() | - neg_integer() | - number() | - list() | - nonempty_improper_list() | nonempty_maybe_improper_list() | - maybe_improper_list() | string() | iolist() | byte() | - module() | - mfa() | - node() | - timeout() | - no_return() | - %% Should not be highlighted - nonempty_() | nonlist() | - erl_scan:bar(34, 92) | t13() | m:f(integer() | <<_:_*16>>)]. - - --type t15() :: {binary(),<<>>,<<_:34>>,<<_:_*42>>, - <<_:3,_:_*14>>,<<>>} | [<<>>|<<_:34>>|<<_:16>>| - <<_:3,_:_*1472>>|<<_:19,_:_*14>>| <<_:34>>| - <<_:34>>|<<_:34>>|<<_:34>>]. --type t16() :: fun(). --type t17() :: fun((...) -> paren()). --type t18() :: fun(() -> t17() | t16()). --type t19() :: fun((t18()) -> t16()) | - fun((nonempty_maybe_improper_list('integer', any())| - 1|2|3|a|b|<<_:3,_:_*14>>|integer()) -> - nonempty_maybe_improper_list('integer', any())| - 1|2|3|a|b|<<_:3,_:_*14>>|integer()). --type t20() :: [t19(), ...]. --type t21() :: tuple(). --type t21(A) :: A. --type t22() :: t21(integer()). --type t23() :: #rec1{}. --type t24() :: #rec2{a :: t23(), b :: [atom()]}. --type t25() :: #rec3{f123 :: [t24() | - 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())]}. --type t26() :: #rec4{ a :: integer() - , b :: any() - }. --type t27() :: { integer() - , atom() - }. --type t99() :: - {t2(),t4(),t5(),t6(),t7(),t8(),t10(),t14(), - t15(),t20(),t21(), t22(),t25()}. --spec t1(FooBar :: t99()) -> t99(); - (t2()) -> t2(); - (t4()) -> t4() when is_subtype(t4(), t24); - (t23()) -> t23() when is_subtype(t23(), atom()), - is_subtype(t23(), t14()); - (t24()) -> t24() when is_subtype(t24(), atom()), - is_subtype(t24(), t14()), - is_subtype(t24(), t4()). - --spec over(I :: integer()) -> R1 :: foo:typen(); - (A :: atom()) -> R2 :: foo:atomen(); - (T :: tuple()) -> R3 :: bar:typen(). - --spec mod:t2() -> any(). - --spec handle_cast(Cast :: {'exchange', node(), [[name(),...]]} - | {'del_member', name(), pid()}, - #state{}) -> {'noreply', #state{}}. - --spec handle_cast(Cast :: - {'exchange', node(), [[name(),...]]} - | {'del_member', name(), pid()}, - #state{}) -> {'noreply', #state{}}. - --spec all(fun((T) -> boolean()), List :: [T]) -> - boolean() when is_subtype(T, term()). % (*) - --spec get_closest_pid(term()) -> - Return :: pid() - | {'error', {'no_process', term()} - | {'no_such_group', term()}}. - --spec add( X :: integer() - , Y :: integer() - ) -> integer(). - --opaque attributes_data() :: - [{'column', column()} | {'line', info_line()} | - {'text', string()}] | {line(),column()}. --record(r,{ - f1 :: attributes_data(), - f222 = foo:bar(34, #rec3{}, 234234234423, - aassdsfsdfsdf, 2234242323) :: - [t24() | 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())], - f333 :: [t24() | 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())], - f3 = x:y(), - f4 = x:z() :: t99(), - f17 :: 'undefined', - f18 :: 1 | 2 | 'undefined', - f19 = 3 :: integer()|undefined, - f5 = 3 :: undefined|integer()}). - --record(state, { - sequence_number = 1 :: integer() - }). - - -highlighting(X) % Function definitions should be highlighted - when is_integer(X) -> % and so should `when' and `is_integer' be - %% Highlighting - %% Various characters (we keep an `atom' after to see that highlighting ends) - $a,atom, % Characters should be marked - "string",atom, % and strings - 'asdasd',atom, % quote should be atoms?? - 'VaV',atom, - 'aVa',atom, - '\'atom',atom, - 'atom\'',atom, - 'at\'om',atom, - '#1',atom, - - $", atom, % atom should be ok - $', atom, - - "string$", atom, "string$", atom, % currently buggy I know... - "string\$", atom, % workaround for bug above - - "char $in string", atom, - - 'atom$', atom, 'atom$', atom, - 'atom\$', atom, - - 'char $in atom', atom, - - $[, ${, $\\, atom, - ?MACRO_1, - ?MACRO_2(foo), - - %% Numerical constants - 16#DD, % AD Should not be highlighted - 32#dd, % AD Should not be highlighted - 32#ddAB, % AD Should not be highlighted - 32#101, % AD Should not be highlighted - 32#ABTR, % AD Should not be highlighted - - %% Variables - Variables = lists:foo(), - _Variables = lists:foo(), % AD - AppSpec = Xyz/2, - Module42 = Xyz(foo, bar), - Module:foo(), - _Module:foo(), % AD - FooÅÅ = lists:reverse([tl,hd,tl,hd]), % AD Should highlight FooÅÅ - _FooÅÅ = 42, % AD Should highlight _FooÅÅ - - %% Bifs - erlang:registered(), - registered(), - hd(tl(tl(hd([a,b,c])))), - erlang:anything(lists), - %% Guards - is_atom(foo), is_float(2.3), is_integer(32), is_number(4323.3), - is_function(Fun), is_pid(self()), - not_a_guard:is_list([]), - %% Other Types - - atom, % not (currently) hightlighted - 234234, - 234.43, - - [list, are, not, higlighted], - {nor, is, tuple}, - ok. - -%%% -%%% Indentation -%%% - -%%% Left - -%% Indented - - % Right - - -indent_basics(X, Y, Z) - when X > 42, - Z < 13; - Y =:= 4711 -> - %% comments - % right comments - case lists:filter(fun(_, AlongName, - B, - C) -> - true - end, - [a,v,b]) - of - [] -> - Y = 5 * 43, - ok; - [_|_] -> - Y = 5 * 43, - ok - end, - Y, - %% List, tuples and binaries - [a, - b, c - ], - [ a, - b, c - ], - - [ - a, - b - ], - {a, - b,c - }, - { a, - b,c - }, - - { - a, - b - }, - - <<1:8, - 2:8 - >>, - << - 1:8, - 2:8 - >>, - << 1:8, - 2:8 - >>, - - (a, - b, - c - ), - - ( a, - b, - c - ), - - - ( - a, - b, - c - ), - - call(2#42423 bor - #4234, - 2#5432, - other_arg), - ok; -indent_basics(Xlongname, - #struct{a=Foo, - b=Bar}, - [X| - Y]) -> - testing_next_clause, - ok; -indent_basics( % AD added clause - X, % not sure how this should look - Y, - Z) - when - X < 42, Z > 13; - Y =:= 4711 -> - foo; -indent_basics(X, Y, Z) when % AD added clause - X < 42, Z > 13; % testing when indentation - Y =:= 4711 -> - foo; -indent_basics(X, Y, Z) % AD added clause - when % testing when indentation - X < 42, Z > 13; % unsure about this one - Y =:= 4711 -> - foo. - - - -indent_nested() -> - [ - {foo, 2, "string"}, - {bar, 3, "another string"} - ]. - - -indent_icr(Z) -> % icr = if case receive - %% If - if Z >= 0 -> - X = 43 div 4, - foo(X); - Z =< 10 -> - X = 43 div 4, - foo(X); - Z == 5 orelse - Z == 7 -> - X = 43 div 4, - foo(X); - true -> - if_works - end, - %% Case - case {Z, foo, bar} of - {Z,_,_} -> - X = 43 div 4, - foo(X); - {Z,_,_} when - Z =:= 42 -> % AD line should be indented as a when - X = 43 div 4, - foo(X); - {Z,_,_} - when Z < 10 -> % AD when should be indented - X = 43 div 4, - foo(X); - {Z,_,_} - when % AD when should be indented - Z < 10 % and the guards should follow when - andalso % unsure about how though - true -> - X = 43 div 4, - foo(X) - end, - %% begin - begin - sune, - X = 74234 + foo(8456) + - 345 div 43, - ok - end, - - - %% receive - receive - {Z,_,_} -> - X = 43 div 4, - foo(X); - Z -> - X = 43 div 4, - foo(X) - end, - receive - {Z,_,_} -> - X = 43 div 4, - foo(X); - Z % AD added clause - when Z =:= 1 -> % This line should be indented by 2 - X = 43 div 4, - foo(X); - Z when % AD added clause - Z =:= 2 -> % This line should be indented by 2 - X = 43 div 4, - foo(X); - Z -> - X = 43 div 4, - foo(X) - after infinity -> - foo(X), - asd(X), - 5*43 - end, - receive - after 10 -> - foo(X), - asd(X), - 5*43 - end, - ok. - -indent_fun() -> - %% Changed fun to one indention level - Var = spawn(fun(X) - when X == 2; - X > 10 -> - hello, - case Hello() of - true when is_atom(X) -> - foo; - false -> - bar - end; - (Foo) when is_atom(Foo), - is_integer(X) -> - X = 6* 45, - Y = true andalso - kalle - end), - %% check EEP37 named funs - Fn1 = fun Fact(N) when N > 0 -> - F = Fact(N-1), - N * F; - Fact(0) -> - 1 - end, - %% check anonymous funs too - Fn2 = fun(0) -> - 1; - (N) -> - N - end, - ok. - -indent_try_catch() -> - try - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2) - catch - exit:{badarg,R} -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R); - error:R % AD added clause - when R =:= 42 -> % when should be indented - foo(R); - error:R % AD added clause - when % when should be indented - R =:= 42 -> % but unsure about this (maybe 2 more) - foo(R); - error:R when % AD added clause - R =:= foo -> % line should be 2 indented (works) - foo(R); - error:R -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R) - after - foo('after'), - file:close(Xfile) - end; -indent_try_catch() -> - try - foo(bar) - of - X when true andalso - kalle -> - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2); - X % AD added clause - when false andalso % when should be 2 indented - bengt -> - gurka(); - X when % AD added clause - false andalso % line should be 2 indented - not bengt -> - gurka(); - X -> - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2) - catch - exit:{badarg,R} -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R); - error:R -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R) - after - foo('after'), - file:close(Xfile), - bar(with_long_arg, - with_second_arg) - end; -indent_try_catch() -> - try foo() - after - foo(), - bar(with_long_arg, - with_second_arg) - end. - -indent_catch() -> - D = B + - float(43.1), - - B = catch oskar(X), - - A = catch (baz + - bax), - catch foo(), - - C = catch B + - float(43.1), - - case catch foo(X) of - A -> - B - end, - - case - catch foo(X) - of - A -> - B - end, - - case - foo(X) - of - A -> - catch B, - X - end, - - try sune of - _ -> foo - catch _:_ -> baf - end, - - try - sune - of - _ -> - X = 5, - (catch foo(X)), - X + 10 - catch _:_ -> baf - end, - - try - (catch sune) - of - _ -> - catch foo() %% BUGBUG can't handle catch inside try without parentheses - catch _:_ -> - baf - end, - - try - (catch exit()) - catch - _ -> - catch baf() - end, - ok. - -indent_binary() -> - X = lists:foldr(fun(M) -> - <<Ma/binary, " ">> - end, [], A), - A = <<X/binary, 0:8>>, - B. - - -indent_comprehensions() -> - %% I don't have a good idea how we want to handle this - %% but they are here to show how they are indented today. - Result1 = [X || - #record{a=X} <- lists:seq(1, 10), - true = (X rem 2) - ], - Result2 = [X || <<X:32,_:32>> <= <<0:512>>, - true = (X rem 2) - ], - - Binary1 = << <<X:8>> || - #record{a=X} <- lists:seq(1, 10), - true = (X rem 2) - >>, - - Binary2 = << <<X:8>> || <<X:32,_:32>> <= <<0:512>>, - true = (X rem 2) - >>, - ok. - -%% This causes an error in earlier erlang-mode versions. -foo() -> - [#foo{ - foo = foo}]. - -%% Record indentation -some_function_with_a_very_long_name() -> - #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b}, - case dummy_function_with_a_very_very_long_name(x) of - #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b} -> - ok; - Var = #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b} -> - Var#'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b}; - #xyz{ - a=1, - b=2} -> - ok - end. - -another_function_with_a_very_very_long_name() -> - #rec{ - field1=1, - field2=1}. - -some_function_name_xyz(xyzzy, #some_record{ - field1=Field1, - field2=Field2}) -> - SomeVariable = f(#'Some-long-record-name'{ - field_a = 1, - 'inter-xyz-parameters' = - #'Some-other-very-long-record-name'{ - field2 = Field1, - field2 = Field2}}), - {ok, SomeVariable}. - -commas_first() -> - {abc, [ {some_var, 1} - , {some_other_var, 2} - , {erlang_ftw, 9} - , {erlang_cookie, 'cookie'} - , {cmds, - [ {one, "sudo ls"} - , {one, "sudo ls"} - , {two, "sudo ls"} - , {three, "sudo ls"} - , {four, "sudo ls"} - , {three, "sudo ls"} - ] } - , {ssh_username, "yow"} - , {cluster, - [ {aaaa, [ {"10.198.55.12" , "" } - , {"10.198.55.13" , "" } - ] } - , {bbbb, [ {"10.198.55.151", "" } - , {"10.198.55.123", "" } - , {"10.198.55.34" , "" } - , {"10.198.55.85" , "" } - , {"10.198.55.67" , "" } - ] } - , {cccc, [ {"10.198.55.68" , "" } - , {"10.198.55.69" , "" } - ] } - ] } - ] - }. - - -%% this used to result in a scan-sexp error -[{ - }]. - -%% this used to result in 2x the correct indentation within the function -%% body, due to the function name being mistaken for a keyword -catcher(N) -> - try generate_exception(N) of - Val -> {N, normal, Val} - catch - throw:X -> {N, caught, thrown, X}; - exit:X -> {N, caught, exited, X}; - error:X -> {N, caught, error, X} - end. diff --git a/lib/tools/emacs/test.erl.orig b/lib/tools/emacs/test.erl.orig deleted file mode 100644 index c0cf1749b6..0000000000 --- a/lib/tools/emacs/test.erl.orig +++ /dev/null @@ -1,784 +0,0 @@ -%% -*- Mode: erlang; indent-tabs-mode: nil -*- -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% - -%%%------------------------------------------------------------------- -%%% File : test.erl -%%% Author : Dan Gudmundsson <[email protected]> -%%% Description : Test emacs mode indention and font-locking -%%% this file is intentionally not indented. -%%% Copy the file and indent it and you should end up with test.erl.indented -%%% Created : 6 Oct 2009 by Dan Gudmundsson <[email protected]> -%%%------------------------------------------------------------------- - -%% Start off with syntax highlighting you have to verify this by looking here -%% and see that the code looks alright - --module(test). --compile(export_all). - -%% Used to cause an "Unbalanced parentheses" error. -foo(M) -> -M#{a :=<<"a">> -,b:=1}. -foo() -> -#{a =><<"a">> -,b=>1}. - -%% Module attributes should be highlighted - --export([t/1]). --record(record1, {a, - b, - c -}). --record(record2, { - a, - b - }). - --record(record3, {a = 8#42423 bor - 8#4234, - b = 8#5432 - bor 2#1010101 - c = 123 + -234, - d}). - --record(record4, { - a = 8#42423 bor - 8#4234, - b = 8#5432 - bor 2#1010101 - c = 123 + - 234, - d}). - --record(record5, { a = 1 :: integer() -, b = foobar :: atom() -}). - --define(MACRO_1, macro). --define(MACRO_2(_), macro). - --spec t(integer()) -> any(). - --type ann() :: Var :: integer(). --type ann2() :: Var :: - 'return' - | 'return_white_spaces' - | 'return_comments' - | 'text' | ann(). --type paren() :: - (ann2()). --type t1() :: atom(). --type t2() :: [t1()]. --type t3(Atom) :: integer(Atom). --type t4() :: t3(foobar). --type t5() :: {t1(), t3(foo)}. --type t6() :: 1 | 2 | 3 | - 'foo' | 'bar'. --type t7() :: []. --type t71() :: [_]. --type t8() :: {any(),none(),pid(),port(), - reference(),float()}. --type t9() :: [1|2|3|foo|bar] | - list(a | b | c) | t71(). --type t10() :: {1|2|3|foo|t9()} | {}. --type t11() :: 1..2. --type t13() :: maybe_improper_list(integer(), t11()). --type t14() :: [erl_scan:foo() | - %% Should be highlighted - term() | - bool() | - byte() | - char() | - non_neg_integer() | nonempty_list() | - pos_integer() | - neg_integer() | - number() | - list() | - nonempty_improper_list() | nonempty_maybe_improper_list() | - maybe_improper_list() | string() | iolist() | byte() | - module() | - mfa() | - node() | - timeout() | - no_return() | - %% Should not be highlighted - nonempty_() | nonlist() | - erl_scan:bar(34, 92) | t13() | m:f(integer() | <<_:_*16>>)]. - - --type t15() :: {binary(),<<>>,<<_:34>>,<<_:_*42>>, - <<_:3,_:_*14>>,<<>>} | [<<>>|<<_:34>>|<<_:16>>| -<<_:3,_:_*1472>>|<<_:19,_:_*14>>| <<_:34>>| -<<_:34>>|<<_:34>>|<<_:34>>]. --type t16() :: fun(). --type t17() :: fun((...) -> paren()). --type t18() :: fun(() -> t17() | t16()). --type t19() :: fun((t18()) -> t16()) | - fun((nonempty_maybe_improper_list('integer', any())| - 1|2|3|a|b|<<_:3,_:_*14>>|integer()) -> -nonempty_maybe_improper_list('integer', any())| -1|2|3|a|b|<<_:3,_:_*14>>|integer()). --type t20() :: [t19(), ...]. --type t21() :: tuple(). --type t21(A) :: A. --type t22() :: t21(integer()). --type t23() :: #rec1{}. --type t24() :: #rec2{a :: t23(), b :: [atom()]}. --type t25() :: #rec3{f123 :: [t24() | -1|2|3|4|a|b|c|d| -nonempty_maybe_improper_list(integer, any())]}. --type t26() :: #rec4{ a :: integer() -, b :: any() -}. --type t27() :: { integer() -, atom() -}. --type t99() :: -{t2(),t4(),t5(),t6(),t7(),t8(),t10(),t14(), -t15(),t20(),t21(), t22(),t25()}. --spec t1(FooBar :: t99()) -> t99(); -(t2()) -> t2(); - (t4()) -> t4() when is_subtype(t4(), t24); -(t23()) -> t23() when is_subtype(t23(), atom()), - is_subtype(t23(), t14()); -(t24()) -> t24() when is_subtype(t24(), atom()), - is_subtype(t24(), t14()), - is_subtype(t24(), t4()). - --spec over(I :: integer()) -> R1 :: foo:typen(); - (A :: atom()) -> R2 :: foo:atomen(); - (T :: tuple()) -> R3 :: bar:typen(). - --spec mod:t2() -> any(). - --spec handle_cast(Cast :: {'exchange', node(), [[name(),...]]} - | {'del_member', name(), pid()}, - #state{}) -> {'noreply', #state{}}. - --spec handle_cast(Cast :: - {'exchange', node(), [[name(),...]]} - | {'del_member', name(), pid()}, - #state{}) -> {'noreply', #state{}}. - --spec all(fun((T) -> boolean()), List :: [T]) -> - boolean() when is_subtype(T, term()). % (*) - --spec get_closest_pid(term()) -> - Return :: pid() - | {'error', {'no_process', term()} - | {'no_such_group', term()}}. - --spec add( X :: integer() -, Y :: integer() -) -> integer(). - --opaque attributes_data() :: -[{'column', column()} | {'line', info_line()} | - {'text', string()}] | {line(),column()}. --record(r,{ - f1 :: attributes_data(), -f222 = foo:bar(34, #rec3{}, 234234234423, - aassdsfsdfsdf, 2234242323) :: -[t24() | 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())], -f333 :: [t24() | 1|2|3|4|a|b|c|d| - nonempty_maybe_improper_list(integer, any())], -f3 = x:y(), -f4 = x:z() :: t99(), -f17 :: 'undefined', -f18 :: 1 | 2 | 'undefined', -f19 = 3 :: integer()|undefined, -f5 = 3 :: undefined|integer()}). - --record(state, { - sequence_number = 1 :: integer() - }). - - -highlighting(X) % Function definitions should be highlighted - when is_integer(X) -> % and so should `when' and `is_integer' be - %% Highlighting - %% Various characters (we keep an `atom' after to see that highlighting ends) - $a,atom, % Characters should be marked - "string",atom, % and strings - 'asdasd',atom, % quote should be atoms?? - 'VaV',atom, - 'aVa',atom, - '\'atom',atom, - 'atom\'',atom, - 'at\'om',atom, - '#1',atom, - - $", atom, % atom should be ok - $', atom, - - "string$", atom, "string$", atom, % currently buggy I know... - "string\$", atom, % workaround for bug above - - "char $in string", atom, - - 'atom$', atom, 'atom$', atom, - 'atom\$', atom, - - 'char $in atom', atom, - - $[, ${, $\\, atom, - ?MACRO_1, - ?MACRO_2(foo), - - %% Numerical constants - 16#DD, % AD Should not be highlighted - 32#dd, % AD Should not be highlighted - 32#ddAB, % AD Should not be highlighted - 32#101, % AD Should not be highlighted - 32#ABTR, % AD Should not be highlighted - - %% Variables - Variables = lists:foo(), - _Variables = lists:foo(), % AD - AppSpec = Xyz/2, - Module42 = Xyz(foo, bar), - Module:foo(), - _Module:foo(), % AD - FooÅÅ = lists:reverse([tl,hd,tl,hd]), % AD Should highlight FooÅÅ - _FooÅÅ = 42, % AD Should highlight _FooÅÅ - - %% Bifs - erlang:registered(), - registered(), - hd(tl(tl(hd([a,b,c])))), - erlang:anything(lists), - %% Guards - is_atom(foo), is_float(2.3), is_integer(32), is_number(4323.3), - is_function(Fun), is_pid(self()), - not_a_guard:is_list([]), - %% Other Types - - atom, % not (currently) hightlighted - 234234, - 234.43, - - [list, are, not, higlighted], - {nor, is, tuple}, - ok. - -%%% -%%% Indentation -%%% - -%%% Left - -%% Indented - -% Right - - -indent_basics(X, Y, Z) - when X > 42, -Z < 13; -Y =:= 4711 -> - %% comments - % right comments - case lists:filter(fun(_, AlongName, - B, - C) -> - true - end, - [a,v,b]) - of - [] -> - Y = 5 * 43, - ok; - [_|_] -> - Y = 5 * 43, - ok - end, - Y, - %% List, tuples and binaries - [a, - b, c - ], - [ a, - b, c - ], - - [ - a, - b -], - {a, - b,c - }, - { a, - b,c - }, - - { - a, - b - }, - -<<1:8, - 2:8 - >>, - << - 1:8, - 2:8 - >>, - << 1:8, - 2:8 - >>, - - (a, - b, - c - ), - - ( a, - b, - c - ), - - - ( - a, - b, - c - ), - - call(2#42423 bor - #4234, - 2#5432, - other_arg), - ok; -indent_basics(Xlongname, - #struct{a=Foo, - b=Bar}, - [X| - Y]) -> - testing_next_clause, - ok; -indent_basics( % AD added clause - X, % not sure how this should look - Y, - Z) - when - X < 42, Z > 13; - Y =:= 4711 -> - foo; -indent_basics(X, Y, Z) when % AD added clause - X < 42, Z > 13; % testing when indentation - Y =:= 4711 -> - foo; -indent_basics(X, Y, Z) % AD added clause - when % testing when indentation - X < 42, Z > 13; % unsure about this one - Y =:= 4711 -> - foo. - - - -indent_nested() -> - [ - {foo, 2, "string"}, - {bar, 3, "another string"} - ]. - - -indent_icr(Z) -> % icr = if case receive - %% If - if Z >= 0 -> - X = 43 div 4, - foo(X); - Z =< 10 -> - X = 43 div 4, - foo(X); - Z == 5 orelse - Z == 7 -> - X = 43 div 4, - foo(X); - true -> - if_works - end, - %% Case - case {Z, foo, bar} of - {Z,_,_} -> - X = 43 div 4, - foo(X); - {Z,_,_} when - Z =:= 42 -> % AD line should be indented as a when - X = 43 div 4, - foo(X); - {Z,_,_} - when Z < 10 -> % AD when should be indented - X = 43 div 4, - foo(X); - {Z,_,_} - when % AD when should be indented - Z < 10 % and the guards should follow when - andalso % unsure about how though - true -> - X = 43 div 4, - foo(X) - end, - %% begin - begin - sune, - X = 74234 + foo(8456) + - 345 div 43, - ok - end, - - - %% receive - receive - {Z,_,_} -> - X = 43 div 4, - foo(X); - Z -> - X = 43 div 4, - foo(X) - end, - receive - {Z,_,_} -> - X = 43 div 4, - foo(X); - Z % AD added clause - when Z =:= 1 -> % This line should be indented by 2 - X = 43 div 4, - foo(X); - Z when % AD added clause - Z =:= 2 -> % This line should be indented by 2 - X = 43 div 4, - foo(X); - Z -> - X = 43 div 4, - foo(X) - after infinity -> - foo(X), - asd(X), - 5*43 - end, - receive - after 10 -> - foo(X), - asd(X), - 5*43 - end, - ok. - -indent_fun() -> - %% Changed fun to one indention level -Var = spawn(fun(X) - when X == 2; - X > 10 -> - hello, - case Hello() of - true when is_atom(X) -> - foo; - false -> - bar - end; - (Foo) when is_atom(Foo), - is_integer(X) -> - X = 6* 45, - Y = true andalso - kalle - end), -%% check EEP37 named funs -Fn1 = fun Fact(N) when N > 0 -> - F = Fact(N-1), - N * F; -Fact(0) -> - 1 - end, -%% check anonymous funs too - Fn2 = fun(0) -> -1; - (N) -> - N - end, - ok. - -indent_try_catch() -> - try - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2) - catch - exit:{badarg,R} -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R); - error:R % AD added clause - when R =:= 42 -> % when should be indented - foo(R); - error:R % AD added clause - when % when should be indented - R =:= 42 -> % but unsure about this (maybe 2 more) - foo(R); - error:R when % AD added clause - R =:= foo -> % line should be 2 indented (works) - foo(R); - error:R -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R) - after - foo('after'), - file:close(Xfile) - end; -indent_try_catch() -> - try - foo(bar) - of - X when true andalso - kalle -> - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2); - X % AD added clause - when false andalso % when should be 2 indented - bengt -> - gurka(); - X when % AD added clause - false andalso % line should be 2 indented - not bengt -> - gurka(); - X -> - io:format(stdout, "Parsing file ~s, ", - [St0#leex.xfile]), - {ok,Line3,REAs,Actions,St3} = - parse_rules(Xfile, Line2, Macs, St2) - catch - exit:{badarg,R} -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R); - error:R -> - foo(R), - io:format(stdout, - "ERROR reason ~p~n", - R) - after - foo('after'), - file:close(Xfile), - bar(with_long_arg, - with_second_arg) - end; - indent_try_catch() -> - try foo() - after - foo(), - bar(with_long_arg, - with_second_arg) - end. - -indent_catch() -> - D = B + - float(43.1), - - B = catch oskar(X), - - A = catch (baz + - bax), - catch foo(), - - C = catch B + - float(43.1), - - case catch foo(X) of - A -> - B - end, - - case - catch foo(X) - of - A -> - B - end, - - case - foo(X) - of - A -> - catch B, - X - end, - - try sune of - _ -> foo - catch _:_ -> baf - end, - - try -sune - of - _ -> - X = 5, - (catch foo(X)), - X + 10 - catch _:_ -> baf - end, - - try - (catch sune) - of - _ -> - catch foo() %% BUGBUG can't handle catch inside try without parentheses - catch _:_ -> - baf - end, - - try -(catch exit()) - catch -_ -> - catch baf() - end, - ok. - -indent_binary() -> - X = lists:foldr(fun(M) -> - <<Ma/binary, " ">> - end, [], A), - A = <<X/binary, 0:8>>, - B. - - -indent_comprehensions() -> -%% I don't have a good idea how we want to handle this -%% but they are here to show how they are indented today. -Result1 = [X || - #record{a=X} <- lists:seq(1, 10), - true = (X rem 2) - ], -Result2 = [X || <<X:32,_:32>> <= <<0:512>>, - true = (X rem 2) - ], - -Binary1 = << <<X:8>> || - #record{a=X} <- lists:seq(1, 10), - true = (X rem 2) - >>, - -Binary2 = << <<X:8>> || <<X:32,_:32>> <= <<0:512>>, - true = (X rem 2) - >>, -ok. - -%% This causes an error in earlier erlang-mode versions. -foo() -> -[#foo{ -foo = foo}]. - -%% Record indentation -some_function_with_a_very_long_name() -> - #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b}, - case dummy_function_with_a_very_very_long_name(x) of - #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b} -> - ok; - Var = #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b} -> - Var#'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ - field1=a, - field2=b}; - #xyz{ - a=1, - b=2} -> - ok - end. - -another_function_with_a_very_very_long_name() -> - #rec{ - field1=1, - field2=1}. - -some_function_name_xyz(xyzzy, #some_record{ - field1=Field1, - field2=Field2}) -> - SomeVariable = f(#'Some-long-record-name'{ - field_a = 1, - 'inter-xyz-parameters' = - #'Some-other-very-long-record-name'{ - field2 = Field1, - field2 = Field2}}), - {ok, SomeVariable}. - -commas_first() -> - {abc, [ {some_var, 1} - , {some_other_var, 2} - , {erlang_ftw, 9} - , {erlang_cookie, 'cookie'} - , {cmds, - [ {one, "sudo ls"} - , {one, "sudo ls"} - , {two, "sudo ls"} - , {three, "sudo ls"} - , {four, "sudo ls"} - , {three, "sudo ls"} - ] } - , {ssh_username, "yow"} - , {cluster, - [ {aaaa, [ {"10.198.55.12" , "" } - , {"10.198.55.13" , "" } - ] } - , {bbbb, [ {"10.198.55.151", "" } - , {"10.198.55.123", "" } - , {"10.198.55.34" , "" } - , {"10.198.55.85" , "" } - , {"10.198.55.67" , "" } - ] } - , {cccc, [ {"10.198.55.68" , "" } - , {"10.198.55.69" , "" } - ] } - ] } -] -}. - - -%% this used to result in a scan-sexp error -[{ -}]. - -%% this used to result in 2x the correct indentation within the function -%% body, due to the function name being mistaken for a keyword -catcher(N) -> -try generate_exception(N) of -Val -> {N, normal, Val} -catch -throw:X -> {N, caught, thrown, X}; -exit:X -> {N, caught, exited, X}; -error:X -> {N, caught, error, X} -end. diff --git a/lib/tools/src/fprof.erl b/lib/tools/src/fprof.erl index fb657c2928..a3b4bfdddf 100644 --- a/lib/tools/src/fprof.erl +++ b/lib/tools/src/fprof.erl @@ -1242,8 +1242,7 @@ spawn_3step(Spawn, FunPrelude, FunAck, FunBody) catch Child ! {Parent, Ref, Go}, Result catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> catch exit(Child, kill), erlang:raise(Class, Reason, Stacktrace) end; diff --git a/lib/tools/src/lcnt.erl b/lib/tools/src/lcnt.erl index 139b3d8a4a..d0152a4915 100644 --- a/lib/tools/src/lcnt.erl +++ b/lib/tools/src/lcnt.erl @@ -218,9 +218,11 @@ raw() -> call(raw). set(Option, Value) -> call({set, Option, Value}). set({Option, Value}) -> call({set, Option, Value}). save(Filename) -> call({save, Filename}). -load(Filename) -> ok = start_internal(), call({load, Filename}). +load(Filename) -> call({load, Filename}). -call(Msg) -> gen_server:call(?MODULE, Msg, infinity). +call(Msg) -> + ok = start_internal(), + gen_server:call(?MODULE, Msg, infinity). %% -------------------------------------------------------------------- %% %% @@ -237,7 +239,6 @@ apply(Fun) when is_function(Fun) -> lcnt:apply(Fun, []). apply(Fun, As) when is_function(Fun) -> - ok = start_internal(), Opt = lcnt:rt_opt({copy_save, true}), lcnt:clear(), Res = erlang:apply(Fun, As), @@ -943,7 +944,7 @@ print_state_information(#state{locks = Locks} = State) -> print(kv("#tries", s(Stats#stats.tries))), print(kv("#colls", s(Stats#stats.colls))), print(kv("wait time", s(Stats#stats.time) ++ " us" ++ " ( " ++ s(Stats#stats.time/1000000) ++ " s)")), - print(kv("percent of duration", s(Stats#stats.time/State#state.duration*100) ++ " %")), + print(kv("percent of duration", s(percent(Stats#stats.time, State#state.duration)) ++ " %")), ok. diff --git a/lib/tools/test/emacs_SUITE.erl b/lib/tools/test/emacs_SUITE.erl index 77a8813db5..f4e78da667 100644 --- a/lib/tools/test/emacs_SUITE.erl +++ b/lib/tools/test/emacs_SUITE.erl @@ -23,10 +23,10 @@ -export([all/0, init_per_testcase/2, end_per_testcase/2]). --export([bif_highlight/1]). +-export([bif_highlight/1, indent/1]). -all() -> - [bif_highlight]. +all() -> + [bif_highlight, indent]. init_per_testcase(_Case, Config) -> ErlangEl = filename:join([code:lib_dir(tools),"emacs","erlang.el"]), @@ -74,4 +74,69 @@ check_bif_highlight(Bin, Tag, Compare) -> [] = Compare -- EmacsIntBifs, [] = EmacsIntBifs -- Compare. - +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +indent(Config) -> + case emacs_version_ok() of + false -> {skip, "Old or no emacs found"}; + true -> + Def = filename:dirname(code:which(?MODULE)) ++ "/" ++ ?MODULE_STRING ++ "_data", + Dir = proplists:get_value(data_dir, Config, Def), + OrigFs = filelib:wildcard(Dir ++ "/*"), + io:format("Dir: ~s~nFs: ~p~n", [Dir, OrigFs]), + Fs = [{File, unindent(File)} || File <- OrigFs, + filename:extension(File) =:= ""], + Indent = fun emacs/1, + [Indent(File) || {_, File} <- Fs], + Res = [diff(Orig, File) || {Orig, File} <- Fs], + [file:delete(File) || {ok, File} <- Res], %% Cleanup + [] = [Fail || {fail, Fail} <- Res], + ok + end. + +unindent(Input) -> + Output = Input ++ ".erl", + {ok, Bin} = file:read_file(Input), + Lines0 = string:split(Bin, "\n", all), + Lines = [string:trim(Line, leading, [$\s,$\t]) || Line <- Lines0], + %% io:format("File: ~s lines: ~w~n", [Input, length(Lines0)]), + %% [io:format("~s~n", [L]) || L <- Lines], + ok = file:write_file(Output, lists:join("\n", Lines)), + Output. + +diff(Orig, File) -> + case os:cmd(["diff ", Orig, " ", File]) of + "" -> {ok, File}; + Diff -> + io:format("Fail: ~s vs ~s~n~s~n~n",[Orig, File, Diff]), + {fail, File} + end. + +emacs_version_ok() -> + case os:cmd("emacs --version | head -1") of + "GNU Emacs " ++ Ver -> + case string:to_float(Ver) of + {Vsn, _} when Vsn >= 24.1 -> + true; + _ -> + io:format("Emacs version fail~n~s~n~n",[Ver]), + false + end; + Res -> + io:format("Emacs version fail~n~s~n~n",[Res]), + false + end. + +emacs(File) -> + EmacsErlDir = filename:join([code:lib_dir(tools), "emacs"]), + Cmd = ["emacs ", + "--batch --quick ", + "--directory ", EmacsErlDir, " ", + "--eval \"(require 'erlang-start)\" ", + File, " ", + "--eval '(indent-region (point-min) (point-max) nil)' ", + "--eval '(save-buffer 0)'" + ], + _Res = os:cmd(Cmd), + % io:format("cmd ~s:~n=> ~s~n", [Cmd, _Res]), + ok. diff --git a/lib/tools/test/emacs_SUITE_data/comments b/lib/tools/test/emacs_SUITE_data/comments new file mode 100644 index 0000000000..ff974ca295 --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/comments @@ -0,0 +1,25 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% 3 comment chars: always left indented +%%% 2 comment chars: Context indented +%%% 1 comment char: Rigth indented + +%%% left +%% context dependent + % rigth + +func() -> +%%% left + %% context dependent + % right indented + case get(foo) of + undefined -> + %% Testing indention + ok; + %% Catch all + Other -> + Other + end, + ok. + diff --git a/lib/tools/test/emacs_SUITE_data/comprehensions b/lib/tools/test/emacs_SUITE_data/comprehensions new file mode 100644 index 0000000000..45279850a5 --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/comprehensions @@ -0,0 +1,47 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% indentation of comprehensions + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + +list() -> + %% I don't have a good idea how we want to handle this + %% but they are here to show how they are indented today. + Result1 = [X || + #record{a=X} <- lists:seq(1, 10), + true = (X rem 2) + ], + Result2 = [X || <<X:32,_:32>> <= <<0:512>>, + true = (X rem 2) + ], + Res = [ func(X, + arg2) + || + #record{a=X} <- lists:seq(1, 10), + true = (X rem 2) + ], + Result1. + +binary(B) -> + Binary1 = << <<X:8>> || + #record{a=X} <- lists:seq(1, 10), + true = (X rem 2) + >>, + + Binary2 = << <<X:8>> || <<X:32,_:32>> <= <<0:512>>, + true = (X rem 2) + >>, + + Bin3 = << + << + X:8, + 34:8 + >> + || <<X:32,_:32>> <= <<0:512>>, + true = (X rem 2) + >>, + ok. diff --git a/lib/tools/test/emacs_SUITE_data/funcs b/lib/tools/test/emacs_SUITE_data/funcs new file mode 100644 index 0000000000..877f982005 --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/funcs @@ -0,0 +1,174 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% Function (and funs) indentation + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + +-export([ + func1/0, + func2/0, + a_function_with_a_very_very_long_name/0, + when1/2 + ]). + +-compile([nowarn_unused_functions, + {inline, [ + func2/2, + func3/2 + ] + } + ]). + +func1() -> + basic. + +func2(A1, + A2) -> + ok. + +func3( + A1, + A2 + ) -> + ok. + +%% Okeefe style +func4(A1 + ,A2 + ,A3 + ) -> + ok. + +func5( + A41 + ,A42) -> + ok. + +a_function_with_a_very_very_long_name() -> + A00 = #record{ + field1=1, + field2=1 + }, + A00. + +when1(W1, W2) + when is_number(W1), + is_number(W2) -> + ok. + +when2(W1,W2,W3) when + W1 > W2, + W2 > W3 -> + ok. + +when3(W1,W2,W3) when + W1 > W2, + W2 > W3 + -> + ok. + +when4(W1,W2,W3) + when + W1 > W2, + W2 > W3 -> + ok. + +match1({[H|T], + Other}, + M1A2) -> + ok. + +match2( + { + [H|T], + Other + }, + M2A2 + ) -> + ok. + +match3({ + M3A1, + [ + H | + T + ], + Other + }, + M3A2 + ) -> + ok. + +match4(<< + M4A:8, + M4B:16/unsigned-integer, + _/binary + >>, + M4C) -> + ok. + +match5(M5A, + #record{ + b=M5B, + c=M5C + } + ) -> + ok. + +match6(M6A, + #{key6a := a6, + key6b := b6 + }) -> + ok. + +funs(1) + when + X -> + %% Changed fun to one indention level + %% 'when' and several clause forces a depth of '4' + Var = spawn(fun(X, _) + when X == 2; + X > 10 -> + hello, + case Hello() of + true when is_atom(X) -> + foo; + false -> + bar + end; + (Foo) when is_atom(Foo), + is_integer(X) -> + X = 6 * 45, + Y = true andalso + kalle + end), + Var; +funs(2) -> + %% check EEP37 named funs + Fn1 = fun + Factory(N) when + N > 0 -> + F = Fact(N-1), + N * F; + Factory(0) -> + 1 + end, + Fn1; +funs(3) -> + %% check anonymous funs too + Fn2 = fun(0) -> + 1; + (N) -> + N + end, + ok; +funs(4) -> + X = lists:foldr(fun(M) -> + <<M/binary, " ">> + end, [], Z), + A = <<X/binary, 0:8>>, + A. diff --git a/lib/tools/test/emacs_SUITE_data/highlight b/lib/tools/test/emacs_SUITE_data/highlight new file mode 100644 index 0000000000..0719f6516a --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/highlight @@ -0,0 +1,78 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% Open this file in your editor and manually check the colors of +%%% different types and calls and builtin words + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + + +highlighting(X) % Function definitions should be highlighted + when is_integer(X) -> % and so should `when' and `is_integer' be + %% Highlighting + %% Various characters (we keep an `atom' after to see that highlighting ends) + $a,atom, % Characters should be marked + "string",atom, % and strings + 'asdasd',atom, % quote should be atoms?? + 'VaV',atom, + 'aVa',atom, + '\'atom',atom, + 'atom\'',atom, + 'at\'om',atom, + '#1',atom, + + $", atom, % atom should be ok + $', atom, + + "string$", atom, "string$", atom, % currently buggy I know... + "string\$", atom, % workaround for bug above + + "char $in string", atom, + + 'atom$', atom, 'atom$', atom, + 'atom\$', atom, + + 'char $in atom', atom, + + $[, ${, $\\, atom, + ?MACRO_1, + ?MACRO_2(foo), + + %% Numerical constants + 16#DD, % Should not be highlighted + 32#dd, % Should not be highlighted + 32#ddAB, % Should not be highlighted + 32#101, % Should not be highlighted + 32#ABTR, % Should not be highlighted + + %% Variables + Variables = lists:foo(), + _Variables = lists:foo(), + AppSpec = Xyz/2, + Module42 = Xyz(foo, bar), + Module:foo(), + _Module:foo(), % + FooÅÅ = lists:reverse([tl,hd,tl,hd]), % Should highlight FooÅÅ + _FooÅÅ = 42, % Should highlight _FooÅÅ + + %% Bifs + erlang:registered(), + registered(), + hd(tl(tl(hd([a,b,c])))), + erlang:anything(lists), + %% Guards + is_atom(foo), is_float(2.3), is_integer(32), is_number(4323.3), + is_function(Fun), is_pid(self()), + not_a_guard:is_list([]), + %% Other Types + + atom, % not (currently) hightlighted + 234234, + 234.43, + + [list, are, not, higlighted], + {nor, is, tuple}, + ok. diff --git a/lib/tools/test/emacs_SUITE_data/icr b/lib/tools/test/emacs_SUITE_data/icr new file mode 100644 index 0000000000..8445c1a74d --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/icr @@ -0,0 +1,157 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% indentation of if case receive statements + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + +indent_if(1, Z) -> + %% If + if Z >= 0 -> + X = 43 div Z, + X; + Z =< 10 -> + X = 43 div Z, + X; + Z == 5 orelse + Z == 7 -> + X = 43 div Z, + X; + is_number(Z), + Z < 32 -> + Z; + is_number(Z); + Z < 32 -> + Z * 32; + true -> + if_works + end; +indent_if(2, Z) -> + %% If + if + Z >= 0 -> + X = 43 div Z, + X + ; Z =< 10 -> + 43 div Z + ; Z == 5 orelse + Z == 7 -> + X = 43 div Z, + X + ; is_number(Z), + Z < 32 -> + Z + ; true -> + if_works + end. + +indent_case(1, Z) -> + %% Case + case {Z, foo, bar} of + {Z,_,_} -> + X = 43 div 4, + foo(X); + {Z,_,_} when + Z =:= 42 -> % line should be indented as a when + X = 43 div 4, + foo(X); + {Z,_,_} + when Z < 10 orelse + Z =:= foo -> % Binary op alignment here !!! + X = 43 div 4, + Bool = Z < 5 orelse % Binary op args align differently after when + Z =:= foo, % and elsewhere ??? + foo(X); + {Z,_,_} + when % when should be indented + Z < 10 % and the guards should follow when + andalso % unsure about how though + true -> + X = 43 div 4, + foo(X) + end; +indent_case(2, Z) -> + %% Case + case {Z, foo, bar} of + {Z,_,_} -> + X = 43 div 4, + foo(X) + ; {Z,_,_} when + Z =:= 42 -> % line should be indented as a when + X = 43 div 4, + foo(X) + ; {Z,_,_} + when Z < 10 -> % when should be indented + X = 43 div 4, + foo(X) + ; {Z,_,_} + when % when should be indented + Z < 10 % and the guards should follow when + andalso % unsure about how though + true -> + X = 43 div 4, + foo(X) + end. + +indent_begin(Z) -> + %% Begin + begin + sune, + Z = 74234 + + foo(8456) + + 345 div 43, + Foo = begin + ok, + foo(234), + begin + io:format("Down here\n") + end + end, + {Foo, + bar} + end. + +indent_receive(1) -> + %% receive + receive + {Z,_,_} -> + X = 43 div 4, + foo(X) + ; Z -> + X = 43 div 4, + foo(X) + end, + ok; +indent_receive(2) -> + receive + {Z,_,_} -> + X = 43 div 4, + foo(X); + Z % added clause + when Z =:= 1 -> % This line should be indented by 2 + X = 43 div 4, + foo(X); + Z when % added clause + Z =:= 2 -> % This line should be indented by 2 + X = 43 div 4, + foo(X); + Z -> + X = 43 div 4, + foo(X) + after infinity -> + foo(X), + asd(X), + 5*43 + end, + ok; +indent_receive() -> + receive + after 10 -> + foo(X), + asd(X), + 5*43 + end, + ok. diff --git a/lib/tools/test/emacs_SUITE_data/macros b/lib/tools/test/emacs_SUITE_data/macros new file mode 100644 index 0000000000..6c874e9187 --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/macros @@ -0,0 +1,31 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% Macros should be indented as code + +-define(M0, ok). + +-define(M1, + case X of + undefined -> error; + _ -> ok + end). + +-define(M2(M2A1, + M2A2), + func(M2A1, + M2A2) + ). + +-define( + M3, + undefined + ). + +-ifdef(DEBUG). +-define(LOG, + logger:log(?MODULE,?LINE) + ). +-else(). +-define(LOG, ok). +-endif(). diff --git a/lib/tools/test/emacs_SUITE_data/records b/lib/tools/test/emacs_SUITE_data/records new file mode 100644 index 0000000000..241582718c --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/records @@ -0,0 +1,35 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%% Test that records are indented correctly + +-record(record0, + { + r0a, + r0b, + r0c + }). + +-record(record1, {r1a, + r1b, + r1c + }). + +-record(record2, { + r2a, + r2b + }). + +-record(record3, {r3a = 8#42423 bor + 8#4234, + r3b = 8#5432 + bor 2#1010101, + r3c = 123 + + 234, + r3d}). + +-record(record5, + { r5a = 1 :: integer() + , r5b = foobar :: atom() + }). + diff --git a/lib/tools/test/emacs_SUITE_data/terms b/lib/tools/test/emacs_SUITE_data/terms new file mode 100644 index 0000000000..352364a73c --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/terms @@ -0,0 +1,174 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% indentation of terms contain builtin types + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + + +list(1) -> + [a, + b, + c + ]; +list(2) -> + [ a, + b, c + ]; +list(3) -> + [ + a, + b, c + ]; +list(4) -> + [ a + , b + , c + ]. + +tuple(1) -> + {a, + b,c + }; +tuple(2) -> + { a, + b,c + }; +tuple(3) -> + { + a, + b,c + }; +tuple(4) -> + { a + , b + ,c + }. + +binary(1) -> + <<1:8, + 2:8 + >>; +binary(2) -> + << + 1:8, + 2:8 + >>; +binary(3) -> + << 1:8, + 2:8 + >>; +binary(4) -> + << + 1:8 + ,2:8 + >>; +binary(5) -> + << 1:8 + , 2:8 + >>. + +record(1) -> + #record{a=1, + b=2 + }; +record(2) -> + #record{ a=1, + b=2 + }; +record(3) -> + #record{ + a=1, + b=2 + }; +record(4) -> + #record{ + a=1 + ,b=2 + }; +record(Record) -> + Record#record{ + a=1 + ,b=2 + }. + +map(1) -> + #{a=>1, + b=>2 + }; +map(2) -> + #{ a=>1, + b=>2 + }; +map(3) -> + #{ + a=>1, + b=>2 + }; +map(4) -> + #{ + a => <<"a">> + ,b => 2 + }; +map(MapVar) -> + MapVar = #{a :=<<"a">> + ,b:=1}. + +deep(Rec) -> + Rec#rec{ atom = 'atom', + map = #{ k1 => {v, + 1}, + k2 => [ + 1, + 2, + 3 + ], + {key, + 3} + => + << + 123:8, + 255:8 + >> + } + }. + +%% Record indentation +some_function_with_a_very_long_name() -> + #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ + field1=a, + field2=b}, + case dummy_function_with_a_very_very_long_name(x) of + #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ + field1=a, + field2=b} -> + ok; + Var = #'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ + field1=a, + field2=b} -> + Var#'a-long-record-name-like-it-sometimes-is-with-asn.1-records'{ + field1=a, + field2=b}; + #xyz{ + a=1, + b=2} -> + ok + end. + +some_function_name_xyz(xyzzy, #some_record{ + field1=Field1, + field2=Field2}) -> + SomeVariable = f(#'Some-long-record-name'{ + field_a = 1, + 'inter-xyz-parameters' = + #'Some-other-very-long-record-name'{ + field2 = Field1, + field2 = Field2}}), + {ok, SomeVariable}. + +foo() -> + [#foo{ + foo = foo}]. diff --git a/lib/tools/test/emacs_SUITE_data/try_catch b/lib/tools/test/emacs_SUITE_data/try_catch new file mode 100644 index 0000000000..0005b2003a --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/try_catch @@ -0,0 +1,166 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%%% Try and catch indentation is hard + +%%% Not everything in these test are set in stone +%%% better indentation rules can be added but by having +%%% these tests we can see what changes in new implementations +%%% and notice when doing unintentional changes + +try_catch() -> + try + io:format(stdout, "Parsing file ~s, ", + [St0#leex.xfile]), + {ok,Line3,REAs,Actions,St3} = + parse_rules(Xfile, Line2, Macs, St2) + catch + exit:{badarg,R} -> + foo(R), + io:format(stdout, + "ERROR reason ~p~n", + R); + error:R + when R =:= 42 -> % when should be indented + foo(R); + error:R + when % when should be indented + R =:= 42 -> % but unsure about this (maybe 2 more) + foo(R); + error:R when + R =:= foo -> % line should be 2 indented (works) + foo(R); + error:R -> + foo(R), + io:format(stdout, + "ERROR reason ~p~n", + R) + after + foo('after'), + file:close(Xfile) + end; +try_catch() -> + try + foo(bar) + of + X when true andalso + kalle -> + io:format(stdout, "Parsing file ~s, ", + [St0#leex.xfile]), + {ok,Line3,REAs,Actions,St3} = + parse_rules(Xfile, Line2, Macs, St2); + X + when false andalso % when should be 2 indented + bengt -> + gurka(); + X when + false andalso % line should be 2 indented + not bengt -> + gurka(); + X -> + io:format(stdout, "Parsing file ~s, ", + [St0#leex.xfile]), + {ok,Line3,REAs,Actions,St3} = + parse_rules(Xfile, Line2, Macs, St2) + catch + exit:{badarg,R} -> + foo(R), + io:format(stdout, + "ERROR reason ~p~n", + R); + error:R -> + foo(R), + io:format(stdout, + "ERROR reason ~p~n", + R) + after + foo('after'), + file:close(Xfile), + bar(with_long_arg, + with_second_arg) + end; +try_catch() -> + try foo() + after + foo(), + bar(with_long_arg, + with_second_arg) + end. + +indent_catch() -> + D = B + + float(43.1), + + B = catch oskar(X), + + A = catch (baz + + bax), + catch foo(), + + C = catch B + + float(43.1), + + case catch foo(X) of + A -> + B + end, + + case + catch foo(X) + of + A -> + B + end, + + case + foo(X) + of + A -> + catch B, + X + end, + + try sune of + _ -> foo + catch _:_ -> baf + end, + + Variable = try + sune + of + _ -> + X = 5, + (catch foo(X)), + X + 10 + catch _:_ -> baf + after cleanup() + end, + + try + (catch sune) + of + _ -> + foo1(), + catch foo() %% BUGBUG can't handle catch inside try without parentheses + catch _:_ -> + baf + end, + + try + (catch exit()) + catch + _ -> + catch baf() + end, + ok. + +%% this used to result in 2x the correct indentation within the function +%% body, due to the function name being mistaken for a keyword +catcher(N) -> + try generate_exception(N) of + Val -> {N, normal, Val} + catch + throw:X -> {N, caught, thrown, X}; + exit:X -> {N, caught, exited, X}; + error:X -> {N, caught, error, X} + end. diff --git a/lib/tools/test/emacs_SUITE_data/type_specs b/lib/tools/test/emacs_SUITE_data/type_specs new file mode 100644 index 0000000000..e71841cc7a --- /dev/null +++ b/lib/tools/test/emacs_SUITE_data/type_specs @@ -0,0 +1,110 @@ +%% -*- Mode: erlang; indent-tabs-mode: nil -*- +%% Copyright Ericsson AB 2017. All Rights Reserved. + +%% Tests how types and specs are indented (also that the editor can parse them) +%% May need improvements + + +-type ann() :: Var :: integer(). +-type ann2() :: + 'return' + | 'return_white_spaces' + | 'return_comments' + | 'text' | ann(). +-type paren() :: + (ann2()). + +-type t6() :: + 1 | 2 | 3 | + 'foo' + | 'bar'. + +-type t8() :: {any(),none(),pid(),port(), + reference(),float()}. + +-type t14() :: [erl_scan:foo() | + %% Should be highlighted + term() | + boolean() | + byte() | + char() | + non_neg_integer() | nonempty_list() | + pos_integer() | + neg_integer() | + number() | + list() | + nonempty_improper_list() | nonempty_maybe_improper_list() | + maybe_improper_list() | string() | iolist() | byte() | + module() | + mfa() | + node() | + timeout() | + no_return() | + %% Should not be highlighted + nonempty_() | nonlist() | + erl_scan:bar(34, 92) | t13() | m:f(integer() | <<_:_*16>>)]. + +-type t15() :: {binary(),<<>>,<<_:34>>,<<_:_*42>>, + <<_:3,_:_*14>>,<<>>} | [<<>>|<<_:34>>|<<_:16>>| + <<_:3,_:_*1472>>|<<_:19,_:_*14>>| <<_:34>>| + <<_:34>>|<<_:34>>|<<_:34>>]. + +-type t18() :: + fun(() -> t17() | t16()). +-type t19() :: + fun((t18()) -> t16()) | + fun((nonempty_maybe_improper_list('integer', any())| + 1|2|3|a|b|<<_:3,_:_*14>>|integer()) + -> + nonempty_maybe_improper_list('integer', any())| %% left to col 16? + 1|2|3|a|b|<<_:3,_:_*14>>|integer()). %% left to col 16? +-type t20() :: [t19(), ...]. +-type t25() :: #rec3{f123 :: [t24() | + 1|2|3|4|a|b|c|d| + nonempty_maybe_improper_list(integer, any())]}. +-type t26() :: #rec4{ a :: integer() + , b :: any() + }. + +%% Spec + +-spec t1(FooBar :: t99()) -> t99(); + (t2()) -> t2(); + (t4()) -> t4() when is_subtype(t4(), t24); + (t23()) -> t23() when is_subtype(t23(), atom()), + is_subtype(t23(), t14()); + (t24()) -> t24() when is_subtype(t24(), atom()), + is_subtype(t24(), t14()), + is_subtype(t24(), t4()). + +-spec over(I :: integer()) -> R1 :: foo:typen(); + (A :: atom()) -> R2 :: foo:atomen(); + (T :: tuple()) -> R3 :: bar:typen(). + +-spec mod:t2() -> any(). + +-spec handle_cast(Cast :: {'exchange', node(), [[name(),...]]} + | {'del_member', name(), pid()}, + #state{}) -> {'noreply', #state{}}. + +-spec handle_cast(Cast :: + {'exchange', node(), [[name(),...]]} + | {'del_member', name(), pid()}, + #state{}) -> + {'noreply', #state{}}. %% left to col 10? + +-spec all(fun((T) -> boolean()), List :: [T]) -> + boolean() when is_subtype(T, term()). % (*) + +-spec get_closest_pid(term()) -> + Return :: pid() %% left to col 10? + | {'error', {'no_process', term()}} %% left to col 10? + | {'no_such_group', term()}. %% left to col 10? + +-spec add( X :: integer() + , Y :: integer() + ) -> integer(). + +-opaque attributes_data() :: + [{'column', column()} | {'line', info_line()} | + {'text', string()}] | {line(),column()}. diff --git a/lib/tools/test/lcnt_SUITE.erl b/lib/tools/test/lcnt_SUITE.erl index 146c915087..a79572a742 100644 --- a/lib/tools/test/lcnt_SUITE.erl +++ b/lib/tools/test/lcnt_SUITE.erl @@ -30,6 +30,8 @@ t_conflicts/1, t_locations/1, t_swap_keys/1, + t_implicit_start/1, + t_crash_before_collect/1, smoke_lcnt/1]). init_per_testcase(_Case, Config) -> @@ -44,8 +46,8 @@ suite() -> {timetrap,{minutes,4}}]. all() -> - [t_load, t_conflicts, t_locations, t_swap_keys, - smoke_lcnt]. + [t_load, t_conflicts, t_locations, t_swap_keys, t_implicit_start, + t_crash_before_collect, smoke_lcnt]. %%---------------------------------------------------------------------- %% Tests @@ -149,6 +151,15 @@ t_swap_keys_file([File|Files]) -> ok = lcnt:stop(), t_swap_keys_file(Files). +%% Prior to OTP-14913 this would crash with 'noproc' as the lcnt server hadn't +%% been started yet. +t_implicit_start(Config) when is_list(Config) -> + ok = lcnt:conflicts(). + +t_crash_before_collect(Config) when is_list(Config) -> + {ok, _} = lcnt:start(), + ok = lcnt:information(). + %% Simple smoke test of actual lock-counting, if running on %% a run-time with lock-counting enabled. smoke_lcnt(Config) -> diff --git a/lib/wx/api_gen/gen_util.erl b/lib/wx/api_gen/gen_util.erl index 5e2f405498..3068a2f4ea 100644 --- a/lib/wx/api_gen/gen_util.erl +++ b/lib/wx/api_gen/gen_util.erl @@ -106,8 +106,8 @@ check_diff(Diff) -> throw:_ -> diff; error:{badmatch,_} -> diff; - _:What -> - io:format("~p:~p: ~p ~p~n", [?MODULE,?LINE, What, erlang:get_stacktrace()]), + _:What:Stacktrace -> + io:format("~p:~p: ~p ~p~n", [?MODULE,?LINE, What, Stacktrace]), diff end. diff --git a/lib/wx/api_gen/gl_gen.erl b/lib/wx/api_gen/gl_gen.erl index 7e3766a43b..6f68517c16 100644 --- a/lib/wx/api_gen/gl_gen.erl +++ b/lib/wx/api_gen/gl_gen.erl @@ -47,9 +47,9 @@ safe(What, QuitOnErr) -> What(), io:format("Completed successfully~n~n", []), QuitOnErr andalso gen_util:halt(0) - catch Err:Reason -> + catch Err:Reason:Stacktrace -> io:format("Error ~p: ~p:~p~n ~p~n", - [get(current_func),Err,Reason,erlang:get_stacktrace()]), + [get(current_func),Err,Reason,Stacktrace]), (catch gen_util:close()), timer:sleep(1999), QuitOnErr andalso gen_util:halt(1) diff --git a/lib/wx/api_gen/gl_gen_erl.erl b/lib/wx/api_gen/gl_gen_erl.erl index 45f5fd8f4c..7e673c2987 100644 --- a/lib/wx/api_gen/gl_gen_erl.erl +++ b/lib/wx/api_gen/gl_gen_erl.erl @@ -246,8 +246,8 @@ gen_types(Where) -> gen_export(F) -> try gen_export_1(F) - catch E:R -> - io:format("Crash ~p:~p in ~p ~n",[E,R, erlang:get_stacktrace()]), + catch E:R:S -> + io:format("Crash ~p:~p in ~p ~n",[E,R,S]), io:format("Func = ~p~n ~p", [F, get(F)]) end. @@ -489,8 +489,8 @@ doc_return_types2(T, Ps) -> doc_arg_type(#arg{name=Name,type=T}) -> try erl_arg_name(Name) ++ " :: " ++ doc_arg_type2(T) - catch _:Error -> - io:format("Error spec: ~p ~p~n~p~n",[Name, Error, erlang:get_stacktrace()]), + catch _:Error:Stacktrace -> + io:format("Error spec: ~p ~p~n~p~n",[Name, Error, Stacktrace]), exit(error) end. diff --git a/lib/wx/api_gen/wx_gen.erl b/lib/wx/api_gen/wx_gen.erl index aadfe4b111..ab70a588ab 100644 --- a/lib/wx/api_gen/wx_gen.erl +++ b/lib/wx/api_gen/wx_gen.erl @@ -47,9 +47,9 @@ safe(What, QuitOnErr) -> What(), io:format("Completed successfully~n~n", []), QuitOnErr andalso gen_util:halt(0) - catch Err:Reason -> + catch Err:Reason:Stacktrace -> io:format("Error in ~p ~p~n", [get(current_class),get(current_func)]), - erlang:display({Err,Reason, erlang:get_stacktrace()}), + erlang:display({Err,Reason,Stacktrace}), catch gen_util:close(), QuitOnErr andalso gen_util:halt(1) end. diff --git a/lib/wx/examples/demo/ex_aui.erl b/lib/wx/examples/demo/ex_aui.erl index d8fc0021f1..97805621ed 100644 --- a/lib/wx/examples/demo/ex_aui.erl +++ b/lib/wx/examples/demo/ex_aui.erl @@ -87,8 +87,7 @@ do_init(Config) -> wxAuiManager:update(Manager), process_flag(trap_exit, true), {Panel, #state{parent=Panel, config=Config, aui=Manager}} - catch Class:Reason -> - ST = erlang:get_stacktrace(), + catch Class:Reason:ST -> io:format("AUI Crashed ~p ~p~n",[Reason, ST]), wxAuiManager:unInit(Manager), wxAuiManager:destroy(Manager), diff --git a/lib/wx/src/wx.erl b/lib/wx/src/wx.erl index 34bf06cf46..6dd3c148db 100644 --- a/lib/wx/src/wx.erl +++ b/lib/wx/src/wx.erl @@ -183,7 +183,7 @@ batch(Fun) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try Fun() catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after @@ -196,7 +196,7 @@ foreach(Fun, List) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try lists:foreach(Fun, List) catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after @@ -209,7 +209,7 @@ map(Fun, List) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try lists:map(Fun, List) catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after @@ -222,7 +222,7 @@ foldl(Fun, Acc, List) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try lists:foldl(Fun, Acc, List) catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after @@ -235,7 +235,7 @@ foldr(Fun, Acc, List) -> ok = wxe_util:cast(?BATCH_BEGIN, <<>>), try lists:foldr(Fun, Acc, List) catch - error:W -> erlang:exit({W, erlang:get_stacktrace()}); + error:W:S -> erlang:exit({W, S}); throw:W -> erlang:throw(W); exit:W -> erlang:exit(W) after diff --git a/lib/wx/src/wxe_server.erl b/lib/wx/src/wxe_server.erl index 58fcaf8f23..d6d9dbd629 100644 --- a/lib/wx/src/wxe_server.erl +++ b/lib/wx/src/wxe_server.erl @@ -283,10 +283,10 @@ invoke_callback(Pid, Ev, Ref) -> Return -> exit({bad_return, Return}) end end - catch _:Reason -> + catch _:Reason:Stacktrace -> wxEvent:skip(Ref), ?log("Callback fun crashed with {'EXIT, ~p, ~p}~n", - [Reason, erlang:get_stacktrace()]) + [Reason, Stacktrace]) end, wxe_util:cast(?WXE_CB_RETURN, <<>>) end, @@ -299,9 +299,9 @@ invoke_callback_fun(Fun) -> Return = Fun(), true = is_binary(Return), Return - catch _:Reason -> + catch _:Reason:Stacktrace -> ?log("Callback fun crashed with {'EXIT, ~p, ~p}~n", - [Reason, erlang:get_stacktrace()]), + [Reason, Stacktrace]), <<>> end, wxe_util:cast(?WXE_CB_RETURN, Res). diff --git a/otp_versions.table b/otp_versions.table index 2e1052264d..b976962c1e 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,5 @@ +OTP-20.2.4 : ssh-4.6.5 # asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 erts-9.2.1 et-1.6.1 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 kernel-5.4.2 megaco-3.18.3 mnesia-4.15.3 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.4 sasl-3.1.1 snmp-5.2.9 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 : +OTP-20.2.3 : erts-9.2.1 kernel-5.4.2 runtime_tools-1.12.4 # asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 et-1.6.1 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 megaco-3.18.3 mnesia-4.15.3 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 sasl-3.1.1 snmp-5.2.9 ssh-4.6.4 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 : OTP-20.2.2 : mnesia-4.15.3 # asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 erts-9.2 et-1.6.1 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 kernel-5.4.1 megaco-3.18.3 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.3 sasl-3.1.1 snmp-5.2.9 ssh-4.6.4 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 : OTP-20.2.1 : ssh-4.6.4 # asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 erts-9.2 et-1.6.1 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 kernel-5.4.1 megaco-3.18.3 mnesia-4.15.2 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.3 sasl-3.1.1 snmp-5.2.9 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 : OTP-20.2 : asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 erts-9.2 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 kernel-5.4.1 megaco-3.18.3 mnesia-4.15.2 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 runtime_tools-1.12.3 sasl-3.1.1 snmp-5.2.9 ssh-4.6.3 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 # et-1.6.1 reltool-0.7.5 : @@ -16,6 +18,7 @@ OTP-20.0.3 : asn1-5.0.2 compiler-7.1.1 erts-9.0.3 ssh-4.5.1 # common_test-1.15.1 OTP-20.0.2 : asn1-5.0.1 erts-9.0.2 kernel-5.3.1 # common_test-1.15.1 compiler-7.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.2 cosTime-1.2.2 cosTransactions-1.3.2 crypto-4.0 debugger-4.2.2 dialyzer-3.2 diameter-2.0 edoc-0.9 eldap-1.2.2 erl_docgen-0.7 erl_interface-3.10 et-1.6 eunit-2.3.3 hipe-3.16 ic-4.4.2 inets-6.4 jinterface-1.8 megaco-3.18.2 mnesia-4.15 observer-2.4 odbc-2.12 orber-3.8.3 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.5 public_key-1.4.1 reltool-0.7.4 runtime_tools-1.12.1 sasl-3.0.4 snmp-5.2.6 ssh-4.5 ssl-8.2 stdlib-3.4.1 syntax_tools-2.1.2 tools-2.10.1 wx-1.8.1 xmerl-1.3.15 : OTP-20.0.1 : common_test-1.15.1 erts-9.0.1 runtime_tools-1.12.1 stdlib-3.4.1 tools-2.10.1 # asn1-5.0 compiler-7.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.2 cosTime-1.2.2 cosTransactions-1.3.2 crypto-4.0 debugger-4.2.2 dialyzer-3.2 diameter-2.0 edoc-0.9 eldap-1.2.2 erl_docgen-0.7 erl_interface-3.10 et-1.6 eunit-2.3.3 hipe-3.16 ic-4.4.2 inets-6.4 jinterface-1.8 kernel-5.3 megaco-3.18.2 mnesia-4.15 observer-2.4 odbc-2.12 orber-3.8.3 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.5 public_key-1.4.1 reltool-0.7.4 sasl-3.0.4 snmp-5.2.6 ssh-4.5 ssl-8.2 syntax_tools-2.1.2 wx-1.8.1 xmerl-1.3.15 : OTP-20.0 : asn1-5.0 common_test-1.15 compiler-7.1 cosProperty-1.2.2 crypto-4.0 debugger-4.2.2 dialyzer-3.2 diameter-2.0 edoc-0.9 erl_docgen-0.7 erl_interface-3.10 erts-9.0 eunit-2.3.3 hipe-3.16 inets-6.4 jinterface-1.8 kernel-5.3 megaco-3.18.2 mnesia-4.15 observer-2.4 orber-3.8.3 parsetools-2.1.5 public_key-1.4.1 reltool-0.7.4 runtime_tools-1.12 sasl-3.0.4 snmp-5.2.6 ssh-4.5 ssl-8.2 stdlib-3.4 syntax_tools-2.1.2 tools-2.10 wx-1.8.1 xmerl-1.3.15 # cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosTime-1.2.2 cosTransactions-1.3.2 eldap-1.2.2 et-1.6 ic-4.4.2 odbc-2.12 os_mon-2.4.2 otp_mibs-1.1.1 : +OTP-19.3.6.6 : ssh-4.4.2.2 ssl-8.1.3.1.1 # asn1-4.0.4 common_test-1.14 compiler-7.0.4.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 erts-8.3.5.4 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3.1 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.14 : OTP-19.3.6.5 : erts-8.3.5.4 mnesia-4.14.3.1 ssh-4.4.2.1 # asn1-4.0.4 common_test-1.14 compiler-7.0.4.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssl-8.1.3.1 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.14 : OTP-19.3.6.4 : ssl-8.1.3.1 # asn1-4.0.4 common_test-1.14 compiler-7.0.4.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 erts-8.3.5.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.14 : OTP-19.3.6.3 : compiler-7.0.4.1 erts-8.3.5.3 # asn1-4.0.4 common_test-1.14 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2 ssl-8.1.3 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.14 : @@ -49,6 +52,7 @@ OTP-19.0.3 : inets-6.3.2 kernel-5.0.1 ssl-8.0.1 # asn1-4.0.3 common_test-1.12.2 OTP-19.0.2 : compiler-7.0.1 erts-8.0.2 stdlib-3.0.1 # asn1-4.0.3 common_test-1.12.2 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7 debugger-4.2 dialyzer-3.0.1 diameter-1.12 edoc-0.7.19 eldap-1.2.2 erl_docgen-0.5 erl_interface-3.9 et-1.6 eunit-2.3 gs-1.6.1 hipe-3.15.1 ic-4.4.1 inets-6.3.1 jinterface-1.7 kernel-5.0 megaco-3.18.1 mnesia-4.14 observer-2.2.1 odbc-2.11.2 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.2 percept-0.9 public_key-1.2 reltool-0.7.1 runtime_tools-1.10 sasl-3.0 snmp-5.2.3 ssh-4.3.1 ssl-8.0 syntax_tools-2.0 tools-2.8.5 typer-0.9.11 wx-1.7 xmerl-1.3.11 : OTP-19.0.1 : dialyzer-3.0.1 erts-8.0.1 inets-6.3.1 observer-2.2.1 ssh-4.3.1 tools-2.8.5 # asn1-4.0.3 common_test-1.12.2 compiler-7.0 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7 debugger-4.2 diameter-1.12 edoc-0.7.19 eldap-1.2.2 erl_docgen-0.5 erl_interface-3.9 et-1.6 eunit-2.3 gs-1.6.1 hipe-3.15.1 ic-4.4.1 jinterface-1.7 kernel-5.0 megaco-3.18.1 mnesia-4.14 odbc-2.11.2 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.2 percept-0.9 public_key-1.2 reltool-0.7.1 runtime_tools-1.10 sasl-3.0 snmp-5.2.3 ssl-8.0 stdlib-3.0 syntax_tools-2.0 typer-0.9.11 wx-1.7 xmerl-1.3.11 : OTP-19.0 : asn1-4.0.3 common_test-1.12.2 compiler-7.0 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7 debugger-4.2 dialyzer-3.0 diameter-1.12 edoc-0.7.19 eldap-1.2.2 erl_docgen-0.5 erl_interface-3.9 erts-8.0 et-1.6 eunit-2.3 gs-1.6.1 hipe-3.15.1 ic-4.4.1 inets-6.3 jinterface-1.7 kernel-5.0 megaco-3.18.1 mnesia-4.14 observer-2.2 odbc-2.11.2 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.2 percept-0.9 public_key-1.2 reltool-0.7.1 runtime_tools-1.10 sasl-3.0 snmp-5.2.3 ssh-4.3 ssl-8.0 stdlib-3.0 syntax_tools-2.0 tools-2.8.4 typer-0.9.11 wx-1.7 xmerl-1.3.11 # : +OTP-18.3.4.8 : ssh-4.2.2.5 # asn1-4.0.2 common_test-1.12.1.1 compiler-6.0.3.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2.1 cosProperty-1.2 cosTime-1.2.1 cosTransactions-1.3.1 crypto-3.6.3.1 debugger-4.1.2 dialyzer-2.9 diameter-1.11.2 edoc-0.7.18 eldap-1.2.1.1 erl_docgen-0.4.2 erl_interface-3.8.2 erts-7.3.1.4 et-1.5.1 eunit-2.2.13 gs-1.6 hipe-3.15 ic-4.4 inets-6.2.4.1 jinterface-1.6.1 kernel-4.2 megaco-3.18 mnesia-4.13.4 observer-2.1.2 odbc-2.11.1 orber-3.8.1 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1.1 percept-0.8.11 public_key-1.1.1 reltool-0.7 runtime_tools-1.9.3 sasl-2.7 snmp-5.2.2 ssl-7.3.3.2 stdlib-2.8 syntax_tools-1.7 test_server-3.10 tools-2.8.3 typer-0.9.10 webtool-0.9.1 wx-1.6.1 xmerl-1.3.10 : OTP-18.3.4.7 : ssl-7.3.3.2 # asn1-4.0.2 common_test-1.12.1.1 compiler-6.0.3.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2.1 cosProperty-1.2 cosTime-1.2.1 cosTransactions-1.3.1 crypto-3.6.3.1 debugger-4.1.2 dialyzer-2.9 diameter-1.11.2 edoc-0.7.18 eldap-1.2.1.1 erl_docgen-0.4.2 erl_interface-3.8.2 erts-7.3.1.4 et-1.5.1 eunit-2.2.13 gs-1.6 hipe-3.15 ic-4.4 inets-6.2.4.1 jinterface-1.6.1 kernel-4.2 megaco-3.18 mnesia-4.13.4 observer-2.1.2 odbc-2.11.1 orber-3.8.1 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1.1 percept-0.8.11 public_key-1.1.1 reltool-0.7 runtime_tools-1.9.3 sasl-2.7 snmp-5.2.2 ssh-4.2.2.4 stdlib-2.8 syntax_tools-1.7 test_server-3.10 tools-2.8.3 typer-0.9.10 webtool-0.9.1 wx-1.6.1 xmerl-1.3.10 : OTP-18.3.4.6 : compiler-6.0.3.1 eldap-1.2.1.1 erts-7.3.1.4 ssh-4.2.2.4 # asn1-4.0.2 common_test-1.12.1.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2.1 cosProperty-1.2 cosTime-1.2.1 cosTransactions-1.3.1 crypto-3.6.3.1 debugger-4.1.2 dialyzer-2.9 diameter-1.11.2 edoc-0.7.18 erl_docgen-0.4.2 erl_interface-3.8.2 et-1.5.1 eunit-2.2.13 gs-1.6 hipe-3.15 ic-4.4 inets-6.2.4.1 jinterface-1.6.1 kernel-4.2 megaco-3.18 mnesia-4.13.4 observer-2.1.2 odbc-2.11.1 orber-3.8.1 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1.1 percept-0.8.11 public_key-1.1.1 reltool-0.7 runtime_tools-1.9.3 sasl-2.7 snmp-5.2.2 ssl-7.3.3.1 stdlib-2.8 syntax_tools-1.7 test_server-3.10 tools-2.8.3 typer-0.9.10 webtool-0.9.1 wx-1.6.1 xmerl-1.3.10 : OTP-18.3.4.5 : crypto-3.6.3.1 erts-7.3.1.3 inets-6.2.4.1 ssh-4.2.2.3 # asn1-4.0.2 common_test-1.12.1.1 compiler-6.0.3 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2.1 cosProperty-1.2 cosTime-1.2.1 cosTransactions-1.3.1 debugger-4.1.2 dialyzer-2.9 diameter-1.11.2 edoc-0.7.18 eldap-1.2.1 erl_docgen-0.4.2 erl_interface-3.8.2 et-1.5.1 eunit-2.2.13 gs-1.6 hipe-3.15 ic-4.4 jinterface-1.6.1 kernel-4.2 megaco-3.18 mnesia-4.13.4 observer-2.1.2 odbc-2.11.1 orber-3.8.1 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1.1 percept-0.8.11 public_key-1.1.1 reltool-0.7 runtime_tools-1.9.3 sasl-2.7 snmp-5.2.2 ssl-7.3.3.1 stdlib-2.8 syntax_tools-1.7 test_server-3.10 tools-2.8.3 typer-0.9.10 webtool-0.9.1 wx-1.6.1 xmerl-1.3.10 : diff --git a/system/doc/design_principles/release_structure.xml b/system/doc/design_principles/release_structure.xml index a0ab21c43f..8e62ba845f 100644 --- a/system/doc/design_principles/release_structure.xml +++ b/system/doc/design_principles/release_structure.xml @@ -208,8 +208,8 @@ releases/ch_rel-1.rel</pre> is therefore now instead duplicated in the tar file so no manual copying is necessary.</p> <p>If a <c>relup</c> file and/or a system configuration file called - <c>sys.config</c> is found, these files are also included in - the release package. See + <c>sys.config</c>, or a <c>sys.config.src</c>, is found, these files + are also included in the release package. See <seealso marker="release_handling#req">Release Handling</seealso>.</p> <p>Options can be set to make the release package include source code and the ERTS binary as well.</p> @@ -240,7 +240,7 @@ $ROOT/lib/App1-AVsn1/ebin <item><c>erts-EVsn/bin</c> - Erlang runtime system executables</item> <item><c>releases/Vsn</c> - <c>.rel</c> file and boot script <c>start.boot</c>; if present in the release package, <c>relup</c> - and/or <c>sys.config</c></item> + and/or <c>sys.config</c> or <c>sys.config.src</c></item> <item><c>bin</c> - Top-level Erlang runtime system executables</item> </list> <p>Applications are not required to be located under directory diff --git a/system/doc/efficiency_guide/advanced.xml b/system/doc/efficiency_guide/advanced.xml index 896eda5f1c..245e7d3674 100644 --- a/system/doc/efficiency_guide/advanced.xml +++ b/system/doc/efficiency_guide/advanced.xml @@ -151,7 +151,7 @@ <row> <cell>Processes</cell> <cell>The maximum number of simultaneously alive Erlang processes - is by default 32,768. This limit can be configured at startup. + is by default 262,144. This limit can be configured at startup. For more information, see the <seealso marker="erts:erl#max_processes"><c>+P</c></seealso> command-line flag in the diff --git a/system/doc/system_principles/create_target.xmlsrc b/system/doc/system_principles/create_target.xmlsrc index f9b27ffc35..dc6cbbe980 100644 --- a/system/doc/system_principles/create_target.xmlsrc +++ b/system/doc/system_principles/create_target.xmlsrc @@ -263,6 +263,14 @@ os> <input>/usr/local/erl-target/bin/erl -boot /usr/local/erl-target/releases/FI current directory create not only the file <c>mysystem.rel</c>, but also file <c>sys.config</c>, the latter file is tacitly put in the appropriate directory.</p> + <p>However, it can also be convenient to replace variables in + within a <c>sys.config</c> on the target after unpacking but + before running the release. If you have a <c>sys.config.src</c> + it will be included and is not required to be a valid Erlang term + file like <c>sys.config</c>. Before running the release you must + have a valid <c>sys.config</c> in the same directory, so using + <c>sys.config.src</c> requires having some tool to populate what is + needed and write <c>sys.config</c> to disk before booting the release.</p> </section> <section> |