diff options
252 files changed, 3531 insertions, 916 deletions
diff --git a/.travis.yml b/.travis.yml index f418f2099f..eee75c9769 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,9 +42,7 @@ script: - ./scripts/build-otp after_success: - - $ERL_TOP/bin/dialyzer --build_plt --apps asn1 compiler crypto dialyzer edoc erts et hipe inets kernel mnesia observer public_key runtime_tools snmp ssh ssl stdlib syntax_tools wx xmerl --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps compiler erts kernel stdlib asn1 crypto dialyzer hipe parsetools public_key runtime_tools sasl tools --statistics - - $ERL_TOP/bin/dialyzer -n --apps common_test debugger edoc inets mnesia observer ssh ssl syntax_tools tools wx xmerl --statistics + - ./scripts/run-dialyzer - ./otp_build tests && make release_docs after_script: diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex b18f96e48e..8d675873fe 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex b18f96e48e..8d675873fe 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_a.beam b/bootstrap/lib/compiler/ebin/beam_a.beam Binary files differindex dbab4e7a10..7cca4b212a 100644 --- a/bootstrap/lib/compiler/ebin/beam_a.beam +++ b/bootstrap/lib/compiler/ebin/beam_a.beam diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 228e0e699a..72f9982ec8 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_block.beam b/bootstrap/lib/compiler/ebin/beam_block.beam Binary files differindex 1714723eca..475cecfa6d 100644 --- a/bootstrap/lib/compiler/ebin/beam_block.beam +++ b/bootstrap/lib/compiler/ebin/beam_block.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bs.beam b/bootstrap/lib/compiler/ebin/beam_bs.beam Binary files differindex 5d4ea1fd1f..b19b7efdde 100644 --- a/bootstrap/lib/compiler/ebin/beam_bs.beam +++ b/bootstrap/lib/compiler/ebin/beam_bs.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bsm.beam b/bootstrap/lib/compiler/ebin/beam_bsm.beam Binary files differindex 82bb7d7104..aeb44c114c 100644 --- a/bootstrap/lib/compiler/ebin/beam_bsm.beam +++ b/bootstrap/lib/compiler/ebin/beam_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_clean.beam b/bootstrap/lib/compiler/ebin/beam_clean.beam Binary files differindex c1272f878d..469c48dafb 100644 --- a/bootstrap/lib/compiler/ebin/beam_clean.beam +++ b/bootstrap/lib/compiler/ebin/beam_clean.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dead.beam b/bootstrap/lib/compiler/ebin/beam_dead.beam Binary files differindex 56c97d2875..38882d6f77 100644 --- a/bootstrap/lib/compiler/ebin/beam_dead.beam +++ b/bootstrap/lib/compiler/ebin/beam_dead.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dict.beam b/bootstrap/lib/compiler/ebin/beam_dict.beam Binary files differindex a25cef7ed2..7e52f634be 100644 --- a/bootstrap/lib/compiler/ebin/beam_dict.beam +++ b/bootstrap/lib/compiler/ebin/beam_dict.beam diff --git a/bootstrap/lib/compiler/ebin/beam_disasm.beam b/bootstrap/lib/compiler/ebin/beam_disasm.beam Binary files differindex f9e8c13823..9468053997 100644 --- a/bootstrap/lib/compiler/ebin/beam_disasm.beam +++ b/bootstrap/lib/compiler/ebin/beam_disasm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_except.beam b/bootstrap/lib/compiler/ebin/beam_except.beam Binary files differindex 0b43291236..9b7984d027 100644 --- a/bootstrap/lib/compiler/ebin/beam_except.beam +++ b/bootstrap/lib/compiler/ebin/beam_except.beam diff --git a/bootstrap/lib/compiler/ebin/beam_flatten.beam b/bootstrap/lib/compiler/ebin/beam_flatten.beam Binary files differindex a30e430f1f..e0eab64ddf 100644 --- a/bootstrap/lib/compiler/ebin/beam_flatten.beam +++ b/bootstrap/lib/compiler/ebin/beam_flatten.beam diff --git a/bootstrap/lib/compiler/ebin/beam_jump.beam b/bootstrap/lib/compiler/ebin/beam_jump.beam Binary files differindex 584b97bace..8996609dde 100644 --- a/bootstrap/lib/compiler/ebin/beam_jump.beam +++ b/bootstrap/lib/compiler/ebin/beam_jump.beam diff --git a/bootstrap/lib/compiler/ebin/beam_listing.beam b/bootstrap/lib/compiler/ebin/beam_listing.beam Binary files differindex ccd68a576f..423c171f2b 100644 --- a/bootstrap/lib/compiler/ebin/beam_listing.beam +++ b/bootstrap/lib/compiler/ebin/beam_listing.beam diff --git a/bootstrap/lib/compiler/ebin/beam_opcodes.beam b/bootstrap/lib/compiler/ebin/beam_opcodes.beam Binary files differindex 3930b41a2a..6b4a8e99fd 100644 --- a/bootstrap/lib/compiler/ebin/beam_opcodes.beam +++ b/bootstrap/lib/compiler/ebin/beam_opcodes.beam diff --git a/bootstrap/lib/compiler/ebin/beam_peep.beam b/bootstrap/lib/compiler/ebin/beam_peep.beam Binary files differindex c34f9ba60e..51f28cd1bd 100644 --- a/bootstrap/lib/compiler/ebin/beam_peep.beam +++ b/bootstrap/lib/compiler/ebin/beam_peep.beam diff --git a/bootstrap/lib/compiler/ebin/beam_receive.beam b/bootstrap/lib/compiler/ebin/beam_receive.beam Binary files differindex 0e1b0c5424..3c8ce31c8b 100644 --- a/bootstrap/lib/compiler/ebin/beam_receive.beam +++ b/bootstrap/lib/compiler/ebin/beam_receive.beam diff --git a/bootstrap/lib/compiler/ebin/beam_record.beam b/bootstrap/lib/compiler/ebin/beam_record.beam Binary files differindex 9e2a451c99..01c2e5a373 100644 --- a/bootstrap/lib/compiler/ebin/beam_record.beam +++ b/bootstrap/lib/compiler/ebin/beam_record.beam diff --git a/bootstrap/lib/compiler/ebin/beam_reorder.beam b/bootstrap/lib/compiler/ebin/beam_reorder.beam Binary files differindex 50d781d8e6..73f42ff1f6 100644 --- a/bootstrap/lib/compiler/ebin/beam_reorder.beam +++ b/bootstrap/lib/compiler/ebin/beam_reorder.beam diff --git a/bootstrap/lib/compiler/ebin/beam_split.beam b/bootstrap/lib/compiler/ebin/beam_split.beam Binary files differindex 5da1d8c4e2..7ca6a53728 100644 --- a/bootstrap/lib/compiler/ebin/beam_split.beam +++ b/bootstrap/lib/compiler/ebin/beam_split.beam diff --git a/bootstrap/lib/compiler/ebin/beam_trim.beam b/bootstrap/lib/compiler/ebin/beam_trim.beam Binary files differindex b7e9817267..4c045a6f8b 100644 --- a/bootstrap/lib/compiler/ebin/beam_trim.beam +++ b/bootstrap/lib/compiler/ebin/beam_trim.beam diff --git a/bootstrap/lib/compiler/ebin/beam_type.beam b/bootstrap/lib/compiler/ebin/beam_type.beam Binary files differindex 124578233d..b60a11b01b 100644 --- a/bootstrap/lib/compiler/ebin/beam_type.beam +++ b/bootstrap/lib/compiler/ebin/beam_type.beam diff --git a/bootstrap/lib/compiler/ebin/beam_utils.beam b/bootstrap/lib/compiler/ebin/beam_utils.beam Binary files differindex ff39e0291e..a72921204f 100644 --- a/bootstrap/lib/compiler/ebin/beam_utils.beam +++ b/bootstrap/lib/compiler/ebin/beam_utils.beam diff --git a/bootstrap/lib/compiler/ebin/beam_validator.beam b/bootstrap/lib/compiler/ebin/beam_validator.beam Binary files differindex 416aeff58b..8d8c9e56cf 100644 --- a/bootstrap/lib/compiler/ebin/beam_validator.beam +++ b/bootstrap/lib/compiler/ebin/beam_validator.beam diff --git a/bootstrap/lib/compiler/ebin/beam_z.beam b/bootstrap/lib/compiler/ebin/beam_z.beam Binary files differindex 6d723ac93a..15c76094a7 100644 --- a/bootstrap/lib/compiler/ebin/beam_z.beam +++ b/bootstrap/lib/compiler/ebin/beam_z.beam diff --git a/bootstrap/lib/compiler/ebin/cerl.beam b/bootstrap/lib/compiler/ebin/cerl.beam Binary files differindex 3839496347..9e79fd9b07 100644 --- a/bootstrap/lib/compiler/ebin/cerl.beam +++ b/bootstrap/lib/compiler/ebin/cerl.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_clauses.beam b/bootstrap/lib/compiler/ebin/cerl_clauses.beam Binary files differindex 4bd91621bf..c6d7a3db0a 100644 --- a/bootstrap/lib/compiler/ebin/cerl_clauses.beam +++ b/bootstrap/lib/compiler/ebin/cerl_clauses.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_inline.beam b/bootstrap/lib/compiler/ebin/cerl_inline.beam Binary files differindex 78c0f41771..cae60e45b6 100644 --- a/bootstrap/lib/compiler/ebin/cerl_inline.beam +++ b/bootstrap/lib/compiler/ebin/cerl_inline.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_sets.beam b/bootstrap/lib/compiler/ebin/cerl_sets.beam Binary files differindex c527517045..601d3dbd87 100644 --- a/bootstrap/lib/compiler/ebin/cerl_sets.beam +++ b/bootstrap/lib/compiler/ebin/cerl_sets.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_trees.beam b/bootstrap/lib/compiler/ebin/cerl_trees.beam Binary files differindex a7dc29f586..322e5b7930 100644 --- a/bootstrap/lib/compiler/ebin/cerl_trees.beam +++ b/bootstrap/lib/compiler/ebin/cerl_trees.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex a41cb3a916..26890187b9 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/core_lib.beam b/bootstrap/lib/compiler/ebin/core_lib.beam Binary files differindex a4ca16d554..33b1d00463 100644 --- a/bootstrap/lib/compiler/ebin/core_lib.beam +++ b/bootstrap/lib/compiler/ebin/core_lib.beam diff --git a/bootstrap/lib/compiler/ebin/core_lint.beam b/bootstrap/lib/compiler/ebin/core_lint.beam Binary files differindex 582b7fbc0a..f6dc90b2cc 100644 --- a/bootstrap/lib/compiler/ebin/core_lint.beam +++ b/bootstrap/lib/compiler/ebin/core_lint.beam diff --git a/bootstrap/lib/compiler/ebin/core_parse.beam b/bootstrap/lib/compiler/ebin/core_parse.beam Binary files differindex d1282160e2..7865efe421 100644 --- a/bootstrap/lib/compiler/ebin/core_parse.beam +++ b/bootstrap/lib/compiler/ebin/core_parse.beam diff --git a/bootstrap/lib/compiler/ebin/core_pp.beam b/bootstrap/lib/compiler/ebin/core_pp.beam Binary files differindex e14f99b3cf..1672897475 100644 --- a/bootstrap/lib/compiler/ebin/core_pp.beam +++ b/bootstrap/lib/compiler/ebin/core_pp.beam diff --git a/bootstrap/lib/compiler/ebin/core_scan.beam b/bootstrap/lib/compiler/ebin/core_scan.beam Binary files differindex b0c0560946..556294691e 100644 --- a/bootstrap/lib/compiler/ebin/core_scan.beam +++ b/bootstrap/lib/compiler/ebin/core_scan.beam diff --git a/bootstrap/lib/compiler/ebin/erl_bifs.beam b/bootstrap/lib/compiler/ebin/erl_bifs.beam Binary files differindex 4cf2c89551..8ac0f6ac21 100644 --- a/bootstrap/lib/compiler/ebin/erl_bifs.beam +++ b/bootstrap/lib/compiler/ebin/erl_bifs.beam diff --git a/bootstrap/lib/compiler/ebin/rec_env.beam b/bootstrap/lib/compiler/ebin/rec_env.beam Binary files differindex 1040c9fdb7..d882555fa2 100644 --- a/bootstrap/lib/compiler/ebin/rec_env.beam +++ b/bootstrap/lib/compiler/ebin/rec_env.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_dsetel.beam b/bootstrap/lib/compiler/ebin/sys_core_dsetel.beam Binary files differindex 45898a11d7..0a74db8040 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_dsetel.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_dsetel.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold.beam b/bootstrap/lib/compiler/ebin/sys_core_fold.beam Binary files differindex a0f771eafe..8df7b2374b 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam Binary files differindex 3968505655..7877d17db5 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_inline.beam b/bootstrap/lib/compiler/ebin/sys_core_inline.beam Binary files differindex 520fb315c3..e5861dd494 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_inline.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_inline.beam diff --git a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam Binary files differindex 2c424cb97f..3a71c094ed 100644 --- a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam +++ b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam diff --git a/bootstrap/lib/compiler/ebin/v3_codegen.beam b/bootstrap/lib/compiler/ebin/v3_codegen.beam Binary files differindex a0af668740..dcc9a51a38 100644 --- a/bootstrap/lib/compiler/ebin/v3_codegen.beam +++ b/bootstrap/lib/compiler/ebin/v3_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/v3_core.beam b/bootstrap/lib/compiler/ebin/v3_core.beam Binary files differindex 863bd3de0e..d2bccf05ca 100644 --- a/bootstrap/lib/compiler/ebin/v3_core.beam +++ b/bootstrap/lib/compiler/ebin/v3_core.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel.beam b/bootstrap/lib/compiler/ebin/v3_kernel.beam Binary files differindex 2762b53960..25f6134fbd 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam Binary files differindex 8cf701f685..790b8df8bc 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam diff --git a/bootstrap/lib/compiler/ebin/v3_life.beam b/bootstrap/lib/compiler/ebin/v3_life.beam Binary files differindex db04f6e73f..cd8bc19620 100644 --- a/bootstrap/lib/compiler/ebin/v3_life.beam +++ b/bootstrap/lib/compiler/ebin/v3_life.beam diff --git a/bootstrap/lib/kernel/ebin/application.beam b/bootstrap/lib/kernel/ebin/application.beam Binary files differindex 39858046d4..4ddd32379b 100644 --- a/bootstrap/lib/kernel/ebin/application.beam +++ b/bootstrap/lib/kernel/ebin/application.beam diff --git a/bootstrap/lib/kernel/ebin/application_controller.beam b/bootstrap/lib/kernel/ebin/application_controller.beam Binary files differindex ab717d3a5a..cc17a3f862 100644 --- a/bootstrap/lib/kernel/ebin/application_controller.beam +++ b/bootstrap/lib/kernel/ebin/application_controller.beam diff --git a/bootstrap/lib/kernel/ebin/application_master.beam b/bootstrap/lib/kernel/ebin/application_master.beam Binary files differindex 1e4a2dd1b7..1d4e23437d 100644 --- a/bootstrap/lib/kernel/ebin/application_master.beam +++ b/bootstrap/lib/kernel/ebin/application_master.beam diff --git a/bootstrap/lib/kernel/ebin/application_starter.beam b/bootstrap/lib/kernel/ebin/application_starter.beam Binary files differindex 203470742e..632fb56fe7 100644 --- a/bootstrap/lib/kernel/ebin/application_starter.beam +++ b/bootstrap/lib/kernel/ebin/application_starter.beam diff --git a/bootstrap/lib/kernel/ebin/auth.beam b/bootstrap/lib/kernel/ebin/auth.beam Binary files differindex d844ace8ee..f315376adc 100644 --- a/bootstrap/lib/kernel/ebin/auth.beam +++ b/bootstrap/lib/kernel/ebin/auth.beam diff --git a/bootstrap/lib/kernel/ebin/code.beam b/bootstrap/lib/kernel/ebin/code.beam Binary files differindex 4044f135d7..13eeb6a469 100644 --- a/bootstrap/lib/kernel/ebin/code.beam +++ b/bootstrap/lib/kernel/ebin/code.beam diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam Binary files differindex 73ef34a491..211c9933b7 100644 --- a/bootstrap/lib/kernel/ebin/code_server.beam +++ b/bootstrap/lib/kernel/ebin/code_server.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log.beam b/bootstrap/lib/kernel/ebin/disk_log.beam Binary files differindex 087bdac527..4ae813bdec 100644 --- a/bootstrap/lib/kernel/ebin/disk_log.beam +++ b/bootstrap/lib/kernel/ebin/disk_log.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_1.beam b/bootstrap/lib/kernel/ebin/disk_log_1.beam Binary files differindex 51d649b3d2..95e6323c34 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_1.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_1.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_server.beam b/bootstrap/lib/kernel/ebin/disk_log_server.beam Binary files differindex b306ad6906..fa5c5691f4 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_server.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_server.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_sup.beam b/bootstrap/lib/kernel/ebin/disk_log_sup.beam Binary files differindex d76d4753d8..cb802b16ac 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_sup.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_sup.beam diff --git a/bootstrap/lib/kernel/ebin/dist_ac.beam b/bootstrap/lib/kernel/ebin/dist_ac.beam Binary files differindex ed2b95d7a6..b1caf1c247 100644 --- a/bootstrap/lib/kernel/ebin/dist_ac.beam +++ b/bootstrap/lib/kernel/ebin/dist_ac.beam diff --git a/bootstrap/lib/kernel/ebin/dist_util.beam b/bootstrap/lib/kernel/ebin/dist_util.beam Binary files differindex 1947d115d5..e0870153f6 100644 --- a/bootstrap/lib/kernel/ebin/dist_util.beam +++ b/bootstrap/lib/kernel/ebin/dist_util.beam diff --git a/bootstrap/lib/kernel/ebin/erl_boot_server.beam b/bootstrap/lib/kernel/ebin/erl_boot_server.beam Binary files differindex 19987c0219..ad18c8a1d1 100644 --- a/bootstrap/lib/kernel/ebin/erl_boot_server.beam +++ b/bootstrap/lib/kernel/ebin/erl_boot_server.beam diff --git a/bootstrap/lib/kernel/ebin/erl_ddll.beam b/bootstrap/lib/kernel/ebin/erl_ddll.beam Binary files differindex 77b0d32dae..b55c290300 100644 --- a/bootstrap/lib/kernel/ebin/erl_ddll.beam +++ b/bootstrap/lib/kernel/ebin/erl_ddll.beam diff --git a/bootstrap/lib/kernel/ebin/erl_distribution.beam b/bootstrap/lib/kernel/ebin/erl_distribution.beam Binary files differindex 5780b5a73f..5ce25edd5d 100644 --- a/bootstrap/lib/kernel/ebin/erl_distribution.beam +++ b/bootstrap/lib/kernel/ebin/erl_distribution.beam diff --git a/bootstrap/lib/kernel/ebin/erl_epmd.beam b/bootstrap/lib/kernel/ebin/erl_epmd.beam Binary files differindex 4bd82f52f4..d19a07d633 100644 --- a/bootstrap/lib/kernel/ebin/erl_epmd.beam +++ b/bootstrap/lib/kernel/ebin/erl_epmd.beam diff --git a/bootstrap/lib/kernel/ebin/erl_reply.beam b/bootstrap/lib/kernel/ebin/erl_reply.beam Binary files differindex db0d69bc35..53298573b4 100644 --- a/bootstrap/lib/kernel/ebin/erl_reply.beam +++ b/bootstrap/lib/kernel/ebin/erl_reply.beam diff --git a/bootstrap/lib/kernel/ebin/erl_signal_handler.beam b/bootstrap/lib/kernel/ebin/erl_signal_handler.beam Binary files differindex ef8a03f86d..063f08877f 100644 --- a/bootstrap/lib/kernel/ebin/erl_signal_handler.beam +++ b/bootstrap/lib/kernel/ebin/erl_signal_handler.beam diff --git a/bootstrap/lib/kernel/ebin/error_handler.beam b/bootstrap/lib/kernel/ebin/error_handler.beam Binary files differindex 1064eea058..568790c444 100644 --- a/bootstrap/lib/kernel/ebin/error_handler.beam +++ b/bootstrap/lib/kernel/ebin/error_handler.beam diff --git a/bootstrap/lib/kernel/ebin/error_logger.beam b/bootstrap/lib/kernel/ebin/error_logger.beam Binary files differindex cf8dce1109..f619c6f6bf 100644 --- a/bootstrap/lib/kernel/ebin/error_logger.beam +++ b/bootstrap/lib/kernel/ebin/error_logger.beam diff --git a/bootstrap/lib/kernel/ebin/erts_debug.beam b/bootstrap/lib/kernel/ebin/erts_debug.beam Binary files differindex 3655e12021..e378694072 100644 --- a/bootstrap/lib/kernel/ebin/erts_debug.beam +++ b/bootstrap/lib/kernel/ebin/erts_debug.beam diff --git a/bootstrap/lib/kernel/ebin/file.beam b/bootstrap/lib/kernel/ebin/file.beam Binary files differindex b90ca03696..d7bc770e98 100644 --- a/bootstrap/lib/kernel/ebin/file.beam +++ b/bootstrap/lib/kernel/ebin/file.beam diff --git a/bootstrap/lib/kernel/ebin/file_io_server.beam b/bootstrap/lib/kernel/ebin/file_io_server.beam Binary files differindex f8f258f1f9..0224c193b4 100644 --- a/bootstrap/lib/kernel/ebin/file_io_server.beam +++ b/bootstrap/lib/kernel/ebin/file_io_server.beam diff --git a/bootstrap/lib/kernel/ebin/file_server.beam b/bootstrap/lib/kernel/ebin/file_server.beam Binary files differindex d251ab10c6..5037daacfc 100644 --- a/bootstrap/lib/kernel/ebin/file_server.beam +++ b/bootstrap/lib/kernel/ebin/file_server.beam diff --git a/bootstrap/lib/kernel/ebin/gen_sctp.beam b/bootstrap/lib/kernel/ebin/gen_sctp.beam Binary files differindex d98d2e7aef..083f43d892 100644 --- a/bootstrap/lib/kernel/ebin/gen_sctp.beam +++ b/bootstrap/lib/kernel/ebin/gen_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/gen_tcp.beam b/bootstrap/lib/kernel/ebin/gen_tcp.beam Binary files differindex 74ddffb84d..d57a66dff7 100644 --- a/bootstrap/lib/kernel/ebin/gen_tcp.beam +++ b/bootstrap/lib/kernel/ebin/gen_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/gen_udp.beam b/bootstrap/lib/kernel/ebin/gen_udp.beam Binary files differindex eb7ef08251..32cd268c8a 100644 --- a/bootstrap/lib/kernel/ebin/gen_udp.beam +++ b/bootstrap/lib/kernel/ebin/gen_udp.beam diff --git a/bootstrap/lib/kernel/ebin/global.beam b/bootstrap/lib/kernel/ebin/global.beam Binary files differindex 2df5e19fba..de0b853da1 100644 --- a/bootstrap/lib/kernel/ebin/global.beam +++ b/bootstrap/lib/kernel/ebin/global.beam diff --git a/bootstrap/lib/kernel/ebin/global_group.beam b/bootstrap/lib/kernel/ebin/global_group.beam Binary files differindex b67e595c38..e6b9c647db 100644 --- a/bootstrap/lib/kernel/ebin/global_group.beam +++ b/bootstrap/lib/kernel/ebin/global_group.beam diff --git a/bootstrap/lib/kernel/ebin/global_search.beam b/bootstrap/lib/kernel/ebin/global_search.beam Binary files differindex 100e6f26bb..743fb6a89f 100644 --- a/bootstrap/lib/kernel/ebin/global_search.beam +++ b/bootstrap/lib/kernel/ebin/global_search.beam diff --git a/bootstrap/lib/kernel/ebin/group.beam b/bootstrap/lib/kernel/ebin/group.beam Binary files differindex a27c29ffc6..ce535a9f0b 100644 --- a/bootstrap/lib/kernel/ebin/group.beam +++ b/bootstrap/lib/kernel/ebin/group.beam diff --git a/bootstrap/lib/kernel/ebin/heart.beam b/bootstrap/lib/kernel/ebin/heart.beam Binary files differindex 409ee06ecb..99b7487044 100644 --- a/bootstrap/lib/kernel/ebin/heart.beam +++ b/bootstrap/lib/kernel/ebin/heart.beam diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam Binary files differindex 55629b0f7d..82fcb51d3f 100644 --- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam +++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam diff --git a/bootstrap/lib/kernel/ebin/inet.beam b/bootstrap/lib/kernel/ebin/inet.beam Binary files differindex 4bc874c905..ddfe94c1b9 100644 --- a/bootstrap/lib/kernel/ebin/inet.beam +++ b/bootstrap/lib/kernel/ebin/inet.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_sctp.beam b/bootstrap/lib/kernel/ebin/inet6_sctp.beam Binary files differindex 2944d5034c..a5ea524671 100644 --- a/bootstrap/lib/kernel/ebin/inet6_sctp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_tcp.beam b/bootstrap/lib/kernel/ebin/inet6_tcp.beam Binary files differindex d7b4eb1694..5d8af46cea 100644 --- a/bootstrap/lib/kernel/ebin/inet6_tcp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam b/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam Binary files differindex 1202b4ee82..23ffe7572a 100644 --- a/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam +++ b/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_udp.beam b/bootstrap/lib/kernel/ebin/inet6_udp.beam Binary files differindex 060efabf6a..70f4997430 100644 --- a/bootstrap/lib/kernel/ebin/inet6_udp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_udp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_config.beam b/bootstrap/lib/kernel/ebin/inet_config.beam Binary files differindex 9da6310703..e803c23433 100644 --- a/bootstrap/lib/kernel/ebin/inet_config.beam +++ b/bootstrap/lib/kernel/ebin/inet_config.beam diff --git a/bootstrap/lib/kernel/ebin/inet_db.beam b/bootstrap/lib/kernel/ebin/inet_db.beam Binary files differindex 07c159560b..e91883b418 100644 --- a/bootstrap/lib/kernel/ebin/inet_db.beam +++ b/bootstrap/lib/kernel/ebin/inet_db.beam diff --git a/bootstrap/lib/kernel/ebin/inet_dns.beam b/bootstrap/lib/kernel/ebin/inet_dns.beam Binary files differindex c535ec37c8..d610a6a662 100644 --- a/bootstrap/lib/kernel/ebin/inet_dns.beam +++ b/bootstrap/lib/kernel/ebin/inet_dns.beam diff --git a/bootstrap/lib/kernel/ebin/inet_gethost_native.beam b/bootstrap/lib/kernel/ebin/inet_gethost_native.beam Binary files differindex 703df26cf4..3e06a72e80 100644 --- a/bootstrap/lib/kernel/ebin/inet_gethost_native.beam +++ b/bootstrap/lib/kernel/ebin/inet_gethost_native.beam diff --git a/bootstrap/lib/kernel/ebin/inet_hosts.beam b/bootstrap/lib/kernel/ebin/inet_hosts.beam Binary files differindex 26974163d4..3882ee4fb5 100644 --- a/bootstrap/lib/kernel/ebin/inet_hosts.beam +++ b/bootstrap/lib/kernel/ebin/inet_hosts.beam diff --git a/bootstrap/lib/kernel/ebin/inet_parse.beam b/bootstrap/lib/kernel/ebin/inet_parse.beam Binary files differindex 6ab66bd8f2..e8fc0876d2 100644 --- a/bootstrap/lib/kernel/ebin/inet_parse.beam +++ b/bootstrap/lib/kernel/ebin/inet_parse.beam diff --git a/bootstrap/lib/kernel/ebin/inet_res.beam b/bootstrap/lib/kernel/ebin/inet_res.beam Binary files differindex 822d5b66aa..f21bdf50b5 100644 --- a/bootstrap/lib/kernel/ebin/inet_res.beam +++ b/bootstrap/lib/kernel/ebin/inet_res.beam diff --git a/bootstrap/lib/kernel/ebin/inet_sctp.beam b/bootstrap/lib/kernel/ebin/inet_sctp.beam Binary files differindex 15e9cd1bf9..993752d44a 100644 --- a/bootstrap/lib/kernel/ebin/inet_sctp.beam +++ b/bootstrap/lib/kernel/ebin/inet_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_tcp.beam b/bootstrap/lib/kernel/ebin/inet_tcp.beam Binary files differindex b930765387..b24dc68ae7 100644 --- a/bootstrap/lib/kernel/ebin/inet_tcp.beam +++ b/bootstrap/lib/kernel/ebin/inet_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam Binary files differindex 26afdfc366..cee1e8a6c0 100644 --- a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam +++ b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam diff --git a/bootstrap/lib/kernel/ebin/inet_udp.beam b/bootstrap/lib/kernel/ebin/inet_udp.beam Binary files differindex 433e3880c2..21fc35c8b7 100644 --- a/bootstrap/lib/kernel/ebin/inet_udp.beam +++ b/bootstrap/lib/kernel/ebin/inet_udp.beam diff --git a/bootstrap/lib/kernel/ebin/kernel.beam b/bootstrap/lib/kernel/ebin/kernel.beam Binary files differindex 5cd4a762ed..24e15fc063 100644 --- a/bootstrap/lib/kernel/ebin/kernel.beam +++ b/bootstrap/lib/kernel/ebin/kernel.beam diff --git a/bootstrap/lib/kernel/ebin/kernel_config.beam b/bootstrap/lib/kernel/ebin/kernel_config.beam Binary files differindex bb2372d5c0..680477f566 100644 --- a/bootstrap/lib/kernel/ebin/kernel_config.beam +++ b/bootstrap/lib/kernel/ebin/kernel_config.beam diff --git a/bootstrap/lib/kernel/ebin/local_tcp.beam b/bootstrap/lib/kernel/ebin/local_tcp.beam Binary files differindex ab6ecc6f8e..a3097efd74 100644 --- a/bootstrap/lib/kernel/ebin/local_tcp.beam +++ b/bootstrap/lib/kernel/ebin/local_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/local_udp.beam b/bootstrap/lib/kernel/ebin/local_udp.beam Binary files differindex 590ab4ef79..8a83529d66 100644 --- a/bootstrap/lib/kernel/ebin/local_udp.beam +++ b/bootstrap/lib/kernel/ebin/local_udp.beam diff --git a/bootstrap/lib/kernel/ebin/net.beam b/bootstrap/lib/kernel/ebin/net.beam Binary files differindex 606efbc494..7bb5453cb8 100644 --- a/bootstrap/lib/kernel/ebin/net.beam +++ b/bootstrap/lib/kernel/ebin/net.beam diff --git a/bootstrap/lib/kernel/ebin/net_adm.beam b/bootstrap/lib/kernel/ebin/net_adm.beam Binary files differindex e296b700d9..cf91942455 100644 --- a/bootstrap/lib/kernel/ebin/net_adm.beam +++ b/bootstrap/lib/kernel/ebin/net_adm.beam diff --git a/bootstrap/lib/kernel/ebin/net_kernel.beam b/bootstrap/lib/kernel/ebin/net_kernel.beam Binary files differindex b36d8799b1..e5813705da 100644 --- a/bootstrap/lib/kernel/ebin/net_kernel.beam +++ b/bootstrap/lib/kernel/ebin/net_kernel.beam diff --git a/bootstrap/lib/kernel/ebin/os.beam b/bootstrap/lib/kernel/ebin/os.beam Binary files differindex 18c777b613..d42c3b0f31 100644 --- a/bootstrap/lib/kernel/ebin/os.beam +++ b/bootstrap/lib/kernel/ebin/os.beam diff --git a/bootstrap/lib/kernel/ebin/pg2.beam b/bootstrap/lib/kernel/ebin/pg2.beam Binary files differindex aa50eb4e8b..e3348158b7 100644 --- a/bootstrap/lib/kernel/ebin/pg2.beam +++ b/bootstrap/lib/kernel/ebin/pg2.beam diff --git a/bootstrap/lib/kernel/ebin/ram_file.beam b/bootstrap/lib/kernel/ebin/ram_file.beam Binary files differindex 83b9c9796a..893cff2458 100644 --- a/bootstrap/lib/kernel/ebin/ram_file.beam +++ b/bootstrap/lib/kernel/ebin/ram_file.beam diff --git a/bootstrap/lib/kernel/ebin/rpc.beam b/bootstrap/lib/kernel/ebin/rpc.beam Binary files differindex 8ffde85f5a..6d8d55d312 100644 --- a/bootstrap/lib/kernel/ebin/rpc.beam +++ b/bootstrap/lib/kernel/ebin/rpc.beam diff --git a/bootstrap/lib/kernel/ebin/seq_trace.beam b/bootstrap/lib/kernel/ebin/seq_trace.beam Binary files differindex c6520409cb..d9e9f5200d 100644 --- a/bootstrap/lib/kernel/ebin/seq_trace.beam +++ b/bootstrap/lib/kernel/ebin/seq_trace.beam diff --git a/bootstrap/lib/kernel/ebin/standard_error.beam b/bootstrap/lib/kernel/ebin/standard_error.beam Binary files differindex 4a29e8d4b2..41f5c2ef47 100644 --- a/bootstrap/lib/kernel/ebin/standard_error.beam +++ b/bootstrap/lib/kernel/ebin/standard_error.beam diff --git a/bootstrap/lib/kernel/ebin/user.beam b/bootstrap/lib/kernel/ebin/user.beam Binary files differindex ce087f15e9..1523c5f6e8 100644 --- a/bootstrap/lib/kernel/ebin/user.beam +++ b/bootstrap/lib/kernel/ebin/user.beam diff --git a/bootstrap/lib/kernel/ebin/user_drv.beam b/bootstrap/lib/kernel/ebin/user_drv.beam Binary files differindex fc2ef312b5..cb695170b9 100644 --- a/bootstrap/lib/kernel/ebin/user_drv.beam +++ b/bootstrap/lib/kernel/ebin/user_drv.beam diff --git a/bootstrap/lib/kernel/ebin/user_sup.beam b/bootstrap/lib/kernel/ebin/user_sup.beam Binary files differindex 354dd24c34..7de9f27b7c 100644 --- a/bootstrap/lib/kernel/ebin/user_sup.beam +++ b/bootstrap/lib/kernel/ebin/user_sup.beam diff --git a/bootstrap/lib/kernel/ebin/wrap_log_reader.beam b/bootstrap/lib/kernel/ebin/wrap_log_reader.beam Binary files differindex cdd8559b00..d8db01e52b 100644 --- a/bootstrap/lib/kernel/ebin/wrap_log_reader.beam +++ b/bootstrap/lib/kernel/ebin/wrap_log_reader.beam diff --git a/bootstrap/lib/stdlib/ebin/array.beam b/bootstrap/lib/stdlib/ebin/array.beam Binary files differindex d70a5239e7..8f5ec723af 100644 --- a/bootstrap/lib/stdlib/ebin/array.beam +++ b/bootstrap/lib/stdlib/ebin/array.beam diff --git a/bootstrap/lib/stdlib/ebin/base64.beam b/bootstrap/lib/stdlib/ebin/base64.beam Binary files differindex 1f60f385a6..eb16f91461 100644 --- a/bootstrap/lib/stdlib/ebin/base64.beam +++ b/bootstrap/lib/stdlib/ebin/base64.beam diff --git a/bootstrap/lib/stdlib/ebin/beam_lib.beam b/bootstrap/lib/stdlib/ebin/beam_lib.beam Binary files differindex 0ec559086e..b8fcf17609 100644 --- a/bootstrap/lib/stdlib/ebin/beam_lib.beam +++ b/bootstrap/lib/stdlib/ebin/beam_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/binary.beam b/bootstrap/lib/stdlib/ebin/binary.beam Binary files differindex 909cadf9ea..6dc0df9ebe 100644 --- a/bootstrap/lib/stdlib/ebin/binary.beam +++ b/bootstrap/lib/stdlib/ebin/binary.beam diff --git a/bootstrap/lib/stdlib/ebin/c.beam b/bootstrap/lib/stdlib/ebin/c.beam Binary files differindex f79cae8ea2..5bdce4fc48 100644 --- a/bootstrap/lib/stdlib/ebin/c.beam +++ b/bootstrap/lib/stdlib/ebin/c.beam diff --git a/bootstrap/lib/stdlib/ebin/calendar.beam b/bootstrap/lib/stdlib/ebin/calendar.beam Binary files differindex dbd5ec957d..6e6c77b3c0 100644 --- a/bootstrap/lib/stdlib/ebin/calendar.beam +++ b/bootstrap/lib/stdlib/ebin/calendar.beam diff --git a/bootstrap/lib/stdlib/ebin/dets.beam b/bootstrap/lib/stdlib/ebin/dets.beam Binary files differindex f5815fae1f..b83b604c5e 100644 --- a/bootstrap/lib/stdlib/ebin/dets.beam +++ b/bootstrap/lib/stdlib/ebin/dets.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_server.beam b/bootstrap/lib/stdlib/ebin/dets_server.beam Binary files differindex a7381dd920..07d44a322f 100644 --- a/bootstrap/lib/stdlib/ebin/dets_server.beam +++ b/bootstrap/lib/stdlib/ebin/dets_server.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_sup.beam b/bootstrap/lib/stdlib/ebin/dets_sup.beam Binary files differindex 9b75387892..29bed64d7c 100644 --- a/bootstrap/lib/stdlib/ebin/dets_sup.beam +++ b/bootstrap/lib/stdlib/ebin/dets_sup.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_utils.beam b/bootstrap/lib/stdlib/ebin/dets_utils.beam Binary files differindex bd19fa49db..1700ef5738 100644 --- a/bootstrap/lib/stdlib/ebin/dets_utils.beam +++ b/bootstrap/lib/stdlib/ebin/dets_utils.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_v9.beam b/bootstrap/lib/stdlib/ebin/dets_v9.beam Binary files differindex 482ba1af9d..1df7208cf8 100644 --- a/bootstrap/lib/stdlib/ebin/dets_v9.beam +++ b/bootstrap/lib/stdlib/ebin/dets_v9.beam diff --git a/bootstrap/lib/stdlib/ebin/dict.beam b/bootstrap/lib/stdlib/ebin/dict.beam Binary files differindex 25e1d2e60e..e9970e878b 100644 --- a/bootstrap/lib/stdlib/ebin/dict.beam +++ b/bootstrap/lib/stdlib/ebin/dict.beam diff --git a/bootstrap/lib/stdlib/ebin/digraph.beam b/bootstrap/lib/stdlib/ebin/digraph.beam Binary files differindex 4e3e7bcdc4..53a2364302 100644 --- a/bootstrap/lib/stdlib/ebin/digraph.beam +++ b/bootstrap/lib/stdlib/ebin/digraph.beam diff --git a/bootstrap/lib/stdlib/ebin/digraph_utils.beam b/bootstrap/lib/stdlib/ebin/digraph_utils.beam Binary files differindex 1a7151beb9..5bb6250241 100644 --- a/bootstrap/lib/stdlib/ebin/digraph_utils.beam +++ b/bootstrap/lib/stdlib/ebin/digraph_utils.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin.beam b/bootstrap/lib/stdlib/ebin/edlin.beam Binary files differindex d46735c226..af842aba99 100644 --- a/bootstrap/lib/stdlib/ebin/edlin.beam +++ b/bootstrap/lib/stdlib/ebin/edlin.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin_expand.beam b/bootstrap/lib/stdlib/ebin/edlin_expand.beam Binary files differindex 0a01568c9b..1636da3701 100644 --- a/bootstrap/lib/stdlib/ebin/edlin_expand.beam +++ b/bootstrap/lib/stdlib/ebin/edlin_expand.beam diff --git a/bootstrap/lib/stdlib/ebin/epp.beam b/bootstrap/lib/stdlib/ebin/epp.beam Binary files differindex 7db2022396..f53cb974b3 100644 --- a/bootstrap/lib/stdlib/ebin/epp.beam +++ b/bootstrap/lib/stdlib/ebin/epp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_anno.beam b/bootstrap/lib/stdlib/ebin/erl_anno.beam Binary files differindex e901b51609..20ce3494a5 100644 --- a/bootstrap/lib/stdlib/ebin/erl_anno.beam +++ b/bootstrap/lib/stdlib/ebin/erl_anno.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_bits.beam b/bootstrap/lib/stdlib/ebin/erl_bits.beam Binary files differindex 38fb5eb4a5..8c15ac2c30 100644 --- a/bootstrap/lib/stdlib/ebin/erl_bits.beam +++ b/bootstrap/lib/stdlib/ebin/erl_bits.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_compile.beam b/bootstrap/lib/stdlib/ebin/erl_compile.beam Binary files differindex 75e838bb81..ef2c7548fe 100644 --- a/bootstrap/lib/stdlib/ebin/erl_compile.beam +++ b/bootstrap/lib/stdlib/ebin/erl_compile.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_eval.beam b/bootstrap/lib/stdlib/ebin/erl_eval.beam Binary files differindex 4b130ba3a4..1f2f215397 100644 --- a/bootstrap/lib/stdlib/ebin/erl_eval.beam +++ b/bootstrap/lib/stdlib/ebin/erl_eval.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_expand_records.beam b/bootstrap/lib/stdlib/ebin/erl_expand_records.beam Binary files differindex 4b09697539..933bce5ada 100644 --- a/bootstrap/lib/stdlib/ebin/erl_expand_records.beam +++ b/bootstrap/lib/stdlib/ebin/erl_expand_records.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_internal.beam b/bootstrap/lib/stdlib/ebin/erl_internal.beam Binary files differindex 55dc3eefd7..dd329206b4 100644 --- a/bootstrap/lib/stdlib/ebin/erl_internal.beam +++ b/bootstrap/lib/stdlib/ebin/erl_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_lint.beam b/bootstrap/lib/stdlib/ebin/erl_lint.beam Binary files differindex 63968ab6cf..a41622fe49 100644 --- a/bootstrap/lib/stdlib/ebin/erl_lint.beam +++ b/bootstrap/lib/stdlib/ebin/erl_lint.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_parse.beam b/bootstrap/lib/stdlib/ebin/erl_parse.beam Binary files differindex 306c0a57e9..ea9534fe0a 100644 --- a/bootstrap/lib/stdlib/ebin/erl_parse.beam +++ b/bootstrap/lib/stdlib/ebin/erl_parse.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam b/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam Binary files differindex bfe55b571f..65c04825be 100644 --- a/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam +++ b/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_pp.beam b/bootstrap/lib/stdlib/ebin/erl_pp.beam Binary files differindex 3e386547e4..e95c141a83 100644 --- a/bootstrap/lib/stdlib/ebin/erl_pp.beam +++ b/bootstrap/lib/stdlib/ebin/erl_pp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_scan.beam b/bootstrap/lib/stdlib/ebin/erl_scan.beam Binary files differindex 7f006ebfd9..45d37ec838 100644 --- a/bootstrap/lib/stdlib/ebin/erl_scan.beam +++ b/bootstrap/lib/stdlib/ebin/erl_scan.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_tar.beam b/bootstrap/lib/stdlib/ebin/erl_tar.beam Binary files differindex 9a3ec9a834..ba71ea136b 100644 --- a/bootstrap/lib/stdlib/ebin/erl_tar.beam +++ b/bootstrap/lib/stdlib/ebin/erl_tar.beam diff --git a/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam b/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam Binary files differindex 00e92ce306..bb7b78a690 100644 --- a/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam +++ b/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam diff --git a/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam b/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam Binary files differindex 37bf96c402..59accf7f51 100644 --- a/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam +++ b/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam diff --git a/bootstrap/lib/stdlib/ebin/escript.beam b/bootstrap/lib/stdlib/ebin/escript.beam Binary files differindex 87eca72bf9..f14e21ac6e 100644 --- a/bootstrap/lib/stdlib/ebin/escript.beam +++ b/bootstrap/lib/stdlib/ebin/escript.beam diff --git a/bootstrap/lib/stdlib/ebin/ets.beam b/bootstrap/lib/stdlib/ebin/ets.beam Binary files differindex 994bb089d6..20b89a952a 100644 --- a/bootstrap/lib/stdlib/ebin/ets.beam +++ b/bootstrap/lib/stdlib/ebin/ets.beam diff --git a/bootstrap/lib/stdlib/ebin/eval_bits.beam b/bootstrap/lib/stdlib/ebin/eval_bits.beam Binary files differindex 0f0427ce94..bb0df11a28 100644 --- a/bootstrap/lib/stdlib/ebin/eval_bits.beam +++ b/bootstrap/lib/stdlib/ebin/eval_bits.beam diff --git a/bootstrap/lib/stdlib/ebin/file_sorter.beam b/bootstrap/lib/stdlib/ebin/file_sorter.beam Binary files differindex c89599e4c9..99e195f664 100644 --- a/bootstrap/lib/stdlib/ebin/file_sorter.beam +++ b/bootstrap/lib/stdlib/ebin/file_sorter.beam diff --git a/bootstrap/lib/stdlib/ebin/filelib.beam b/bootstrap/lib/stdlib/ebin/filelib.beam Binary files differindex fdaea73286..23487969dd 100644 --- a/bootstrap/lib/stdlib/ebin/filelib.beam +++ b/bootstrap/lib/stdlib/ebin/filelib.beam diff --git a/bootstrap/lib/stdlib/ebin/filename.beam b/bootstrap/lib/stdlib/ebin/filename.beam Binary files differindex af5a9b727e..91eeb2cb90 100644 --- a/bootstrap/lib/stdlib/ebin/filename.beam +++ b/bootstrap/lib/stdlib/ebin/filename.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_sets.beam b/bootstrap/lib/stdlib/ebin/gb_sets.beam Binary files differindex fa333fb0e7..04c8ca9997 100644 --- a/bootstrap/lib/stdlib/ebin/gb_sets.beam +++ b/bootstrap/lib/stdlib/ebin/gb_sets.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_trees.beam b/bootstrap/lib/stdlib/ebin/gb_trees.beam Binary files differindex b9cb66e3ef..3240e162bc 100644 --- a/bootstrap/lib/stdlib/ebin/gb_trees.beam +++ b/bootstrap/lib/stdlib/ebin/gb_trees.beam diff --git a/bootstrap/lib/stdlib/ebin/gen.beam b/bootstrap/lib/stdlib/ebin/gen.beam Binary files differindex 812c636b51..7b7a373d7c 100644 --- a/bootstrap/lib/stdlib/ebin/gen.beam +++ b/bootstrap/lib/stdlib/ebin/gen.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_event.beam b/bootstrap/lib/stdlib/ebin/gen_event.beam Binary files differindex aa1fca4e7e..6984704d22 100644 --- a/bootstrap/lib/stdlib/ebin/gen_event.beam +++ b/bootstrap/lib/stdlib/ebin/gen_event.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_fsm.beam b/bootstrap/lib/stdlib/ebin/gen_fsm.beam Binary files differindex 5538b9a80a..29ef47b629 100644 --- a/bootstrap/lib/stdlib/ebin/gen_fsm.beam +++ b/bootstrap/lib/stdlib/ebin/gen_fsm.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_server.beam b/bootstrap/lib/stdlib/ebin/gen_server.beam Binary files differindex c09943167d..1cebc6084e 100644 --- a/bootstrap/lib/stdlib/ebin/gen_server.beam +++ b/bootstrap/lib/stdlib/ebin/gen_server.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_statem.beam b/bootstrap/lib/stdlib/ebin/gen_statem.beam Binary files differindex b4f2d97231..27be2c302a 100644 --- a/bootstrap/lib/stdlib/ebin/gen_statem.beam +++ b/bootstrap/lib/stdlib/ebin/gen_statem.beam diff --git a/bootstrap/lib/stdlib/ebin/io.beam b/bootstrap/lib/stdlib/ebin/io.beam Binary files differindex 6998269f87..6cf3a1f177 100644 --- a/bootstrap/lib/stdlib/ebin/io.beam +++ b/bootstrap/lib/stdlib/ebin/io.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib.beam b/bootstrap/lib/stdlib/ebin/io_lib.beam Binary files differindex fced342da5..f0bde71883 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_format.beam b/bootstrap/lib/stdlib/ebin/io_lib_format.beam Binary files differindex 93a877a85a..55c85df1bd 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_format.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_format.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_fread.beam b/bootstrap/lib/stdlib/ebin/io_lib_fread.beam Binary files differindex c9714afe80..777e3e9b06 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_fread.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_fread.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam Binary files differindex 76a7a675cc..5dd445c5d6 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam diff --git a/bootstrap/lib/stdlib/ebin/lib.beam b/bootstrap/lib/stdlib/ebin/lib.beam Binary files differindex 0a4d019e48..a60c9af6b4 100644 --- a/bootstrap/lib/stdlib/ebin/lib.beam +++ b/bootstrap/lib/stdlib/ebin/lib.beam diff --git a/bootstrap/lib/stdlib/ebin/lists.beam b/bootstrap/lib/stdlib/ebin/lists.beam Binary files differindex c6f67c47f6..1278758a58 100644 --- a/bootstrap/lib/stdlib/ebin/lists.beam +++ b/bootstrap/lib/stdlib/ebin/lists.beam diff --git a/bootstrap/lib/stdlib/ebin/log_mf_h.beam b/bootstrap/lib/stdlib/ebin/log_mf_h.beam Binary files differindex a5e8dfea83..f25858537b 100644 --- a/bootstrap/lib/stdlib/ebin/log_mf_h.beam +++ b/bootstrap/lib/stdlib/ebin/log_mf_h.beam diff --git a/bootstrap/lib/stdlib/ebin/maps.beam b/bootstrap/lib/stdlib/ebin/maps.beam Binary files differindex 7079935dae..d405932b1d 100644 --- a/bootstrap/lib/stdlib/ebin/maps.beam +++ b/bootstrap/lib/stdlib/ebin/maps.beam diff --git a/bootstrap/lib/stdlib/ebin/math.beam b/bootstrap/lib/stdlib/ebin/math.beam Binary files differindex 07ecabcca2..40ed5460ce 100644 --- a/bootstrap/lib/stdlib/ebin/math.beam +++ b/bootstrap/lib/stdlib/ebin/math.beam diff --git a/bootstrap/lib/stdlib/ebin/ms_transform.beam b/bootstrap/lib/stdlib/ebin/ms_transform.beam Binary files differindex 1e422c0ab1..eff213fcf4 100644 --- a/bootstrap/lib/stdlib/ebin/ms_transform.beam +++ b/bootstrap/lib/stdlib/ebin/ms_transform.beam diff --git a/bootstrap/lib/stdlib/ebin/orddict.beam b/bootstrap/lib/stdlib/ebin/orddict.beam Binary files differindex 1f2013a4e3..bcf62fef7b 100644 --- a/bootstrap/lib/stdlib/ebin/orddict.beam +++ b/bootstrap/lib/stdlib/ebin/orddict.beam diff --git a/bootstrap/lib/stdlib/ebin/ordsets.beam b/bootstrap/lib/stdlib/ebin/ordsets.beam Binary files differindex 7d3067de0b..9e05e9d729 100644 --- a/bootstrap/lib/stdlib/ebin/ordsets.beam +++ b/bootstrap/lib/stdlib/ebin/ordsets.beam diff --git a/bootstrap/lib/stdlib/ebin/otp_internal.beam b/bootstrap/lib/stdlib/ebin/otp_internal.beam Binary files differindex 12d2e0d119..04ede72f53 100644 --- a/bootstrap/lib/stdlib/ebin/otp_internal.beam +++ b/bootstrap/lib/stdlib/ebin/otp_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/pool.beam b/bootstrap/lib/stdlib/ebin/pool.beam Binary files differindex 3dacc8e54e..ca7c91cd1f 100644 --- a/bootstrap/lib/stdlib/ebin/pool.beam +++ b/bootstrap/lib/stdlib/ebin/pool.beam diff --git a/bootstrap/lib/stdlib/ebin/proc_lib.beam b/bootstrap/lib/stdlib/ebin/proc_lib.beam Binary files differindex c60f6c590d..781a4f7eb3 100644 --- a/bootstrap/lib/stdlib/ebin/proc_lib.beam +++ b/bootstrap/lib/stdlib/ebin/proc_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/proplists.beam b/bootstrap/lib/stdlib/ebin/proplists.beam Binary files differindex 3e543e9b23..e566d3a985 100644 --- a/bootstrap/lib/stdlib/ebin/proplists.beam +++ b/bootstrap/lib/stdlib/ebin/proplists.beam diff --git a/bootstrap/lib/stdlib/ebin/qlc.beam b/bootstrap/lib/stdlib/ebin/qlc.beam Binary files differindex 691cdbf912..400837a60f 100644 --- a/bootstrap/lib/stdlib/ebin/qlc.beam +++ b/bootstrap/lib/stdlib/ebin/qlc.beam diff --git a/bootstrap/lib/stdlib/ebin/qlc_pt.beam b/bootstrap/lib/stdlib/ebin/qlc_pt.beam Binary files differindex 78cce4b16b..9b8be5280a 100644 --- a/bootstrap/lib/stdlib/ebin/qlc_pt.beam +++ b/bootstrap/lib/stdlib/ebin/qlc_pt.beam diff --git a/bootstrap/lib/stdlib/ebin/queue.beam b/bootstrap/lib/stdlib/ebin/queue.beam Binary files differindex 0e4101ceb4..544b34922c 100644 --- a/bootstrap/lib/stdlib/ebin/queue.beam +++ b/bootstrap/lib/stdlib/ebin/queue.beam diff --git a/bootstrap/lib/stdlib/ebin/rand.beam b/bootstrap/lib/stdlib/ebin/rand.beam Binary files differindex b12ae5be99..8f09b8c509 100644 --- a/bootstrap/lib/stdlib/ebin/rand.beam +++ b/bootstrap/lib/stdlib/ebin/rand.beam diff --git a/bootstrap/lib/stdlib/ebin/random.beam b/bootstrap/lib/stdlib/ebin/random.beam Binary files differindex 2371a8be4b..fa572500b4 100644 --- a/bootstrap/lib/stdlib/ebin/random.beam +++ b/bootstrap/lib/stdlib/ebin/random.beam diff --git a/bootstrap/lib/stdlib/ebin/re.beam b/bootstrap/lib/stdlib/ebin/re.beam Binary files differindex 61e40bb79e..b133cbb0f2 100644 --- a/bootstrap/lib/stdlib/ebin/re.beam +++ b/bootstrap/lib/stdlib/ebin/re.beam diff --git a/bootstrap/lib/stdlib/ebin/sets.beam b/bootstrap/lib/stdlib/ebin/sets.beam Binary files differindex c7c2a13a2d..9d7af9104a 100644 --- a/bootstrap/lib/stdlib/ebin/sets.beam +++ b/bootstrap/lib/stdlib/ebin/sets.beam diff --git a/bootstrap/lib/stdlib/ebin/shell.beam b/bootstrap/lib/stdlib/ebin/shell.beam Binary files differindex 5a6351b25b..1a3e3c67b5 100644 --- a/bootstrap/lib/stdlib/ebin/shell.beam +++ b/bootstrap/lib/stdlib/ebin/shell.beam diff --git a/bootstrap/lib/stdlib/ebin/shell_default.beam b/bootstrap/lib/stdlib/ebin/shell_default.beam Binary files differindex 84b03ca306..be0490f286 100644 --- a/bootstrap/lib/stdlib/ebin/shell_default.beam +++ b/bootstrap/lib/stdlib/ebin/shell_default.beam diff --git a/bootstrap/lib/stdlib/ebin/slave.beam b/bootstrap/lib/stdlib/ebin/slave.beam Binary files differindex d2b1e0902d..0b007135c0 100644 --- a/bootstrap/lib/stdlib/ebin/slave.beam +++ b/bootstrap/lib/stdlib/ebin/slave.beam diff --git a/bootstrap/lib/stdlib/ebin/sofs.beam b/bootstrap/lib/stdlib/ebin/sofs.beam Binary files differindex bd1059d121..a03c98f7bf 100644 --- a/bootstrap/lib/stdlib/ebin/sofs.beam +++ b/bootstrap/lib/stdlib/ebin/sofs.beam diff --git a/bootstrap/lib/stdlib/ebin/string.beam b/bootstrap/lib/stdlib/ebin/string.beam Binary files differindex bd34b03dc1..f141044d21 100644 --- a/bootstrap/lib/stdlib/ebin/string.beam +++ b/bootstrap/lib/stdlib/ebin/string.beam diff --git a/bootstrap/lib/stdlib/ebin/supervisor.beam b/bootstrap/lib/stdlib/ebin/supervisor.beam Binary files differindex 4c94a81a37..f2f4f3fd68 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor.beam diff --git a/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam b/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam Binary files differindex 78658a1c50..69aec60291 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam diff --git a/bootstrap/lib/stdlib/ebin/sys.beam b/bootstrap/lib/stdlib/ebin/sys.beam Binary files differindex 64c0e9709a..6855ce434d 100644 --- a/bootstrap/lib/stdlib/ebin/sys.beam +++ b/bootstrap/lib/stdlib/ebin/sys.beam diff --git a/bootstrap/lib/stdlib/ebin/timer.beam b/bootstrap/lib/stdlib/ebin/timer.beam Binary files differindex ebc68f1558..d6af12921a 100644 --- a/bootstrap/lib/stdlib/ebin/timer.beam +++ b/bootstrap/lib/stdlib/ebin/timer.beam diff --git a/bootstrap/lib/stdlib/ebin/unicode.beam b/bootstrap/lib/stdlib/ebin/unicode.beam Binary files differindex 357ab8a08e..9d0258c79a 100644 --- a/bootstrap/lib/stdlib/ebin/unicode.beam +++ b/bootstrap/lib/stdlib/ebin/unicode.beam diff --git a/bootstrap/lib/stdlib/ebin/win32reg.beam b/bootstrap/lib/stdlib/ebin/win32reg.beam Binary files differindex 4d1312bbf9..5e90246722 100644 --- a/bootstrap/lib/stdlib/ebin/win32reg.beam +++ b/bootstrap/lib/stdlib/ebin/win32reg.beam diff --git a/bootstrap/lib/stdlib/ebin/zip.beam b/bootstrap/lib/stdlib/ebin/zip.beam Binary files differindex 2ed3801d92..394b4f9a8a 100644 --- a/bootstrap/lib/stdlib/ebin/zip.beam +++ b/bootstrap/lib/stdlib/ebin/zip.beam diff --git a/configure.in b/configure.in index e44a873805..229d77863f 100644 --- a/configure.in +++ b/configure.in @@ -354,14 +354,12 @@ fi AC_SUBST(DEFAULT_VERBOSITY) if test X${enable_m64_build} = Xyes; then - enable_hipe=no CFLAGS="-m64 $CFLAGS" export CFLAGS LDFLAGS="-m64 $LDFLAGS" export LDFLAGS fi if test X${enable_m32_build} = Xyes; then - enable_hipe=no CFLAGS="-m32 $CFLAGS" export CFLAGS LDFLAGS="-m32 $LDFLAGS" diff --git a/erts/Makefile.in b/erts/Makefile.in index cddabbecee..3052dc3065 100644 --- a/erts/Makefile.in +++ b/erts/Makefile.in @@ -75,10 +75,12 @@ local_setup: $(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe \ $(ERL_TOP)/bin/escript $(ERL_TOP)/bin/escript.exe \ $(ERL_TOP)/bin/dialyzer $(ERL_TOP)/bin/dialyzer.exe \ + $(ERL_TOP)/bin/typer $(ERL_TOP)/bin/typer.exe \ $(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/ct_run.exe \ $(ERL_TOP)/bin/start*.boot $(ERL_TOP)/bin/start*.script @if [ "X$(TARGET)" = "Xwin32" ]; then \ cp $(ERL_TOP)/bin/$(TARGET)/dialyzer.exe $(ERL_TOP)/bin/dialyzer.exe; \ + cp $(ERL_TOP)/bin/$(TARGET)/typer.exe $(ERL_TOP)/bin/typer.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/ct_run.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erlc.exe $(ERL_TOP)/bin/erlc.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erl.exe $(ERL_TOP)/bin/erl.exe; \ @@ -98,6 +100,7 @@ local_setup: -e "s;%VSN%;$(VSN);" \ $(ERL_TOP)/erts/etc/unix/cerl.src > $(ERL_TOP)/bin/cerl; \ cp $(ERL_TOP)/bin/$(TARGET)/dialyzer $(ERL_TOP)/bin/dialyzer; \ + cp $(ERL_TOP)/bin/$(TARGET)/typer $(ERL_TOP)/bin/typer; \ cp $(ERL_TOP)/bin/$(TARGET)/ct_run $(ERL_TOP)/bin/ct_run; \ cp $(ERL_TOP)/bin/$(TARGET)/erlc $(ERL_TOP)/bin/erlc; \ cp $(ERL_TOP)/bin/$(TARGET)/escript $(ERL_TOP)/bin/escript; \ diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index ec00955ccd..e49c8c32e9 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -365,7 +365,7 @@ <c>{cons,LINE,Rep(E_h),Rep(E_t)}</c>.</p> </item> <item> - <p>>If E is a fun expression <c>fun Name/Arity</c>, then Rep(E) = + <p>If E is a fun expression <c>fun Name/Arity</c>, then Rep(E) = <c>{'fun',LINE,{function,Name,Arity}}</c>.</p> </item> <item> diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 37d8699a4b..3f4a1f9e80 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -308,7 +308,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) BIF_RET(am_undef); } } - code_ptr = erts_codemfa_to_code(cmfa); + code_ptr = (BeamInstr*)erts_code_to_codeinfo(erts_codemfa_to_code(cmfa)); } else { goto error; } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 4140938210..66e5dc2988 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -393,6 +393,7 @@ bif erl_ddll:demonitor/1 # # Bifs in the re module # +bif re:version/0 bif re:compile/1 bif re:compile/2 bif re:run/2 diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 925d99e0de..a72697eb80 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -512,6 +512,17 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con * Compile BIFs */ +BIF_RETTYPE +re_version_0(BIF_ALIST_0) +{ + Eterm ret; + size_t version_size = 0; + byte *version = (byte *) erts_pcre_version(); + version_size = strlen((const char *) version); + ret = new_binary(BIF_P, version, version_size); + BIF_RET(ret); +} + static BIF_RETTYPE re_compile(Process* p, Eterm arg1, Eterm arg2) { diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 267ca7e8a6..a7c90867f0 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -280,6 +280,7 @@ schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp, } } +#ifdef DEBUG static ERTS_INLINE Uint32 schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) { @@ -290,6 +291,7 @@ schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) #endif return res; } +#endif static ERTS_INLINE void schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp, @@ -11393,7 +11395,7 @@ erts_execute_dirty_system_task(Process *c_p) switch (st->type) { case ERTS_PSTT_CLA: - ASSERT(is_value(st_res)); + ASSERT(is_value(cla_res)); st_res = cla_res; break; case ERTS_PSTT_GC_MAJOR: diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 00c70268df..b6d71b3ec5 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -2306,7 +2306,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) case ERTS_EV_TYPE_NIF: { /* Requested via enif_select()... */ struct erts_nif_select_event in = {NIL}; struct erts_nif_select_event out = {NIL}; - ErtsResource* resource; + ErtsResource* resource = NULL; ErtsPollEvents revents = pollres[i].events; if (revents & ERTS_POLL_EV_ERR) { diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index f80f136d79..f7ee9bace0 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -550,11 +550,13 @@ void sys_sigrelease(int sig) sigprocmask(SIG_UNBLOCK, &mask, (sigset_t *)NULL); } +#ifdef ERTS_HAVE_TRY_CATCH void erts_sys_sigsegv_handler(int signo) { if (signo == SIGSEGV) { longjmp(erts_sys_sigsegv_jmp, 1); } } +#endif /* * Function returns 1 if we can read from all values in between diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 8c8c73aa3e..ca81c9915e 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -121,7 +121,6 @@ MODULES= \ port_trace_SUITE \ unique_SUITE \ z_SUITE \ - old_mod \ long_timers_test \ ignore_cores \ dgawd_handler \ diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 6994bfef83..4f98cd6cee 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -19,7 +19,7 @@ %% -module(distribution_SUITE). --compile(r15). +-compile(r16). -define(VERSION_MAGIC, 131). @@ -48,7 +48,7 @@ dist_parallel_send/1, atom_roundtrip/1, unicode_atom_roundtrip/1, - atom_roundtrip_r15b/1, + atom_roundtrip_r16b/1, contended_atom_cache_entry/1, contended_unicode_atom_cache_entry/1, bad_dist_structure/1, @@ -78,7 +78,8 @@ all() -> link_to_dead_new_node, applied_monitor_node, ref_port_roundtrip, nil_roundtrip, stop_dist, {group, trap_bif}, {group, dist_auto_connect}, - dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip, atom_roundtrip_r15b, + dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip, + atom_roundtrip_r16b, contended_atom_cache_entry, contended_unicode_atom_cache_entry, bad_dist_structure, {group, bad_dist_ext}, start_epmd_false, epmd_module]. @@ -1032,21 +1033,21 @@ atom_roundtrip(Config) when is_list(Config) -> stop_node(Node), ok. -atom_roundtrip_r15b(Config) when is_list(Config) -> - case test_server:is_release_available("r15b") of +atom_roundtrip_r16b(Config) when is_list(Config) -> + case test_server:is_release_available("r16b") of true -> ct:timetrap({minutes, 6}), - AtomData = atom_data(), + AtomData = unicode_atom_data(), verify_atom_data(AtomData), - case start_node(Config, [], "r15b") of + case start_node(Config, [], "r16b") of {ok, Node} -> do_atom_roundtrip(Node, AtomData), stop_node(Node); {error, timeout} -> - {skip,"Unable to start OTP R15B release"} + {skip,"Unable to start OTP R16B release"} end; false -> - {skip,"No OTP R15B available"} + {skip,"No OTP R16B available"} end. unicode_atom_roundtrip(Config) when is_list(Config) -> diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index 291d3ee30d..30518a5b27 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -45,7 +45,6 @@ ets_refc/1, match_spec_refc/1, timer_refc/1, - otp_4715/1, pid_wrap/1, port_wrap/1, bad_nc/1, @@ -62,7 +61,7 @@ all() -> [term_to_binary_to_term_eq, round_trip_eq, cmp, ref_eq, node_table_gc, dist_link_refc, dist_monitor_refc, node_controller_refc, ets_refc, match_spec_refc, - timer_refc, otp_4715, pid_wrap, port_wrap, bad_nc, + timer_refc, pid_wrap, port_wrap, bad_nc, unique_pid, iter_max_procs, magic_ref]. init_per_suite(Config) -> @@ -684,35 +683,6 @@ timer_refc(Config) when is_list(Config) -> nc_refc_check(node()), ok. -otp_4715(Config) when is_list(Config) -> - case test_server:is_release_available("r9b") of - true -> otp_4715_1(Config); - false -> {skip,"No R9B found"} - end. - -otp_4715_1(Config) -> - case erlang:system_info(compat_rel) of - 9 -> - run_otp_4715(Config); - _ -> - Pa = filename:dirname(code:which(?MODULE)), - test_server:run_on_shielded_node(fun () -> - run_otp_4715(Config) - end, - "+R9 -pa " ++ Pa) - end. - -run_otp_4715(Config) when is_list(Config) -> - erts_debug:set_internal_state(available_internal_state, true), - PidList = [mk_pid({a@b, 1}, 4710, 2), - mk_pid({a@b, 1}, 4712, 1), - mk_pid({c@b, 1}, 4711, 1), - mk_pid({b@b, 3}, 4711, 1), - mk_pid({b@b, 2}, 4711, 1)], - - R9Sorted = old_mod:sort_on_old_node(PidList), - R9Sorted = lists:sort(PidList). - pid_wrap(Config) when is_list(Config) -> pp_wrap(pid). port_wrap(Config) when is_list(Config) -> diff --git a/erts/emulator/test/old_mod.erl b/erts/emulator/test/old_mod.erl deleted file mode 100644 index 866aba79bb..0000000000 --- a/erts/emulator/test/old_mod.erl +++ /dev/null @@ -1,48 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(old_mod). --compile(r10). - --export([sort_on_old_node/1, sorter/3]). - --include_lib("common_test/include/ct.hrl"). - -sorter(Receiver, Ref, List) -> - Receiver ! {Ref, lists:sort(List)}. - -sort_on_old_node(List) when is_list(List) -> - OldVersion = "r10", - Pa = filename:dirname(code:which(?MODULE)), - {X, Y, Z} = now(), - NodeName = list_to_atom(OldVersion - ++ "_" - ++ integer_to_list(X) - ++ integer_to_list(Y) - ++ integer_to_list(Z)), - {ok, Node} = test_server:start_node(NodeName, - peer, - [{args, " -pa " ++ Pa}, - {erl, [{release, OldVersion++"b_patched"}]}]), - Ref = make_ref(), - spawn_link(Node, ?MODULE, sorter, [self(), Ref, List]), - SortedPids = receive {Ref, SP} -> SP end, - true = test_server:stop_node(Node), - SortedPids. diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in index d3af634729..b4fa0d4c79 100644 --- a/erts/etc/common/Makefile.in +++ b/erts/etc/common/Makefile.in @@ -143,7 +143,7 @@ MC_OUTPUTS=$(OBJDIR)/erlsrv_logmess.h $(OBJDIR)/erlsrv_logmess.res MT_FLAG="-MD" endif INET_GETHOST = $(BINDIR)/inet_gethost.exe -INSTALL_EMBEDDED_PROGS += $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/ct_run.exe +INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/ct_run.exe INSTALL_SRC = $(WINETC)/start_erl.c $(WINETC)/Nmakefile.start_erl ERLEXECDIR=. INSTALL_LIBS = @@ -176,7 +176,7 @@ ENTRY_OBJ= ERLSRV_OBJECTS= MC_OUTPUTS= INET_GETHOST = $(BINDIR)/inet_gethost@EXEEXT@ -INSTALL_EMBEDDED_PROGS += $(BINDIR)/dialyzer@EXEEXT@ \ +INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer@EXEEXT@ $(BINDIR)/dialyzer@EXEEXT@ \ $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ $(BINDIR)/ct_run@EXEEXT@ \ $(BINDIR)/run_erl $(BINDIR)/to_erl $(BINDIR)/dyn_erl INSTALL_EMBEDDED_DATA = $(UXETC)/start.src $(UXETC)/start_erl.src @@ -238,6 +238,7 @@ endif rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/reclaim.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/run_erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/to_erl.o + rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/typer.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/dyn_erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/safe_string.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/run_erl.o @@ -360,7 +361,7 @@ Install.ini: ../$(TARGET)/Install.src ../../vsn.mk $(TARGET)/Makefile else -RC_GENERATED = +RC_GENERATED = $(ERL_TOP)/erts/$(TARGET)/config.h endif #--------------------------------------------------------- # End of windows specific targets. @@ -432,6 +433,12 @@ $(BINDIR)/dialyzer@EXEEXT@: $(OBJDIR)/dialyzer.o $(ERTS_LIB) $(OBJDIR)/dialyzer.o: dialyzer.c $(RC_GENERATED) $(V_CC) $(CFLAGS) -o $@ -c dialyzer.c +$(BINDIR)/typer@EXEEXT@: $(OBJDIR)/typer.o $(ERTS_LIB) + $(ld_verbose)$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/typer.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) + +$(OBJDIR)/typer.o: typer.c $(RC_GENERATED) + $(V_CC) $(CFLAGS) -o $@ -c typer.c + $(BINDIR)/escript@EXEEXT@: $(OBJDIR)/escript.o $(ERTS_LIB) $(ld_verbose)$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/escript.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) diff --git a/erts/etc/common/typer.c b/erts/etc/common/typer.c new file mode 100644 index 0000000000..77a95ccded --- /dev/null +++ b/erts/etc/common/typer.c @@ -0,0 +1,455 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * Purpose: Typer front-end. + */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#ifdef __WIN32__ +#include <winbase.h> +#endif + +#include <ctype.h> + +#define NO 0 +#define YES 1 + +#define ASIZE(a) (sizeof(a)/sizeof(a[0])) + +static int debug = 0; /* Bit flags for debug printouts. */ + +static char** eargv_base; /* Base of vector. */ +static char** eargv; /* First argument for erl. */ + +static int eargc; /* Number of arguments in eargv. */ + +#ifdef __WIN32__ +# define QUOTE(s) possibly_quote(s) +# define IS_DIRSEP(c) ((c) == '/' || (c) == '\\') +# define ERL_NAME "erl.exe" +#else +# define QUOTE(s) s +# define IS_DIRSEP(c) ((c) == '/') +# define ERL_NAME "erl" +#endif + +#define UNSHIFT(s) eargc++, eargv--; eargv[0] = QUOTE(s) +#define PUSH(s) eargv[eargc++] = QUOTE(s) +#define PUSH2(s, t) PUSH(s); PUSH(t) +#define PUSH3(s, t, u) PUSH2(s, t); PUSH(u) + +/* + * Local functions. + */ + +static void error(char* format, ...); +static void* emalloc(size_t size); +static char* strsave(char* string); +static void push_words(char* src); +static int run_erlang(char* name, char** argv); +static char* get_default_emulator(char* progname); +#ifdef __WIN32__ +static char* possibly_quote(char* arg); +static void* erealloc(void *p, size_t size); +#endif + +/* + * Supply a strerror() function if libc doesn't. + */ +#ifndef HAVE_STRERROR + +extern int sys_nerr; + +#ifndef SYS_ERRLIST_DECLARED +extern const char * const sys_errlist[]; +#endif /* !SYS_ERRLIST_DECLARED */ + +char *strerror(int errnum) +{ + static char *emsg[1024]; + + if (errnum != 0) { + if (errnum > 0 && errnum < sys_nerr) + sprintf((char *) &emsg[0], "(%s)", sys_errlist[errnum]); + else + sprintf((char *) &emsg[0], "errnum = %d ", errnum); + } + else { + emsg[0] = '\0'; + } + return (char *) &emsg[0]; +} +#endif /* !HAVE_STRERROR */ + +#ifdef __WIN32__ +int wmain(int argc, wchar_t **wcargv) +{ + char** argv; +#else +int +main(int argc, char** argv) +{ +#endif + int eargv_size; + int eargc_base; /* How many arguments in the base of eargv. */ + char* emulator; + int need_shell = 0; + +#ifdef __WIN32__ + int i; + int len; + /* Convert argv to utf8 */ + argv = emalloc((argc+1) * sizeof(char*)); + for (i=0; i<argc; i++) { + len = WideCharToMultiByte(CP_UTF8, 0, wcargv[i], -1, NULL, 0, NULL, NULL); + argv[i] = emalloc(len*sizeof(char)); + WideCharToMultiByte(CP_UTF8, 0, wcargv[i], -1, argv[i], len, NULL, NULL); + } + argv[argc] = NULL; +#endif + + emulator = get_default_emulator(argv[0]); + + /* + * Allocate the argv vector to be used for arguments to Erlang. + * Arrange for starting to pushing information in the middle of + * the array, to allow easy addition of commands in the beginning. + */ + + eargv_size = argc*4+100; + eargv_base = (char **) emalloc(eargv_size*sizeof(char*)); + eargv = eargv_base; + eargc = 0; + push_words(emulator); + eargc_base = eargc; + eargv = eargv + eargv_size/2; + eargc = 0; + + /* + * Push initial arguments. + */ + + if (argc > 1 && strcmp(argv[1], "-smp") == 0) { + PUSH("-smpauto"); + argc--, argv++; + } + + PUSH("+B"); + PUSH2("-boot", "start_clean"); + PUSH3("-run", "typer", "start"); + PUSH("-extra"); + + /* + * Push everything except --shell. + */ + + while (argc > 1) { + if (strcmp(argv[1], "--shell") == 0) { + need_shell = 1; + } else { + PUSH(argv[1]); + } + argc--, argv++; + } + + if (!need_shell) { + UNSHIFT("-noinput"); + } + + /* + * Move up the commands for invoking the emulator and adjust eargv + * accordingly. + */ + + while (--eargc_base >= 0) { + UNSHIFT(eargv_base[eargc_base]); + } + + /* + * Invoke Erlang with the collected options. + */ + + PUSH(NULL); + return run_erlang(eargv[0], eargv); +} + +static void +push_words(char* src) +{ + char sbuf[MAXPATHLEN]; + char* dst; + + dst = sbuf; + while ((*dst++ = *src++) != '\0') { + if (isspace((int)*src)) { + *dst = '\0'; + PUSH(strsave(sbuf)); + dst = sbuf; + do { + src++; + } while (isspace((int)*src)); + } + } + if (sbuf[0]) + PUSH(strsave(sbuf)); +} +#ifdef __WIN32__ +wchar_t *make_commandline(char **argv) +{ + static wchar_t *buff = NULL; + static int siz = 0; + int num = 0, len; + char **arg; + wchar_t *p; + + if (*argv == NULL) { + return L""; + } + for (arg = argv; *arg != NULL; ++arg) { + num += strlen(*arg)+1; + } + if (!siz) { + siz = num; + buff = (wchar_t *) emalloc(siz*sizeof(wchar_t)); + } else if (siz < num) { + siz = num; + buff = (wchar_t *) erealloc(buff,siz*sizeof(wchar_t)); + } + p = buff; + num=0; + for (arg = argv; *arg != NULL; ++arg) { + len = MultiByteToWideChar(CP_UTF8, 0, *arg, -1, p, siz); + p+=(len-1); + *p++=L' '; + } + *(--p) = L'\0'; + + if (debug) { + printf("Processed command line:%S\n",buff); + } + return buff; +} + +int my_spawnvp(char **argv) +{ + STARTUPINFOW siStartInfo; + PROCESS_INFORMATION piProcInfo; + DWORD ec; + + memset(&siStartInfo,0,sizeof(STARTUPINFOW)); + siStartInfo.cb = sizeof(STARTUPINFOW); + siStartInfo.dwFlags = STARTF_USESTDHANDLES; + siStartInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + siStartInfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + siStartInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE); + + if (!CreateProcessW(NULL, + make_commandline(argv), + NULL, + NULL, + TRUE, + 0, + NULL, + NULL, + &siStartInfo, + &piProcInfo)) { + return -1; + } + CloseHandle(piProcInfo.hThread); + + WaitForSingleObject(piProcInfo.hProcess,INFINITE); + if (!GetExitCodeProcess(piProcInfo.hProcess,&ec)) { + return 0; + } + return (int) ec; +} +#endif /* __WIN32__ */ + + +static int +run_erlang(char* progname, char** argv) +{ +#ifdef __WIN32__ + int status; +#endif + + if (debug) { + int i = 0; + while (argv[i] != NULL) + printf(" %s", argv[i++]); + printf("\n"); + } + +#ifdef __WIN32__ + /* + * Alas, we must wait here for the program to finish. + * Otherwise, the shell from which we were executed will think + * we are finished and print a prompt and read keyboard input. + */ + + status = my_spawnvp(argv)/*_spawnvp(_P_WAIT,progname,argv)*/; + if (status == -1) { + fprintf(stderr, "typer: Error executing '%s': %d", progname, + GetLastError()); + } + return status; +#else + execvp(progname, argv); + error("Error %d executing \'%s\'.", errno, progname); + return 2; +#endif +} + +static void +error(char* format, ...) +{ + char sbuf[1024]; + va_list ap; + + va_start(ap, format); + erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); + va_end(ap); + fprintf(stderr, "typer: %s\n", sbuf); + exit(1); +} + +static void* +emalloc(size_t size) +{ + void *p = malloc(size); + if (p == NULL) + error("Insufficient memory"); + return p; +} + +#ifdef __WIN32__ +static void * +erealloc(void *p, size_t size) +{ + void *res = realloc(p, size); + if (res == NULL) + error("Insufficient memory"); + return res; +} +#endif + +static char* +strsave(char* string) +{ + char* p = emalloc(strlen(string)+1); + strcpy(p, string); + return p; +} + +static int +file_exists(char *progname) +{ +#ifdef __WIN32__ + wchar_t wcsbuf[MAXPATHLEN]; + MultiByteToWideChar(CP_UTF8, 0, progname, -1, wcsbuf, MAXPATHLEN); + return (_waccess(wcsbuf, 0) != -1); +#else + return (access(progname, 1) != -1); +#endif +} + +static char* +get_default_emulator(char* progname) +{ + char sbuf[MAXPATHLEN]; + char* s; + + if (strlen(progname) >= sizeof(sbuf)) + return ERL_NAME; + + strcpy(sbuf, progname); + for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { + if (IS_DIRSEP(*s)) { + strcpy(s+1, ERL_NAME); + if(file_exists(sbuf)) + return strsave(sbuf); + break; + } + } + return ERL_NAME; +} + +#ifdef __WIN32__ +static char* +possibly_quote(char* arg) +{ + int mustQuote = NO; + int n = 0; + char* s; + char* narg; + + if (arg == NULL) { + return arg; + } + + /* + * Scan the string to find out if it needs quoting and return + * the original argument if not. + */ + + for (s = arg; *s; s++, n++) { + switch(*s) { + case ' ': + mustQuote = YES; + continue; + case '"': + mustQuote = YES; + n++; + continue; + case '\\': + if(s[1] == '"') + n++; + continue; + default: + continue; + } + } + if (!mustQuote) { + return arg; + } + + /* + * Insert the quotes and put a backslash in front of every quote + * inside the string. + */ + + s = narg = emalloc(n+2+1); + for (*s++ = '"'; *arg; arg++, s++) { + if (*arg == '"' || (*arg == '\\' && arg[1] == '"')) { + *s++ = '\\'; + } + *s = *arg; + } + if (s[-1] == '\\') { + *s++ ='\\'; + } + *s++ = '"'; + *s = '\0'; + return narg; +} +#endif /* __WIN32__ */ diff --git a/erts/etc/unix/Install.src b/erts/etc/unix/Install.src index 8be696b16f..e71308edbe 100644 --- a/erts/etc/unix/Install.src +++ b/erts/etc/unix/Install.src @@ -89,6 +89,7 @@ cd "$ERL_ROOT/bin" cp -p "$ERL_ROOT/erts-%I_VSN%/bin/erl" . cp -p "$ERL_ROOT/erts-%I_VSN%/bin/erlc" . cp -p "$ERL_ROOT/erts-%I_VSN%/bin/dialyzer" . +cp -p "$ERL_ROOT/erts-%I_VSN%/bin/typer" . cp -p "$ERL_ROOT/erts-%I_VSN%/bin/ct_run" . cp -p "$ERL_ROOT/erts-%I_VSN%/bin/escript" . diff --git a/erts/etc/unix/to_erl.c b/erts/etc/unix/to_erl.c index 0bd469727c..0ccd7ead3e 100644 --- a/erts/etc/unix/to_erl.c +++ b/erts/etc/unix/to_erl.c @@ -416,7 +416,7 @@ int main(int argc, char **argv) if (len) { #ifdef DEBUG - if(write(1, buf, len)); + (void)write(1, buf, len); #endif if (write_all(wfd, buf, len) != len) { fprintf(stderr, "Error in writing to FIFO.\n"); diff --git a/erts/etc/win32/Install.c b/erts/etc/win32/Install.c index 04522a0779..5701df35a3 100644 --- a/erts/etc/win32/Install.c +++ b/erts/etc/win32/Install.c @@ -49,6 +49,7 @@ int wmain(int argc, wchar_t **argv) HANDLE module = GetModuleHandle(NULL); wchar_t *binaries[] = { L"erl.exe", L"werl.exe", L"erlc.exe", L"dialyzer.exe", + L"typer.exe", L"escript.exe", L"ct_run.exe", NULL }; wchar_t *scripts[] = { L"start_clean.boot", L"start_sasl.boot", L"no_dot_erlang.boot", NULL }; wchar_t fromname[MAX_PATH]; diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c index 8186463b9c..2ffb23bf54 100644 --- a/erts/lib_src/common/erl_misc_utils.c +++ b/erts/lib_src/common/erl_misc_utils.c @@ -842,7 +842,7 @@ read_topology(erts_cpu_info_t *cpuinfo) cpuinfo->topology[ix].logical = -1; } - ix = -1; + ix = 0; if (realpath(ERTS_SYS_NODE_PATH, npath)) { ndir = opendir(npath); @@ -886,6 +886,7 @@ read_topology(erts_cpu_info_t *cpuinfo) cdir = NULL; break; } + if (sscanf(cde->d_name, "cpu%d", &cpu_id) == 1) { char buf[50]; /* Much more than enough for an integer */ int processor_id, core_id; @@ -906,23 +907,33 @@ read_topology(erts_cpu_info_t *cpuinfo) if (sscanf(buf, "%d", &core_id) != 1) continue; + /* + * The number of CPUs that proc fs presents is greater + * then the number of CPUs configured in sysconf. + * This has been known to happen in docker. When this + * happens we refuse to give a CPU topology. + */ + if (ix >= cpuinfo->configured) + goto error; + /* * We now know node id, processor id, and * core id of the logical processor with * the cpu id 'cpu_id'. */ - ix++; cpuinfo->topology[ix].node = node_id; cpuinfo->topology[ix].processor = processor_id; cpuinfo->topology[ix].processor_node = -1; /* Fixed later */ cpuinfo->topology[ix].core = core_id; cpuinfo->topology[ix].thread = 0; /* we'll numerate later */ cpuinfo->topology[ix].logical = cpu_id; + ix++; + } } } while (got_nodes); - res = ix+1; + res = ix; if (!res || res < cpuinfo->online) res = 0; diff --git a/lib/common_test/src/test_server_node.erl b/lib/common_test/src/test_server_node.erl index 0b406c54cc..92c610730e 100644 --- a/lib/common_test/src/test_server_node.erl +++ b/lib/common_test/src/test_server_node.erl @@ -18,11 +18,11 @@ %% %CopyrightEnd% %% -module(test_server_node). --compile(r12). +-compile(r16). %%% %%% The same compiled code for this module must be possible to load -%%% in R12B and later. +%%% in R16B and later. %%% %% Test Controller interface diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index 2b5d558ee4..7e9a243ada 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -26,6 +26,8 @@ -import(lists, [filter/2,foldl/3,keyfind/3,member/2, reverse/1,reverse/2,sort/1]). +-define(UNICODE_INT, {integer,{0,16#10FFFF}}). + -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -494,6 +496,10 @@ update({test,test_arity,_Fail,[Src,Arity]}, Ts0) -> tdb_update([{Src,{tuple,Arity,[]}}], Ts0); update({test,is_map,_Fail,[Src]}, Ts0) -> tdb_update([{Src,map}], Ts0); +update({get_map_elements,_,Src,{list,Elems0}}, Ts0) -> + {_Ss,Ds} = beam_utils:split_even(Elems0), + Elems = [{Dst,kill} || Dst <- Ds], + tdb_update([{Src,map}|Elems], Ts0); update({test,is_nonempty_list,_Fail,[Src]}, Ts0) -> tdb_update([{Src,nonempty_list}], Ts0); update({test,is_eq_exact,_,[Reg,{atom,_}=Atom]}, Ts) -> @@ -507,10 +513,39 @@ update({test,is_eq_exact,_,[Reg,{atom,_}=Atom]}, Ts) -> end; update({test,is_record,_Fail,[Src,Tag,{integer,Arity}]}, Ts) -> tdb_update([{Src,{tuple,Arity,[Tag]}}], Ts); -update({test,_Test,_Fail,_Other}, Ts) -> - Ts; + +%% Binary matching + update({test,bs_get_integer2,_,_,Args,Dst}, Ts) -> tdb_update([{Dst,get_bs_integer_type(Args)}], Ts); +update({test,bs_get_utf8,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,?UNICODE_INT}], Ts); +update({test,bs_get_utf16,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,?UNICODE_INT}], Ts); +update({test,bs_get_utf32,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,?UNICODE_INT}], Ts); +update({bs_init,_,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,kill}], Ts); +update({bs_put,_,_,_}, Ts) -> + Ts; +update({bs_save2,_,_}, Ts) -> + Ts; +update({bs_restore2,_,_}, Ts) -> + Ts; +update({bs_context_to_binary,Dst}, Ts) -> + tdb_update([{Dst,kill}], Ts); +update({test,bs_start_match2,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,kill}], Ts); +update({test,bs_get_binary2,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,kill}], Ts); +update({test,bs_get_float2,_,_,_,Dst}, Ts) -> + tdb_update([{Dst,float}], Ts); + +update({test,_Test,_Fail,_Other}, Ts) -> + Ts; + +%% Calls + update({call_ext,Ar,{extfunc,math,Math,Ar}}, Ts) -> case is_math_bif(Math, Ar) of true -> tdb_update([{{x,0},float}], Ts); @@ -537,9 +572,10 @@ update({call_ext,3,{extfunc,erlang,setelement,3}}, Ts0) -> update({call,_Arity,_Func}, Ts) -> tdb_kill_xregs(Ts); update({call_ext,_Arity,_Func}, Ts) -> tdb_kill_xregs(Ts); update({make_fun2,_,_,_,_}, Ts) -> tdb_kill_xregs(Ts); +update({call_fun, _}, Ts) -> tdb_kill_xregs(Ts); +update({apply, _}, Ts) -> tdb_kill_xregs(Ts); + update({line,_}, Ts) -> Ts; -update({bs_save2,_,_}, Ts) -> Ts; -update({bs_restore2,_,_}, Ts) -> Ts; %% The instruction is unknown. Kill all information. update(_I, _Ts) -> tdb_new(). diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 03b52932d1..019d8ba864 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -213,14 +213,6 @@ expand_opt(report, Os) -> [report_errors,report_warnings|Os]; expand_opt(return, Os) -> [return_errors,return_warnings|Os]; -expand_opt(r12, Os) -> - [no_recv_opt,no_line_info,no_utf8_atoms|Os]; -expand_opt(r13, Os) -> - [no_record_opt,no_recv_opt,no_line_info,no_utf8_atoms|Os]; -expand_opt(r14, Os) -> - [no_record_opt,no_line_info,no_utf8_atoms|Os]; -expand_opt(r15, Os) -> - [no_record_opt,no_utf8_atoms|Os]; expand_opt(r16, Os) -> [no_record_opt,no_utf8_atoms|Os]; expand_opt(r17, Os) -> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 1287ec6176..765998b85d 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -35,7 +35,6 @@ -export([rand_plugin_next/1]). -export([rand_plugin_uniform/1]). -export([rand_plugin_uniform/2]). --export([rand_plugin_jump/1]). -export([rand_uniform/2]). -export([block_encrypt/3, block_decrypt/3, block_encrypt/4, block_decrypt/4]). -export([next_iv/2, next_iv/3]). @@ -316,11 +315,10 @@ rand_seed() -> rand_seed_s() -> {#{ type => ?MODULE, - max => infinity, + bits => 64, next => fun ?MODULE:rand_plugin_next/1, uniform => fun ?MODULE:rand_plugin_uniform/1, - uniform_n => fun ?MODULE:rand_plugin_uniform/2, - jump => fun ?MODULE:rand_plugin_jump/1}, + uniform_n => fun ?MODULE:rand_plugin_uniform/2}, no_seed}. rand_plugin_next(Seed) -> @@ -332,8 +330,6 @@ rand_plugin_uniform(State) -> rand_plugin_uniform(Max, State) -> {bytes_to_integer(strong_rand_range(Max)) + 1, State}. -rand_plugin_jump(State) -> - State. strong_rand_range(Range) when is_integer(Range), Range > 0 -> BinRange = int_to_bin(Range), diff --git a/lib/dialyzer/doc/src/Makefile b/lib/dialyzer/doc/src/Makefile index 77d0a6fc68..8fe6cd30eb 100644 --- a/lib/dialyzer/doc/src/Makefile +++ b/lib/dialyzer/doc/src/Makefile @@ -34,7 +34,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # Target Specs # ---------------------------------------------------- XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = dialyzer.xml +XML_REF3_FILES = dialyzer.xml typer.xml XML_PART_FILES = part.xml part_notes.xml XML_CHAPTER_FILES = dialyzer_chapter.xml notes.xml diff --git a/lib/dialyzer/doc/src/ref_man.xml b/lib/dialyzer/doc/src/ref_man.xml index ddac047f2e..d820fc5e00 100644 --- a/lib/dialyzer/doc/src/ref_man.xml +++ b/lib/dialyzer/doc/src/ref_man.xml @@ -31,5 +31,6 @@ <description> </description> <xi:include href="dialyzer.xml"/> + <xi:include href="typer.xml"/> </application> diff --git a/lib/dialyzer/doc/src/typer.xml b/lib/dialyzer/doc/src/typer.xml new file mode 100644 index 0000000000..abd7f07ccf --- /dev/null +++ b/lib/dialyzer/doc/src/typer.xml @@ -0,0 +1,157 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2006</year><year>2016</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>typer</title> + <prepared></prepared> + <docno></docno> + <date>2017-04-13</date> + <rev></rev> + <file>type.xml</file> + </header> + <module>typer</module> + <modulesummary>Typer, a Type annotator for ERlang programs. + </modulesummary> + <description> + <p>TypEr shows type information for Erlang modules to the user. + Additionally, it can annotate the code of files with such type + information.</p> + </description> + + <section> + <marker id="command_line"></marker> + <title>Using TypEr from the Command Line</title> + <p>TypEr is used from the command-line. This section provides a + brief description of the options. The same information can be + obtained by writing the following in a shell:</p> + + <code type="none"> +typer --help</code> + + <p><em>Usage:</em></p> + + <code type="none"> +typer [--help] [--version] [--plt PLT] [--edoc] + [--show | --show-exported | --annotate | --annotate-inc-files] + [-Ddefine]* [-I include_dir]* [-pa dir]* [-pz dir]* + [-T application]* [-r] file*</code> + + <note> + <p>* denotes that multiple occurrences of the option are possible.</p> + </note> + + <p><em>Options:</em></p> + + <taglist> + + <tag><c>--r</c></tag> + <item> + <p>Search directories recursively for .erl files below them.</p> + </item> + <tag><c>--show</c></tag> + <item> + <p>Print type specifications for all functions on stdout. + (This is the default behaviour; this option is not really + needed.)</p> + </item> + + <tag><c>--show-exported</c> (or <c>show_exported</c>)</tag> + <item> + <p>Same as <c>--show</c>, but print specifications for + exported functions only. Specs are displayed sorted + alphabetically on the function's name.</p> + </item> + + <tag><c>--annotate</c></tag> + <item> + <p>Annotate the specified files with type specifications.</p> + </item> + + <tag><c>--annotate-inc-files</c></tag> + <item> + <p>Same as <c>--annotate</c> but annotates all + <c>-include()</c> files as well as all .erl files. (Use this + option with caution - it has not been tested much).</p> + </item> + + <tag><c>--edoc</c></tag> + <item> + <p>Print type information as Edoc <c>@spec</c> comments, not + as type specs.</p> + </item> + + <tag><c>--plt</c></tag> + <item> + <p>Use the specified dialyzer PLT file rather than the default one.</p> + </item> + + <tag><c>-T file*</c></tag> + <item> + <p>The specified file(s) already contain type specifications + and these are to be trusted in order to print specs for the + rest of the files. (Multiple files or dirs, separated by + spaces, can be specified.)</p> + </item> + + <tag><c>-Dname</c> (or <c>-Dname=value</c>)</tag> + <item> + <p>Pass the defined name(s) to TypEr. (**)</p> + </item> + + <tag><c>-I</c></tag> + <item> + <p>Pass the include_dir to TypEr. (**)</p> + </item> + + <tag><c>-pa dir</c></tag> + <item> + <p>Include <c>dir</c> in the path for Erlang. This is useful + when analyzing files that have <c>-include_lib()</c> + directives or use parse transforms.</p> + </item> + + <tag><c>-pz dir</c></tag> + <item> + <p>Include <c>dir</c> in the path for Erlang. This is useful + when analyzing files that have <c>-include_lib()</c> + directives or use parse transforms.</p> + </item> + + <tag><c>--version</c> (or <c>-v</c>)</tag> + <item> + <p>Print the TypEr version and some more information and + exit.</p> + </item> + + </taglist> + + <note> + <p>** options <c>-D</c> and <c>-I</c> work both + from the command line and in the TypEr GUI; the syntax of + defines and includes is the same as that used by + <seealso marker="erts:erlc">erlc(1)</seealso>.</p> + </note> + + </section> + +</erlref> diff --git a/lib/dialyzer/src/Makefile b/lib/dialyzer/src/Makefile index 256f20f549..28f74ed441 100644 --- a/lib/dialyzer/src/Makefile +++ b/lib/dialyzer/src/Makefile @@ -68,7 +68,8 @@ MODULES = \ dialyzer_typesig \ dialyzer_coordinator \ dialyzer_worker \ - dialyzer_utils + dialyzer_utils \ + typer HRL_FILES= dialyzer.hrl dialyzer_gui_wx.hrl ERL_FILES= $(MODULES:%=%.erl) @@ -117,6 +118,9 @@ $(EBIN)/dialyzer_plt.$(EMULATOR): dialyzer_plt.erl ../vsn.mk $(EBIN)/dialyzer_gui_wx.$(EMULATOR): dialyzer_gui_wx.erl ../vsn.mk $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -DVSN="\"v$(VSN)\"" -o$(EBIN) dialyzer_gui_wx.erl +$(EBIN)/typer.$(EMULATOR): typer.erl ../vsn.mk + $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -DVSN="\"v$(VSN)\"" -o$(EBIN) typer.erl + $(APP_TARGET): $(APP_SRC) ../vsn.mk $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ diff --git a/lib/dialyzer/src/dialyzer.app.src b/lib/dialyzer/src/dialyzer.app.src index f517c51ec1..5f803875b0 100644 --- a/lib/dialyzer/src/dialyzer.app.src +++ b/lib/dialyzer/src/dialyzer.app.src @@ -43,7 +43,8 @@ dialyzer_typesig, dialyzer_utils, dialyzer_timing, - dialyzer_worker]}, + dialyzer_worker, + typer]}, {registered, []}, {applications, [compiler, hipe, kernel, stdlib, wx]}, {env, []}, diff --git a/lib/dialyzer/src/typer.erl b/lib/dialyzer/src/typer.erl new file mode 100644 index 0000000000..18c4fe902d --- /dev/null +++ b/lib/dialyzer/src/typer.erl @@ -0,0 +1,1110 @@ +%% -*- erlang-indent-level: 2 -*- +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%----------------------------------------------------------------------- +%% File : typer.erl +%% Author(s) : The first version of typer was written by Bingwen He +%% with guidance from Kostis Sagonas and Tobias Lindahl. +%% Since June 2008 typer is maintained by Kostis Sagonas. +%% Description : An Erlang/OTP application that shows type information +%% for Erlang modules to the user. Additionally, it can +%% annotate the code of files with such type information. +%%----------------------------------------------------------------------- + +-module(typer). + +-export([start/0]). + +%%----------------------------------------------------------------------- + +-define(SHOW, show). +-define(SHOW_EXPORTED, show_exported). +-define(ANNOTATE, annotate). +-define(ANNOTATE_INC_FILES, annotate_inc_files). + +-type mode() :: ?SHOW | ?SHOW_EXPORTED | ?ANNOTATE | ?ANNOTATE_INC_FILES. + +%%----------------------------------------------------------------------- + +-type files() :: [file:filename()]. +-type callgraph() :: dialyzer_callgraph:callgraph(). +-type codeserver() :: dialyzer_codeserver:codeserver(). +-type plt() :: dialyzer_plt:plt(). + +-record(analysis, + {mode :: mode() | 'undefined', + macros = [] :: [{atom(), term()}], + includes = [] :: files(), + codeserver = dialyzer_codeserver:new():: codeserver(), + callgraph = dialyzer_callgraph:new() :: callgraph(), + files = [] :: files(), % absolute names + plt = none :: 'none' | file:filename(), + no_spec = false :: boolean(), + show_succ = false :: boolean(), + %% For choosing between specs or edoc @spec comments + edoc = false :: boolean(), + %% Files in 'fms' are compilable with option 'to_pp'; we keep them + %% as {FileName, ModuleName} in case the ModuleName is different + fms = [] :: [{file:filename(), module()}], + ex_func = map__new() :: map_dict(), + record = map__new() :: map_dict(), + func = map__new() :: map_dict(), + inc_func = map__new() :: map_dict(), + trust_plt = dialyzer_plt:new() :: plt()}). +-type analysis() :: #analysis{}. + +-record(args, {files = [] :: files(), + files_r = [] :: files(), + trusted = [] :: files()}). +-type args() :: #args{}. + +%%-------------------------------------------------------------------- + +-spec start() -> no_return(). + +start() -> + {Args, Analysis} = process_cl_args(), + %% io:format("Args: ~p\n", [Args]), + %% io:format("Analysis: ~p\n", [Analysis]), + Timer = dialyzer_timing:init(false), + TrustedFiles = filter_fd(Args#args.trusted, [], fun is_erl_file/1), + Analysis2 = extract(Analysis, TrustedFiles), + All_Files = get_all_files(Args), + %% io:format("All_Files: ~p\n", [All_Files]), + Analysis3 = Analysis2#analysis{files = All_Files}, + Analysis4 = collect_info(Analysis3), + %% io:format("Final: ~p\n", [Analysis4#analysis.fms]), + TypeInfo = get_type_info(Analysis4), + dialyzer_timing:stop(Timer), + show_or_annotate(TypeInfo), + %% io:format("\nTyper analysis finished\n"), + erlang:halt(0). + +%%-------------------------------------------------------------------- + +-spec extract(analysis(), files()) -> analysis(). + +extract(#analysis{macros = Macros, + includes = Includes, + trust_plt = TrustPLT} = Analysis, TrustedFiles) -> + %% io:format("--- Extracting trusted typer_info... "), + Ds = [{d, Name, Value} || {Name, Value} <- Macros], + CodeServer = dialyzer_codeserver:new(), + Fun = + fun(File, CS) -> + %% We include one more dir; the one above the one we are trusting + %% E.g, for /home/tests/typer_ann/test.ann.erl, we should include + %% /home/tests/ rather than /home/tests/typer_ann/ + AllIncludes = [filename:dirname(filename:dirname(File)) | Includes], + Is = [{i, Dir} || Dir <- AllIncludes], + CompOpts = dialyzer_utils:src_compiler_opts() ++ Is ++ Ds, + case dialyzer_utils:get_abstract_code_from_src(File, CompOpts) of + {ok, AbstractCode} -> + case dialyzer_utils:get_record_and_type_info(AbstractCode) of + {ok, RecDict} -> + Mod = list_to_atom(filename:basename(File, ".erl")), + case dialyzer_utils:get_spec_info(Mod, AbstractCode, RecDict) of + {ok, SpecDict, CbDict} -> + CS1 = dialyzer_codeserver:store_temp_records(Mod, RecDict, CS), + dialyzer_codeserver:store_temp_contracts(Mod, SpecDict, CbDict, CS1); + {error, Reason} -> compile_error([Reason]) + end; + {error, Reason} -> compile_error([Reason]) + end; + {error, Reason} -> compile_error(Reason) + end + end, + CodeServer1 = lists:foldl(Fun, CodeServer, TrustedFiles), + %% Process remote types + NewCodeServer = + try + CodeServer2 = + dialyzer_utils:merge_types(CodeServer1, + TrustPLT), % XXX change to the PLT? + NewExpTypes = dialyzer_codeserver:get_temp_exported_types(CodeServer1), + case sets:size(NewExpTypes) of 0 -> ok end, + CodeServer3 = dialyzer_codeserver:finalize_exported_types(NewExpTypes, CodeServer2), + CodeServer4 = dialyzer_utils:process_record_remote_types(CodeServer3), + dialyzer_contracts:process_contract_remote_types(CodeServer4) + catch + throw:{error, ErrorMsg} -> + compile_error(ErrorMsg) + end, + %% Create TrustPLT + ContractsDict = dialyzer_codeserver:get_contracts(NewCodeServer), + Contracts = orddict:from_list(dict:to_list(ContractsDict)), + NewTrustPLT = dialyzer_plt:insert_contract_list(TrustPLT, Contracts), + Analysis#analysis{trust_plt = NewTrustPLT}. + +%%-------------------------------------------------------------------- + +-spec get_type_info(analysis()) -> analysis(). + +get_type_info(#analysis{callgraph = CallGraph, + trust_plt = TrustPLT, + codeserver = CodeServer} = Analysis) -> + StrippedCallGraph = remove_external(CallGraph, TrustPLT), + %% io:format("--- Analyzing callgraph... "), + try + NewMiniPlt = dialyzer_succ_typings:analyze_callgraph(StrippedCallGraph, + TrustPLT, + CodeServer), + NewPlt = dialyzer_plt:restore_full_plt(NewMiniPlt), + Analysis#analysis{callgraph = StrippedCallGraph, trust_plt = NewPlt} + catch + error:What -> + fatal_error(io_lib:format("Analysis failed with message: ~p", + [{What, erlang:get_stacktrace()}])); + throw:{dialyzer_succ_typing_error, Msg} -> + fatal_error(io_lib:format("Analysis failed with message: ~s", [Msg])) + end. + +-spec remove_external(callgraph(), plt()) -> callgraph(). + +remove_external(CallGraph, PLT) -> + {StrippedCG0, Ext} = dialyzer_callgraph:remove_external(CallGraph), + case get_external(Ext, PLT) of + [] -> ok; + Externals -> + msg(io_lib:format(" Unknown functions: ~p\n", [lists:usort(Externals)])), + ExtTypes = rcv_ext_types(), + case ExtTypes of + [] -> ok; + _ -> msg(io_lib:format(" Unknown types: ~p\n", [ExtTypes])) + end + end, + StrippedCG0. + +-spec get_external([{mfa(), mfa()}], plt()) -> [mfa()]. + +get_external(Exts, Plt) -> + Fun = fun ({_From, To = {M, F, A}}, Acc) -> + case dialyzer_plt:contains_mfa(Plt, To) of + false -> + case erl_bif_types:is_known(M, F, A) of + true -> Acc; + false -> [To|Acc] + end; + true -> Acc + end + end, + lists:foldl(Fun, [], Exts). + +%%-------------------------------------------------------------------- +%% Showing type information or annotating files with such information. +%%-------------------------------------------------------------------- + +-define(TYPER_ANN_DIR, "typer_ann"). + +-type line() :: non_neg_integer(). +-type fa() :: {atom(), arity()}. +-type func_info() :: {line(), atom(), arity()}. + +-record(info, {records = maps:new() :: erl_types:type_table(), + functions = [] :: [func_info()], + types = map__new() :: map_dict(), + edoc = false :: boolean()}). +-record(inc, {map = map__new() :: map_dict(), filter = [] :: files()}). +-type inc() :: #inc{}. + +-spec show_or_annotate(analysis()) -> 'ok'. + +show_or_annotate(#analysis{mode = Mode, fms = Files} = Analysis) -> + case Mode of + ?SHOW -> show(Analysis); + ?SHOW_EXPORTED -> show(Analysis); + ?ANNOTATE -> + Fun = fun ({File, Module}) -> + Info = get_final_info(File, Module, Analysis), + write_typed_file(File, Info) + end, + lists:foreach(Fun, Files); + ?ANNOTATE_INC_FILES -> + IncInfo = write_and_collect_inc_info(Analysis), + write_inc_files(IncInfo) + end. + +write_and_collect_inc_info(Analysis) -> + Fun = fun ({File, Module}, Inc) -> + Info = get_final_info(File, Module, Analysis), + write_typed_file(File, Info), + IncFuns = get_functions(File, Analysis), + collect_imported_functions(IncFuns, Info#info.types, Inc) + end, + NewInc = lists:foldl(Fun, #inc{}, Analysis#analysis.fms), + clean_inc(NewInc). + +write_inc_files(Inc) -> + Fun = + fun (File) -> + Val = map__lookup(File, Inc#inc.map), + %% Val is function with its type info + %% in form [{{Line,F,A},Type}] + Functions = [Key || {Key, _} <- Val], + Val1 = [{{F,A},Type} || {{_Line,F,A},Type} <- Val], + Info = #info{types = map__from_list(Val1), + records = maps:new(), + %% Note we need to sort functions here! + functions = lists:keysort(1, Functions)}, + %% io:format("Types ~p\n", [Info#info.types]), + %% io:format("Functions ~p\n", [Info#info.functions]), + %% io:format("Records ~p\n", [Info#info.records]), + write_typed_file(File, Info) + end, + lists:foreach(Fun, dict:fetch_keys(Inc#inc.map)). + +show(Analysis) -> + Fun = fun ({File, Module}) -> + Info = get_final_info(File, Module, Analysis), + show_type_info(File, Info) + end, + lists:foreach(Fun, Analysis#analysis.fms). + +get_final_info(File, Module, Analysis) -> + Records = get_records(File, Analysis), + Types = get_types(Module, Analysis, Records), + Functions = get_functions(File, Analysis), + Edoc = Analysis#analysis.edoc, + #info{records = Records, functions = Functions, types = Types, edoc = Edoc}. + +collect_imported_functions(Functions, Types, Inc) -> + %% Coming from other sourses, including: + %% FIXME: How to deal with yecc-generated file???? + %% --.yrl (yecc-generated file)??? + %% -- yeccpre.hrl (yecc-generated file)??? + %% -- other cases + Fun = fun ({File, _} = Obj, I) -> + case is_yecc_gen(File, I) of + {true, NewI} -> NewI; + {false, NewI} -> + check_imported_functions(Obj, NewI, Types) + end + end, + lists:foldl(Fun, Inc, Functions). + +-spec is_yecc_gen(file:filename(), inc()) -> {boolean(), inc()}. + +is_yecc_gen(File, #inc{filter = Fs} = Inc) -> + case lists:member(File, Fs) of + true -> {true, Inc}; + false -> + case filename:extension(File) of + ".yrl" -> + Rootname = filename:rootname(File, ".yrl"), + Obj = Rootname ++ ".erl", + case lists:member(Obj, Fs) of + true -> {true, Inc}; + false -> + NewInc = Inc#inc{filter = [Obj|Fs]}, + {true, NewInc} + end; + _ -> + case filename:basename(File) of + "yeccpre.hrl" -> {true, Inc}; + _ -> {false, Inc} + end + end + end. + +check_imported_functions({File, {Line, F, A}}, Inc, Types) -> + IncMap = Inc#inc.map, + FA = {F, A}, + Type = get_type_info(FA, Types), + case map__lookup(File, IncMap) of + none -> %% File is not added. Add it + Obj = {File,[{FA, {Line, Type}}]}, + NewMap = map__insert(Obj, IncMap), + Inc#inc{map = NewMap}; + Val -> %% File is already in. Check. + case lists:keyfind(FA, 1, Val) of + false -> + %% Function is not in; add it + Obj = {File, Val ++ [{FA, {Line, Type}}]}, + NewMap = map__insert(Obj, IncMap), + Inc#inc{map = NewMap}; + Type -> + %% Function is in and with same type + Inc; + _ -> + %% Function is in but with diff type + inc_warning(FA, File), + Elem = lists:keydelete(FA, 1, Val), + NewMap = case Elem of + [] -> map__remove(File, IncMap); + _ -> map__insert({File, Elem}, IncMap) + end, + Inc#inc{map = NewMap} + end + end. + +inc_warning({F, A}, File) -> + io:format(" ***Warning: Skip function ~p/~p ", [F, A]), + io:format("in file ~p because of inconsistent type\n", [File]). + +clean_inc(Inc) -> + Inc1 = remove_yecc_generated_file(Inc), + normalize_obj(Inc1). + +remove_yecc_generated_file(#inc{filter = Filter} = Inc) -> + Fun = fun (Key, #inc{map = Map} = I) -> + I#inc{map = map__remove(Key, Map)} + end, + lists:foldl(Fun, Inc, Filter). + +normalize_obj(TmpInc) -> + Fun = fun (Key, Val, Inc) -> + NewVal = [{{Line,F,A},Type} || {{F,A},{Line,Type}} <- Val], + map__insert({Key, NewVal}, Inc) + end, + TmpInc#inc{map = map__fold(Fun, map__new(), TmpInc#inc.map)}. + +get_records(File, Analysis) -> + map__lookup(File, Analysis#analysis.record). + +get_types(Module, Analysis, Records) -> + TypeInfoPlt = Analysis#analysis.trust_plt, + TypeInfo = + case dialyzer_plt:lookup_module(TypeInfoPlt, Module) of + none -> []; + {value, List} -> List + end, + CodeServer = Analysis#analysis.codeserver, + TypeInfoList = + case Analysis#analysis.show_succ of + true -> + [convert_type_info(I) || I <- TypeInfo]; + false -> + [get_type(I, CodeServer, Records) || I <- TypeInfo] + end, + map__from_list(TypeInfoList). + +convert_type_info({{_M, F, A}, Range, Arg}) -> + {{F, A}, {Range, Arg}}. + +get_type({{M, F, A} = MFA, Range, Arg}, CodeServer, Records) -> + case dialyzer_codeserver:lookup_mfa_contract(MFA, CodeServer) of + error -> + {{F, A}, {Range, Arg}}; + {ok, {_FileLine, Contract, _Xtra}} -> + Sig = erl_types:t_fun(Arg, Range), + case dialyzer_contracts:check_contract(Contract, Sig) of + ok -> {{F, A}, {contract, Contract}}; + {error, {extra_range, _, _}} -> + {{F, A}, {contract, Contract}}; + {error, {overlapping_contract, []}} -> + {{F, A}, {contract, Contract}}; + {error, invalid_contract} -> + CString = dialyzer_contracts:contract_to_string(Contract), + SigString = dialyzer_utils:format_sig(Sig, Records), + Msg = io_lib:format("Error in contract of function ~w:~w/~w\n" + "\t The contract is: " ++ CString ++ "\n" ++ + "\t but the inferred signature is: ~s", + [M, F, A, SigString]), + fatal_error(Msg); + {error, ErrorStr} when is_list(ErrorStr) -> % ErrorStr is a string() + Msg = io_lib:format("Error in contract of function ~w:~w/~w: ~s", + [M, F, A, ErrorStr]), + fatal_error(Msg) + end + end. + +get_functions(File, Analysis) -> + case Analysis#analysis.mode of + ?SHOW -> + Funcs = map__lookup(File, Analysis#analysis.func), + Inc_Funcs = map__lookup(File, Analysis#analysis.inc_func), + remove_module_info(Funcs) ++ normalize_incFuncs(Inc_Funcs); + ?SHOW_EXPORTED -> + Ex_Funcs = map__lookup(File, Analysis#analysis.ex_func), + remove_module_info(Ex_Funcs); + ?ANNOTATE -> + Funcs = map__lookup(File, Analysis#analysis.func), + remove_module_info(Funcs); + ?ANNOTATE_INC_FILES -> + map__lookup(File, Analysis#analysis.inc_func) + end. + +normalize_incFuncs(Functions) -> + [FunInfo || {_FileName, FunInfo} <- Functions]. + +-spec remove_module_info([func_info()]) -> [func_info()]. + +remove_module_info(FunInfoList) -> + F = fun ({_,module_info,0}) -> false; + ({_,module_info,1}) -> false; + ({Line,F,A}) when is_integer(Line), is_atom(F), is_integer(A) -> true + end, + lists:filter(F, FunInfoList). + +write_typed_file(File, Info) -> + io:format(" Processing file: ~p\n", [File]), + Dir = filename:dirname(File), + RootName = filename:basename(filename:rootname(File)), + Ext = filename:extension(File), + TyperAnnDir = filename:join(Dir, ?TYPER_ANN_DIR), + TmpNewFilename = lists:concat([RootName, ".ann", Ext]), + NewFileName = filename:join(TyperAnnDir, TmpNewFilename), + case file:make_dir(TyperAnnDir) of + {error, Reason} -> + case Reason of + eexist -> %% TypEr dir exists; remove old typer files if they exist + case file:delete(NewFileName) of + ok -> ok; + {error, enoent} -> ok; + {error, _} -> + Msg = io_lib:format("Error in deleting file ~s\n", [NewFileName]), + fatal_error(Msg) + end, + write_typed_file(File, Info, NewFileName); + enospc -> + Msg = io_lib:format("Not enough space in ~p\n", [Dir]), + fatal_error(Msg); + eacces -> + Msg = io_lib:format("No write permission in ~p\n", [Dir]), + fatal_error(Msg); + _ -> + Msg = io_lib:format("Unhandled error ~s when writing ~p\n", + [Reason, Dir]), + fatal_error(Msg) + end; + ok -> %% Typer dir does NOT exist + write_typed_file(File, Info, NewFileName) + end. + +write_typed_file(File, Info, NewFileName) -> + {ok, Binary} = file:read_file(File), + Chars = binary_to_list(Binary), + write_typed_file(Chars, NewFileName, Info, 1, []), + io:format(" Saved as: ~p\n", [NewFileName]). + +write_typed_file(Chars, File, #info{functions = []}, _LNo, _Acc) -> + ok = file:write_file(File, list_to_binary(Chars), [append]); +write_typed_file([Ch|Chs] = Chars, File, Info, LineNo, Acc) -> + [{Line,F,A}|RestFuncs] = Info#info.functions, + case Line of + 1 -> %% This will happen only for inc files + ok = raw_write(F, A, Info, File, []), + NewInfo = Info#info{functions = RestFuncs}, + NewAcc = [], + write_typed_file(Chars, File, NewInfo, Line, NewAcc); + _ -> + case Ch of + 10 -> + NewLineNo = LineNo + 1, + {NewInfo, NewAcc} = + case NewLineNo of + Line -> + ok = raw_write(F, A, Info, File, [Ch|Acc]), + {Info#info{functions = RestFuncs}, []}; + _ -> + {Info, [Ch|Acc]} + end, + write_typed_file(Chs, File, NewInfo, NewLineNo, NewAcc); + _ -> + write_typed_file(Chs, File, Info, LineNo, [Ch|Acc]) + end + end. + +raw_write(F, A, Info, File, Content) -> + TypeInfo = get_type_string(F, A, Info, file), + ContentList = lists:reverse(Content) ++ TypeInfo ++ "\n", + ContentBin = list_to_binary(ContentList), + file:write_file(File, ContentBin, [append]). + +get_type_string(F, A, Info, Mode) -> + Type = get_type_info({F,A}, Info#info.types), + TypeStr = + case Type of + {contract, C} -> + dialyzer_contracts:contract_to_string(C); + {RetType, ArgType} -> + Sig = erl_types:t_fun(ArgType, RetType), + dialyzer_utils:format_sig(Sig, Info#info.records) + end, + case Info#info.edoc of + false -> + case {Mode, Type} of + {file, {contract, _}} -> ""; + _ -> + Prefix = lists:concat(["-spec ", erl_types:atom_to_string(F)]), + lists:concat([Prefix, TypeStr, "."]) + end; + true -> + Prefix = lists:concat(["%% @spec ", F]), + lists:concat([Prefix, TypeStr, "."]) + end. + +show_type_info(File, Info) -> + io:format("\n%% File: ~p\n%% ", [File]), + OutputString = lists:concat(["~.", length(File)+8, "c~n"]), + io:fwrite(OutputString, [$-]), + Fun = fun ({_LineNo, F, A}) -> + TypeInfo = get_type_string(F, A, Info, show), + io:format("~s\n", [TypeInfo]) + end, + lists:foreach(Fun, Info#info.functions). + +get_type_info(Func, Types) -> + case map__lookup(Func, Types) of + none -> + %% Note: Typeinfo of any function should exist in + %% the result offered by dialyzer, otherwise there + %% *must* be something wrong with the analysis + Msg = io_lib:format("No type info for function: ~p\n", [Func]), + fatal_error(Msg); + {contract, _Fun} = C -> C; + {_RetType, _ArgType} = RA -> RA + end. + +%%-------------------------------------------------------------------- +%% Processing of command-line options and arguments. +%%-------------------------------------------------------------------- + +-spec process_cl_args() -> {args(), analysis()}. + +process_cl_args() -> + ArgList = init:get_plain_arguments(), + %% io:format("Args is ~p\n", [ArgList]), + {Args, Analysis} = analyze_args(ArgList, #args{}, #analysis{}), + %% if the mode has not been set, set it to the default mode (show) + {Args, case Analysis#analysis.mode of + undefined -> Analysis#analysis{mode = ?SHOW}; + Mode when is_atom(Mode) -> Analysis + end}. + +analyze_args([], Args, Analysis) -> + {Args, Analysis}; +analyze_args(ArgList, Args, Analysis) -> + {Result, Rest} = cl(ArgList), + {NewArgs, NewAnalysis} = analyze_result(Result, Args, Analysis), + analyze_args(Rest, NewArgs, NewAnalysis). + +cl(["-h"|_]) -> help_message(); +cl(["--help"|_]) -> help_message(); +cl(["-v"|_]) -> version_message(); +cl(["--version"|_]) -> version_message(); +cl(["--edoc"|Opts]) -> {edoc, Opts}; +cl(["--show"|Opts]) -> {{mode, ?SHOW}, Opts}; +cl(["--show_exported"|Opts]) -> {{mode, ?SHOW_EXPORTED}, Opts}; +cl(["--show-exported"|Opts]) -> {{mode, ?SHOW_EXPORTED}, Opts}; +cl(["--show_success_typings"|Opts]) -> {show_succ, Opts}; +cl(["--show-success-typings"|Opts]) -> {show_succ, Opts}; +cl(["--annotate"|Opts]) -> {{mode, ?ANNOTATE}, Opts}; +cl(["--annotate-inc-files"|Opts]) -> {{mode, ?ANNOTATE_INC_FILES}, Opts}; +cl(["--no_spec"|Opts]) -> {no_spec, Opts}; +cl(["--plt",Plt|Opts]) -> {{plt, Plt}, Opts}; +cl(["-D"++Def|Opts]) -> + case Def of + "" -> fatal_error("no variable name specified after -D"); + _ -> + DefPair = process_def_list(re:split(Def, "=", [{return, list}])), + {{def, DefPair}, Opts} + end; +cl(["-I",Dir|Opts]) -> {{inc, Dir}, Opts}; +cl(["-I"++Dir|Opts]) -> + case Dir of + "" -> fatal_error("no include directory specified after -I"); + _ -> {{inc, Dir}, Opts} + end; +cl(["-T"|Opts]) -> + {Files, RestOpts} = dialyzer_cl_parse:collect_args(Opts), + case Files of + [] -> fatal_error("no file or directory specified after -T"); + [_|_] -> {{trusted, Files}, RestOpts} + end; +cl(["-r"|Opts]) -> + {Files, RestOpts} = dialyzer_cl_parse:collect_args(Opts), + {{files_r, Files}, RestOpts}; +cl(["-pa",Dir|Opts]) -> {{pa,Dir}, Opts}; +cl(["-pz",Dir|Opts]) -> {{pz,Dir}, Opts}; +cl(["-"++H|_]) -> fatal_error("unknown option -"++H); +cl(Opts) -> + {Files, RestOpts} = dialyzer_cl_parse:collect_args(Opts), + {{files, Files}, RestOpts}. + +process_def_list(L) -> + case L of + [Name, Value] -> + {ok, Tokens, _} = erl_scan:string(Value ++ "."), + {ok, ErlValue} = erl_parse:parse_term(Tokens), + {list_to_atom(Name), ErlValue}; + [Name] -> + {list_to_atom(Name), true} + end. + +%% Get information about files that the user trusts and wants to analyze +analyze_result({files, Val}, Args, Analysis) -> + NewVal = Args#args.files ++ Val, + {Args#args{files = NewVal}, Analysis}; +analyze_result({files_r, Val}, Args, Analysis) -> + NewVal = Args#args.files_r ++ Val, + {Args#args{files_r = NewVal}, Analysis}; +analyze_result({trusted, Val}, Args, Analysis) -> + NewVal = Args#args.trusted ++ Val, + {Args#args{trusted = NewVal}, Analysis}; +analyze_result(edoc, Args, Analysis) -> + {Args, Analysis#analysis{edoc = true}}; +%% Get useful information for actual analysis +analyze_result({mode, Mode}, Args, Analysis) -> + case Analysis#analysis.mode of + undefined -> {Args, Analysis#analysis{mode = Mode}}; + OldMode -> mode_error(OldMode, Mode) + end; +analyze_result({def, Val}, Args, Analysis) -> + NewVal = Analysis#analysis.macros ++ [Val], + {Args, Analysis#analysis{macros = NewVal}}; +analyze_result({inc, Val}, Args, Analysis) -> + NewVal = Analysis#analysis.includes ++ [Val], + {Args, Analysis#analysis{includes = NewVal}}; +analyze_result({plt, Plt}, Args, Analysis) -> + {Args, Analysis#analysis{plt = Plt}}; +analyze_result(show_succ, Args, Analysis) -> + {Args, Analysis#analysis{show_succ = true}}; +analyze_result(no_spec, Args, Analysis) -> + {Args, Analysis#analysis{no_spec = true}}; +analyze_result({pa, Dir}, Args, Analysis) -> + true = code:add_patha(Dir), + {Args, Analysis}; +analyze_result({pz, Dir}, Args, Analysis) -> + true = code:add_pathz(Dir), + {Args, Analysis}. + +%%-------------------------------------------------------------------- +%% File processing. +%%-------------------------------------------------------------------- + +-spec get_all_files(args()) -> [file:filename(),...]. + +get_all_files(#args{files = Fs, files_r = Ds}) -> + case filter_fd(Fs, Ds, fun test_erl_file_exclude_ann/1) of + [] -> fatal_error("no file(s) to analyze"); + AllFiles -> AllFiles + end. + +-spec test_erl_file_exclude_ann(file:filename()) -> boolean(). + +test_erl_file_exclude_ann(File) -> + case is_erl_file(File) of + true -> %% Exclude files ending with ".ann.erl" + case re:run(File, "[\.]ann[\.]erl$") of + {match, _} -> false; + nomatch -> true + end; + false -> false + end. + +-spec is_erl_file(file:filename()) -> boolean(). + +is_erl_file(File) -> + filename:extension(File) =:= ".erl". + +-type test_file_fun() :: fun((file:filename()) -> boolean()). + +-spec filter_fd(files(), files(), test_file_fun()) -> files(). + +filter_fd(File_Dir, Dir_R, Fun) -> + All_File_1 = process_file_and_dir(File_Dir, Fun), + All_File_2 = process_dir_rec(Dir_R, Fun), + remove_dup(All_File_1 ++ All_File_2). + +-spec process_file_and_dir(files(), test_file_fun()) -> files(). + +process_file_and_dir(File_Dir, TestFun) -> + Fun = + fun (Elem, Acc) -> + case filelib:is_regular(Elem) of + true -> process_file(Elem, TestFun, Acc); + false -> check_dir(Elem, false, Acc, TestFun) + end + end, + lists:foldl(Fun, [], File_Dir). + +-spec process_dir_rec(files(), test_file_fun()) -> files(). + +process_dir_rec(Dirs, TestFun) -> + Fun = fun (Dir, Acc) -> check_dir(Dir, true, Acc, TestFun) end, + lists:foldl(Fun, [], Dirs). + +-spec check_dir(file:filename(), boolean(), files(), test_file_fun()) -> files(). + +check_dir(Dir, Recursive, Acc, Fun) -> + case file:list_dir(Dir) of + {ok, Files} -> + {TmpDirs, TmpFiles} = split_dirs_and_files(Files, Dir), + case Recursive of + false -> + FinalFiles = process_file_and_dir(TmpFiles, Fun), + Acc ++ FinalFiles; + true -> + TmpAcc1 = process_file_and_dir(TmpFiles, Fun), + TmpAcc2 = process_dir_rec(TmpDirs, Fun), + Acc ++ TmpAcc1 ++ TmpAcc2 + end; + {error, eacces} -> + fatal_error("no access permission to dir \""++Dir++"\""); + {error, enoent} -> + fatal_error("cannot access "++Dir++": No such file or directory"); + {error, _Reason} -> + fatal_error("error involving a use of file:list_dir/1") + end. + +%% Same order as the input list +-spec process_file(file:filename(), test_file_fun(), files()) -> files(). + +process_file(File, TestFun, Acc) -> + case TestFun(File) of + true -> Acc ++ [File]; + false -> Acc + end. + +%% Same order as the input list +-spec split_dirs_and_files(files(), file:filename()) -> {files(), files()}. + +split_dirs_and_files(Elems, Dir) -> + Test_Fun = + fun (Elem, {DirAcc, FileAcc}) -> + File = filename:join(Dir, Elem), + case filelib:is_regular(File) of + false -> {[File|DirAcc], FileAcc}; + true -> {DirAcc, [File|FileAcc]} + end + end, + {Dirs, Files} = lists:foldl(Test_Fun, {[], []}, Elems), + {lists:reverse(Dirs), lists:reverse(Files)}. + +%% Removes duplicate filenames but keeps the order of the input list +-spec remove_dup(files()) -> files(). + +remove_dup(Files) -> + Test_Dup = fun (File, Acc) -> + case lists:member(File, Acc) of + true -> Acc; + false -> [File|Acc] + end + end, + Reversed_Elems = lists:foldl(Test_Dup, [], Files), + lists:reverse(Reversed_Elems). + +%%-------------------------------------------------------------------- +%% Collect information. +%%-------------------------------------------------------------------- + +-type inc_file_info() :: {file:filename(), func_info()}. + +-record(tmpAcc, {file :: file:filename(), + module :: atom(), + funcAcc = [] :: [func_info()], + incFuncAcc = [] :: [inc_file_info()], + dialyzerObj = [] :: [{mfa(), {_, _}}]}). + +-spec collect_info(analysis()) -> analysis(). + +collect_info(Analysis) -> + NewPlt = + try get_dialyzer_plt(Analysis) of + DialyzerPlt -> + dialyzer_plt:merge_plts([Analysis#analysis.trust_plt, DialyzerPlt]) + catch + throw:{dialyzer_error,_Reason} -> + fatal_error("Dialyzer's PLT is missing or is not up-to-date; please (re)create it") + end, + NewAnalysis = lists:foldl(fun collect_one_file_info/2, + Analysis#analysis{trust_plt = NewPlt}, + Analysis#analysis.files), + %% Process Remote Types + TmpCServer = NewAnalysis#analysis.codeserver, + NewCServer = + try + TmpCServer1 = dialyzer_utils:merge_types(TmpCServer, NewPlt), + NewExpTypes = dialyzer_codeserver:get_temp_exported_types(TmpCServer), + OldExpTypes = dialyzer_plt:get_exported_types(NewPlt), + MergedExpTypes = sets:union(NewExpTypes, OldExpTypes), + TmpCServer2 = + dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1), + TmpCServer3 = dialyzer_utils:process_record_remote_types(TmpCServer2), + dialyzer_contracts:process_contract_remote_types(TmpCServer3) + catch + throw:{error, ErrorMsg} -> + fatal_error(ErrorMsg) + end, + NewAnalysis#analysis{codeserver = NewCServer}. + +collect_one_file_info(File, Analysis) -> + Ds = [{d,Name,Val} || {Name,Val} <- Analysis#analysis.macros], + %% Current directory should also be included in "Includes". + Includes = [filename:dirname(File)|Analysis#analysis.includes], + Is = [{i,Dir} || Dir <- Includes], + Options = dialyzer_utils:src_compiler_opts() ++ Is ++ Ds, + case dialyzer_utils:get_abstract_code_from_src(File, Options) of + {error, Reason} -> + %% io:format("File=~p\n,Options=~p\n,Error=~p\n", [File,Options,Reason]), + compile_error(Reason); + {ok, AbstractCode} -> + case dialyzer_utils:get_core_from_abstract_code(AbstractCode, Options) of + error -> compile_error(["Could not get core erlang for "++File]); + {ok, Core} -> + case dialyzer_utils:get_record_and_type_info(AbstractCode) of + {error, Reason} -> compile_error([Reason]); + {ok, Records} -> + Mod = cerl:concrete(cerl:module_name(Core)), + case dialyzer_utils:get_spec_info(Mod, AbstractCode, Records) of + {error, Reason} -> compile_error([Reason]); + {ok, SpecInfo, CbInfo} -> + ExpTypes = get_exported_types_from_core(Core), + analyze_core_tree(Core, Records, SpecInfo, CbInfo, + ExpTypes, Analysis, File) + end + end + end + end. + +analyze_core_tree(Core, Records, SpecInfo, CbInfo, ExpTypes, Analysis, File) -> + Module = cerl:concrete(cerl:module_name(Core)), + TmpTree = cerl:from_records(Core), + CS1 = Analysis#analysis.codeserver, + NextLabel = dialyzer_codeserver:get_next_core_label(CS1), + {Tree, NewLabel} = cerl_trees:label(TmpTree, NextLabel), + CS2 = dialyzer_codeserver:insert(Module, Tree, CS1), + CS3 = dialyzer_codeserver:set_next_core_label(NewLabel, CS2), + CS4 = dialyzer_codeserver:store_temp_records(Module, Records, CS3), + CS5 = + case Analysis#analysis.no_spec of + true -> CS4; + false -> + dialyzer_codeserver:store_temp_contracts(Module, SpecInfo, CbInfo, CS4) + end, + OldExpTypes = dialyzer_codeserver:get_temp_exported_types(CS5), + MergedExpTypes = sets:union(ExpTypes, OldExpTypes), + CS6 = dialyzer_codeserver:insert_temp_exported_types(MergedExpTypes, CS5), + Ex_Funcs = [{0,F,A} || {_,_,{F,A}} <- cerl:module_exports(Tree)], + CG = Analysis#analysis.callgraph, + {V, E} = dialyzer_callgraph:scan_core_tree(Tree, CG), + dialyzer_callgraph:add_edges(E, V, CG), + Fun = fun analyze_one_function/2, + All_Defs = cerl:module_defs(Tree), + Acc = lists:foldl(Fun, #tmpAcc{file = File, module = Module}, All_Defs), + Exported_FuncMap = map__insert({File, Ex_Funcs}, Analysis#analysis.ex_func), + %% we must sort all functions in the file which + %% originate from this file by *numerical order* of lineNo + Sorted_Functions = lists:keysort(1, Acc#tmpAcc.funcAcc), + FuncMap = map__insert({File, Sorted_Functions}, Analysis#analysis.func), + %% we do not need to sort functions which are imported from included files + IncFuncMap = map__insert({File, Acc#tmpAcc.incFuncAcc}, + Analysis#analysis.inc_func), + FMs = Analysis#analysis.fms ++ [{File, Module}], + RecordMap = map__insert({File, Records}, Analysis#analysis.record), + Analysis#analysis{fms = FMs, + callgraph = CG, + codeserver = CS6, + ex_func = Exported_FuncMap, + inc_func = IncFuncMap, + record = RecordMap, + func = FuncMap}. + +analyze_one_function({Var, FunBody} = Function, Acc) -> + F = cerl:fname_id(Var), + A = cerl:fname_arity(Var), + TmpDialyzerObj = {{Acc#tmpAcc.module, F, A}, Function}, + NewDialyzerObj = Acc#tmpAcc.dialyzerObj ++ [TmpDialyzerObj], + Anno = cerl:get_ann(FunBody), + LineNo = get_line(Anno), + FileName = get_file(Anno), + BaseName = filename:basename(FileName), + FuncInfo = {LineNo, F, A}, + OriginalName = Acc#tmpAcc.file, + {FuncAcc, IncFuncAcc} = + case (FileName =:= OriginalName) orelse (BaseName =:= OriginalName) of + true -> %% Coming from original file + %% io:format("Added function ~p\n", [{LineNo, F, A}]), + {Acc#tmpAcc.funcAcc ++ [FuncInfo], Acc#tmpAcc.incFuncAcc}; + false -> + %% Coming from other sourses, including: + %% -- .yrl (yecc-generated file) + %% -- yeccpre.hrl (yecc-generated file) + %% -- other cases + {Acc#tmpAcc.funcAcc, Acc#tmpAcc.incFuncAcc ++ [{FileName, FuncInfo}]} + end, + Acc#tmpAcc{funcAcc = FuncAcc, + incFuncAcc = IncFuncAcc, + dialyzerObj = NewDialyzerObj}. + +get_line([Line|_]) when is_integer(Line) -> Line; +get_line([_|T]) -> get_line(T); +get_line([]) -> none. + +get_file([{file,File}|_]) -> File; +get_file([_|T]) -> get_file(T); +get_file([]) -> "no_file". % should not happen + +-spec get_dialyzer_plt(analysis()) -> plt(). + +get_dialyzer_plt(#analysis{plt = PltFile0}) -> + PltFile = + case PltFile0 =:= none of + true -> dialyzer_plt:get_default_plt(); + false -> PltFile0 + end, + dialyzer_plt:from_file(PltFile). + +%% Exported Types + +get_exported_types_from_core(Core) -> + Attrs = cerl:module_attrs(Core), + ExpTypes1 = [cerl:concrete(L2) || {L1, L2} <- Attrs, + cerl:is_literal(L1), + cerl:is_literal(L2), + cerl:concrete(L1) =:= 'export_type'], + ExpTypes2 = lists:flatten(ExpTypes1), + M = cerl:atom_val(cerl:module_name(Core)), + sets:from_list([{M, F, A} || {F, A} <- ExpTypes2]). + +%%-------------------------------------------------------------------- +%% Utilities for error reporting. +%%-------------------------------------------------------------------- + +-spec fatal_error(string()) -> no_return(). + +fatal_error(Slogan) -> + msg(io_lib:format("typer: ~s\n", [Slogan])), + erlang:halt(1). + +-spec mode_error(mode(), mode()) -> no_return(). + +mode_error(OldMode, NewMode) -> + Msg = io_lib:format("Mode was previously set to '~s'; " + "can not set it to '~s' now", + [OldMode, NewMode]), + fatal_error(Msg). + +-spec compile_error([string()]) -> no_return(). + +compile_error(Reason) -> + JoinedString = lists:flatten([X ++ "\n" || X <- Reason]), + Msg = "Analysis failed with error report:\n" ++ JoinedString, + fatal_error(Msg). + +-spec msg(string()) -> 'ok'. + +msg(Msg) -> + io:format(standard_error, "~s", [Msg]). + +%%-------------------------------------------------------------------- +%% Version and help messages. +%%-------------------------------------------------------------------- + +-spec version_message() -> no_return(). + +version_message() -> + io:format("TypEr version "++?VSN++"\n"), + erlang:halt(0). + +-spec help_message() -> no_return(). + +help_message() -> + S = <<" Usage: typer [--help] [--version] [--plt PLT] [--edoc] + [--show | --show-exported | --annotate | --annotate-inc-files] + [-Ddefine]* [-I include_dir]* [-pa dir]* [-pz dir]* + [-T application]* [-r] file* + + Options: + -r dir* + search directories recursively for .erl files below them + --show + Prints type specifications for all functions on stdout. + (this is the default behaviour; this option is not really needed) + --show-exported (or --show_exported) + Same as --show, but prints specifications for exported functions only + Specs are displayed sorted alphabetically on the function's name + --annotate + Annotates the specified files with type specifications + --annotate-inc-files + Same as --annotate but annotates all -include() files as well as + all .erl files (use this option with caution - has not been tested much) + --edoc + Prints type information as Edoc @spec comments, not as type specs + --plt PLT + Use the specified dialyzer PLT file rather than the default one + -T file* + The specified file(s) already contain type specifications and these + are to be trusted in order to print specs for the rest of the files + (Multiple files or dirs, separated by spaces, can be specified.) + -Dname (or -Dname=value) + pass the defined name(s) to TypEr + (The syntax of defines is the same as that used by \"erlc\".) + -I include_dir + pass the include_dir to TypEr + (The syntax of includes is the same as that used by \"erlc\".) + -pa dir + -pz dir + Set code path options to TypEr + (This is useful for files that use parse tranforms.) + --version (or -v) + prints the Typer version and exits + --help (or -h) + prints this message and exits + + Note: + * denotes that multiple occurrences of these options are possible. +">>, + io:put_chars(S), + erlang:halt(0). + +%%-------------------------------------------------------------------- +%% Handle messages. +%%-------------------------------------------------------------------- + +rcv_ext_types() -> + Self = self(), + Self ! {Self, done}, + rcv_ext_types(Self, []). + +rcv_ext_types(Self, ExtTypes) -> + receive + {Self, ext_types, ExtType} -> + rcv_ext_types(Self, [ExtType|ExtTypes]); + {Self, done} -> + lists:usort(ExtTypes) + end. + +%%-------------------------------------------------------------------- +%% A convenient abstraction of a Key-Value mapping data structure +%% specialized for the uses in this module +%%-------------------------------------------------------------------- + +-type map_dict() :: dict:dict(). + +-spec map__new() -> map_dict(). +map__new() -> + dict:new(). + +-spec map__insert({term(), term()}, map_dict()) -> map_dict(). +map__insert(Object, Map) -> + {Key, Value} = Object, + dict:store(Key, Value, Map). + +-spec map__lookup(term(), map_dict()) -> term(). +map__lookup(Key, Map) -> + try dict:fetch(Key, Map) catch error:_ -> none end. + +-spec map__from_list([{fa(), term()}]) -> map_dict(). +map__from_list(List) -> + dict:from_list(List). + +-spec map__remove(term(), map_dict()) -> map_dict(). +map__remove(Key, Dict) -> + dict:erase(Key, Dict). + +-spec map__fold(fun((term(), term(), term()) -> map_dict()), map_dict(), map_dict()) -> map_dict(). +map__fold(Fun, Acc0, Dict) -> + dict:fold(Fun, Acc0, Dict). diff --git a/lib/dialyzer/test/Makefile b/lib/dialyzer/test/Makefile index 0d8fba438c..43c8a61ce1 100644 --- a/lib/dialyzer/test/Makefile +++ b/lib/dialyzer/test/Makefile @@ -13,7 +13,8 @@ AUXILIARY_FILES=\ file_utils.erl\ dialyzer_SUITE.erl\ abstract_SUITE.erl\ - plt_SUITE.erl + plt_SUITE.erl\ + typer_SUITE.erl # ---------------------------------------------------- # Release directory specification diff --git a/lib/dialyzer/test/typer_SUITE.erl b/lib/dialyzer/test/typer_SUITE.erl new file mode 100644 index 0000000000..da5b961643 --- /dev/null +++ b/lib/dialyzer/test/typer_SUITE.erl @@ -0,0 +1,158 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(typer_SUITE). + +-export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, + init_per_group/2,end_per_group/2, + smoke/1]). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [smoke]. + +groups() -> + []. + +init_per_suite(Config) -> + OutDir = proplists:get_value(priv_dir, Config), + case dialyzer_common:check_plt(OutDir) of + fail -> {skip, "Plt creation/check failed."}; + ok -> [{dialyzer_options, []}|Config] + end. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +smoke(Config) -> + Code = <<"-module(typer_test_module). + -compile([export_all,nowarn_export_all]). + a(L) -> + L ++ [1,2,3].">>, + PrivDir = proplists:get_value(priv_dir, Config), + Src = filename:join(PrivDir, "typer_test_module.erl"), + ok = file:write_file(Src, Code), + Args = "--plt " ++ PrivDir ++ "dialyzer_plt", + Res = ["^$", + "^%% File:", + "^%% ----", + "^-spec a", + "^_OK_"], + run(Config, Args, Src, Res), + ok. + +typer() -> + case os:find_executable("typer") of + false -> + ct:fail("Can't find typer"); + Typer -> + Typer + end. + +%% Runs a command. + +run(Config, Args0, Name, Expect) -> + Args = Args0 ++ " " ++ Name, + Result = run_command(Config, Args), + verify_result(Result, Expect). + +verify_result(Result, Expect) -> + Messages = split(Result, [], []), + io:format("Result: ~p", [Messages]), + io:format("Expected: ~p", [Expect]), + match_messages(Messages, Expect). + +split([$\n|Rest], Current, Lines) -> + split(Rest, [], [lists:reverse(Current)|Lines]); +split([$\r|Rest], Current, Lines) -> + split(Rest, Current, Lines); +split([Char|Rest], Current, Lines) -> + split(Rest, [Char|Current], Lines); +split([], [], Lines) -> + lists:reverse(Lines); +split([], Current, Lines) -> + split([], [], [lists:reverse(Current)|Lines]). + +match_messages([Msg|Rest1], [Regexp|Rest2]) -> + case re:run(Msg, Regexp, [{capture,none}, unicode]) of + match -> + ok; + nomatch -> + io:format("Not matching: ~s\n", [Msg]), + io:format("Regexp : ~s\n", [Regexp]), + ct:fail(message_mismatch) + end, + match_messages(Rest1, Rest2); +match_messages([], [Expect|Rest]) -> + ct:fail({too_few_messages, [Expect|Rest]}); +match_messages([Msg|Rest], []) -> + ct:fail({too_many_messages, [Msg|Rest]}); +match_messages([], []) -> + ok. + +%% Runs the command using os:cmd/1. +%% +%% Returns the output from the command (as a list of characters with +%% embedded newlines). The very last line will indicate the +%% exit status of the command, where _OK_ means zero, and _ERROR_ +%% a non-zero exit status. + +run_command(Config, Args) -> + TmpDir = filename:join(proplists:get_value(priv_dir, Config), "tmp"), + file:make_dir(TmpDir), + {RunFile, Run, Script} = run_command(TmpDir, os:type(), Args), + ok = file:write_file(filename:join(TmpDir, RunFile), + unicode:characters_to_binary(Script)), + io:format("~ts\n", [Script]), + os:cmd(Run). + +run_command(Dir, {win32, _}, Args) -> + BatchFile = filename:join(Dir, "run.bat"), + Run = re:replace(filename:rootname(BatchFile), "/", "\\", + [global,{return,list}]), + Typer = typer(), + {BatchFile, + Run, + ["@echo off\r\n", + "\"",Typer,"\" ",Args, "\r\n", + "if errorlevel 1 echo _ERROR_\r\n", + "if not errorlevel 1 echo _OK_\r\n"]}; +run_command(Dir, {unix, _}, Args) -> + TyperDir = filename:dirname(typer()), + Name = filename:join(Dir, "run"), + {Name, + "/bin/sh " ++ Name, + ["#!/bin/sh\n", + "PATH=\"",TyperDir,":$PATH\"\n", + "typer ",Args,"\n", + "case $? in\n", + " 0) echo '_OK_';;\n", + " *) echo '_ERROR_';;\n", + "esac\n"]}; +run_command(_Dir, Other, _Args) -> + ct:fail("Don't know how to test exit code for ~p", [Other]). diff --git a/lib/kernel/test/prim_file_SUITE.erl b/lib/kernel/test/prim_file_SUITE.erl index 8be94f1e57..0c84a7d1cc 100644 --- a/lib/kernel/test/prim_file_SUITE.erl +++ b/lib/kernel/test/prim_file_SUITE.erl @@ -19,8 +19,8 @@ %% -module(prim_file_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_group/2,end_per_group/2, - read_write_file/1]). + init_per_group/2,end_per_group/2, init_per_testcase/2, end_per_testcase/2, + read_write_file/1, free_memory/0]). -export([cur_dir_0a/1, cur_dir_0b/1, cur_dir_1a/1, cur_dir_1b/1, make_del_dir_a/1, make_del_dir_b/1, @@ -115,6 +115,18 @@ groups() -> [make_link_a, make_link_b, read_link_info_for_non_link, symlinks_a, symlinks_b, list_dir_error]}]. +init_per_testcase(large_write, Config) -> + {ok, Started} = application:ensure_all_started(os_mon), + [{started, Started}|Config]; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(large_write, Config) -> + [application:stop(App) || App <- lists:reverse(proplists:get_value(started, Config))], + ok; +end_per_testcase(_, _Config) -> + ok. + init_per_group(_GroupName, Config) -> Config. @@ -2022,11 +2034,13 @@ run_large_file_test(Config, Run, Name) -> {{unix,sunos},OsVersion} when OsVersion < {5,5,1} -> {skip,"Only supported on Win32, Unix or SunOS >= 5.5.1"}; {{unix,_},_} -> - N = unix_free(proplists:get_value(priv_dir, Config)), - io:format("Free disk: ~w KByte~n", [N]), - if N < 5 bsl 20 -> + DiscFree = unix_free(proplists:get_value(priv_dir, Config)), + MemFree = free_memory(), + io:format("Free disk: ~w KByte~n", [DiscFree]), + io:format("Free mem: ~w MByte~n", [MemFree]), + if DiscFree < 5 bsl 20; MemFree < 5 bsl 10 -> %% Less than 5 GByte free - {skip,"Less than 5 GByte free disk"}; + {skip,"Less than 5 GByte free disk/mem"}; true -> do_run_large_file_test(Config, Run, Name) end; @@ -2079,6 +2093,27 @@ zip_data([], Bs) -> zip_data(As, []) -> As. +%% Stolen from emulator -> alloc_SUITE +free_memory() -> + %% Free memory in MB. + try + SMD = memsup:get_system_memory_data(), + {value, {free_memory, Free}} = lists:keysearch(free_memory, 1, SMD), + TotFree = (Free + + case lists:keysearch(cached_memory, 1, SMD) of + {value, {cached_memory, Cached}} -> Cached; + false -> 0 + end + + case lists:keysearch(buffered_memory, 1, SMD) of + {value, {buffered_memory, Buffed}} -> Buffed; + false -> 0 + end), + TotFree div (1024*1024) + catch + error : undef -> + ct:fail({"os_mon not built"}) + end. + %%%----------------------------------------------------------------- %%% Utilities rm_rf(Mod,Dir) -> diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl index e57c8162e4..fce15bca89 100644 --- a/lib/observer/test/crashdump_helper.erl +++ b/lib/observer/test/crashdump_helper.erl @@ -20,7 +20,7 @@ -module(crashdump_helper). -export([n1_proc/2,remote_proc/2]). --compile(r13). +-compile(r18). -include_lib("common_test/include/ct.hrl"). n1_proc(N2,Creator) -> diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 0c98232467..1a8bd3f607 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -87,25 +87,25 @@ end_per_group(_GroupName, Config) -> init_per_suite(Config) when is_list(Config) -> + %% To use in end_per_testcase + Path = code:get_path(), + {ok,Cwd} = file:get_cwd(), + %% Make of copy of the data directory. DataDir = ?datadir, PrivDir = ?privdir, CopyDir = fname(PrivDir, "datacopy"), + ok = file:make_dir(CopyDir), TarFile = fname(PrivDir, "datacopy.tgz"), - {ok, Tar} = erl_tar:open(TarFile, [write, compressed]), - ok = erl_tar:add(Tar, DataDir, CopyDir, [compressed]), - ok = erl_tar:close(Tar), - ok = erl_tar:extract(TarFile, [compressed]), + ok = file:set_cwd(DataDir), + ok = erl_tar:create(TarFile, ["."], [compressed]), + ok = erl_tar:extract(TarFile, [compressed, {cwd,CopyDir}]), ok = file:delete(TarFile), %% Compile source files in the copy directory. Sources = filelib:wildcard(fname([CopyDir,'*','*','*','*','*.erl'])), lists:foreach(fun compile_source/1, Sources), - %% To use in end_per_testcase - Path = code:get_path(), - {ok,Cwd} = file:get_cwd(), - [{copy_dir, CopyDir}, {cwd,Cwd}, {path,Path} | Config]. compile_source(File) -> diff --git a/lib/ssl/src/dtls_record.erl b/lib/ssl/src/dtls_record.erl index 0ee51c24b6..049f83e49e 100644 --- a/lib/ssl/src/dtls_record.erl +++ b/lib/ssl/src/dtls_record.erl @@ -439,43 +439,59 @@ encode_dtls_cipher_text(Type, {MajVer, MinVer}, Fragment, encode_plain_text(Type, Version, Data, #{compression_state := CompS0, epoch := Epoch, sequence_number := Seq, + cipher_state := CipherS0, security_parameters := #security_parameters{ cipher_type = ?AEAD, + bulk_cipher_algorithm = + BulkCipherAlgo, compression_algorithm = CompAlg} } = WriteState0) -> {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), - WriteState1 = WriteState0#{compression_state => CompS1}, AAD = calc_aad(Type, Version, Epoch, Seq), - ssl_record:cipher_aead(dtls_v1:corresponding_tls_version(Version), Comp, WriteState1, AAD); -encode_plain_text(Type, Version, Data, #{compression_state := CompS0, + TLSVersion = dtls_v1:corresponding_tls_version(Version), + {CipherFragment, CipherS1} = + ssl_cipher:cipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, Comp, TLSVersion), + {CipherFragment, WriteState0#{compression_state => CompS1, + cipher_state => CipherS1}}; +encode_plain_text(Type, Version, Fragment, #{compression_state := CompS0, epoch := Epoch, sequence_number := Seq, + cipher_state := CipherS0, security_parameters := - #security_parameters{compression_algorithm = CompAlg} + #security_parameters{compression_algorithm = CompAlg, + bulk_cipher_algorithm = + BulkCipherAlgo} }= WriteState0) -> - {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), + {Comp, CompS1} = ssl_record:compress(CompAlg, Fragment, CompS0), WriteState1 = WriteState0#{compression_state => CompS1}, - MacHash = calc_mac_hash(Type, Version, WriteState1, Epoch, Seq, Comp), - ssl_record:cipher(dtls_v1:corresponding_tls_version(Version), Comp, WriteState1, MacHash). + MAC = calc_mac_hash(Type, Version, WriteState1, Epoch, Seq, Comp), + TLSVersion = dtls_v1:corresponding_tls_version(Version), + {CipherFragment, CipherS1} = + ssl_cipher:cipher(BulkCipherAlgo, CipherS0, MAC, Fragment, TLSVersion), + {CipherFragment, WriteState0#{cipher_state => CipherS1}}. decode_cipher_text(#ssl_tls{type = Type, version = Version, epoch = Epoch, sequence_number = Seq, fragment = CipherFragment} = CipherText, #{compression_state := CompressionS0, + cipher_state := CipherS0, security_parameters := #security_parameters{ cipher_type = ?AEAD, + bulk_cipher_algorithm = + BulkCipherAlgo, compression_algorithm = CompAlg}} = ReadState0, ConnnectionStates0) -> AAD = calc_aad(Type, Version, Epoch, Seq), - case ssl_record:decipher_aead(dtls_v1:corresponding_tls_version(Version), - CipherFragment, ReadState0, AAD) of - {PlainFragment, ReadState1} -> + TLSVersion = dtls_v1:corresponding_tls_version(Version), + case ssl_cipher:decipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, CipherFragment, TLSVersion) of + {PlainFragment, CipherState} -> {Plain, CompressionS1} = ssl_record:uncompress(CompAlg, PlainFragment, CompressionS0), - ReadState = ReadState1#{compression_state => CompressionS1}, + ReadState = ReadState0#{compression_state => CompressionS1, + cipher_state => CipherState}, ConnnectionStates = set_connection_state_by_epoch(ReadState, Epoch, ConnnectionStates0, read), {CipherText#ssl_tls{fragment = Plain}, ConnnectionStates}; #alert{} = Alert -> @@ -528,5 +544,4 @@ mac_hash(Version, MacAlg, MacSecret, SeqNo, Type, Length, Fragment) -> Length, Fragment). calc_aad(Type, {MajVer, MinVer}, Epoch, SeqNo) -> - NewSeq = (Epoch bsl 48) + SeqNo, - <<NewSeq:64/integer, ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. + <<?UINT16(Epoch), ?UINT48(SeqNo), ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 8e6860e9dc..d04f09efdc 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -40,7 +40,7 @@ ec_keyed_suites/0, anonymous_suites/1, psk_suites/1, srp_suites/0, rc4_suites/1, des_suites/1, openssl_suite/1, openssl_suite_name/1, filter/2, filter_suites/1, hash_algorithm/1, sign_algorithm/1, is_acceptable_hash/2, is_fallback/1, - random_bytes/1, calc_aad/3, calc_mac_hash/4, + random_bytes/1, calc_mac_hash/4, is_stream_ciphersuite/1]). -export_type([cipher_suite/0, @@ -157,7 +157,7 @@ cipher_aead(?CHACHA20_POLY1305, CipherState, SeqNo, AAD, Fragment, Version) -> aead_cipher(chacha20_poly1305, #cipher_state{key=Key} = CipherState, SeqNo, AAD0, Fragment, _Version) -> CipherLen = erlang:iolist_size(Fragment), AAD = <<AAD0/binary, ?UINT16(CipherLen)>>, - Nonce = <<SeqNo:64/integer>>, + Nonce = ?uint64(SeqNo), {Content, CipherTag} = crypto:block_encrypt(chacha20_poly1305, Key, Nonce, {AAD, Fragment}), {<<Content/binary, CipherTag/binary>>, CipherState}; aead_cipher(Type, #cipher_state{key=Key, iv = IV0, nonce = Nonce} = CipherState, _SeqNo, AAD0, Fragment, _Version) -> @@ -280,7 +280,7 @@ aead_ciphertext_to_state(chacha20_poly1305, SeqNo, _IV, AAD0, Fragment, _Version CipherLen = size(Fragment) - 16, <<CipherText:CipherLen/bytes, CipherTag:16/bytes>> = Fragment, AAD = <<AAD0/binary, ?UINT16(CipherLen)>>, - Nonce = <<SeqNo:64/integer>>, + Nonce = ?uint64(SeqNo), {Nonce, AAD, CipherText, CipherTag}; aead_ciphertext_to_state(_, _SeqNo, <<Salt:4/bytes, _/binary>>, AAD0, Fragment, _Version) -> CipherLen = size(Fragment) - 24, @@ -1531,10 +1531,6 @@ is_fallback(CipherSuites)-> random_bytes(N) -> crypto:strong_rand_bytes(N). -calc_aad(Type, {MajVer, MinVer}, - #{sequence_number := SeqNo}) -> - <<SeqNo:64/integer, ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. - calc_mac_hash(Type, Version, PlainFragment, #{sequence_number := SeqNo, mac_secret := MacSecret, diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 539e189c4f..24e52655b0 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -45,11 +45,7 @@ -export([compress/3, uncompress/3, compressions/0]). %% Payload encryption/decryption --export([cipher/4, decipher/4, is_correct_mac/2, - cipher_aead/4, decipher_aead/4]). - -%% Encoding --export([encode_plain_text/4]). +-export([cipher/4, decipher/4, cipher_aead/4, is_correct_mac/2]). -export_type([ssl_version/0, ssl_atom_version/0, connection_states/0, connection_state/0]). @@ -271,26 +267,6 @@ set_pending_cipher_state(#{pending_read := Read, pending_read => Read#{cipher_state => ServerState}, pending_write => Write#{cipher_state => ClientState}}. -encode_plain_text(Type, Version, Data, #{compression_state := CompS0, - security_parameters := - #security_parameters{ - cipher_type = ?AEAD, - compression_algorithm = CompAlg} - } = WriteState0) -> - {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), - WriteState1 = WriteState0#{compression_state => CompS1}, - AAD = ssl_cipher:calc_aad(Type, Version, WriteState1), - ssl_record:cipher_aead(Version, Comp, WriteState1, AAD); -encode_plain_text(Type, Version, Data, #{compression_state := CompS0, - security_parameters := - #security_parameters{compression_algorithm = CompAlg} - }= WriteState0) -> - {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), - WriteState1 = WriteState0#{compression_state => CompS1}, - MacHash = ssl_cipher:calc_mac_hash(Type, Version, Comp, WriteState1), - ssl_record:cipher(Version, Comp, WriteState1, MacHash); -encode_plain_text(_,_,_,CS) -> - exit({cs, CS}). uncompress(?NULL, Data, CS) -> {Data, CS}. @@ -322,12 +298,12 @@ cipher(Version, Fragment, {CipherFragment, CipherS1} = ssl_cipher:cipher(BulkCipherAlgo, CipherS0, MacHash, Fragment, Version), {CipherFragment, WriteState0#{cipher_state => CipherS1}}. -%%-------------------------------------------------------------------- --spec cipher_aead(ssl_version(), iodata(), connection_state(), MacHash::binary()) -> - {CipherFragment::binary(), connection_state()}. -%% -%% Description: Payload encryption -%%-------------------------------------------------------------------- +%% %%-------------------------------------------------------------------- +%% -spec cipher_aead(ssl_version(), iodata(), connection_state(), MacHash::binary()) -> +%% {CipherFragment::binary(), connection_state()}. +%% %% +%% %% Description: Payload encryption +%% %%-------------------------------------------------------------------- cipher_aead(Version, Fragment, #{cipher_state := CipherS0, sequence_number := SeqNo, @@ -341,7 +317,8 @@ cipher_aead(Version, Fragment, {CipherFragment, WriteState0#{cipher_state => CipherS1}}. %%-------------------------------------------------------------------- --spec decipher(ssl_version(), binary(), connection_state(), boolean()) -> {binary(), binary(), connection_state} | #alert{}. +-spec decipher(ssl_version(), binary(), connection_state(), boolean()) -> + {binary(), binary(), connection_state} | #alert{}. %% %% Description: Payload decryption %%-------------------------------------------------------------------- @@ -359,26 +336,7 @@ decipher(Version, CipherFragment, #alert{} = Alert -> Alert end. -%%-------------------------------------------------------------------- --spec decipher_aead(ssl_version(), binary(), connection_state(), binary()) -> - {binary(), binary(), connection_state()} | #alert{}. -%% -%% Description: Payload decryption -%%-------------------------------------------------------------------- -decipher_aead(Version, CipherFragment, - #{sequence_number := SeqNo, - security_parameters := - #security_parameters{bulk_cipher_algorithm = - BulkCipherAlgo}, - cipher_state := CipherS0 - } = ReadState, AAD) -> - case ssl_cipher:decipher_aead(BulkCipherAlgo, CipherS0, SeqNo, AAD, CipherFragment, Version) of - {PlainFragment, CipherS1} -> - CS1 = ReadState#{cipher_state => CipherS1}, - {PlainFragment, CS1}; - #alert{} = Alert -> - Alert - end. + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- diff --git a/lib/ssl/src/tls_record.erl b/lib/ssl/src/tls_record.erl index 993a1622fe..065c6dc8a7 100644 --- a/lib/ssl/src/tls_record.erl +++ b/lib/ssl/src/tls_record.erl @@ -372,7 +372,7 @@ get_tls_records_aux(Data, Acc) -> end. encode_plain_text(Type, Version, Data, #{current_write := Write0} = ConnectionStates) -> - {CipherFragment, Write1} = ssl_record:encode_plain_text(Type, Version, Data, Write0), + {CipherFragment, Write1} = do_encode_plain_text(Type, Version, Data, Write0), {CipherText, Write} = encode_tls_cipher_text(Type, Version, CipherFragment, Write1), {CipherText, ConnectionStates#{current_write => Write}}. @@ -446,19 +446,24 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, #{current_read := #{compression_state := CompressionS0, sequence_number := Seq, + cipher_state := CipherS0, security_parameters := #security_parameters{ cipher_type = ?AEAD, + bulk_cipher_algorithm = + BulkCipherAlgo, compression_algorithm = CompAlg} } = ReadState0} = ConnnectionStates0, _) -> - AAD = ssl_cipher:calc_aad(Type, Version, ReadState0), - case ssl_record:decipher_aead(Version, CipherFragment, ReadState0, AAD) of - {PlainFragment, ReadState1} -> + AAD = calc_aad(Type, Version, ReadState0), + case ssl_cipher:decipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, CipherFragment, Version) of + {PlainFragment, CipherS1} -> {Plain, CompressionS1} = ssl_record:uncompress(CompAlg, PlainFragment, CompressionS0), ConnnectionStates = ConnnectionStates0#{ - current_read => ReadState1#{sequence_number => Seq + 1, - compression_state => CompressionS1}}, + current_read => ReadState0#{ + cipher_state => CipherS1, + sequence_number => Seq + 1, + compression_state => CompressionS1}}, {CipherText#ssl_tls{fragment = Plain}, ConnnectionStates}; #alert{} = Alert -> Alert @@ -489,4 +494,29 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, end; #alert{} = Alert -> Alert - end. + end. + +do_encode_plain_text(Type, Version, Data, #{compression_state := CompS0, + security_parameters := + #security_parameters{ + cipher_type = ?AEAD, + compression_algorithm = CompAlg} + } = WriteState0) -> + {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), + WriteState1 = WriteState0#{compression_state => CompS1}, + AAD = calc_aad(Type, Version, WriteState1), + ssl_record:cipher_aead(Version, Comp, WriteState1, AAD); +do_encode_plain_text(Type, Version, Data, #{compression_state := CompS0, + security_parameters := + #security_parameters{compression_algorithm = CompAlg} + }= WriteState0) -> + {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), + WriteState1 = WriteState0#{compression_state => CompS1}, + MacHash = ssl_cipher:calc_mac_hash(Type, Version, Comp, WriteState1), + ssl_record:cipher(Version, Comp, WriteState1, MacHash); +do_encode_plain_text(_,_,_,CS) -> + exit({cs, CS}). + +calc_aad(Type, {MajVer, MinVer}, + #{sequence_number := SeqNo}) -> + <<?UINT64(SeqNo), ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index b77f909dfa..b05e2c74db 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -91,11 +91,7 @@ init_per_suite(Config0) -> end_per_suite(Config0), try crypto:start() of ok -> - %% make rsa certs using oppenssl - Config1 = ssl_test_lib:make_rsa_cert(Config0), - Config2 = ssl_test_lib:make_ecdsa_cert(Config1), - Config = ssl_test_lib:make_ecdh_rsa_cert(Config2), - ssl_test_lib:cert_options(Config) + Config0 catch _:_ -> {skip, "Crypto did not start"} end. diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index 66b0c09b73..45bcdf1f78 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -74,7 +74,7 @@ tests() -> cert_expired, invalid_signature_client, invalid_signature_server, - extended_key_usage_verify_client, + extended_key_usage_verify_both, extended_key_usage_verify_server, critical_extension_verify_client, critical_extension_verify_server, @@ -88,18 +88,14 @@ error_handling_tests()-> unknown_server_ca_accept_verify_peer, unknown_server_ca_accept_backwardscompatibility, no_authority_key_identifier, - no_authority_key_identifier_and_nonstandard_encoding]. + no_authority_key_identifier_keyEncipherment]. -init_per_suite(Config0) -> +init_per_suite(Config) -> catch crypto:stop(), try crypto:start() of ok -> - ssl_test_lib:clean_start(), - %% make rsa certs using oppenssl - {ok, _} = make_certs:all(proplists:get_value(data_dir, Config0), - proplists:get_value(priv_dir, Config0)), - Config = ssl_test_lib:make_dsa_cert(Config0), - ssl_test_lib:cert_options(Config) + ssl_test_lib:clean_start(), + ssl_test_lib:make_rsa_cert(Config) catch _:_ -> {skip, "Crypto did not start"} end. @@ -108,49 +104,39 @@ end_per_suite(_Config) -> ssl:stop(), application:stop(crypto). -init_per_group(tls, Config) -> +init_per_group(tls, Config0) -> Version = tls_record:protocol_version(tls_record:highest_protocol_version([])), ssl:stop(), application:load(ssl), application:set_env(ssl, protocol_version, Version), - application:set_env(ssl, bypass_pem_cache, Version), ssl:start(), - NewConfig = proplists:delete(protocol, Config), - [{protocol, tls}, {version, tls_record:protocol_version(Version)} | NewConfig]; + Config = proplists:delete(protocol, Config0), + [{protocol, tls}, {version, tls_record:protocol_version(Version)} | Config]; -init_per_group(dtls, Config) -> +init_per_group(dtls, Config0) -> Version = dtls_record:protocol_version(dtls_record:highest_protocol_version([])), ssl:stop(), application:load(ssl), application:set_env(ssl, protocol_version, Version), - application:set_env(ssl, bypass_pem_cache, Version), ssl:start(), - NewConfig = proplists:delete(protocol_opts, proplists:delete(protocol, Config)), - [{protocol, dtls}, {protocol_opts, [{protocol, dtls}]}, {version, dtls_record:protocol_version(Version)} | NewConfig]; + Config = proplists:delete(protocol_opts, proplists:delete(protocol, Config0)), + [{protocol, dtls}, {protocol_opts, [{protocol, dtls}]}, {version, dtls_record:protocol_version(Version)} | Config]; init_per_group(active, Config) -> - [{active, true}, {receive_function, send_recv_result_active} | Config]; + [{active, true}, {receive_function, send_recv_result_active} | Config]; init_per_group(active_once, Config) -> - [{active, once}, {receive_function, send_recv_result_active_once} | Config]; + [{active, once}, {receive_function, send_recv_result_active_once} | Config]; init_per_group(passive, Config) -> - [{active, false}, {receive_function, send_recv_result} | Config]; + [{active, false}, {receive_function, send_recv_result} | Config]; +init_per_group(error_handling, Config) -> + [{active, false}, {receive_function, send_recv_result} | Config]; + init_per_group(_, Config) -> Config. end_per_group(_GroupName, Config) -> Config. -init_per_testcase(TestCase, Config) when TestCase == cert_expired; - TestCase == invalid_signature_client; - TestCase == invalid_signature_server; - TestCase == extended_key_usage_verify_none; - TestCase == extended_key_usage_verify_peer; - TestCase == critical_extension_verify_none; - TestCase == critical_extension_verify_peer; - TestCase == no_authority_key_identifier; - TestCase == no_authority_key_identifier_and_nonstandard_encoding-> - ssl:clear_pem_cache(), - init_per_testcase(common, Config); init_per_testcase(_TestCase, Config) -> ssl:stop(), ssl:start(), @@ -168,23 +154,23 @@ end_per_testcase(_TestCase, Config) -> verify_peer() -> [{doc,"Test option verify_peer"}]. verify_peer(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, - {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active}, {verify, verify_peer} - | ServerOpts]}]), + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active}, {verify, verify_peer} + | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active} | ClientOpts]}]), - + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active}, {verify, verify_peer} | ClientOpts]}]), + ssl_test_lib:check_result(Server, ok, Client, ok), ssl_test_lib:close(Server), ssl_test_lib:close(Client). @@ -194,23 +180,24 @@ verify_none() -> [{doc,"Test option verify_none"}]. verify_none(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, - {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active}, {verify, verify_none} - | ServerOpts]}]), + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active}, {verify, verify_none} + | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active} | ClientOpts]}]), + {from, self()}, + {mfa, {ssl_test_lib, ReceiveFunction, []}}, + {options, [{active, Active}, + {verify, verify_none} | ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client, ok), ssl_test_lib:close(Server), @@ -222,8 +209,8 @@ server_verify_client_once() -> [{doc,"Test server option verify_client_once"}]. server_verify_client_once(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, []), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, []), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), @@ -239,7 +226,7 @@ server_verify_client_once(Config) when is_list(Config) -> {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{active, Active} | ClientOpts]}]), + {options, [{active, Active} | ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client0, ok), Server ! {listen, {mfa, {ssl_test_lib, no_result, []}}}, @@ -261,8 +248,8 @@ server_require_peer_cert_ok() -> server_require_peer_cert_ok(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -290,20 +277,21 @@ server_require_peer_cert_fail() -> server_require_peer_cert_fail(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], BadClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), + Active = proplists:get_value(active, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, - {options, [{active, false} | ServerOpts]}]), + {options, [{active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, - {options, [{active, false} | BadClientOpts]}]), + {options, [{active, Active} | BadClientOpts]}]), receive {Server, {error, {tls_alert, "handshake failure"}}} -> receive @@ -321,24 +309,25 @@ server_require_peer_cert_partial_chain() -> server_require_peer_cert_partial_chain(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + Active = proplists:get_value(active, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), {ok, ClientCAs} = file:read_file(proplists:get_value(cacertfile, ClientOpts)), - [{_,RootCA,_}, {_, _, _}] = public_key:pem_decode(ClientCAs), + [{_,RootCA,_} | _] = public_key:pem_decode(ClientCAs), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, no_result, []}}, - {options, [{active, false} | ServerOpts]}]), + {options, [{active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, no_result, []}}, - {options, [{active, false}, + {options, [{active, Active}, {cacerts, [RootCA]} | proplists:delete(cacertfile, ClientOpts)]}]), receive @@ -356,14 +345,14 @@ server_require_peer_cert_allow_partial_chain() -> server_require_peer_cert_allow_partial_chain(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), {ok, ClientCAs} = file:read_file(proplists:get_value(cacertfile, ClientOpts)), - [{_,_,_}, {_, IntermidiateCA, _}] = public_key:pem_decode(ClientCAs), + [{_,_,_}, {_, IntermidiateCA, _} | _] = public_key:pem_decode(ClientCAs), PartialChain = fun(CertChain) -> case lists:member(IntermidiateCA, CertChain) of @@ -398,12 +387,12 @@ server_require_peer_cert_do_not_allow_partial_chain() -> server_require_peer_cert_do_not_allow_partial_chain(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), {ok, ServerCAs} = file:read_file(proplists:get_value(cacertfile, ServerOpts)), - [{_,_,_}, {_, IntermidiateCA, _}] = public_key:pem_decode(ServerCAs), + [{_,_,_}, {_, IntermidiateCA, _} | _] = public_key:pem_decode(ServerCAs), PartialChain = fun(_CertChain) -> unknown_ca @@ -439,12 +428,12 @@ server_require_peer_cert_partial_chain_fun_fail() -> server_require_peer_cert_partial_chain_fun_fail(Config) when is_list(Config) -> ServerOpts = [{verify, verify_peer}, {fail_if_no_peer_cert, true} - | ssl_test_lib:ssl_options(server_verification_opts, Config)], - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + | ssl_test_lib:ssl_options(server_rsa_opts, Config)], + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), {ok, ServerCAs} = file:read_file(proplists:get_value(cacertfile, ServerOpts)), - [{_,_,_}, {_, IntermidiateCA, _}] = public_key:pem_decode(ServerCAs), + [{_,_,_}, {_, IntermidiateCA, _} | _] = public_key:pem_decode(ServerCAs), PartialChain = fun(_CertChain) -> ture = false %% crash on purpose @@ -479,8 +468,8 @@ verify_fun_always_run_client() -> [{doc,"Verify that user verify_fun is always run (for valid and valid_peer not only unknown_extension)"}]. verify_fun_always_run_client(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, @@ -524,8 +513,8 @@ verify_fun_always_run_client(Config) when is_list(Config) -> verify_fun_always_run_server() -> [{doc,"Verify that user verify_fun is always run (for valid and valid_peer not only unknown_extension)"}]. verify_fun_always_run_server(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), %% If user verify fun is called correctly we fail the connection. @@ -573,63 +562,28 @@ cert_expired() -> [{doc,"Test server with expired certificate"}]. cert_expired(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), - - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join(PrivDir, "server/expired_cert.pem"), - [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(ServerCertFile), - OTPCert = public_key:pkix_decode_cert(DerCert, otp), - OTPTbsCert = OTPCert#'OTPCertificate'.tbsCertificate, - {Year, Month, Day} = date(), - {Hours, Min, Sec} = time(), - NotBeforeStr = lists:flatten(io_lib:format("~p~s~s~s~s~sZ",[Year-2, - two_digits_str(Month), - two_digits_str(Day), - two_digits_str(Hours), - two_digits_str(Min), - two_digits_str(Sec)])), - NotAfterStr = lists:flatten(io_lib:format("~p~s~s~s~s~sZ",[Year-1, - two_digits_str(Month), - two_digits_str(Day), - two_digits_str(Hours), - two_digits_str(Min), - two_digits_str(Sec)])), - NewValidity = {'Validity', {generalTime, NotBeforeStr}, {generalTime, NotAfterStr}}, - - ct:log("Validity: ~p ~n NewValidity: ~p ~n", - [OTPTbsCert#'OTPTBSCertificate'.validity, NewValidity]), - - NewOTPTbsCert = OTPTbsCert#'OTPTBSCertificate'{validity = NewValidity}, - NewServerDerCert = public_key:pkix_sign(NewOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewServerCertFile, [{'Certificate', NewServerDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - + Active = proplists:get_value(active, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_ca_0, + [{validity, {{Year-2, Month, Day}, + {Year-1, Month, Day}}}]}], + Config, "_expired"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, - {options, NewServerOpts}]), + {options, [{active, Active}| ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, - {options, [{verify, verify_peer} | ClientOpts]}]), - receive - {Client, {error, {tls_alert, "certificate expired"}}} -> - receive - {Server, {error, {tls_alert, "certificate expired"}}} -> - ok; - {Server, {error, closed}} -> - ok - end - end. + {options, [{verify, verify_peer}, {active, Active} | ClientOpts]}]), + + tcp_delivery_workaround(Server, {error, {tls_alert, "certificate expired"}}, + Client, {error, {tls_alert, "certificate expired"}}). two_digits_str(N) when N < 10 -> lists:flatten(io_lib:format("0~p", [N])); @@ -638,60 +592,32 @@ two_digits_str(N) -> %%-------------------------------------------------------------------- extended_key_usage_verify_server() -> - [{doc,"Test cert that has a critical extended_key_usage extension in verify_peer mode for server"}]. - -extended_key_usage_verify_server(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), + [{doc,"Test cert that has a critical extended_key_usage extension in server cert"}]. + +extended_key_usage_verify_server(Config) when is_list(Config) -> + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, + [{?'id-ce-extKeyUsage', + [?'id-kp-serverAuth'], true}] + }]}], Config, "_keyusage_server"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join(PrivDir, "server/new_cert.pem"), - [{'Certificate', ServerDerCert, _}] = ssl_test_lib:pem_to_der(ServerCertFile), - ServerOTPCert = public_key:pkix_decode_cert(ServerDerCert, otp), - ServerExtKeyUsageExt = {'Extension', ?'id-ce-extKeyUsage', true, [?'id-kp-serverAuth']}, - ServerOTPTbsCert = ServerOTPCert#'OTPCertificate'.tbsCertificate, - ServerExtensions = ServerOTPTbsCert#'OTPTBSCertificate'.extensions, - NewServerOTPTbsCert = ServerOTPTbsCert#'OTPTBSCertificate'{extensions = - [ServerExtKeyUsageExt | - ServerExtensions]}, - NewServerDerCert = public_key:pkix_sign(NewServerOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewServerCertFile, [{'Certificate', NewServerDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join(PrivDir, "client/new_cert.pem"), - [{'Certificate', ClientDerCert, _}] = ssl_test_lib:pem_to_der(ClientCertFile), - ClientOTPCert = public_key:pkix_decode_cert(ClientDerCert, otp), - ClientExtKeyUsageExt = {'Extension', ?'id-ce-extKeyUsage', true, [?'id-kp-clientAuth']}, - ClientOTPTbsCert = ClientOTPCert#'OTPCertificate'.tbsCertificate, - ClientExtensions = ClientOTPTbsCert#'OTPTBSCertificate'.extensions, - NewClientOTPTbsCert = ClientOTPTbsCert#'OTPTBSCertificate'{extensions = - [ClientExtKeyUsageExt | - ClientExtensions]}, - NewClientDerCert = public_key:pkix_sign(NewClientOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewClientCertFile, [{'Certificate', NewClientDerCert, not_encrypted}]), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_peer}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | - NewClientOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | + ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client, ok), @@ -699,60 +625,35 @@ extended_key_usage_verify_server(Config) when is_list(Config) -> ssl_test_lib:close(Client). %%-------------------------------------------------------------------- -extended_key_usage_verify_client() -> +extended_key_usage_verify_both() -> [{doc,"Test cert that has a critical extended_key_usage extension in client verify_peer mode"}]. -extended_key_usage_verify_client(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), +extended_key_usage_verify_both(Config) when is_list(Config) -> + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, [{?'id-ce-extKeyUsage', + [?'id-kp-serverAuth'], true}] + }]}, + {client_peer_opts, + [{extensions, [{?'id-ce-extKeyUsage', + [?'id-kp-clientAuth'], true}] + }]}], Config, "_keyusage_both"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join(PrivDir, "server/new_cert.pem"), - [{'Certificate', ServerDerCert, _}] = ssl_test_lib:pem_to_der(ServerCertFile), - ServerOTPCert = public_key:pkix_decode_cert(ServerDerCert, otp), - ServerExtKeyUsageExt = {'Extension', ?'id-ce-extKeyUsage', true, [?'id-kp-serverAuth']}, - ServerOTPTbsCert = ServerOTPCert#'OTPCertificate'.tbsCertificate, - ServerExtensions = ServerOTPTbsCert#'OTPTBSCertificate'.extensions, - NewServerOTPTbsCert = ServerOTPTbsCert#'OTPTBSCertificate'{extensions = - [ServerExtKeyUsageExt | - ServerExtensions]}, - NewServerDerCert = public_key:pkix_sign(NewServerOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewServerCertFile, [{'Certificate', NewServerDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join(PrivDir, "client/new_cert.pem"), - [{'Certificate', ClientDerCert, _}] = ssl_test_lib:pem_to_der(ClientCertFile), - ClientOTPCert = public_key:pkix_decode_cert(ClientDerCert, otp), - ClientExtKeyUsageExt = {'Extension', ?'id-ce-extKeyUsage', true, [?'id-kp-clientAuth']}, - ClientOTPTbsCert = ClientOTPCert#'OTPCertificate'.tbsCertificate, - ClientExtensions = ClientOTPTbsCert#'OTPTBSCertificate'.extensions, - NewClientOTPTbsCert = ClientOTPTbsCert#'OTPTBSCertificate'{extensions = - [ClientExtKeyUsageExt | - ClientExtensions]}, - NewClientDerCert = public_key:pkix_sign(NewClientOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewClientCertFile, [{'Certificate', NewClientDerCert, not_encrypted}]), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewClientOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client, ok), @@ -764,132 +665,103 @@ critical_extension_verify_server() -> [{doc,"Test cert that has a critical unknown extension in verify_peer mode"}]. critical_extension_verify_server(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{client_peer_opts, + [{extensions, [{{2,16,840,1,113730,1,1}, + <<3,2,6,192>>, true}] + }]}], Config, "_client_unknown_extension"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - NewCertName = integer_to_list(erlang:unique_integer()) ++ ".pem", - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join([PrivDir, "server", NewCertName]), - add_critical_netscape_cert_type(ServerCertFile, NewServerCertFile, KeyFile), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join([PrivDir, "client", NewCertName]), - add_critical_netscape_cert_type(ClientCertFile, NewClientCertFile, KeyFile), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error( [{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_peer}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error( [{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewClientOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ClientOpts]}]), %% This certificate has a critical extension that we don't - %% understand. Therefore, verification should fail. - tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}}, - Client, {error, {tls_alert, "unsupported certificate"}}), + %% understand. Therefore, verification should fail. - ssl_test_lib:close(Server), - ok. + tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}}, + Client, {error, {tls_alert, "unsupported certificate"}}), + + ssl_test_lib:close(Server). %%-------------------------------------------------------------------- critical_extension_verify_client() -> [{doc,"Test cert that has a critical unknown extension in verify_peer mode"}]. critical_extension_verify_client(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, [{{2,16,840,1,113730,1,1}, + <<3,2,6,192>>, true}] + }]}], Config, "_server_unknown_extensions"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - NewCertName = integer_to_list(erlang:unique_integer()) ++ ".pem", - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join([PrivDir, "server", NewCertName]), - add_critical_netscape_cert_type(ServerCertFile, NewServerCertFile, KeyFile), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join([PrivDir, "client", NewCertName]), - add_critical_netscape_cert_type(ClientCertFile, NewClientCertFile, KeyFile), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error( [{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client_error( [{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_peer}, {active, Active} | NewClientOpts]}]), + {options, [{verify, verify_peer}, {active, Active} | ClientOpts]}]), %% This certificate has a critical extension that we don't %% understand. Therefore, verification should fail. - tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}}, - Client, {error, {tls_alert, "unsupported certificate"}}), + ssl_test_lib:check_result(Server, {error, {tls_alert, "unsupported certificate"}}, + Client, {error, {tls_alert, "unsupported certificate"}}), + + ssl_test_lib:close(Server). - ssl_test_lib:close(Server), - ok. %%-------------------------------------------------------------------- critical_extension_verify_none() -> [{doc,"Test cert that has a critical unknown extension in verify_none mode"}]. critical_extension_verify_none(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{client_peer_opts, + [{extensions, + [{{2,16,840,1,113730,1,1}, + <<3,2,6,192>>, true}] + }]}], Config, "_unknown_extensions"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), Active = proplists:get_value(active, Config), ReceiveFunction = proplists:get_value(receive_function, Config), - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - NewCertName = integer_to_list(erlang:unique_integer()) ++ ".pem", - - ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join([PrivDir, "server", NewCertName]), - add_critical_netscape_cert_type(ServerCertFile, NewServerCertFile, KeyFile), - NewServerOpts = [{certfile, NewServerCertFile} | proplists:delete(certfile, ServerOpts)], - - ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join([PrivDir, "client", NewCertName]), - add_critical_netscape_cert_type(ClientCertFile, NewClientCertFile, KeyFile), - NewClientOpts = [{certfile, NewClientCertFile} | proplists:delete(certfile, ClientOpts)], - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server( [{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewServerOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client( [{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, ReceiveFunction, []}}, - {options, [{verify, verify_none}, {active, Active} | NewClientOpts]}]), + {options, [{verify, verify_none}, {active, Active} | ClientOpts]}]), %% This certificate has a critical extension that we don't %% understand. But we're using `verify_none', so verification @@ -897,28 +769,7 @@ critical_extension_verify_none(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok, Client, ok), ssl_test_lib:close(Server), - ssl_test_lib:close(Client), - ok. - -add_critical_netscape_cert_type(CertFile, NewCertFile, KeyFile) -> - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(CertFile), - OTPCert = public_key:pkix_decode_cert(DerCert, otp), - %% This is the "Netscape Cert Type" extension, telling us that the - %% certificate can be used for SSL clients and SSL servers. - NetscapeCertTypeExt = #'Extension'{ - extnID = {2,16,840,1,113730,1,1}, - critical = true, - extnValue = <<3,2,6,192>>}, - OTPTbsCert = OTPCert#'OTPCertificate'.tbsCertificate, - Extensions = OTPTbsCert#'OTPTBSCertificate'.extensions, - NewOTPTbsCert = OTPTbsCert#'OTPTBSCertificate'{ - extensions = [NetscapeCertTypeExt] ++ Extensions}, - NewDerCert = public_key:pkix_sign(NewOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewCertFile, [{'Certificate', NewDerCert, not_encrypted}]), - ok. + ssl_test_lib:close(Client). %%-------------------------------------------------------------------- no_authority_key_identifier() -> @@ -926,35 +777,21 @@ no_authority_key_identifier() -> " but are present in trusted certs db."}]. no_authority_key_identifier(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), - - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - CertFile = proplists:get_value(certfile, ServerOpts), - NewCertFile = filename:join(PrivDir, "server/new_cert.pem"), - [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(CertFile), - OTPCert = public_key:pkix_decode_cert(DerCert, otp), - OTPTbsCert = OTPCert#'OTPCertificate'.tbsCertificate, - Extensions = OTPTbsCert#'OTPTBSCertificate'.extensions, - NewExtensions = delete_authority_key_extension(Extensions, []), - NewOTPTbsCert = OTPTbsCert#'OTPTBSCertificate'{extensions = NewExtensions}, - - ct:log("Extensions ~p~n, NewExtensions: ~p~n", [Extensions, NewExtensions]), - - NewDerCert = public_key:pkix_sign(NewOTPTbsCert, Key), - ssl_test_lib:der_to_pem(NewCertFile, [{'Certificate', NewDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewCertFile} | proplists:delete(certfile, ServerOpts)], + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, [{auth_key_id, undefined}] + }]}, + {client_peer_opts, + [{extensions, [{auth_key_id, undefined}] + }]}], Config, "_peer_no_auth_key_id"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result_active, []}}, - {options, NewServerOpts}]), + {options, ServerOpts}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -970,53 +807,35 @@ no_authority_key_identifier(Config) when is_list(Config) -> delete_authority_key_extension([], Acc) -> lists:reverse(Acc); delete_authority_key_extension([#'Extension'{extnID = ?'id-ce-authorityKeyIdentifier'} | Rest], - Acc) -> + Acc) -> delete_authority_key_extension(Rest, Acc); delete_authority_key_extension([Head | Rest], Acc) -> delete_authority_key_extension(Rest, [Head | Acc]). %%-------------------------------------------------------------------- -no_authority_key_identifier_and_nonstandard_encoding() -> - [{doc, "Test cert with nonstandard encoding that does not have" - " authorityKeyIdentifier extension but are present in trusted certs db."}]. - -no_authority_key_identifier_and_nonstandard_encoding(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - PrivDir = proplists:get_value(priv_dir, Config), - - KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), - [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), - - CertFile = proplists:get_value(certfile, ServerOpts), - NewCertFile = filename:join(PrivDir, "server/new_cert.pem"), - [{'Certificate', DerCert, _}] = ssl_test_lib:pem_to_der(CertFile), - ServerCert = public_key:pkix_decode_cert(DerCert, plain), - ServerTbsCert = ServerCert#'Certificate'.tbsCertificate, - Extensions0 = ServerTbsCert#'TBSCertificate'.extensions, - %% need to remove authorityKeyIdentifier extension to cause DB lookup by signature - Extensions = delete_authority_key_extension(Extensions0, []), - NewExtensions = replace_key_usage_extension(Extensions, []), - NewServerTbsCert = ServerTbsCert#'TBSCertificate'{extensions = NewExtensions}, - - ct:log("Extensions ~p~n, NewExtensions: ~p~n", [Extensions, NewExtensions]), - - TbsDer = public_key:pkix_encode('TBSCertificate', NewServerTbsCert, plain), - Sig = public_key:sign(TbsDer, md5, Key), - NewServerCert = ServerCert#'Certificate'{tbsCertificate = NewServerTbsCert, signature = Sig}, - NewDerCert = public_key:pkix_encode('Certificate', NewServerCert, plain), - ssl_test_lib:der_to_pem(NewCertFile, [{'Certificate', NewDerCert, not_encrypted}]), - NewServerOpts = [{certfile, NewCertFile} | proplists:delete(certfile, ServerOpts)], - +no_authority_key_identifier_keyEncipherment() -> + [{doc, "Test cert with keyEncipherment key_usage an no" + " authorityKeyIdentifier extension, but are present in trusted certs db."}]. + +no_authority_key_identifier_keyEncipherment(Config) when is_list(Config) -> + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{server_peer_opts, + [{extensions, [{auth_key_id, undefined}, + {key_usage, [digitalSignature, + keyEncipherment]}] + }]}, + {client_peer_opts, + [{extensions, [{auth_key_id, undefined}] + }]}], Config, "_peer_keyEncipherment"), + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result_active, []}}, - {options, [{active, true} | NewServerOpts]}]), + {options, [{active, true} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -1028,14 +847,6 @@ no_authority_key_identifier_and_nonstandard_encoding(Config) when is_list(Config ssl_test_lib:close(Server), ssl_test_lib:close(Client). -replace_key_usage_extension([], Acc) -> - lists:reverse(Acc); -replace_key_usage_extension([#'Extension'{extnID = ?'id-ce-keyUsage'} = E | Rest], Acc) -> - %% A nonstandard DER encoding of [digitalSignature, keyEncipherment] - Val = <<3, 2, 0, 16#A0>>, - replace_key_usage_extension(Rest, [E#'Extension'{extnValue = Val} | Acc]); -replace_key_usage_extension([Head | Rest], Acc) -> - replace_key_usage_extension(Rest, [Head | Acc]). %%-------------------------------------------------------------------- @@ -1043,16 +854,16 @@ invalid_signature_server() -> [{doc,"Test client with invalid signature"}]. invalid_signature_server(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), PrivDir = proplists:get_value(priv_dir, Config), - KeyFile = filename:join(PrivDir, "server/key.pem"), + KeyFile = proplists:get_value(keyfile, ServerOpts), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ServerCertFile = proplists:get_value(certfile, ServerOpts), - NewServerCertFile = filename:join(PrivDir, "server/invalid_cert.pem"), + NewServerCertFile = filename:join(PrivDir, "server_invalid_cert.pem"), [{'Certificate', ServerDerCert, _}] = ssl_test_lib:pem_to_der(ServerCertFile), ServerOTPCert = public_key:pkix_decode_cert(ServerDerCert, otp), ServerOTPTbsCert = ServerOTPCert#'OTPCertificate'.tbsCertificate, @@ -1071,8 +882,8 @@ invalid_signature_server(Config) when is_list(Config) -> {from, self()}, {options, [{verify, verify_peer} | ClientOpts]}]), - tcp_delivery_workaround(Server, {error, {tls_alert, "bad certificate"}}, - Client, {error, {tls_alert, "bad certificate"}}). + tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}}, + Client, {error, {tls_alert, "unknown ca"}}). %%-------------------------------------------------------------------- @@ -1080,16 +891,16 @@ invalid_signature_client() -> [{doc,"Test server with invalid signature"}]. invalid_signature_client(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), PrivDir = proplists:get_value(priv_dir, Config), - KeyFile = filename:join(PrivDir, "client/key.pem"), + KeyFile = proplists:get_value(keyfile, ClientOpts), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ClientCertFile = proplists:get_value(certfile, ClientOpts), - NewClientCertFile = filename:join(PrivDir, "client/invalid_cert.pem"), + NewClientCertFile = filename:join(PrivDir, "client_invalid_cert.pem"), [{'Certificate', ClientDerCert, _}] = ssl_test_lib:pem_to_der(ClientCertFile), ClientOTPCert = public_key:pkix_decode_cert(ClientDerCert, otp), ClientOTPTbsCert = ClientOTPCert#'OTPCertificate'.tbsCertificate, @@ -1108,8 +919,8 @@ invalid_signature_client(Config) when is_list(Config) -> {from, self()}, {options, NewClientOpts}]), - tcp_delivery_workaround(Server, {error, {tls_alert, "bad certificate"}}, - Client, {error, {tls_alert, "bad certificate"}}). + tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}}, + Client, {error, {tls_alert, "unknown ca"}}). %%-------------------------------------------------------------------- @@ -1118,8 +929,14 @@ client_with_cert_cipher_suites_handshake() -> [{doc, "Test that client with a certificate without keyEncipherment usage " " extension can connect to a server with restricted cipher suites "}]. client_with_cert_cipher_suites_handshake(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts_digital_signature_only, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + {ClientOpts0, ServerOpts0} = ssl_test_lib:make_rsa_cert_chains([{client_peer_opts, + [{extensions, + [{key_usage, [digitalSignature]}] + }]}], Config, "_sign_only_extensions"), + + + ClientOpts = ssl_test_lib:ssl_options(ClientOpts0, Config), + ServerOpts = ssl_test_lib:ssl_options(ServerOpts0, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -1148,7 +965,7 @@ client_with_cert_cipher_suites_handshake(Config) when is_list(Config) -> server_verify_no_cacerts() -> [{doc,"Test server must have cacerts if it wants to verify client"}]. server_verify_no_cacerts(Config) when is_list(Config) -> - ServerOpts = proplists:delete(cacertfile, ssl_test_lib:ssl_options(server_opts, Config)), + ServerOpts = proplists:delete(cacertfile, ssl_test_lib:ssl_options(server_rsa_opts, Config)), {_, ServerNode, _} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1163,7 +980,7 @@ unknown_server_ca_fail() -> [{doc,"Test that the client fails if the ca is unknown in verify_peer mode"}]. unknown_server_ca_fail(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1207,7 +1024,7 @@ unknown_server_ca_accept_verify_none() -> [{doc,"Test that the client succeds if the ca is unknown in verify_none mode"}]. unknown_server_ca_accept_verify_none(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1232,7 +1049,7 @@ unknown_server_ca_accept_verify_peer() -> " with a verify_fun that accepts the unknown ca error"}]. unknown_server_ca_accept_verify_peer(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, @@ -1271,7 +1088,7 @@ unknown_server_ca_accept_backwardscompatibility() -> [{doc,"Test that old style verify_funs will work"}]. unknown_server_ca_accept_backwardscompatibility(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(empty_client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 302b5178a5..b8fd5dc975 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -485,6 +485,18 @@ make_dsa_cert(Config) -> {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]} | Config]. +make_rsa_cert_chains(ChainConf, Config, Suffix) -> + CryptoSupport = crypto:supports(), + KeyGenSpec = key_gen_info(rsa, rsa), + ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "rsa" ++ Suffix]), + ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "rsa" ++ Suffix]), + GenCertData = x509_test:gen_test_certs([{digest, appropriate_sha(CryptoSupport)} | KeyGenSpec] ++ ChainConf), + [{server_config, ServerConf}, + {client_config, ClientConf}] = + x509_test:gen_pem_config_files(GenCertData, ClientFileBase, ServerFileBase), + {[{verify, verify_peer} | ClientConf], + [{reuseaddr, true}, {verify, verify_peer} | ServerConf] + }. make_ec_cert_chains(ClientChainType, ServerChainType, Config) -> CryptoSupport = crypto:supports(), @@ -524,6 +536,11 @@ key_gen_spec(Role, ecdhe_rsa) -> [{list_to_atom(Role ++ "_key_gen"), hardcode_rsa_key(1)}, {list_to_atom(Role ++ "_key_gen_chain"), [hardcode_rsa_key(2), hardcode_rsa_key(3)]} + ]; +key_gen_spec(Role, rsa) -> + [{list_to_atom(Role ++ "_key_gen"), hardcode_rsa_key(1)}, + {list_to_atom(Role ++ "_key_gen_chain"), [hardcode_rsa_key(2), + hardcode_rsa_key(3)]} ]. make_ecdsa_cert(Config) -> CryptoSupport = crypto:supports(), @@ -571,7 +588,8 @@ make_rsa_cert(Config) -> {server_rsa_verify_opts, [{ssl_imp, new}, {reuseaddr, true}, {verify, verify_peer} | ServerConf]}, - {client_rsa_opts, ClientConf} + {client_rsa_opts, ClientConf}, + {client_rsa_verify_opts, [{verify, verify_peer} |ClientConf]} | Config]; false -> Config @@ -935,9 +953,9 @@ available_suites(Version) -> rsa_non_signed_suites(Version) -> lists:filter(fun({rsa, _, _}) -> - true; + false; (_) -> - false + true end, available_suites(Version)). @@ -1398,10 +1416,13 @@ do_supports_ssl_tls_version(Port) -> true end. -ssl_options(Option, Config) -> +ssl_options(Option, Config) when is_atom(Option) -> ProtocolOpts = proplists:get_value(protocol_opts, Config, []), Opts = proplists:get_value(Option, Config, []), - Opts ++ ProtocolOpts. + Opts ++ ProtocolOpts; +ssl_options(Options, Config) -> + ProtocolOpts = proplists:get_value(protocol_opts, Config, []), + Options ++ ProtocolOpts. protocol_version(Config) -> protocol_version(Config, atom). diff --git a/lib/ssl/test/x509_test.erl b/lib/ssl/test/x509_test.erl index 13f8dfdaa9..c36e96013b 100644 --- a/lib/ssl/test/x509_test.erl +++ b/lib/ssl/test/x509_test.erl @@ -96,7 +96,7 @@ gen_pem_config_files(GenCertData, ClientBase, ServerBase) -> public_key:generate_key(KeyGen) end. - root_cert(Role, PrivKey, Opts) -> +root_cert(Role, PrivKey, Opts) -> TBS = cert_template(), Issuer = issuer("root", Role, " ROOT CA"), OTPTBS = TBS#'OTPTBSCertificate'{ @@ -105,7 +105,7 @@ gen_pem_config_files(GenCertData, ClientBase, ServerBase) -> validity = validity(Opts), subject = Issuer, subjectPublicKeyInfo = public_key(PrivKey), - extensions = extensions(Opts) + extensions = extensions(ca, Opts) }, public_key:pkix_sign(OTPTBS, PrivKey). @@ -175,32 +175,31 @@ validity(Opts) -> #'Validity'{notBefore={generalTime, Format(DefFrom)}, notAfter ={generalTime, Format(DefTo)}}. -extensions(Opts) -> - case proplists:get_value(extensions, Opts, []) of - false -> - asn1_NOVALUE; - Exts -> - lists:flatten([extension(Ext) || Ext <- default_extensions(Exts)]) - end. +extensions(Type, Opts) -> + Exts = proplists:get_value(extensions, Opts, []), + lists:flatten([extension(Ext) || Ext <- default_extensions(Type, Exts)]). + +%% Common extension: name_constraints, policy_constraints, ext_key_usage, inhibit_any, +%% auth_key_id, subject_key_id, policy_mapping, + +default_extensions(ca, Exts) -> + Def = [{key_usage, [keyCertSign, cRLSign]}, + {basic_constraints, default}], + add_default_extensions(Def, Exts); -default_extensions(Exts) -> - Def = [{key_usage,undefined}, - {subject_altname, undefined}, - {issuer_altname, undefined}, - {basic_constraints, default}, - {name_constraints, undefined}, - {policy_constraints, undefined}, - {ext_key_usage, undefined}, - {inhibit_any, undefined}, - {auth_key_id, undefined}, - {subject_key_id, undefined}, - {policy_mapping, undefined}], +default_extensions(peer, Exts) -> + Def = [{key_usage, [digitalSignature, keyAgreement]}], + add_default_extensions(Def, Exts). + +add_default_extensions(Def, Exts) -> Filter = fun({Key, _}, D) -> - lists:keydelete(Key, 1, D) + lists:keydelete(Key, 1, D); + ({Key, _, _}, D) -> + lists:keydelete(Key, 1, D) end, Exts ++ lists:foldl(Filter, Def, Exts). - -extension({_, undefined}) -> + +extension({_, undefined}) -> []; extension({basic_constraints, Data}) -> case Data of @@ -218,6 +217,17 @@ extension({basic_constraints, Data}) -> #'Extension'{extnID = ?'id-ce-basicConstraints', extnValue = Data} end; +extension({auth_key_id, {Oid, Issuer, SNr}}) -> + #'Extension'{extnID = ?'id-ce-authorityKeyIdentifier', + extnValue = #'AuthorityKeyIdentifier'{ + keyIdentifier = Oid, + authorityCertIssuer = Issuer, + authorityCertSerialNumber = SNr}, + critical = false}; +extension({key_usage, Value}) -> + #'Extension'{extnID = ?'id-ce-keyUsage', + extnValue = Value, + critical = false}; extension({Id, Data, Critical}) -> #'Extension'{extnID = Id, extnValue = Data, critical = Critical}. @@ -277,24 +287,31 @@ cert_chain(Role, Root, RootKey, Opts, Keys) -> cert_chain(Role, Root, RootKey, Opts, Keys, 0, []). cert_chain(Role, IssuerCert, IssuerKey, Opts, [Key], _, Acc) -> + PeerOpts = list_to_atom(atom_to_list(Role) ++ "_peer_opts"), Cert = cert(Role, public_key:pkix_decode_cert(IssuerCert, otp), - IssuerKey, Key, "admin", " Peer cert", Opts), + IssuerKey, Key, "admin", " Peer cert", Opts, PeerOpts, peer), [{Cert, Key}, {IssuerCert, IssuerKey} | Acc]; cert_chain(Role, IssuerCert, IssuerKey, Opts, [Key | Keys], N, Acc) -> + CAOpts = list_to_atom(atom_to_list(Role) ++ "_ca_" ++ integer_to_list(N)), Cert = cert(Role, public_key:pkix_decode_cert(IssuerCert, otp), IssuerKey, Key, "webadmin", - " Intermidiate CA " ++ integer_to_list(N), Opts), + " Intermidiate CA " ++ integer_to_list(N), Opts, CAOpts, ca), cert_chain(Role, Cert, Key, Opts, Keys, N+1, [{IssuerCert, IssuerKey} | Acc]). -cert(Role, #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{subject = Issuer}}, - PrivKey, Key, Contact, Name, Opts) -> +cert(Role, #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{subject = Issuer, + serialNumber = SNr + }}, + PrivKey, Key, Contact, Name, Opts, CertOptsName, Type) -> + CertOpts = proplists:get_value(CertOptsName, Opts, []), TBS = cert_template(), OTPTBS = TBS#'OTPTBSCertificate'{ signature = sign_algorithm(PrivKey, Opts), issuer = Issuer, - validity = validity(Opts), + validity = validity(CertOpts), subject = subject(Contact, atom_to_list(Role) ++ Name), subjectPublicKeyInfo = public_key(Key), - extensions = extensions(Opts) + extensions = extensions(Type, + add_default_extensions([{auth_key_id, {auth_key_oid(Role), Issuer, SNr}}], + CertOpts)) }, public_key:pkix_sign(OTPTBS, PrivKey). @@ -319,3 +336,8 @@ default_key_gen() -> [{namedCurve, hd(tls_v1:ecc_curves(0))}, {namedCurve, hd(tls_v1:ecc_curves(0))}] end. + +auth_key_oid(server) -> + ?'id-kp-serverAuth'; +auth_key_oid(client) -> + ?'id-kp-clientAuth'. diff --git a/lib/stdlib/doc/src/erl_tar.xml b/lib/stdlib/doc/src/erl_tar.xml index f28d8b425b..fab7c832d5 100644 --- a/lib/stdlib/doc/src/erl_tar.xml +++ b/lib/stdlib/doc/src/erl_tar.xml @@ -292,6 +292,10 @@ <c>Fd</c> is assumed to be a file descriptor returned from function <c>file:open/2</c>.</p> <p>Otherwise, <c>Name</c> is to be a filename.</p> + <note><p>Leading slashes in tar member names will be removed before + writing the file. That is, absolute paths will be turned into + relative paths. There will be an info message written to the error + logger when paths are changed in this way.</p></note> </desc> </func> diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml index 2ddf3021ac..e06d7e467d 100644 --- a/lib/stdlib/doc/src/rand.xml +++ b/lib/stdlib/doc/src/rand.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2015</year><year>2016</year> + <year>2015</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -50,26 +50,73 @@ <p>The following algorithms are provided:</p> <taglist> - <tag><c>exsplus</c></tag> + <tag><c>exrop</c></tag> <item> - <p>Xorshift116+, 58 bits precision and period of 2^116-1</p> + <p>Xoroshiro116+, 58 bits precision and period of 2^116-1</p> <p>Jump function: equivalent to 2^64 calls</p> </item> - <tag><c>exs64</c></tag> - <item> - <p>Xorshift64*, 64 bits precision and a period of 2^64-1</p> - <p>Jump function: not available</p> - </item> - <tag><c>exs1024</c></tag> + <tag><c>exs1024s</c></tag> <item> <p>Xorshift1024*, 64 bits precision and a period of 2^1024-1</p> <p>Jump function: equivalent to 2^512 calls</p> </item> + <tag><c>exsp</c></tag> + <item> + <p>Xorshift116+, 58 bits precision and period of 2^116-1</p> + <p>Jump function: equivalent to 2^64 calls</p> + <p> + This is a corrected version of the previous default algorithm, + that now has been superseeded by Xoroshiro116+ (<c>exrop</c>). + Since there is no native 58 bit rotate instruction this + algorithm executes a little (say < 15%) faster than <c>exrop</c>. + See the + <url href="http://xorshift.di.unimi.it">algorithms' homepage</url>. + </p> + </item> </taglist> - <p>The default algorithm is <c>exsplus</c>. If a specific algorithm is + <p> + The default algorithm is <c>exrop</c> (Xoroshiro116+). + If a specific algorithm is required, ensure to always use <seealso marker="#seed-1"> - <c>seed/1</c></seealso> to initialize the state.</p> + <c>seed/1</c></seealso> to initialize the state. + </p> + + <p> + Undocumented (old) algorithms are deprecated but still implemented + so old code relying on them will produce + the same pseudo random sequences as before. + </p> + + <note> + <p> + There were a number of problems in the implementation + of the now undocumented algorithms, which is why + they are deprecated. The new algorithms are a bit slower + but do not have these problems: + </p> + <p> + Uniform integer ranges had a skew in the probability distribution + that was not noticable for small ranges but for large ranges + less than the generator's precision the probability to produce + a low number could be twice the probability for a high. + </p> + <p> + Uniform integer ranges larger than or equal to the generator's + precision used a floating point fallback that only calculated + with 52 bits which is smaller than the requested range + and therefore were not all numbers in the requested range + even possible to produce. + </p> + <p> + Uniform floats had a non-uniform density so small values + i.e less than 0.5 had got smaller intervals decreasing + as the generated value approached 0.0 although still uniformly + distributed for sufficiently large subranges. The new algorithms + produces uniformly distributed floats on the form N * 2.0^(-53) + hence equally spaced. + </p> + </note> <p>Every time a random number is requested, a state is used to calculate it and a new state is produced. The state can either be @@ -99,19 +146,19 @@ R1 = rand:uniform(),</pre> <p>Use a specified algorithm:</p> <pre> -_ = rand:seed(exs1024), +_ = rand:seed(exs1024s), R2 = rand:uniform(),</pre> <p>Use a specified algorithm with a constant seed:</p> <pre> -_ = rand:seed(exs1024, {123, 123534, 345345}), +_ = rand:seed(exs1024s, {123, 123534, 345345}), R3 = rand:uniform(),</pre> <p>Use the functional API with a non-constant seed:</p> <pre> -S0 = rand:seed_s(exsplus), +S0 = rand:seed_s(exrop), {R4, S1} = rand:uniform_s(S0),</pre> <p>Create a standard normal deviate:</p> @@ -119,6 +166,11 @@ S0 = rand:seed_s(exsplus), <pre> {SND0, S2} = rand:normal_s(S1),</pre> + <p>Create a normal deviate with mean -3 and variance 0.5:</p> + + <pre> +{ND0, S3} = rand:normal_s(-3, 0.5, S2),</pre> + <note> <p>The builtin random number generator algorithms are not cryptographically strong. If a cryptographically strong @@ -127,6 +179,39 @@ S0 = rand:seed_s(exsplus), </p> </note> + <p> + For all these generators the lowest bit(s) has got + a slightly less random behaviour than all other bits. + 1 bit for <c>exrop</c> (and <c>exsp</c>), + and 3 bits for <c>exs1024s</c>. + See for example the explanation in the + <url href="http://xoroshiro.di.unimi.it/xoroshiro128plus.c"> + Xoroshiro128+ + </url> + generator source code: + </p> + <pre> +Beside passing BigCrush, this generator passes the PractRand test suite +up to (and included) 16TB, with the exception of binary rank tests, +which fail due to the lowest bit being an LFSR; all other bits pass all +tests. We suggest to use a sign test to extract a random Boolean value.</pre> + <p> + If this is a problem; to generate a boolean + use something like this: + </p> + <pre>(rand:uniform(16) > 8)</pre> + <p> + And for a general range, with <c>N = 1</c> for <c>exrop</c>, + and <c>N = 3</c> for <c>exs1024s</c>: + </p> + <pre>(((rand:uniform(Range bsl N) - 1) bsr N) + 1)</pre> + <p> + The floating point generating functions in this module + waste the lowest bits when converting from an integer + so they avoid this snag. + </p> + + </description> <datatypes> <datatype> @@ -142,6 +227,18 @@ S0 = rand:seed_s(exsplus), <name name="alg_state"/> </datatype> <datatype> + <name name="state"/> + <desc><p>Algorithm-dependent state.</p></desc> + </datatype> + <datatype> + <name name="export_state"/> + <desc> + <p> + Algorithm-dependent state that can be printed or saved to file. + </p> + </desc> + </datatype> + <datatype> <name name="exs64_state"/> <desc><p>Algorithm specific internal state</p></desc> </datatype> @@ -154,16 +251,8 @@ S0 = rand:seed_s(exsplus), <desc><p>Algorithm specific internal state</p></desc> </datatype> <datatype> - <name name="state"/> - <desc><p>Algorithm-dependent state.</p></desc> - </datatype> - <datatype> - <name name="export_state"/> - <desc> - <p> - Algorithm-dependent state that can be printed or saved to file. - </p> - </desc> + <name name="exrop_state"/> + <desc><p>Algorithm specific internal state</p></desc> </datatype> </datatypes> @@ -224,6 +313,15 @@ S0 = rand:seed_s(exsplus), </func> <func> + <name name="normal" arity="2"/> + <fsummary>Return a normal distributed random float.</fsummary> + <desc> + <p>Returns a normal N(Mean, Variance) deviate float + and updates the state in the process dictionary.</p> + </desc> + </func> + + <func> <name name="normal_s" arity="1"/> <fsummary>Return a standard normal distributed random float.</fsummary> <desc> @@ -234,6 +332,15 @@ S0 = rand:seed_s(exsplus), </func> <func> + <name name="normal_s" arity="3"/> + <fsummary>Return a normal distributed random float.</fsummary> + <desc> + <p>Returns, for a specified state, a normal N(Mean, Variance) + deviate float and a new state.</p> + </desc> + </func> + + <func> <name name="seed" arity="1"/> <fsummary>Seed random number generator.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/re.xml b/lib/stdlib/doc/src/re.xml index aadb5beaba..078ca0e38c 100644 --- a/lib/stdlib/doc/src/re.xml +++ b/lib/stdlib/doc/src/re.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2007</year> - <year>2016</year> + <year>2017</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -79,6 +79,14 @@ <funcs> <func> + <name name="version" arity="0"/> + <fsummary>Gives the PCRE version of the system in a string format</fsummary> + <desc> + <p>The return of this function is a string with the PCRE version of the system that was used in the Erlang/OTP compilation.</p> + </desc> + </func> + + <func> <name name="compile" arity="1"/> <fsummary>Compile a regular expression into a match program</fsummary> <desc> diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl index a54df939bf..168ea4002c 100644 --- a/lib/stdlib/src/erl_tar.erl +++ b/lib/stdlib/src/erl_tar.erl @@ -69,6 +69,8 @@ format_error(invalid_gnu_1_0_sparsemap) -> "Invalid GNU sparse map (version 1.0)"; format_error({invalid_gnu_0_1_sparsemap, Format}) -> lists:flatten(io_lib:format("Invalid GNU sparse map (version ~s)", [Format])); +format_error(unsafe_path) -> + "The path points above the current working directory"; format_error({Name,Reason}) -> lists:flatten(io_lib:format("~ts: ~ts", [Name,format_error(Reason)])); format_error(Atom) when is_atom(Atom) -> @@ -120,26 +122,38 @@ do_extract(Handle, Opts) when is_list(Opts) -> extract1(eof, Reader, _, Acc) when is_list(Acc) -> {ok, {ok, lists:reverse(Acc)}, Reader}; +extract1(eof, Reader, _, leading_slash) -> + error_logger:info_msg("erl_tar: removed leading '/' from member names\n"), + {ok, ok, Reader}; extract1(eof, Reader, _, Acc) -> {ok, Acc, Reader}; -extract1(#tar_header{name=Name,size=Size}=Header, Reader, Opts, Acc) -> +extract1(#tar_header{name=Name,size=Size}=Header, Reader0, Opts, Acc0) -> case check_extract(Name, Opts) of true -> - case do_read(Reader, Size) of - {ok, Bin, Reader2} -> - case write_extracted_element(Header, Bin, Opts) of - ok -> - {ok, Acc, Reader2}; - {ok, NameBin} when is_list(Acc) -> - {ok, [NameBin | Acc], Reader2}; - {error, _} = Err -> - throw(Err) - end; + case do_read(Reader0, Size) of + {ok, Bin, Reader1} -> + Acc = extract2(Header, Bin, Opts, Acc0), + {ok, Acc, Reader1}; {error, _} = Err -> throw(Err) end; false -> - {ok, Acc, skip_file(Reader)} + {ok, Acc0, skip_file(Reader0)} + end. + +extract2(Header, Bin, Opts, Acc) -> + case write_extracted_element(Header, Bin, Opts) of + ok -> + case Header of + #tar_header{name="/"++_} -> + leading_slash; + #tar_header{} -> + Acc + end; + {ok, NameBin} when is_list(Acc) -> + [NameBin | Acc]; + {error, _} = Err -> + throw(Err) end. %% Checks if the file Name should be extracted. @@ -1052,14 +1066,11 @@ unpack_modern(Format, #header_v7{}=V7, Bin, #tar_header{}=Header0) safe_join_path([], Name) -> - strip_slashes(Name, both); + filename:join([Name]); safe_join_path(Prefix, []) -> - strip_slashes(Prefix, right); + filename:join([Prefix]); safe_join_path(Prefix, Name) -> - filename:join(strip_slashes(Prefix, right), strip_slashes(Name, both)). - -strip_slashes(Str, Direction) -> - string:strip(Str, Direction, $/). + filename:join(Prefix, Name). new_sparse_file_reader(Reader, Sparsemap, RealSize) -> true = validate_sparse_entries(Sparsemap, RealSize), @@ -1557,7 +1568,7 @@ write_extracted_element(#tar_header{name=Name,typeflag=Type}, ok end; write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> - Name1 = filename:absname(Name0, Opts#read_opts.cwd), + Name1 = make_safe_path(Name0, Opts), Created = case typeflag(Header#tar_header.typeflag) of regular -> @@ -1585,6 +1596,16 @@ write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> not_written -> ok end. +make_safe_path([$/|Path], Opts) -> + make_safe_path(Path, Opts); +make_safe_path(Path, #read_opts{cwd=Cwd}) -> + case filename:safe_relative_path(Path) of + unsafe -> + throw({error,{Path,unsafe_path}}); + SafePath -> + filename:absname(SafePath, Cwd) + end. + create_regular(Name, NameInArchive, Bin, Opts) -> case write_extracted_file(Name, Bin, Opts) of not_written -> diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl index dfd102f9ef..ab9731180f 100644 --- a/lib/stdlib/src/rand.erl +++ b/lib/stdlib/src/rand.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2016. All Rights Reserved. +%% Copyright Ericsson AB 2015-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,6 +20,9 @@ %% ===================================================================== %% Multiple PRNG module for Erlang/OTP %% Copyright (c) 2015-2016 Kenji Rikitake +%% +%% exrop (xoroshiro116+) added and statistical distribution +%% improvements by the Erlang/OTP team 2017 %% ===================================================================== -module(rand). @@ -28,48 +31,179 @@ export_seed/0, export_seed_s/1, uniform/0, uniform/1, uniform_s/1, uniform_s/2, jump/0, jump/1, - normal/0, normal_s/1 + normal/0, normal/2, normal_s/1, normal_s/3 ]). -compile({inline, [exs64_next/1, exsplus_next/1, - exsplus_jump/1, exs1024_next/1, exs1024_calc/2, - exs1024_jump/1, + exrop_next/1, exrop_next_s/2, get_52/1, normal_kiwi/1]}). --define(DEFAULT_ALG_HANDLER, exsplus). +-define(DEFAULT_ALG_HANDLER, exrop). -define(SEED_DICT, rand_seed). %% ===================================================================== +%% Bit fiddling macros +%% ===================================================================== + +-define(BIT(Bits), (1 bsl (Bits))). +-define(MASK(Bits), (?BIT(Bits) - 1)). +-define(MASK(Bits, X), ((X) band ?MASK(Bits))). +-define( + BSL(Bits, X, N), + %% N is evaluated 2 times + (?MASK((Bits)-(N), (X)) bsl (N))). +-define( + ROTL(Bits, X, N), + %% Bits is evaluated 2 times + %% X is evaluated 2 times + %% N i evaluated 3 times + (?BSL((Bits), (X), (N)) bor ((X) bsr ((Bits)-(N))))). + +%%-define(TWO_POW_MINUS53, (math:pow(2, -53))). +-define(TWO_POW_MINUS53, 1.11022302462515657e-16). + +%% ===================================================================== %% Types %% ===================================================================== +-type uint64() :: 0..?MASK(64). +-type uint58() :: 0..?MASK(58). + %% This depends on the algorithm handler function -type alg_state() :: - exs64_state() | exsplus_state() | exs1024_state() | term(). + exs64_state() | exsplus_state() | exs1024_state() | + exrop_state() | term(). -%% This is the algorithm handler function within this module +%% This is the algorithm handling definition within this module, +%% and the type to use for plugins. +%% +%% The 'type' field must be recognized by the module that implements +%% the algorithm, to interpret an exported state. +%% +%% The 'bits' field indicates how many bits the integer +%% returned from 'next' has got, i.e 'next' shall return +%% an random integer in the range 0..(2^Bits - 1). +%% At least 53 bits is required for the floating point +%% producing fallbacks. This field is only used when +%% the 'uniform' or 'uniform_n' fields are not defined. +%% +%% The fields 'next', 'uniform' and 'uniform_n' +%% implement the algorithm. If 'uniform' or 'uinform_n' +%% is not present there is a fallback using 'next' and either +%% 'bits' or the deprecated 'max'. +%% -type alg_handler() :: #{type := alg(), - max := integer() | infinity, + bits => non_neg_integer(), + weak_low_bits => non_neg_integer(), + max => non_neg_integer(), % Deprecated next := - fun((alg_state()) -> {non_neg_integer(), alg_state()}), - uniform := - fun((state()) -> {float(), state()}), - uniform_n := - fun((pos_integer(), state()) -> {pos_integer(), state()}), - jump := - fun((state()) -> state())}. + fun ((alg_state()) -> {non_neg_integer(), alg_state()}), + uniform => + fun ((state()) -> {float(), state()}), + uniform_n => + fun ((pos_integer(), state()) -> {pos_integer(), state()}), + jump => + fun ((state()) -> state())}. %% Algorithm state -type state() :: {alg_handler(), alg_state()}. --type builtin_alg() :: exs64 | exsplus | exs1024. +-type builtin_alg() :: exs64 | exsplus | exsp | exs1024 | exs1024s | exrop. -type alg() :: builtin_alg() | atom(). -type export_state() :: {alg(), alg_state()}. -export_type( [builtin_alg/0, alg/0, alg_handler/0, alg_state/0, state/0, export_state/0]). --export_type([exs64_state/0, exsplus_state/0, exs1024_state/0]). +-export_type( + [exs64_state/0, exsplus_state/0, exs1024_state/0, exrop_state/0]). + +%% ===================================================================== +%% Range macro and helper +%% ===================================================================== + +-define( + uniform_range(Range, Alg, R, V, MaxMinusRange, I), + if + 0 =< (MaxMinusRange) -> + if + %% Really work saving in odd cases; + %% large ranges in particular + (V) < (Range) -> + {(V) + 1, {(Alg), (R)}}; + true -> + (I) = (V) rem (Range), + if + (V) - (I) =< (MaxMinusRange) -> + {(I) + 1, {(Alg), (R)}}; + true -> + %% V in the truncated top range + %% - try again + ?FUNCTION_NAME((Range), {(Alg), (R)}) + end + end; + true -> + uniform_range((Range), (Alg), (R), (V)) + end). + +%% For ranges larger than the algorithm bit size +uniform_range(Range, #{next:=Next, bits:=Bits} = Alg, R, V) -> + WeakLowBits = + case Alg of + #{weak_low_bits:=WLB} -> WLB; + #{} -> 0 + end, + %% Maybe waste the lowest bit(s) when shifting in new bits + Shift = Bits - WeakLowBits, + ShiftMask = bnot ?MASK(WeakLowBits), + RangeMinus1 = Range - 1, + if + (Range band RangeMinus1) =:= 0 -> % Power of 2 + %% Generate at least the number of bits for the range + {V1, R1, _} = + uniform_range( + Range bsr Bits, Next, R, V, ShiftMask, Shift, Bits), + {(V1 band RangeMinus1) + 1, {Alg, R1}}; + true -> + %% Generate a value with at least two bits more than the range + %% and try that for a fit, otherwise recurse + %% + %% Just one bit more should ensure that the generated + %% number range is at least twice the size of the requested + %% range, which would make the probability to draw a good + %% number better than 0.5. And repeating that until + %% success i guess would take 2 times statistically amortized. + %% But since the probability for fairly many attemtpts + %% is not that low, use two bits more than the range which + %% should make the probability to draw a bad number under 0.25, + %% which decreases the bad case probability a lot. + {V1, R1, B} = + uniform_range( + Range bsr (Bits - 2), Next, R, V, ShiftMask, Shift, Bits), + I = V1 rem Range, + if + (V1 - I) =< (1 bsl B) - Range -> + {I + 1, {Alg, R1}}; + true -> + %% V1 drawn from the truncated top range + %% - try again + {V2, R2} = Next(R1), + uniform_range(Range, Alg, R2, V2) + end + end. +%% +uniform_range(Range, Next, R, V, ShiftMask, Shift, B) -> + if + Range =< 1 -> + {V, R, B}; + true -> + {V1, R1} = Next(R), + %% Waste the lowest bit(s) when shifting in new bits + uniform_range( + Range bsr Shift, Next, R1, + ((V band ShiftMask) bsl Shift) bor V1, + ShiftMask, Shift, B + Shift) + end. %% ===================================================================== %% API @@ -156,7 +290,16 @@ uniform(N) -> -spec uniform_s(State :: state()) -> {X :: float(), NewState :: state()}. uniform_s(State = {#{uniform:=Uniform}, _}) -> - Uniform(State). + Uniform(State); +uniform_s({#{bits:=Bits, next:=Next} = Alg, R0}) -> + {V, R1} = Next(R0), + %% Produce floats on the form N * 2^(-53) + {(V bsr (Bits - 53)) * ?TWO_POW_MINUS53, {Alg, R1}}; +uniform_s({#{max:=Max, next:=Next} = Alg, R0}) -> + {V, R1} = Next(R0), + %% Old broken algorithm with non-uniform density + {V / (Max + 1), {Alg, R1}}. + %% uniform_s/2: given an integer N >= 1 and a state, uniform_s/2 %% uniform_s/2 returns a random integer X where 1 =< X =< N, @@ -164,13 +307,26 @@ uniform_s(State = {#{uniform:=Uniform}, _}) -> -spec uniform_s(N :: pos_integer(), State :: state()) -> {X :: pos_integer(), NewState :: state()}. -uniform_s(N, State = {#{uniform_n:=Uniform, max:=Max}, _}) - when 0 < N, N =< Max -> - Uniform(N, State); -uniform_s(N, State0 = {#{uniform:=Uniform}, _}) - when is_integer(N), 0 < N -> - {F, State} = Uniform(State0), - {trunc(F * N) + 1, State}. +uniform_s(N, State = {#{uniform_n:=UniformN}, _}) + when is_integer(N), 1 =< N -> + UniformN(N, State); +uniform_s(N, {#{bits:=Bits, next:=Next} = Alg, R0}) + when is_integer(N), 1 =< N -> + {V, R1} = Next(R0), + MaxMinusN = ?BIT(Bits) - N, + ?uniform_range(N, Alg, R1, V, MaxMinusN, I); +uniform_s(N, {#{max:=Max, next:=Next} = Alg, R0}) + when is_integer(N), 1 =< N -> + %% Old broken algorithm with skewed probability + %% and gap in ranges > Max + {V, R1} = Next(R0), + if + N =< Max -> + {(V rem N) + 1, {Alg, R1}}; + true -> + F = V / (Max + 1), + {trunc(F * N) + 1, {Alg, R1}} + end. %% jump/1: given a state, jump/1 %% returns a new state which is equivalent to that @@ -179,7 +335,10 @@ uniform_s(N, State0 = {#{uniform:=Uniform}, _}) -spec jump(state()) -> NewState :: state(). jump(State = {#{jump:=Jump}, _}) -> - Jump(State). + Jump(State); +jump({#{}, _}) -> + erlang:error(not_implemented). + %% jump/0: read the internal state and %% apply the jump function for the state as in jump/1 @@ -187,7 +346,6 @@ jump(State = {#{jump:=Jump}, _}) -> %% then returns the new value. -spec jump() -> NewState :: state(). - jump() -> seed_put(jump(seed_get())). @@ -200,6 +358,13 @@ normal() -> _ = seed_put(Seed), X. +%% normal/2: returns a random float with N(μ, σ²) normal distribution +%% updating the state in the process dictionary. + +-spec normal(Mean :: number(), Variance :: number()) -> float(). +normal(Mean, Variance) -> + Mean + (math:sqrt(Variance) * normal()). + %% normal_s/1: returns a random float with standard normal distribution %% The Ziggurat Method for generating random variables - Marsaglia and Tsang %% Paper and reference code: http://www.jstatsoft.org/v05/i08/ @@ -207,7 +372,7 @@ normal() -> -spec normal_s(State :: state()) -> {float(), NewState :: state()}. normal_s(State0) -> {Sign, R, State} = get_52(State0), - Idx = R band 16#FF, + Idx = ?MASK(8, R), Idx1 = Idx+1, {Ki, Wi} = normal_kiwi(Idx1), X = R * Wi, @@ -220,18 +385,15 @@ normal_s(State0) -> false -> normal_s(Idx, Sign, -X, State) end. -%% ===================================================================== -%% Internal functions +%% normal_s/3: returns a random float with normal N(μ, σ²) distribution --define(UINT21MASK, 16#00000000001fffff). --define(UINT32MASK, 16#00000000ffffffff). --define(UINT33MASK, 16#00000001ffffffff). --define(UINT39MASK, 16#0000007fffffffff). --define(UINT58MASK, 16#03ffffffffffffff). --define(UINT64MASK, 16#ffffffffffffffff). +-spec normal_s(Mean :: number(), Variance :: number(), state()) -> {float(), NewS :: state()}. +normal_s(Mean, Variance, State0) when Variance > 0 -> + {X, State} = normal_s(State0), + {Mean + (math:sqrt(Variance) * X), State}. --type uint64() :: 0..16#ffffffffffffffff. --type uint58() :: 0..16#03ffffffffffffff. +%% ===================================================================== +%% Internal functions -spec seed_put(state()) -> state(). seed_put(Seed) -> @@ -246,20 +408,30 @@ seed_get() -> %% Setup alg record mk_alg(exs64) -> - {#{type=>exs64, max=>?UINT64MASK, next=>fun exs64_next/1, - uniform=>fun exs64_uniform/1, uniform_n=>fun exs64_uniform/2, - jump=>fun exs64_jump/1}, + {#{type=>exs64, max=>?MASK(64), next=>fun exs64_next/1}, fun exs64_seed/1}; mk_alg(exsplus) -> - {#{type=>exsplus, max=>?UINT58MASK, next=>fun exsplus_next/1, - uniform=>fun exsplus_uniform/1, uniform_n=>fun exsplus_uniform/2, + {#{type=>exsplus, max=>?MASK(58), next=>fun exsplus_next/1, + jump=>fun exsplus_jump/1}, + fun exsplus_seed/1}; +mk_alg(exsp) -> + {#{type=>exsp, bits=>58, weak_low_bits=>1, next=>fun exsplus_next/1, + uniform=>fun exsp_uniform/1, uniform_n=>fun exsp_uniform/2, jump=>fun exsplus_jump/1}, fun exsplus_seed/1}; mk_alg(exs1024) -> - {#{type=>exs1024, max=>?UINT64MASK, next=>fun exs1024_next/1, - uniform=>fun exs1024_uniform/1, uniform_n=>fun exs1024_uniform/2, + {#{type=>exs1024, max=>?MASK(64), next=>fun exs1024_next/1, jump=>fun exs1024_jump/1}, - fun exs1024_seed/1}. + fun exs1024_seed/1}; +mk_alg(exs1024s) -> + {#{type=>exs1024s, bits=>64, weak_low_bits=>3, next=>fun exs1024_next/1, + jump=>fun exs1024_jump/1}, + fun exs1024_seed/1}; +mk_alg(exrop) -> + {#{type=>exrop, bits=>58, weak_low_bits=>1, next=>fun exrop_next/1, + uniform=>fun exrop_uniform/1, uniform_n=>fun exrop_uniform/2, + jump=>fun exrop_jump/1}, + fun exrop_seed/1}. %% ===================================================================== %% exs64 PRNG: Xorshift64* @@ -270,29 +442,18 @@ mk_alg(exs1024) -> -opaque exs64_state() :: uint64(). exs64_seed({A1, A2, A3}) -> - {V1, _} = exs64_next(((A1 band ?UINT32MASK) * 4294967197 + 1)), - {V2, _} = exs64_next(((A2 band ?UINT32MASK) * 4294967231 + 1)), - {V3, _} = exs64_next(((A3 band ?UINT32MASK) * 4294967279 + 1)), - ((V1 * V2 * V3) rem (?UINT64MASK - 1)) + 1. + {V1, _} = exs64_next((?MASK(32, A1) * 4294967197 + 1)), + {V2, _} = exs64_next((?MASK(32, A2) * 4294967231 + 1)), + {V3, _} = exs64_next((?MASK(32, A3) * 4294967279 + 1)), + ((V1 * V2 * V3) rem (?MASK(64) - 1)) + 1. %% Advance xorshift64* state for one step and generate 64bit unsigned integer -spec exs64_next(exs64_state()) -> {uint64(), exs64_state()}. exs64_next(R) -> R1 = R bxor (R bsr 12), - R2 = R1 bxor ((R1 band ?UINT39MASK) bsl 25), + R2 = R1 bxor ?BSL(64, R1, 25), R3 = R2 bxor (R2 bsr 27), - {(R3 * 2685821657736338717) band ?UINT64MASK, R3}. - -exs64_uniform({Alg, R0}) -> - {V, R1} = exs64_next(R0), - {V / 18446744073709551616, {Alg, R1}}. - -exs64_uniform(Max, {Alg, R}) -> - {V, R1} = exs64_next(R), - {(V rem Max) + 1, {Alg, R1}}. - -exs64_jump(_) -> - erlang:error(not_implemented). + {?MASK(64, R3 * 2685821657736338717), R3}. %% ===================================================================== %% exsplus PRNG: Xorshift116+ @@ -307,10 +468,12 @@ exs64_jump(_) -> -dialyzer({no_improper_lists, exsplus_seed/1}). exsplus_seed({A1, A2, A3}) -> - {_, R1} = exsplus_next([(((A1 * 4294967197) + 1) band ?UINT58MASK)| - (((A2 * 4294967231) + 1) band ?UINT58MASK)]), - {_, R2} = exsplus_next([(((A3 * 4294967279) + 1) band ?UINT58MASK)| - tl(R1)]), + {_, R1} = exsplus_next( + [?MASK(58, (A1 * 4294967197) + 1)| + ?MASK(58, (A2 * 4294967231) + 1)]), + {_, R2} = exsplus_next( + [?MASK(58, (A3 * 4294967279) + 1)| + tl(R1)]), R2. -dialyzer({no_improper_lists, exsplus_next/1}). @@ -319,17 +482,22 @@ exsplus_seed({A1, A2, A3}) -> -spec exsplus_next(exsplus_state()) -> {uint58(), exsplus_state()}. exsplus_next([S1|S0]) -> %% Note: members s0 and s1 are swapped here - S11 = (S1 bxor (S1 bsl 24)) band ?UINT58MASK, + S11 = S1 bxor ?BSL(58, S1, 24), S12 = S11 bxor S0 bxor (S11 bsr 11) bxor (S0 bsr 41), - {(S0 + S12) band ?UINT58MASK, [S0|S12]}. + {?MASK(58, S0 + S12), [S0|S12]}. + -exsplus_uniform({Alg, R0}) -> +exsp_uniform({Alg, R0}) -> {I, R1} = exsplus_next(R0), - {I / (?UINT58MASK+1), {Alg, R1}}. + %% Waste the lowest bit since it is of lower + %% randomness quality than the others + {(I bsr (58-53)) * ?TWO_POW_MINUS53, {Alg, R1}}. -exsplus_uniform(Max, {Alg, R}) -> +exsp_uniform(Range, {Alg, R}) -> {V, R1} = exsplus_next(R), - {(V rem Max) + 1, {Alg, R1}}. + MaxMinusRange = ?BIT(58) - Range, + ?uniform_range(Range, Alg, R1, V, MaxMinusRange, I). + %% This is the jump function for the exsplus generator, equivalent %% to 2^64 calls to next/1; it can be used to generate 2^52 @@ -357,7 +525,7 @@ exsplus_jump(S, AS, _, 0) -> {S, AS}; exsplus_jump(S, [AS0|AS1], J, N) -> {_, NS} = exsplus_next(S), - case (J band 1) of + case ?MASK(1, J) of 1 -> [S0|S1] = S, exsplus_jump(NS, [(AS0 bxor S0)|(AS1 bxor S1)], J bsr 1, N-1); @@ -374,9 +542,9 @@ exsplus_jump(S, [AS0|AS1], J, N) -> -opaque exs1024_state() :: {list(uint64()), list(uint64())}. exs1024_seed({A1, A2, A3}) -> - B1 = (((A1 band ?UINT21MASK) + 1) * 2097131) band ?UINT21MASK, - B2 = (((A2 band ?UINT21MASK) + 1) * 2097133) band ?UINT21MASK, - B3 = (((A3 band ?UINT21MASK) + 1) * 2097143) band ?UINT21MASK, + B1 = ?MASK(21, (?MASK(21, A1) + 1) * 2097131), + B2 = ?MASK(21, (?MASK(21, A2) + 1) * 2097133), + B3 = ?MASK(21, (?MASK(21, A3) + 1) * 2097143), {exs1024_gen1024((B1 bsl 43) bor (B2 bsl 22) bor (B3 bsl 1) bor 1), []}. @@ -399,11 +567,11 @@ exs1024_gen1024(N, R, L) -> %% X: random number output -spec exs1024_calc(uint64(), uint64()) -> {uint64(), uint64()}. exs1024_calc(S0, S1) -> - S11 = S1 bxor ((S1 band ?UINT33MASK) bsl 31), + S11 = S1 bxor ?BSL(64, S1, 31), S12 = S11 bxor (S11 bsr 11), S01 = S0 bxor (S0 bsr 30), NS1 = S01 bxor S12, - {(NS1 * 1181783497276652981) band ?UINT64MASK, NS1}. + {?MASK(64, NS1 * 1181783497276652981), NS1}. %% Advance xorshift1024* state for one step and generate 64bit unsigned integer -spec exs1024_next(exs1024_state()) -> {uint64(), exs1024_state()}. @@ -414,13 +582,6 @@ exs1024_next({[H], RL}) -> NL = [H|lists:reverse(RL)], exs1024_next({NL, []}). -exs1024_uniform({Alg, R0}) -> - {V, R1} = exs1024_next(R0), - {V / 18446744073709551616, {Alg, R1}}. - -exs1024_uniform(Max, {Alg, R}) -> - {V, R1} = exs1024_next(R), - {(V rem Max) + 1, {Alg, R1}}. %% This is the jump function for the exs1024 generator, equivalent %% to 2^512 calls to next(); it can be used to generate 2^512 @@ -467,7 +628,7 @@ exs1024_jump(S, AS, [H|T], _, 0, TN) -> exs1024_jump(S, AS, T, H, ?JUMPELEMLEN, TN); exs1024_jump({L, RL}, AS, JL, J, N, TN) -> {_, NS} = exs1024_next({L, RL}), - case (J band 1) of + case ?MASK(1, J) of 1 -> AS2 = lists:zipwith(fun(X, Y) -> X bxor Y end, AS, L ++ lists:reverse(RL)), @@ -477,15 +638,149 @@ exs1024_jump({L, RL}, AS, JL, J, N, TN) -> end. %% ===================================================================== +%% exrop PRNG: Xoroshiro116+ +%% +%% Reference URL: http://xorshift.di.unimi.it/ +%% +%% 58 bits fits into an immediate on 64bits Erlang and is thus much faster. +%% In fact, an immediate number is 60 bits signed in Erlang so you can +%% add two positive 58 bit numbers and get a 59 bit number that still is +%% a positive immediate, which is a property we utilize here... +%% +%% Modification of the original Xororhiro128+ algorithm to 116 bits +%% by Sebastiano Vigna. A lot of thanks for his help and work. +%% ===================================================================== +%% (a, b, c) = (24, 2, 35) +%% JUMP Polynomial = 0x9863200f83fcd4a11293241fcb12a (116 bit) +%% +%% From http://xoroshiro.di.unimi.it/xoroshiro116plus.c: +%% --------------------------------------------------------------------- +%% /* Written in 2017 by Sebastiano Vigna ([email protected]). +%% +%% To the extent possible under law, the author has dedicated all copyright +%% and related and neighboring rights to this software to the public domain +%% worldwide. This software is distributed without any warranty. +%% +%% See <http://creativecommons.org/publicdomain/zero/1.0/>. */ +%% +%% #include <stdint.h> +%% +%% #define UINT58MASK (uint64_t)((UINT64_C(1) << 58) - 1) +%% +%% uint64_t s[2]; +%% +%% static inline uint64_t rotl58(const uint64_t x, int k) { +%% return (x << k) & UINT58MASK | (x >> (58 - k)); +%% } +%% +%% uint64_t next(void) { +%% uint64_t s1 = s[1]; +%% const uint64_t s0 = s[0]; +%% const uint64_t result = (s0 + s1) & UINT58MASK; +%% +%% s1 ^= s0; +%% s[0] = rotl58(s0, 24) ^ s1 ^ ((s1 << 2) & UINT58MASK); // a, b +%% s[1] = rotl58(s1, 35); // c +%% return result; +%% } +%% +%% void jump(void) { +%% static const uint64_t JUMP[] = +%% { 0x4a11293241fcb12a, 0x0009863200f83fcd }; +%% +%% uint64_t s0 = 0; +%% uint64_t s1 = 0; +%% for(int i = 0; i < sizeof JUMP / sizeof *JUMP; i++) +%% for(int b = 0; b < 64; b++) { +%% if (JUMP[i] & UINT64_C(1) << b) { +%% s0 ^= s[0]; +%% s1 ^= s[1]; +%% } +%% next(); +%% } +%% s[0] = s0; +%% s[1] = s1; +%% } + +-opaque exrop_state() :: nonempty_improper_list(uint58(), uint58()). + +-dialyzer({no_improper_lists, exrop_seed/1}). +exrop_seed({A1, A2, A3}) -> + [_|S1] = + exrop_next_s( + ?MASK(58, (A1 * 4294967197) + 1), + ?MASK(58, (A2 * 4294967231) + 1)), + exrop_next_s(?MASK(58, (A3 * 4294967279) + 1), S1). + +-dialyzer({no_improper_lists, exrop_next_s/2}). +%% Advance xoroshiro116+ state one step +%% [a, b, c] = [24, 2, 35] +-define( + exrop_next_s(S0, S1, S1_a), + begin + S1_a = S1 bxor S0, + [?ROTL(58, S0, 24) bxor S1_a bxor ?BSL(58, S1_a, 2)| % a, b + ?ROTL(58, S1_a, 35)] % c + end). +exrop_next_s(S0, S1) -> + ?exrop_next_s(S0, S1, S1_a). + +-dialyzer({no_improper_lists, exrop_next/1}). +%% Advance xoroshiro116+ state one step, generate 58 bit unsigned integer, +%% and waste the lowest bit since it is of lower randomness quality +exrop_next([S0|S1]) -> + {?MASK(58, S0 + S1), ?exrop_next_s(S0, S1, S1_a)}. + +exrop_uniform({Alg, R}) -> + {V, R1} = exrop_next(R), + %% Waste the lowest bit since it is of lower + %% randomness quality than the others + {(V bsr (58-53)) * ?TWO_POW_MINUS53, {Alg, R1}}. + +exrop_uniform(Range, {Alg, R}) -> + {V, R1} = exrop_next(R), + MaxMinusRange = ?BIT(58) - Range, + ?uniform_range(Range, Alg, R1, V, MaxMinusRange, I). + +%% Split a 116 bit constant into two '1'++58 bit words, +%% the top '1' marks the top of the word +-define( + JUMP_116(Jump), + [?BIT(58) bor ?MASK(58, (Jump)),?BIT(58) bor ((Jump) bsr 58)]). +%% +exrop_jump({Alg,S}) -> + [J|Js] = ?JUMP_116(16#9863200f83fcd4a11293241fcb12a), + {Alg, exrop_jump(S, 0, 0, J, Js)}. +%% +-dialyzer({no_improper_lists, exrop_jump/5}). +exrop_jump(_S, S0, S1, 1, []) -> % End of jump constant + [S0|S1]; +exrop_jump(S, S0, S1, 1, [J|Js]) -> % End of the word + exrop_jump(S, S0, S1, J, Js); +exrop_jump([S__0|S__1] = _S, S0, S1, J, Js) -> + case ?MASK(1, J) of + 1 -> + NewS = exrop_next_s(S__0, S__1), + exrop_jump(NewS, S0 bxor S__0, S1 bxor S__1, J bsr 1, Js); + 0 -> + NewS = exrop_next_s(S__0, S__1), + exrop_jump(NewS, S0, S1, J bsr 1, Js) + end. + +%% ===================================================================== %% Ziggurat cont %% ===================================================================== -define(NOR_R, 3.6541528853610087963519472518). -define(NOR_INV_R, 1/?NOR_R). %% return a {sign, Random51bits, State} +get_52({Alg=#{bits:=Bits, next:=Next}, S0}) -> + %% Use the high bits + {Int,S1} = Next(S0), + {?BIT(Bits - 51 - 1) band Int, Int bsr (Bits - 51), {Alg, S1}}; get_52({Alg=#{next:=Next}, S0}) -> {Int,S1} = Next(S0), - {((1 bsl 51) band Int), Int band ((1 bsl 51)-1), {Alg, S1}}. + {?BIT(51) band Int, ?MASK(51, Int), {Alg, S1}}. %% Slow path normal_s(0, Sign, X0, State0) -> diff --git a/lib/stdlib/src/re.erl b/lib/stdlib/src/re.erl index 52d3c35608..28aab7b590 100644 --- a/lib/stdlib/src/re.erl +++ b/lib/stdlib/src/re.erl @@ -33,7 +33,12 @@ %%% BIFs --export([compile/1, compile/2, run/2, run/3, inspect/2]). +-export([version/0, compile/1, compile/2, run/2, run/3, inspect/2]). + +-spec version() -> binary(). + +version() -> + erlang:nif_error(undef). -spec compile(Regexp) -> {ok, MP} | {error, ErrSpec} when Regexp :: iodata(), diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index 4cc4e3292c..b795cb0b61 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -18,7 +18,6 @@ %% %CopyrightEnd% %% -module(io_proto_SUITE). --compile(r12). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 098eefeb61..36bc283aec 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2016. All Rights Reserved. +%% Copyright Ericsson AB 2000-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ -export([interval_int/1, interval_float/1, seed/1, api_eq/1, reference/1, basic_stats_uniform_1/1, basic_stats_uniform_2/1, + basic_stats_standard_normal/1, basic_stats_normal/1, plugin/1, measure/1, reference_jump_state/1, reference_jump_procdict/1]). @@ -52,7 +53,8 @@ all() -> groups() -> [{basic_stats, [parallel], - [basic_stats_uniform_1, basic_stats_uniform_2, basic_stats_normal]}, + [basic_stats_uniform_1, basic_stats_uniform_2, + basic_stats_standard_normal, basic_stats_normal]}, {reference_jump, [parallel], [reference_jump_state, reference_jump_procdict]}]. @@ -66,18 +68,19 @@ group(reference_jump) -> %% A simple helper to test without test_server during dev test() -> Tests = all(), - lists:foreach(fun(Test) -> - try - ok = ?MODULE:Test([]), - io:format("~p: ok~n", [Test]) - catch _:Reason -> - io:format("Failed: ~p: ~p ~p~n", - [Test, Reason, erlang:get_stacktrace()]) - end - end, Tests). + lists:foreach( + fun (Test) -> + try + ok = ?MODULE:Test([]), + io:format("~p: ok~n", [Test]) + catch _:Reason -> + io:format("Failed: ~p: ~p ~p~n", + [Test, Reason, erlang:get_stacktrace()]) + end + end, Tests). algs() -> - [exs64, exsplus, exs1024]. + [exs64, exsplus, exsp, exrop, exs1024, exs1024s]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -226,10 +229,10 @@ interval_float_1(0) -> ok; interval_float_1(N) -> X = rand:uniform(), if - 0.0 < X, X < 1.0 -> + 0.0 =< X, X < 1.0 -> ok; true -> - io:format("X=~p 0<~p<1.0~n", [X,X]), + io:format("X=~p 0=<~p<1.0~n", [X,X]), exit({X, rand:export_seed()}) end, interval_float_1(N-1). @@ -246,6 +249,8 @@ reference_1(Alg) -> Testval = gen(Alg), case Refval =:= Testval of true -> ok; + false when Refval =:= not_implemented -> + exit({not_implemented,Alg}); false -> io:format("Failed: ~p~n",[Alg]), io:format("Length ~p ~p~n",[length(Refval), length(Testval)]), @@ -254,25 +259,29 @@ reference_1(Alg) -> end. gen(Algo) -> - Seed = case Algo of - exsplus -> %% Printed with orig 'C' code and this seed - rand:seed_s({exsplus, [12345678|12345678]}); - exs64 -> %% Printed with orig 'C' code and this seed - rand:seed_s({exs64, 12345678}); - exs1024 -> %% Printed with orig 'C' code and this seed - rand:seed_s({exs1024, {lists:duplicate(16, 12345678), []}}); - _ -> - rand:seed(Algo, {100, 200, 300}) - end, - gen(?LOOP, Seed, []). - -gen(N, State0 = {#{max:=Max}, _}, Acc) when N > 0 -> + State = + case Algo of + exs64 -> %% Printed with orig 'C' code and this seed + rand:seed_s({exs64, 12345678}); + _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> + %% Printed with orig 'C' code and this seed + rand:seed_s({Algo, [12345678|12345678]}); + _ when Algo =:= exs1024; Algo =:= exs1024s -> + %% Printed with orig 'C' code and this seed + rand:seed_s({Algo, {lists:duplicate(16, 12345678), []}}); + _ -> + rand:seed(Algo, {100, 200, 300}) + end, + Max = range(State), + gen(?LOOP, State, Max, []). + +gen(N, State0, Max, Acc) when N > 0 -> {Random, State} = rand:uniform_s(Max, State0), case N rem (?LOOP div 100) of - 0 -> gen(N-1, State, [Random|Acc]); - _ -> gen(N-1, State, Acc) + 0 -> gen(N-1, State, Max, [Random|Acc]); + _ -> gen(N-1, State, Max, Acc) end; -gen(_, _, Acc) -> lists:reverse(Acc). +gen(_, _, _, Acc) -> lists:reverse(Acc). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% This just tests the basics so we have not made any serious errors @@ -294,12 +303,35 @@ basic_stats_uniform_2(Config) when is_list(Config) -> || Alg <- algs()], ok. -basic_stats_normal(Config) when is_list(Config) -> +basic_stats_standard_normal(Config) when is_list(Config) -> ct:timetrap({minutes,6}), %% valgrind needs a lot of time - io:format("Testing normal~n",[]), - [basic_normal_1(?LOOP, rand:seed_s(Alg), 0, 0) || Alg <- algs()], + io:format("Testing standard normal~n",[]), + IntendedMean = 0, + IntendedVariance = 1, + [basic_normal_1(?LOOP, IntendedMean, IntendedVariance, + rand:seed_s(Alg), 0, 0) + || Alg <- algs()], ok. +basic_stats_normal(Config) when is_list(Config) -> + IntendedMeans = [-1.0e6, -50, -math:pi(), -math:exp(-1), + 0.12345678, math:exp(1), 100, 1.0e6], + IntendedVariances = [1.0e-6, math:exp(-1), 1, math:pi(), 1.0e6], + IntendedMeanVariancePairs = + [{Mean, Variance} || Mean <- IntendedMeans, + Variance <- IntendedVariances], + + ct:timetrap({minutes, 6 * length(IntendedMeanVariancePairs)}), %% valgrind needs a lot of time + lists:foreach( + fun ({IntendedMean, IntendedVariance}) -> + io:format("Testing normal(~.2f, ~.2f)~n", + [float(IntendedMean), float(IntendedVariance)]), + [basic_normal_1(?LOOP, IntendedMean, IntendedVariance, + rand:seed_s(Alg), 0, 0) + || Alg <- algs()] + end, + IntendedMeanVariancePairs). + basic_uniform_1(N, S0, Sum, A0) when N > 0 -> {X,S} = rand:uniform_s(S0), I = trunc(X*100), @@ -307,11 +339,11 @@ basic_uniform_1(N, S0, Sum, A0) when N > 0 -> basic_uniform_1(N-1, S, Sum+X, A); basic_uniform_1(0, {#{type:=Alg}, _}, Sum, A) -> AverN = Sum / ?LOOP, - io:format("~.10w: Average: ~.4f~n", [Alg, AverN]), + io:format("~.12w: Average: ~.4f~n", [Alg, AverN]), Counters = array:to_list(A), Min = lists:min(Counters), Max = lists:max(Counters), - io:format("~.10w: Min: ~p Max: ~p~n", [Alg, Min, Max]), + io:format("~.12w: Min: ~p Max: ~p~n", [Alg, Min, Max]), %% Verify that the basic statistics are ok %% be gentle we don't want to see to many failing tests @@ -326,11 +358,11 @@ basic_uniform_2(N, S0, Sum, A0) when N > 0 -> basic_uniform_2(N-1, S, Sum+X, A); basic_uniform_2(0, {#{type:=Alg}, _}, Sum, A) -> AverN = Sum / ?LOOP, - io:format("~.10w: Average: ~.4f~n", [Alg, AverN]), + io:format("~.12w: Average: ~.4f~n", [Alg, AverN]), Counters = tl(array:to_list(A)), Min = lists:min(Counters), Max = lists:max(Counters), - io:format("~.10w: Min: ~p Max: ~p~n", [Alg, Min, Max]), + io:format("~.12w: Min: ~p Max: ~p~n", [Alg, Min, Max]), %% Verify that the basic statistics are ok %% be gentle we don't want to see to many failing tests @@ -339,19 +371,33 @@ basic_uniform_2(0, {#{type:=Alg}, _}, Sum, A) -> abs(?LOOP div 100 - Max) < 1000 orelse ct:fail({max, Alg, Max}), ok. -basic_normal_1(N, S0, Sum, Sq) when N > 0 -> - {X,S} = rand:normal_s(S0), - basic_normal_1(N-1, S, X+Sum, X*X+Sq); -basic_normal_1(0, {#{type:=Alg}, _}, Sum, SumSq) -> - Mean = Sum / ?LOOP, - StdDev = math:sqrt((SumSq - (Sum*Sum/?LOOP))/(?LOOP - 1)), - io:format("~.10w: Average: ~7.4f StdDev ~6.4f~n", [Alg, Mean, StdDev]), +basic_normal_1(N, IntendedMean, IntendedVariance, S0, StandardSum, StandardSq) when N > 0 -> + {X,S} = normal_s(IntendedMean, IntendedVariance, S0), + % We now shape X into a standard normal distribution (in case it wasn't already) + % in order to minimise the accumulated error on Sum / SumSq; + % otherwise said error would prevent us of making a fair judgment on + % the overall distribution when targeting large means and variances. + StandardX = (X - IntendedMean) / math:sqrt(IntendedVariance), + basic_normal_1(N-1, IntendedMean, IntendedVariance, S, + StandardX+StandardSum, StandardX*StandardX+StandardSq); +basic_normal_1(0, _IntendedMean, _IntendedVariance, {#{type:=Alg}, _}, StandardSum, StandardSumSq) -> + StandardMean = StandardSum / ?LOOP, + StandardVariance = (StandardSumSq - (StandardSum*StandardSum/?LOOP))/(?LOOP - 1), + StandardStdDev = math:sqrt(StandardVariance), + io:format("~.12w: Standardised Average: ~7.4f, Standardised StdDev ~6.4f~n", + [Alg, StandardMean, StandardStdDev]), %% Verify that the basic statistics are ok %% be gentle we don't want to see to many failing tests - abs(Mean) < 0.005 orelse ct:fail({average, Alg, Mean}), - abs(StdDev - 1.0) < 0.005 orelse ct:fail({stddev, Alg, StdDev}), + abs(StandardMean) < 0.005 orelse ct:fail({average, Alg, StandardMean}), + abs(StandardStdDev - 1.0) < 0.005 orelse ct:fail({stddev, Alg, StandardStdDev}), ok. +normal_s(Mean, Variance, State0) when Mean == 0, Variance == 1 -> + % Make sure we're also testing the standard normal interface + rand:normal_s(State0); +normal_s(Mean, Variance, State0) -> + rand:normal_s(Mean, Variance, State0). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Test that the user can write algorithms. @@ -365,7 +411,7 @@ plugin(Config) when is_list(Config) -> {V2, S2} = rand:uniform_s(S1), true = is_float(V2), S2 - end, crypto_seed(), lists:seq(1, 200)), + end, crypto64_seed(), lists:seq(1, 200)), ok catch error:low_entropy -> @@ -375,86 +421,220 @@ plugin(Config) when is_list(Config) -> end. %% Test implementation -crypto_seed() -> - {#{type=>crypto, - max=>(1 bsl 64)-1, - next=>fun crypto_next/1, - uniform=>fun crypto_uniform/1, - uniform_n=>fun crypto_uniform_n/2}, +crypto64_seed() -> + {#{type=>crypto64, + bits=>64, + next=>fun crypto64_next/1, + uniform=>fun crypto64_uniform/1, + uniform_n=>fun crypto64_uniform_n/2}, <<>>}. %% Be fair and create bignums i.e. 64bits otherwise use 58bits -crypto_next(<<Num:64, Bin/binary>>) -> +crypto64_next(<<Num:64, Bin/binary>>) -> {Num, Bin}; -crypto_next(_) -> - crypto_next(crypto:strong_rand_bytes((64 div 8)*100)). +crypto64_next(_) -> + crypto64_next(crypto:strong_rand_bytes((64 div 8)*100)). -crypto_uniform({Api, Data0}) -> - {Int, Data} = crypto_next(Data0), +crypto64_uniform({Api, Data0}) -> + {Int, Data} = crypto64_next(Data0), {Int / (1 bsl 64), {Api, Data}}. -crypto_uniform_n(N, {Api, Data0}) when N < (1 bsl 64) -> - {Int, Data} = crypto_next(Data0), +crypto64_uniform_n(N, {Api, Data0}) when N < (1 bsl 64) -> + {Int, Data} = crypto64_next(Data0), {(Int rem N)+1, {Api, Data}}; -crypto_uniform_n(N, State0) -> - {F,State} = crypto_uniform(State0), +crypto64_uniform_n(N, State0) -> + {F,State} = crypto64_uniform(State0), {trunc(F * N) + 1, State}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Not a test but measures the time characteristics of the different algorithms -measure(Suite) when is_atom(Suite) -> []; -measure(_Config) -> - ct:timetrap({minutes,15}), %% valgrind needs a lot of time +measure(Config) -> + ct:timetrap({minutes,30}), %% valgrind needs a lot of time + case ct:get_timetrap_info() of + {_,{_,1}} -> % No scaling + do_measure(Config); + {_,{_,Scale}} -> + {skip,{will_not_run_in_scaled_time,Scale}} + end. + +do_measure(_Config) -> Algos = try crypto:strong_rand_bytes(1) of - <<_>> -> [crypto64] + <<_>> -> [crypto64, crypto] catch error:low_entropy -> []; error:undef -> [] end ++ algs(), - io:format("RNG uniform integer performance~n",[]), - _ = measure_1(random, fun(State) -> {int, random:uniform_s(10000, State)} end), - _ = [measure_1(Algo, fun(State) -> {int, rand:uniform_s(10000, State)} end) || Algo <- Algos], - io:format("RNG uniform float performance~n",[]), - _ = measure_1(random, fun(State) -> {uniform, random:uniform_s(State)} end), - _ = [measure_1(Algo, fun(State) -> {uniform, rand:uniform_s(State)} end) || Algo <- Algos], - io:format("RNG normal float performance~n",[]), - io:format("~.10w: not implemented (too few bits)~n", [random]), - _ = [measure_1(Algo, fun(State) -> {normal, rand:normal_s(State)} end) || Algo <- Algos], + %% + ct:pal("RNG uniform integer performance~n",[]), + TMark1 = + measure_1( + random, + fun (_) -> 10000 end, + undefined, + fun (Range, State) -> + {int, random:uniform_s(Range, State)} + end), + _ = + [measure_1( + Algo, + fun (_) -> 10000 end, + TMark1, + fun (Range, State) -> + {int, rand:uniform_s(Range, State)} + end) || Algo <- Algos], + %% + ct:pal("~nRNG uniform integer 2^(N-1) performance~n",[]), + RangeTwoPowFun = fun (State) -> quart_range(State) bsl 1 end, + TMark2 = + measure_1( + random, + RangeTwoPowFun, + undefined, + fun (Range, State) -> + {int, random:uniform_s(Range, State)} + end), + _ = + [measure_1( + Algo, + RangeTwoPowFun, + TMark2, + fun (Range, State) -> + {int, rand:uniform_s(Range, State)} + end) || Algo <- Algos], + %% + ct:pal("~nRNG uniform integer 3*2^(N-2)+1 performance~n",[]), + RangeLargeFun = fun (State) -> 3 * quart_range(State) + 1 end, + TMark3 = + measure_1( + random, + RangeLargeFun, + undefined, + fun (Range, State) -> + {int, random:uniform_s(Range, State)} + end), + _ = + [measure_1( + Algo, + RangeLargeFun, + TMark3, + fun (Range, State) -> + {int, rand:uniform_s(Range, State)} + end) || Algo <- Algos], + %% + ct:pal("~nRNG uniform integer 2^128 performance~n",[]), + TMark4 = + measure_1( + random, + fun (_) -> 1 bsl 128 end, + undefined, + fun (Range, State) -> + {int, random:uniform_s(Range, State)} + end), + _ = + [measure_1( + Algo, + fun (_) -> 1 bsl 128 end, + TMark4, + fun (Range, State) -> + {int, rand:uniform_s(Range, State)} + end) || Algo <- Algos], + %% + ct:pal("~nRNG uniform integer 2^128 + 1 performance~n",[]), + TMark5 = + measure_1( + random, + fun (_) -> (1 bsl 128) + 1 end, + undefined, + fun (Range, State) -> + {int, random:uniform_s(Range, State)} + end), + _ = + [measure_1( + Algo, + fun (_) -> (1 bsl 128) + 1 end, + TMark5, + fun (Range, State) -> + {int, rand:uniform_s(Range, State)} + end) || Algo <- Algos], + %% + ct:pal("~nRNG uniform float performance~n",[]), + TMark6 = + measure_1( + random, + fun (_) -> 0 end, + undefined, + fun (_, State) -> + {uniform, random:uniform_s(State)} + end), + _ = + [measure_1( + Algo, + fun (_) -> 0 end, + TMark6, + fun (_, State) -> + {uniform, rand:uniform_s(State)} + end) || Algo <- Algos], + %% + ct:pal("~nRNG normal float performance~n",[]), + io:format("~.12w: not implemented (too few bits)~n", [random]), + _ = [measure_1( + Algo, + fun (_) -> 0 end, + TMark6, + fun (_, State) -> + {normal, rand:normal_s(State)} + end) || Algo <- Algos], ok. -measure_1(Algo, Gen) -> +measure_1(Algo, RangeFun, TMark, Gen) -> Parent = self(), - Seed = fun(crypto64) -> crypto_seed(); - (random) -> random:seed(os:timestamp()), get(random_seed); - (Alg) -> rand:seed_s(Alg) - end, - - Pid = spawn_link(fun() -> - Fun = fun() -> measure_2(?LOOP, Seed(Algo), Gen) end, - {Time, ok} = timer:tc(Fun), - io:format("~.10w: ~pµs~n", [Algo, Time]), - Parent ! {self(), ok}, - normal - end), + Seed = + case Algo of + crypto64 -> + crypto64_seed(); + crypto -> + crypto:rand_seed_s(); + random -> + random:seed(os:timestamp()), get(random_seed); + _ -> + rand:seed_s(Algo) + end, + Range = RangeFun(Seed), + Pid = spawn_link( + fun() -> + Fun = fun() -> measure_2(?LOOP, Range, Seed, Gen) end, + {Time, ok} = timer:tc(Fun), + Percent = + case TMark of + undefined -> 100; + _ -> (Time * 100 + 50) div TMark + end, + io:format( + "~.12w: ~p ns ~p% [16#~.16b]~n", + [Algo, (Time * 1000 + 500) div ?LOOP, Percent, Range]), + Parent ! {self(), Time}, + normal + end), receive {Pid, Msg} -> Msg end. -measure_2(N, State0, Fun) when N > 0 -> - case Fun(State0) of +measure_2(N, Range, State0, Fun) when N > 0 -> + case Fun(Range, State0) of {int, {Random, State}} - when is_integer(Random), Random >= 1, Random =< 100000 -> - measure_2(N-1, State, Fun); - {uniform, {Random, State}} when is_float(Random), Random > 0, Random < 1 -> - measure_2(N-1, State, Fun); + when is_integer(Random), Random >= 1, Random =< Range -> + measure_2(N-1, Range, State, Fun); + {uniform, {Random, State}} + when is_float(Random), 0.0 =< Random, Random < 1.0 -> + measure_2(N-1, Range, State, Fun); {normal, {Random, State}} when is_float(Random) -> - measure_2(N-1, State, Fun); + measure_2(N-1, Range, State, Fun); Res -> exit({error, Res, State0}) end; -measure_2(0, _, _) -> ok. +measure_2(0, _, _, _) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% The jump sequence tests has two parts @@ -479,36 +659,43 @@ reference_jump_1(Alg) -> io:format("Failed: ~p~n",[Alg]), io:format("Length ~p ~p~n",[length(Refval), length(Testval)]), io:format("Head ~p ~p~n",[hd(Refval), hd(Testval)]), + io:format("Vals ~p ~p~n",[Refval, Testval]), exit(wrong_value) end. gen_jump_1(Algo) -> - Seed = case Algo of - exsplus -> %% Printed with orig 'C' code and this seed - rand:seed_s({exsplus, [12345678|12345678]}); - exs1024 -> %% Printed with orig 'C' code and this seed - rand:seed_s({exs1024, {lists:duplicate(16, 12345678), []}}); - exs64 -> %% Test exception of not_implemented notice - try rand:jump(rand:seed_s(exs64)) - catch - error:not_implemented -> not_implemented - end; - _ -> % unimplemented - not_implemented - end, - case Seed of + State = + case Algo of + exs64 -> %% Test exception of not_implemented notice + try rand:jump(rand:seed_s(exs64)) + catch + error:not_implemented -> not_implemented + end; + _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> + %% Printed with orig 'C' code and this seed + rand:seed_s({Algo, [12345678|12345678]}); + _ when Algo =:= exs1024; Algo =:= exs1024s -> + %% Printed with orig 'C' code and this seed + rand:seed_s({Algo, {lists:duplicate(16, 12345678), []}}); + _ -> % unimplemented + not_implemented + end, + case State of not_implemented -> [not_implemented]; - S -> gen_jump_1(?LOOP_JUMP, S, []) + _ -> + Max = range(State), + gen_jump_1(?LOOP_JUMP, State, Max, []) end. -gen_jump_1(N, State0 = {#{max:=Max}, _}, Acc) when N > 0 -> +gen_jump_1(N, State0, Max, Acc) when N > 0 -> {_, State1} = rand:uniform_s(Max, State0), {Random, State2} = rand:uniform_s(Max, rand:jump(State1)), case N rem (?LOOP_JUMP div 100) of - 0 -> gen_jump_1(N-1, State2, [Random|Acc]); - _ -> gen_jump_1(N-1, State2, Acc) + 0 -> gen_jump_1(N-1, State2, Max, [Random|Acc]); + _ -> gen_jump_1(N-1, State2, Max, Acc) end; -gen_jump_1(_, _, Acc) -> lists:reverse(Acc). +gen_jump_1(_, _, _, Acc) -> lists:reverse(Acc). + %% Check if each algorithm generates the proper jump sequence %% with the internal state in the process dictionary. @@ -530,25 +717,26 @@ reference_jump_0(Alg) -> gen_jump_0(Algo) -> Seed = case Algo of - exsplus -> %% Printed with orig 'C' code and this seed - rand:seed({exsplus, [12345678|12345678]}); - exs1024 -> %% Printed with orig 'C' code and this seed - rand:seed({exs1024, {lists:duplicate(16, 12345678), []}}); exs64 -> %% Test exception of not_implemented notice - try - _ = rand:seed(exs64), - rand:jump() - catch - error:not_implemented -> not_implemented - end; + try + _ = rand:seed(exs64), + rand:jump() + catch + error:not_implemented -> not_implemented + end; + _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> + %% Printed with orig 'C' code and this seed + rand:seed({Algo, [12345678|12345678]}); + _ when Algo =:= exs1024; Algo =:= exs1024s -> + %% Printed with orig 'C' code and this seed + rand:seed({Algo, {lists:duplicate(16, 12345678), []}}); _ -> % unimplemented not_implemented end, case Seed of not_implemented -> [not_implemented]; - S -> - {Seedmap=#{}, _} = S, - Max = maps:get(max, Seedmap), + _ -> + Max = range(Seed), gen_jump_0(?LOOP_JUMP, Max, []) end. @@ -643,9 +831,77 @@ reference_val(exsplus) -> 16#6c6145ffa1169d,16#18ec2c393d45359,16#1f1a5f256e7130c,16#131cc2f49b8004f, 16#36f715a249f4ec2,16#1c27629826c50d3,16#914d9a6648726a,16#27f5bf5ce2301e8, 16#3dd493b8012970f,16#be13bed1e00e5c,16#ceef033b74ae10,16#3da38c6a50abe03, - 16#15cbd1a421c7a8c,16#22794e3ec6ef3b1,16#26154d26e7ea99f,16#3a66681359a6ab6]. + 16#15cbd1a421c7a8c,16#22794e3ec6ef3b1,16#26154d26e7ea99f,16#3a66681359a6ab6]; + +reference_val(exsp) -> + reference_val(exsplus); +reference_val(exs1024s) -> + reference_val(exs1024); +reference_val(exrop) -> +%% #include <stdint.h> +%% #include <stdio.h> +%% +%% uint64_t s[2]; +%% uint64_t next(void); +%% /* Xoroshiro116+ PRNG here */ +%% +%% int main(char *argv[]) { +%% int n; +%% uint64_t r; +%% s[0] = 12345678; +%% s[1] = 12345678; +%% +%% for (n = 1000000; n > 0; n--) { +%% r = next(); +%% if ((n % 10000) == 0) { +%% printf("%llu,", (unsigned long long) (r + 1)); +%% } +%% } +%% printf("\n"); +%% } + [24691357,29089185972758626,135434857127264790, + 277209758236304485,101045429972817342, + 241950202080388093,283018380268425711,268233672110762489, + 173241488791227202,245038518481669421, + 253627577363613736,234979870724373477,115607127954560275, + 96445882796968228,166106849348423677, + 83614184550774836,109634510785746957,68415533259662436, + 12078288820568786,246413981014863011, + 96953486962147513,138629231038332640,206078430370986460, + 11002780552565714,238837272913629203, + 60272901610411077,148828243883348685,203140738399788939, + 131001610760610046,30717739120305678, + 262903815608472425,31891125663924935,107252017522511256, + 241577109487224033,263801934853180827, + 155517416581881714,223609336630639997,112175917931581716, + 16523497284706825,201453767973653420, + 35912153101632769,211525452750005043,96678037860996922, + 70962216125870068,107383886372877124, + 223441708670831233,247351119445661499,233235283318278995, + 280646255087307741,232948506631162445, + %% + 117394974124526779,55395923845250321,274512622756597759, + 31754154862553492,222645458401498438, + 161643932692872858,11771755227312868,93933211280589745, + 92242631276348831,197206910466548143, + 150370169849735808,229903773212075765,264650708561842793, + 30318996509793571,158249985447105184, + 220423733894955738,62892844479829080,112941952955911674, + 203157000073363030,54175707830615686, + 50121351829191185,115891831802446962,62298417197154985, + 6569598473421167,69822368618978464, + 176271134892968134,160793729023716344,271997399244980560, + 59100661824817999,150500611720118722, + 23707133151561128,25156834940231911,257788052162304719, + 176517852966055005,247173855600850875, + 83440973524473396,94711136045581604,154881198769946042, + 236537934330658377,152283781345006019, + 250789092615679985,78848633178610658,72059442721196128, + 98223942961505519,191144652663779840, + 102425686803727694,89058927716079076,80721467542933080, + 8462479817391645,2774921106204163]. -%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% reference_jump_val(exsplus) -> [82445318862816932, 145810727464480743, 16514517716894509, 247642377064868650, @@ -701,4 +957,93 @@ reference_jump_val(exs1024) -> 17936751184378118743, 4224632875737239207, 15888641556987476199, 9586888813112229805, 9476861567287505094, 14909536929239540332, 17996844556292992842, 2699310519182298856]; -reference_jump_val(exs64) -> [not_implemented]. +reference_jump_val(exsp) -> + reference_jump_val(exsplus); +reference_jump_val(exs1024s) -> + reference_jump_val(exs1024); +reference_jump_val(exs64) -> [not_implemented]; +reference_jump_val(exrop) -> +%% #include <stdint.h> +%% #include <stdio.h> +%% +%% uint64_t s[2]; +%% uint64_t next(void); +%% /* Xoroshiro116+ PRNG here */ +%% +%% int main(char *argv[]) { +%% int n; +%% uint64_t r; +%% s[0] = 12345678; +%% s[1] = 12345678; + +%% for (n = 1000; n > 0; n--) { +%% next(); +%% jump(); +%% r = next(); +%% if ((n % 10) == 0) { +%% printf("%llu,", (unsigned long long) (r + 1)); +%% } +%% } +%% printf("\n"); +%% } + [60301713907476001,135397949584721850,4148159712710727, + 110297784509908316,18753463199438866, + 106699913259182846,2414728156662676,237591345910610406, + 48519427605486503,38071665570452612, + 235484041375354592,45428997361037927,112352324717959775, + 226084403445232507,270797890380258829, + 160587966336947922,80453153271416820,222758573634013699, + 195715386237881435,240975253876429810, + 93387593470886224,23845439014202236,235376123357642262, + 22286175195310374,239068556844083490, + 120126027410954482,250690865061862527,113265144383673111, + 57986825640269127,206087920253971490, + 265971029949338955,40654558754415167,185972161822891882, + 72224917962819036,116613804322063968, + 129103518989198416,236110607653724474,98446977363728314, + 122264213760984600,55635665885245081, + 42625530794327559,288031254029912894,81654312180555835, + 261800844953573559,144734008151358432, + 77095621402920587,286730580569820386,274596992060316466, + 97977034409404188,5517946553518132, + %% + 56460292644964432,252118572460428657,38694442746260303, + 165653145330192194,136968555571402812, + 64905200201714082,257386366768713186,22702362175273017, + 208480936480037395,152926769756967697, + 256751159334239189,130982960476845557,21613531985982870, + 87016962652282927,130446710536726404, + 188769410109327420,282891129440391928,251807515151187951, + 262029034126352975,30694713572208714, + 46430187445005589,176983177204884508,144190360369444480, + 14245137612606100,126045457407279122, + 169277107135012393,42599413368851184,130940158341360014, + 113412693367677211,119353175256553456, + 96339829771832349,17378172025472134,110141940813943768, + 253735613682893347,234964721082540068, + 85668779779185140,164542570671430062,18205512302089755, + 282380693509970845,190996054681051049, + 250227633882474729,171181147785250210,55437891969696407, + 241227318715885854,77323084015890802, + 1663590009695191,234064400749487599,222983191707424780, + 254956809144783896,203898972156838252]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%% The old algorithms used a range 2^N - 1 for their reference val +%% tests, which was incorrect but works as long as you do not draw +%% the value 2^N, which is very unlikely. It was not possible +%% to simply correct the range to 2^N due to another incorrectness +%% in that the old algorithms changed to using the broken +%% (multiply a float approach with too few bits) approach for +%% ranges >= 2^N. This function digs out the range to use +%% for the reference tests for old and new algorithms. +range({#{bits:=Bits}, _}) -> 1 bsl Bits; +range({#{max:=Max}, _}) -> Max; %% Old incorrect range +range({_, _, _}) -> 51. % random + + +quart_range({#{bits:=Bits}, _}) -> 1 bsl (Bits - 2); +quart_range({#{max:=Max}, _}) -> (Max bsr 2) + 1; +quart_range({#{}, _}) -> 1 bsl 62; % crypto +quart_range({_, _, _}) -> 1 bsl 49. % random diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index a76ded5f60..71f86e32e5 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ run_options/1,combined_options/1,replace_autogen/1, global_capture/1,replace_input_types/1,replace_return/1, split_autogen/1,split_options/1,split_specials/1, - error_handling/1,pcre_cve_2008_2371/1, + error_handling/1,pcre_cve_2008_2371/1,re_version/1, pcre_compile_workspace_overflow/1,re_infinite_loop/1, re_backwards_accented/1,opt_dupnames/1,opt_all_names/1,inspect/1, opt_no_start_optimize/1,opt_never_utf/1,opt_ucp/1, @@ -45,7 +45,7 @@ all() -> pcre_compile_workspace_overflow, re_infinite_loop, re_backwards_accented, opt_dupnames, opt_all_names, inspect, opt_no_start_optimize,opt_never_utf,opt_ucp, - match_limit, sub_binaries]. + match_limit, sub_binaries, re_version]. groups() -> []. @@ -194,6 +194,14 @@ run_options(Config) when is_list(Config) -> +%% Test the version is retorned correctly +re_version(_Config) -> + Version = re:version(), + {match,[Version]} = re:run(Version,"^[0-9]\\.[0-9]{2} 20[0-9]{2}-[0-9]{2}-[0-9]{2}",[{capture,all,binary}]), + ok. + + + %% Test compile options given directly to run. combined_options(Config) when is_list(Config) -> ok = crtest("ABDabcdABCD","abcd",[],true,{match,[{3,4}]}), diff --git a/lib/stdlib/test/tar_SUITE.erl b/lib/stdlib/test/tar_SUITE.erl index 2e1ae7bcff..76fa760fa8 100644 --- a/lib/stdlib/test/tar_SUITE.erl +++ b/lib/stdlib/test/tar_SUITE.erl @@ -27,7 +27,7 @@ extract_from_binary_compressed/1, extract_filtered/1, extract_from_open_file/1, symlinks/1, open_add_close/1, cooked_compressed/1, memory/1,unicode/1,read_other_implementations/1, - sparse/1, init/1]). + sparse/1, init/1, leading_slash/1, dotdot/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). @@ -41,7 +41,7 @@ all() -> extract_filtered, symlinks, open_add_close, cooked_compressed, memory, unicode, read_other_implementations, - sparse,init]. + sparse,init,leading_slash,dotdot]. groups() -> []. @@ -920,6 +920,39 @@ unicode_create_files() -> [] end]. +leading_slash(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Dir = filename:join(PrivDir, ?FUNCTION_NAME), + TarFile = filename:join(Dir, "leading_slash.tar"), + ok = filelib:ensure_dir(TarFile), + {ok,Fd} = erl_tar:open(TarFile, [write]), + TarMemberName = "e/d/c/b/a_member", + TarMemberNameAbs = "/" ++ TarMemberName, + Contents = <<"contents\n">>, + ok = erl_tar:add(Fd, Contents, TarMemberNameAbs, [verbose]), + ok = erl_tar:close(Fd), + + ok = erl_tar:extract(TarFile, [{cwd,Dir}]), + + {ok,Contents} = file:read_file(filename:join(Dir, TarMemberName)), + ok. + +dotdot(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Dir = filename:join(PrivDir, ?FUNCTION_NAME), + ok = file:make_dir(Dir), + Tar = filename:join(Dir, "dotdot.tar"), + {ok,Fd} = erl_tar:open(Tar, [write]), + BeamFile = code:which(?MODULE), + ok = erl_tar:add(Fd, BeamFile, "a/./../../some_file", []), + ok = erl_tar:close(Fd), + + {error,{_,unsafe_path=Error}} = erl_tar:extract(Tar, [{cwd,Dir}]), + false = filelib:is_regular(filename:join(PrivDir, "some_file")), + io:format("~s\n", [erl_tar:format_error(Error)]), + + ok. + %% Delete the given list of files. delete_files([]) -> ok; delete_files([Item|Rest]) -> diff --git a/lib/tools/src/lcnt.erl b/lib/tools/src/lcnt.erl index 22db947e7a..864d62f6c5 100644 --- a/lib/tools/src/lcnt.erl +++ b/lib/tools/src/lcnt.erl @@ -698,19 +698,47 @@ stats2record([{{File,Line},{Tries,Colls,{S,Ns,N}}}|Stats]) -> nt = N} | stats2record(Stats)]; stats2record([]) -> []. + clean_id_creation(Id) when is_pid(Id) -> Bin = term_to_binary(Id), - <<H:3/binary, L:16, Node:L/binary, Ids:8/binary, _Creation/binary>> = Bin, - Bin2 = list_to_binary([H, bytes16(L), Node, Ids, 0]), + <<H:3/binary, Rest/binary>> = Bin, + <<131, PidTag, AtomTag>> = H, + LL = atomlen_bits(AtomTag), + CL = creation_bits(PidTag), + <<L:LL, Node:L/binary, Ids:8/binary, _Creation/binary>> = Rest, + Bin2 = list_to_binary([H, <<L:LL>>, Node, Ids, <<0:CL>>]), binary_to_term(Bin2); clean_id_creation(Id) when is_port(Id) -> Bin = term_to_binary(Id), - <<H:3/binary, L:16, Node:L/binary, Ids:4/binary, _Creation/binary>> = Bin, - Bin2 = list_to_binary([H, bytes16(L), Node, Ids, 0]), + <<H:3/binary, Rest/binary>> = Bin, + <<131, PortTag, AtomTag>> = H, + LL = atomlen_bits(AtomTag), + CL = creation_bits(PortTag), + <<L:LL, Node:L/binary, Ids:4/binary, _Creation/binary>> = Rest, + Bin2 = list_to_binary([H, <<L:LL>>, Node, Ids, <<0:CL>>]), binary_to_term(Bin2); clean_id_creation(Id) -> Id. +-define(PID_EXT, $g). +-define(NEW_PID_EXT, $X). +-define(PORT_EXT, $f). +-define(NEW_PORT_EXT, $Y). +-define(ATOM_EXT, $d). +-define(SMALL_ATOM_EXT, $s). +-define(ATOM_UTF8_EXT, $v). +-define(SMALL_ATOM_UTF8_EXT, $w). + +atomlen_bits(?ATOM_EXT) -> 16; +atomlen_bits(?SMALL_ATOM_EXT) -> 8; +atomlen_bits(?ATOM_UTF8_EXT) -> 16; +atomlen_bits(?SMALL_ATOM_UTF8_EXT) -> 8. + +creation_bits(?PID_EXT) -> 8; +creation_bits(?NEW_PID_EXT) -> 32; +creation_bits(?PORT_EXT) -> 8; +creation_bits(?NEW_PORT_EXT) -> 32. + %% serializer state_default(Field) -> proplists:get_value(Field, state2list(#state{})). @@ -932,9 +960,6 @@ strings([{space, N, S} | Ss], Out) -> strings(Ss, Out ++ term2string(term2 strings([{left, N, S} | Ss], Out) -> strings(Ss, Out ++ term2string(term2string(" ~~s~~~ws", [N]), [S,""])); strings([S|Ss], Out) -> strings(Ss, Out ++ term2string("~ts", [S])). --define(SMALL_ATOM_UTF8_EXT, $w). --define(ATOM_UTF8_EXT, $v). --define(ATOM_EXT, $d). term2string({M,F,A}) when is_atom(M), is_atom(F), is_integer(A) -> term2string("~p:~p/~p", [M,F,A]); term2string(Term) when is_port(Term) -> diff --git a/lib/tools/test/xref_SUITE.erl b/lib/tools/test/xref_SUITE.erl index f308ea1204..b8e7113595 100644 --- a/lib/tools/test/xref_SUITE.erl +++ b/lib/tools/test/xref_SUITE.erl @@ -89,11 +89,11 @@ init_per_suite(Conf) when is_list(Conf) -> DataDir = ?datadir, PrivDir = ?privdir, CopyDir = fname(PrivDir, "datacopy"), + ok = file:make_dir(CopyDir), TarFile = fname(PrivDir, "datacopy.tgz"), - {ok, Tar} = erl_tar:open(TarFile, [write, compressed]), - ok = erl_tar:add(Tar, DataDir, CopyDir, [compressed]), - ok = erl_tar:close(Tar), - ok = erl_tar:extract(TarFile, [compressed]), + ok = file:set_cwd(DataDir), + ok = erl_tar:create(TarFile, ["."], [compressed]), + ok = erl_tar:extract(TarFile, [compressed, {cwd,CopyDir}]), ok = file:delete(TarFile), [{copy_dir, CopyDir}|Conf]. diff --git a/scripts/run-dialyzer b/scripts/run-dialyzer new file mode 100755 index 0000000000..383ae2301d --- /dev/null +++ b/scripts/run-dialyzer @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +$ERL_TOP/bin/dialyzer --build_plt --apps asn1 compiler crypto dialyzer edoc erts et hipe inets kernel mnesia observer public_key runtime_tools snmp ssh ssl stdlib syntax_tools wx xmerl --statistics +$ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps compiler erts kernel stdlib asn1 crypto dialyzer hipe parsetools public_key runtime_tools sasl tools --statistics +$ERL_TOP/bin/dialyzer -n --apps common_test debugger edoc inets mnesia observer ssh ssl syntax_tools tools wx xmerl --statistics |