diff options
104 files changed, 2788 insertions, 841 deletions
diff --git a/OTP_VERSION b/OTP_VERSION index 39626521cb..5f8e72c36b 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -18.1.4 +18.1.5 diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index 42b6a3bfef..e717fc0c4e 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -2811,7 +2811,7 @@ ERL_DRV_MAP int sz </func> <func> - <name><ret>int</ret><nametext>erl_drv_putenv(char *key, char *value)</nametext></name> + <name><ret>int</ret><nametext>erl_drv_putenv(const char *key, char *value)</nametext></name> <fsummary>Set the value of an environment variable</fsummary> <desc> <marker id="erl_drv_putenv"></marker> @@ -2840,7 +2840,7 @@ ERL_DRV_MAP int sz </desc> </func> <func> - <name><ret>int</ret><nametext>erl_drv_getenv(char *key, char *value, size_t *value_size)</nametext></name> + <name><ret>int</ret><nametext>erl_drv_getenv(const char *key, char *value, size_t *value_size)</nametext></name> <fsummary>Get the value of an environment variable</fsummary> <desc> <marker id="erl_drv_getenv"></marker> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index dae14b8d08..2d8706169f 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -791,6 +791,10 @@ typedef enum { and return true, or return false if <c>term</c> is not an unsigned integer or is outside the bounds of type <c>unsigned long</c>.</p></desc> </func> + <func><name><ret>int</ret><nametext>enif_getenv(const char* key, char* value, size_t *value_size)</nametext></name> + <fsummary>Get the value of an environment variable</fsummary> + <desc><p>Same as <seealso marker="erl_driver#erl_drv_getenv">erl_drv_getenv</seealso>.</p></desc> + </func> <func><name><ret>int</ret><nametext>enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason)</nametext></name> <fsummary>Check if an exception has been raised</fsummary> <desc><p>Return true if a pending exception is associated diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 0bd46a2dae..a8cc19ee1f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4231,6 +4231,7 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) goto bad; enp = erts_find_or_insert_node(dep->sysname, dep->creation); + ASSERT(enp != erts_this_node); etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 0bbcc5f966..4846133aa6 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -2576,7 +2576,9 @@ int distribution_info(int to, void *arg) /* Called by break handler */ } for (dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - info_dist_entry(to, arg, dep, 0, 0); + if (dep != erts_this_dist_entry) { + info_dist_entry(to, arg, dep, 0, 0); + } } return(0); @@ -2654,13 +2656,8 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) if (!net_kernel) goto error; - /* By setting dist_entry==erts_this_dist_entry and DISTRIBUTION on - net_kernel do_net_exist will be called when net_kernel - is terminated !! */ - (void) ERTS_PROC_SET_DIST_ENTRY(net_kernel, - ERTS_PROC_LOCK_MAIN, - erts_this_dist_entry); - erts_refc_inc(&erts_this_dist_entry->refc, 2); + /* By setting F_DISTRIBUTION on net_kernel, + * do_net_exist will be called when net_kernel is terminated !! */ net_kernel->flags |= F_DISTRIBUTION; if (net_kernel != BIF_P) @@ -3021,11 +3018,11 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); - ASSERT(erts_no_of_not_connected_dist_entries >= 0); + ASSERT(erts_no_of_not_connected_dist_entries > 0); ASSERT(erts_no_of_hidden_dist_entries >= 0); ASSERT(erts_no_of_visible_dist_entries >= 0); if(not_connected) - length += erts_no_of_not_connected_dist_entries; + length += (erts_no_of_not_connected_dist_entries - 1); if(hidden) length += erts_no_of_hidden_dist_entries; if(visible) @@ -3047,8 +3044,10 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) #endif if(not_connected) for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - result = CONS(hp, dep->sysname, result); - hp += 2; + if (dep != erts_this_dist_entry) { + result = CONS(hp, dep->sysname, result); + hp += 2; + } } if(hidden) for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 55c164bf11..c3f4fe5a63 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -134,6 +134,7 @@ static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; static ErtsAllocatorState_t fix_alloc_state; +static ErtsAllocatorState_t test_alloc_state; typedef struct { erts_smp_atomic32_t refc; @@ -233,6 +234,7 @@ typedef struct { struct au_init std_low_alloc; struct au_init ll_low_alloc; #endif + struct au_init test_alloc; } erts_alc_hndl_args_init_t; #define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} @@ -432,6 +434,33 @@ set_default_fix_alloc_opts(struct au_init *ip, ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } +static void +set_default_test_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 0; /* Disabled by default */ + ip->thr_spec = -1 * erts_no_schedulers; + ip->atype = AOFIRSTFIT; + ip->init.aoff.flavor = AOFF_BF; + ip->init.util.name_prefix = "test_"; + ip->init.util.alloc_no = ERTS_ALC_A_TEST; + ip->init.util.mmbcs = 0; /* Main carrier size */ + ip->init.util.ts = ERTS_ALC_MTA_TEST; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; + + /* Use a constant minimal MBC size */ +#if ERTS_SA_MB_CARRIERS + ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.sbct = ERTS_SACRR_UNIT_SZ; +#else + ip->init.util.smbcs = 1 << 12; + ip->init.util.lmbcs = 1 << 12; + ip->init.util.sbct = 1 << 12; +#endif +} + + #ifdef ERTS_SMP static void @@ -613,6 +642,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_driver_alloc_opts(&init.driver_alloc); set_default_fix_alloc_opts(&init.fix_alloc, fix_type_sizes); + set_default_test_alloc_opts(&init.test_alloc); if (argc && argv) handle_args(argc, argv, &init); @@ -772,6 +802,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu); set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) @@ -833,6 +864,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) &init.fix_alloc, &fix_alloc_state); + start_au_allocator(ERTS_ALC_A_TEST, + &init.test_alloc, + &test_alloc_state); + erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); @@ -1418,6 +1453,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->fix_alloc, &init->sl_alloc, &init->temp_alloc + /* test_alloc not affected by +Mea??? or +Mu??? */ }; int aui_sz = (int) sizeof(aui)/sizeof(aui[0]); char *arg; @@ -1508,6 +1544,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'T': handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0); break; + case 'Z': + handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0); + break; case 'Y': { /* sys_alloc */ if (has_prefix("tt", param+2)) { /* set trim threshold */ @@ -2222,11 +2261,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators *have* to be enabled */ + /* All alloc_util allocators *have* to be enabled, except test_alloc */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: if (!erts_allctrs_info[ai].enabled @@ -2267,6 +2307,8 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) * contain any allocated memory. */ continue; + case ERTS_ALC_A_TEST: + continue; case ERTS_ALC_A_EHEAP: save = &size.processes; break; @@ -2675,14 +2717,17 @@ erts_alloc_util_allocators(void *proc) /* * Currently all allocators except sys_alloc are * alloc_util allocators. + * Also hide test_alloc which is disabled by default + * and only intended for our own testing. */ - sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2; + sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2; ASSERT(sz > 0); hp = HAlloc((Process *) proc, sz); res = NIL; for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) { switch (i) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: { char *alc_str = (char *) ERTS_ALC_A2AD(i); @@ -3566,6 +3611,41 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) #else case 0xf13: return (UWord) 0; #endif + case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1); + + case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; + + case 0xf16: { + Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + Uint offset = ts->allctr[0]->mbc_header_size; + void* orig_creating_mbc = ts->allctr[0]->creating_mbc; + void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc; + void* new_creating_mbc = *(void**)a2; /* inout arg */ + void* new_destroying_mbc = *(void**)a3; /* inout arg */ + int i; + + for (i=0; i < ts->size; i++) { + Allctr_t* ap = ts->allctr[i]; + if (ap->mbc_header_size != offset + || ap->creating_mbc != orig_creating_mbc + || ap->destroying_mbc != orig_destroying_mbc + || ap->mbc_list.first != NULL) + return -1; + } + for (i=0; i < ts->size; i++) { + ts->allctr[i]->mbc_header_size += extra_hdr_sz; + ts->allctr[i]->creating_mbc = new_creating_mbc; + ts->allctr[i]->destroying_mbc = new_destroying_mbc; + } + *(void**)a2 = orig_creating_mbc; + *(void**)a3 = orig_destroying_mbc; + return offset; + } + case 0xf17: { + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + return ts->allctr[0]->largest_mbc_size; + } default: break; } diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index b1d511ab78..1ecebdeb07 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -112,7 +112,7 @@ allocator STANDARD_LOW false std_low_alloc allocator BINARY true binary_alloc allocator DRIVER true driver_alloc - +allocator TEST true test_alloc # --- Class declarations ----------------------------------------------------- # @@ -456,4 +456,7 @@ type CON_VPRINTF_BUF TEMPORARY SYSTEM con_vprintf_buf +endif +# This type should only be used for test +type TEST TEST SYSTEM testing + # ---------------------------------------------------------------------------- diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 236ee35d18..8229a15824 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1437,6 +1437,16 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned); +static ERTS_INLINE void +dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(IS_MB_CARRIER(crr)); + if (allctr->destroying_mbc) + allctr->destroying_mbc(allctr, crr); + + dealloc_carrier(allctr, crr, 1); +} + #ifdef ERTS_SMP static ERTS_INLINE Allctr_t* @@ -3149,7 +3159,7 @@ cpool_fetch(Allctr_t *allctr, UWord size) cpool_entrance = sentinel; cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev)); if (cpdp == sentinel) - return NULL; + goto check_dc_list; } has_passed_sentinel = 0; @@ -3160,18 +3170,18 @@ cpool_fetch(Allctr_t *allctr, UWord size) if (cpool_entrance == sentinel) { cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) - return NULL; + break; } i = 0; /* Last one to inspect */ } else if (cpdp == sentinel) { if (has_passed_sentinel) { /* We been here before. cpool_entrance must have been removed */ - return NULL; + break; } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) - return NULL; + break; has_passed_sentinel = 1; } crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); @@ -3195,10 +3205,12 @@ cpool_fetch(Allctr_t *allctr, UWord size) return NULL; } +check_dc_list: /* Last; check our own pending dealloc carrier list... */ crr = allctr->cpool.dc_list.last; while (crr) { if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + Block_t* blk; unlink_carrier(&allctr->cpool.dc_list, crr); #ifdef ERTS_ALC_CPOOL_DEBUG ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, @@ -3207,6 +3219,9 @@ cpool_fetch(Allctr_t *allctr, UWord size) #else erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); #endif + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(FBLK_TO_MBC(blk) == crr); + allctr->link_free_block(allctr, blk); return crr; } crr = crr->prev; @@ -3237,7 +3252,7 @@ check_pending_dealloc_carrier(Allctr_t *allctr, dcrr = crr; crr = crr->next; - dealloc_carrier(allctr, dcrr, 1); + dealloc_mbc(allctr, dcrr); i++; } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); @@ -3268,18 +3283,20 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; + Block_t *blk; int check_pending_dealloc; erts_aint_t max_size; + ASSERT(IS_MB_CARRIER(crr)); + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); return; } orig_allctr = crr->cpool.orig_allctr; if (allctr != orig_allctr) { - Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); int cinit = orig_allctr->dd.ix - allctr->dd.ix; /* @@ -3296,6 +3313,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * since the block is an mbc block that is free and last * in the carrier. */ + blk = MBC_TO_FIRST_BLK(allctr, crr); ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); @@ -3315,11 +3333,13 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); return; } - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + max_size = (erts_aint_t) MBC_FBLK_SZ(blk); erts_atomic_set_nob(&crr->cpool.max_size, max_size); crr->next = NULL; @@ -3894,9 +3914,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) } #endif - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr); - #ifdef ERTS_SMP if (busy_pcrr_pp && *busy_pcrr_pp) { ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); @@ -3920,12 +3937,15 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) else #endif STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + + if (allctr->remove_mbc) + allctr->remove_mbc(allctr, crr); } #ifdef ERTS_SMP schedule_dealloc_carrier(allctr, crr); #else - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); #endif } } @@ -6054,6 +6074,16 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x023: return (UWord) 0; case 0x024: return (UWord) 0; #endif + case 0x025: /* UMEM2BLK_TEST*/ +#ifdef DEBUG +# ifdef HARD_DEBUG + return (UWord)UMEM2BLK(a1-3*sizeof(UWord)); +# else + return (UWord)UMEM2BLK(a1-2*sizeof(UWord)); +# endif +#else + return (UWord)UMEM2BLK(a1); +#endif default: ASSERT(0); return ~((UWord) 0); } diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index df1f0aa65a..f4a2ae7ff3 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -277,7 +277,7 @@ typedef struct ErtsDoubleLink_t_ { typedef struct { erts_atomic_t next; erts_atomic_t prev; - Allctr_t *orig_allctr; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ ErtsThrPrgrVal thr_prgr; erts_atomic_t max_size; UWord abandon_limit; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 7c2a5c3323..19420af8ab 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -209,7 +209,9 @@ static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint); static void aoff_link_free_block(Allctr_t *, Block_t*); static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del); static void aoff_creating_mbc(Allctr_t*, Carrier_t*); +#ifdef DEBUG static void aoff_destroying_mbc(Allctr_t*, Carrier_t*); +#endif static void aoff_add_mbc(Allctr_t*, Carrier_t*); static void aoff_remove_mbc(Allctr_t*, Carrier_t*); static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); @@ -271,7 +273,11 @@ erts_aoffalc_start(AOFFAllctr_t *alc, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = aoff_creating_mbc; +#ifdef DEBUG allctr->destroying_mbc = aoff_destroying_mbc; +#else + allctr->destroying_mbc = NULL; +#endif allctr->add_mbc = aoff_add_mbc; allctr->remove_mbc = aoff_remove_mbc; allctr->largest_fblk_in_mbc = aoff_largest_fblk_in_mbc; @@ -885,17 +891,18 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) HARD_CHECK_TREE(NULL, 0, *root, 0); } +#define IS_CRR_IN_TREE(CRR,ROOT) \ + ((CRR)->rbt_node.parent || (ROOT) == &(CRR)->rbt_node) + +#ifdef DEBUG static void aoff_destroying_mbc(Allctr_t *allctr, Carrier_t *carrier) { AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; - AOFF_RBTree_t *root = alc->mbc_root; - if (crr->rbt_node.parent || &crr->rbt_node == root) { - aoff_remove_mbc(allctr, carrier); - } - /*else already removed */ + ASSERT(!IS_CRR_IN_TREE(crr, alc->mbc_root)); } +#endif static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) { @@ -903,6 +910,7 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; + ASSERT(!IS_CRR_IN_TREE(crr, *root)); HARD_CHECK_TREE(NULL, 0, *root, 0); /* Link carrier in address order tree @@ -919,6 +927,10 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_RBTree_t **root = &alc->mbc_root; ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + + if (!IS_CRR_IN_TREE(crr,*root)) + return; + HARD_CHECK_TREE(NULL, 0, *root, 0); rbt_delete(root, &crr->rbt_node); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 6b406d069c..dbb4d719c1 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -696,8 +696,8 @@ EXTERN int driver_dl_close(void *); EXTERN char *driver_dl_error(void); /* environment */ -EXTERN int erl_drv_putenv(char *key, char *value); -EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); +EXTERN int erl_drv_putenv(const char *key, char *value); +EXTERN int erl_drv_getenv(const char *key, char *value, size_t *value_size); #ifdef __OSE__ typedef ErlDrvUInt ErlDrvOseEventId; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d2604f1595..2f21111a2e 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1237,6 +1237,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) Uint oh_size = (char *) OLD_HTOP(p) - oh; Uint n; Uint new_sz; + int done; /* * Do a fullsweep GC. First figure out the size of the heap @@ -1440,6 +1441,8 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) *recl += size_before - (HEAP_TOP(p) - HEAP_START(p)); + remove_message_buffers(p); + { ErlMessage *msgp; @@ -1458,15 +1461,21 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } } - adjust_after_fullsweep(p, need, objv, nobj); - -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); + if (MBUF(p)) { + /* This is a very rare case when distributed messages copied above + * contained maps so big they did not fit on the heap causing the + * factory to create heap frags. + * Solution: Trigger a minor gc (without tenuring) + */ + HIGH_WATER(p) = HEAP_START(p); + done = 0; + } else { + adjust_after_fullsweep(p, need, objv, nobj); + done = 1; + } ErtsGcQuickSanityCheck(p); - return 1; /* We are done. */ + return done; } static void @@ -1955,7 +1964,18 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, if (p->dictionary != NULL) { disallow_heap_frag_ref(p, n_htop, p->dictionary->data, p->dictionary->used); } - disallow_heap_frag_ref_in_heap(p); + /* OTP-18: Actually we do allow references from heap to heap fragments now. + This can happen when doing "binary_to_term" with a "fat" map contained + in another term. A "fat" map is a hashmap with higher heap demand than + first estimated by "binary_to_term" causing the factory to allocate + additional heap (fragments) for the hashmap tree nodes. + Run map_SUITE:t_gc_rare_map_overflow to provoke this. + + Inverted references like this does not matter however. The copy done + below by move_one_area() with move markers in the fragments and the + sweeping done later by the GC should make everything ok in the end. + */ + /***disallow_heap_frag_ref_in_heap(p);***/ #endif /* diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index add4a66f90..d7a2076d85 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1173,6 +1173,7 @@ ErlNifTid enif_thread_self(void) { return erl_drv_thread_self(); } int enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2) { return erl_drv_equal_tids(tid1,tid2); } void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); } int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); } +int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); } int enif_fprintf(void* filep, const char* format, ...) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 7d880126f8..5e39343e9b 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -48,9 +48,10 @@ ** add ErlNifEntry options ** add ErlNifFunc flags ** 2.8: 18.0 add enif_has_pending_exception +** 2.9: 18.2 enif_getenv */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 8 +#define ERL_NIF_MINOR_VERSION 9 /* * The emulator will refuse to load a nif-lib with a major version @@ -231,6 +232,7 @@ typedef enum { # define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS typedef struct { # include "erl_nif_api_funcs.h" + void* erts_alc_test; } TWinDynNifCallbacks; extern TWinDynNifCallbacks WinDynNifCallbacks; # undef ERL_NIF_API_FUNC_DECL diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 2f2180e1aa..08b9afc6af 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -159,6 +159,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[])); ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason)); +ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* value_size)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -310,6 +311,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); # define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif) # define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception) # define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception) +# define enif_getenv ERL_NIF_API_FUNC_MACRO(enif_getenv) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 2fb790b953..a7d0511bf9 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -37,18 +37,18 @@ erts_smp_rwmtx_t erts_node_table_rwmtx; DistEntry *erts_hidden_dist_entries; DistEntry *erts_visible_dist_entries; -DistEntry *erts_not_connected_dist_entries; +DistEntry *erts_not_connected_dist_entries; /* including erts_this_dist_entry */ Sint erts_no_of_hidden_dist_entries; Sint erts_no_of_visible_dist_entries; -Sint erts_no_of_not_connected_dist_entries; +Sint erts_no_of_not_connected_dist_entries; /* including erts_this_dist_entry */ DistEntry *erts_this_dist_entry; ErlNode *erts_this_node; char erts_this_node_sysname_BUFFER[256], *erts_this_node_sysname = "uninitialized yet"; -static Uint node_entries; -static Uint dist_entries; +static Uint node_entries = 0; +static Uint dist_entries = 0; static int references_atoms_need_init = 1; @@ -91,9 +91,6 @@ dist_table_alloc(void *dep_tmpl) erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) - return dep_tmpl; - sysname = ((DistEntry *) dep_tmpl)->sysname; chnl_nr = make_small((Uint) atom_val(sysname)); dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); @@ -132,7 +129,9 @@ dist_table_alloc(void *dep_tmpl) /* Link in */ - /* All new dist entries are "not connected" */ + /* All new dist entries are "not connected". + * erts_this_dist_entry is also always included among "not connected" + */ dep->next = erts_not_connected_dist_entries; if(erts_not_connected_dist_entries) { ASSERT(erts_not_connected_dist_entries->prev == NULL); @@ -149,9 +148,6 @@ dist_table_free(void *vdep) { DistEntry *dep = (DistEntry *) vdep; - if(dep == erts_this_dist_entry) - return; - ASSERT(is_nil(dep->cid)); ASSERT(dep->nlinks == NULL); ASSERT(dep->node_links == NULL); @@ -186,7 +182,7 @@ dist_table_free(void *vdep) #endif erts_free(ERTS_ALC_T_DIST_ENTRY, (void *) dep); - ASSERT(dist_entries > 1); + ASSERT(dist_entries > 0); dist_entries--; } @@ -306,7 +302,7 @@ static void try_delete_dist_entry(void *vdep) * thread incremented refc twice. Once for the new reference * and once for this thread. * - * If refc reach -1, noone has used the entry since we + * If refc reach -1, no one has used the entry since we * set up the timer. Delete the entry. * * If refc reach 0, the entry is currently not in use @@ -369,8 +365,7 @@ erts_dist_table_size(void) ASSERT(dist_entries == (erts_no_of_visible_dist_entries + erts_no_of_hidden_dist_entries - + erts_no_of_not_connected_dist_entries - + 1 /* erts_this_dist_entry */)); + + erts_no_of_not_connected_dist_entries)); #endif res = (hash_table_sz(&erts_dist_table) @@ -543,9 +538,6 @@ node_table_alloc(void *venp_tmpl) { ErlNode *enp; - if(((ErlNode *) venp_tmpl) == erts_this_node) - return venp_tmpl; - enp = (ErlNode *) erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode)); node_entries++; @@ -563,8 +555,7 @@ node_table_free(void *venp) { ErlNode *enp = (ErlNode *) venp; - if(enp == erts_this_node) - return; + ERTS_SMP_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking()); erts_deref_dist_entry(enp->dist_entry); #ifdef DEBUG @@ -572,7 +563,7 @@ node_table_free(void *venp) #endif erts_free(ERTS_ALC_T_NODE_ENTRY, venp); - ASSERT(node_entries > 1); + ASSERT(node_entries > 0); node_entries--; } @@ -650,7 +641,7 @@ static void try_delete_node(void *venp) * thread incremented refc twice. Once for the new reference * and once for this thread. * - * If refc reach -1, noone has used the entry since we + * If refc reach -1, no one has used the entry since we * set up the timer. Delete the entry. * * If refc reach 0, the entry is currently not in use @@ -747,25 +738,20 @@ void erts_print_node_info(int to, void erts_set_this_node(Eterm sysname, Uint creation) { - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); + ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2)); - (void) hash_erase(&erts_dist_table, (void *) erts_this_dist_entry); - erts_this_dist_entry->sysname = sysname; - erts_this_dist_entry->creation = creation; - (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry); + if (erts_refc_dectest(&erts_this_node->refc, 0) == 0) + try_delete_node(erts_this_node); - (void) hash_erase(&erts_node_table, (void *) erts_this_node); - erts_this_node->sysname = sysname; - erts_this_node->creation = creation; - erts_this_node_sysname = erts_this_node_sysname_BUFFER; - erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), - "%T", sysname); - (void) hash_put(&erts_node_table, (void *) erts_this_node); + if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) + try_delete_dist_entry(erts_this_dist_entry); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_this_node = NULL; /* to make sure refc is bumped for this node */ + erts_this_node = erts_find_or_insert_node(sysname, creation); + erts_this_dist_entry = erts_this_node->dist_entry; + erts_refc_inc(&erts_this_dist_entry->refc, 2); } Uint @@ -782,6 +768,7 @@ void erts_init_node_tables(int dd_sec) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; + ErlNode node_tmpl; if (dd_sec == ERTS_NODE_TAB_DELAY_GC_INFINITY) node_tab_delete_delay = (ErtsMonotonicTime) -1; @@ -793,16 +780,21 @@ void erts_init_node_tables(int dd_sec) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); + f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; f.alloc = (HALLOC_FUN) dist_table_alloc; f.free = (HFREE_FUN) dist_table_free; - - erts_this_dist_entry = erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); - dist_entries = 1; - hash_init(ERTS_ALC_T_DIST_TABLE, &erts_dist_table, "dist_table", 11, f); + f.hash = (H_FUN) node_table_hash; + f.cmp = (HCMP_FUN) node_table_cmp; + f.alloc = (HALLOC_FUN) node_table_alloc; + f.free = (HFREE_FUN) node_table_free; + hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f); + erts_hidden_dist_entries = NULL; erts_visible_dist_entries = NULL; erts_not_connected_dist_entries = NULL; @@ -810,69 +802,23 @@ void erts_init_node_tables(int dd_sec) erts_no_of_visible_dist_entries = 0; erts_no_of_not_connected_dist_entries = 0; - erts_this_dist_entry->next = NULL; - erts_this_dist_entry->prev = NULL; - erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - - erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, - &rwmtx_opt, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->sysname = am_Noname; - erts_this_dist_entry->cid = NIL; - erts_this_dist_entry->connection_id = 0; - erts_this_dist_entry->status = 0; - erts_this_dist_entry->flags = 0; - erts_this_dist_entry->version = 0; - - erts_smp_mtx_init_x(&erts_this_dist_entry->lnk_mtx, - "dist_entry_links", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->node_links = NULL; - erts_this_dist_entry->nlinks = NULL; - erts_this_dist_entry->monitors = NULL; - - erts_smp_mtx_init_x(&erts_this_dist_entry->qlock, - "dist_entry_out_queue", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->qflgs = 0; - erts_this_dist_entry->qsize = 0; - erts_this_dist_entry->out_queue.first = NULL; - erts_this_dist_entry->out_queue.last = NULL; - erts_this_dist_entry->suspended = NULL; - - erts_this_dist_entry->finalized_out_queue.first = NULL; - erts_this_dist_entry->finalized_out_queue.last = NULL; - erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0); - erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd); - erts_this_dist_entry->send = NULL; - erts_this_dist_entry->cache = NULL; - - (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry); - - f.hash = (H_FUN) node_table_hash; - f.cmp = (HCMP_FUN) node_table_cmp; - f.alloc = (HALLOC_FUN) node_table_alloc; - f.free = (HFREE_FUN) node_table_free; + node_tmpl.sysname = am_Noname; + node_tmpl.creation = 0; + erts_this_node = hash_put(&erts_node_table, &node_tmpl); + /* +1 for erts_this_node */ + erts_refc_init(&erts_this_node->refc, 1); - hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f); + ASSERT(erts_this_node->dist_entry != NULL); + erts_this_dist_entry = erts_this_node->dist_entry; + /* +1 for erts_this_dist_entry */ + /* +1 for erts_this_node->dist_entry */ + erts_refc_init(&erts_this_dist_entry->refc, 2); - erts_this_node = erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode)); - node_entries = 1; - erts_refc_init(&erts_this_node->refc, 1); /* The system itself */ - erts_this_node->sysname = am_Noname; - erts_this_node->creation = 0; - erts_this_node->dist_entry = erts_this_dist_entry; erts_this_node_sysname = erts_this_node_sysname_BUFFER; erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), "%T", erts_this_node->sysname); - (void) hash_put(&erts_node_table, (void *) erts_this_node); - - erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); - erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); - references_atoms_need_init = 1; } @@ -1410,6 +1356,10 @@ setup_reference_table(void) SYSTEM_REF, TUPLE2(&heap[0], AM_system, am_undefined)); + insert_dist_entry(erts_this_dist_entry, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, am_undefined), + erts_this_node->creation); UnUseTmpHeapNoproc(3); max = erts_ptab_max(&erts_proc); @@ -1472,12 +1422,6 @@ setup_reference_table(void) insert_links(ERTS_P_LINKS(proc), proc->common.id); if (ERTS_P_MONITORS(proc)) insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); - /* Insert controller */ - { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); - if (dep) - insert_dist_entry(dep, CTRL_REF, proc->common.id, 0); - } } } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 3b1b593d1c..d583118e7b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -625,11 +625,6 @@ erts_pre_init_process(void) erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks = ERTS_PSD_SCHED_ID_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks - = ERTS_PSD_DIST_ENTRY_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks - = ERTS_PSD_DIST_ENTRY_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks @@ -12373,9 +12368,7 @@ erts_continue_exit_process(Process *p) erts_proc_dec_refc(p); } - dep = ((p->flags & F_DISTRIBUTION) - ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) - : NULL); + dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, ERTS_PROC_LOCKS_ALL, NULL); @@ -12387,8 +12380,6 @@ erts_continue_exit_process(Process *p) if (dep) { erts_do_net_exits(dep, reason); - if(dep) - erts_deref_dist_entry(dep); } /* diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 20ffe7ea7c..10c6fa4a67 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -801,12 +801,11 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_ERROR_HANDLER 0 #define ERTS_PSD_SAVED_CALLS_BUF 1 #define ERTS_PSD_SCHED_ID 2 -#define ERTS_PSD_DIST_ENTRY 3 -#define ERTS_PSD_CALL_TIME_BP 4 -#define ERTS_PSD_DELAYED_GC_TASK_QS 5 -#define ERTS_PSD_NIF_TRAP_EXPORT 6 +#define ERTS_PSD_CALL_TIME_BP 3 +#define ERTS_PSD_DELAYED_GC_TASK_QS 4 +#define ERTS_PSD_NIF_TRAP_EXPORT 5 -#define ERTS_PSD_SIZE 7 +#define ERTS_PSD_SIZE 6 typedef struct { void *data[ERTS_PSD_SIZE]; @@ -824,9 +823,6 @@ typedef struct { #define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS #define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS -#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN - #define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN @@ -1887,11 +1883,6 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #define ERTS_PROC_SCHED_ID(P, L, ID) \ ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) -#define ERTS_PROC_GET_DIST_ENTRY(P) \ - ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) -#define ERTS_PROC_SET_DIST_ENTRY(P, L, D) \ - ((DistEntry *) erts_psd_set((P), (L), ERTS_PSD_DIST_ENTRY, (void *) (D))) - #define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \ ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF)) #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 900616c981..c64c8802b9 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -7596,15 +7596,15 @@ int null_func(void) } int -erl_drv_putenv(char *key, char *value) +erl_drv_putenv(const char *key, char *value) { - return erts_sys_putenv_raw(key, value); + return erts_sys_putenv_raw((char*)key, value); } int -erl_drv_getenv(char *key, char *value, size_t *value_size) +erl_drv_getenv(const char *key, char *value, size_t *value_size) { - return erts_sys_getenv_raw(key, value, value_size); + return erts_sys_getenv_raw((char*)key, value, value_size); } /* get heart_port diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index bb871b05ba..ec94e3a596 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -634,7 +634,6 @@ Uint erts_sys_misc_mem_sz(void); /* Io constants to erts_print and erts_putc */ #define ERTS_PRINT_STDERR (2) #define ERTS_PRINT_STDOUT (1) -#define ERTS_PRINT_INVALID (0) /* Don't want to use 0 since CBUF was 0 */ #define ERTS_PRINT_FILE (-1) #define ERTS_PRINT_SBUF (-2) #define ERTS_PRINT_SNBUF (-3) diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index e9d7c91ac9..5286391746 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -399,9 +399,6 @@ erts_print(int to, void *arg, char *format, ...) case ERTS_PRINT_DSBUF: res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list); break; - case ERTS_PRINT_INVALID: - res = -EINVAL; - break; default: res = erts_vfdprintf((int) to, format, arg_list); break; diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 8aff6c1865..3b6abec25e 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -2581,7 +2581,6 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) case FILE_CLOSE_ON_PORT_EXIT: /* See file_stop. However this is never invoked after the port is killed. */ free_data(data); - EF_FREE(desc); desc = NULL; /* This is it for this port, so just send dtrace and return, avoid doing anything to the freed data */ DTRACE6(efile_drv_return, sched_i1, sched_i2, sched_utag, diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c index bb8a3f041f..69d4ea10c2 100644 --- a/erts/emulator/hipe/hipe_x86_signal.c +++ b/erts/emulator/hipe/hipe_x86_signal.c @@ -198,7 +198,7 @@ static void do_init(void) #define INIT() do { if (!init_done()) do_init(); } while (0) #endif /* __DARWIN__ */ -#if !defined(__GLIBC__) && !defined(__DARWIN__) && !defined(__NetBSD__) +#if defined(__sun__) /* * Assume Solaris/x86 2.8. * There is a number of sigaction() procedures in libc: @@ -232,7 +232,34 @@ static void do_init(void) } #define _NSIG NSIG #define INIT() do { if (!init_done()) do_init(); } while (0) -#endif /* not glibc or darwin */ +#endif /* __sun__ */ + +#if !(defined(__GLIBC__) || defined(__DARWIN__) || defined(__NetBSD__) || defined(__sun__)) +/* + * Unknown libc -- assume musl. Note: musl deliberately does not provide a musl-specific + * feature test macro, so we cannot check for it. + * + * sigaction is a weak alias for __sigaction, which is a wrapper for __libc_sigaction. + * There are libc-internal calls to __libc_sigaction which install handlers, so we must + * override __libc_sigaction rather than __sigaction. + */ +#include <dlfcn.h> +static int (*__next_sigaction)(int, const struct sigaction*, struct sigaction*); +#define init_done() (__next_sigaction != 0) +#define __SIGACTION __libc_sigaction +static void do_init(void) +{ + __next_sigaction = dlsym(RTLD_NEXT, "__libc_sigaction"); + if (__next_sigaction != 0) + return; + perror("dlsym"); + abort(); +} +#ifndef _NSIG +#define _NSIG NSIG +#endif +#define INIT() do { if (!init_done()) do_init(); } while (0) +#endif /* !(__GLIBC__ || __DARWIN__ || __NetBSD__ || __sun__) */ #if !defined(__NetBSD__) /* diff --git a/erts/emulator/sys/win32/erl_win32_sys_ddll.c b/erts/emulator/sys/win32/erl_win32_sys_ddll.c index 9a5557e93d..7c24a77e31 100644 --- a/erts/emulator/sys/win32/erl_win32_sys_ddll.c +++ b/erts/emulator/sys/win32/erl_win32_sys_ddll.c @@ -52,7 +52,8 @@ void erl_sys_ddll_init(void) { #define ERL_NIF_API_FUNC_DECL(RET,NAME,ARGS) nif_callbacks.NAME = NAME #include "erl_nif_api_funcs.h" #undef ERL_NIF_API_FUNC_DECL - + nif_callbacks.erts_alc_test = erts_alc_test; + return; } diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h index 5e62320be4..baac7c903e 100644 --- a/erts/emulator/sys/win32/erl_win_dyn_driver.h +++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h @@ -145,8 +145,8 @@ WDD_TYPEDEF(ErlDrvTid, erl_drv_thread_self, (void)); WDD_TYPEDEF(int, erl_drv_equal_tids, (ErlDrvTid tid1, ErlDrvTid tid2)); WDD_TYPEDEF(void, erl_drv_thread_exit, (void *resp)); WDD_TYPEDEF(int, erl_drv_thread_join, (ErlDrvTid, void **respp)); -WDD_TYPEDEF(int, erl_drv_putenv, (char *key, char *value)); -WDD_TYPEDEF(int, erl_drv_getenv, (char *key, char *value, size_t *value_size)); +WDD_TYPEDEF(int, erl_drv_putenv, (const char *key, char *value)); +WDD_TYPEDEF(int, erl_drv_getenv, (const char *key, char *value, size_t *value_size)); typedef struct { WDD_FTYPE(null_func) *null_func; diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 7c7ddde5d4..aa6a1fbcdc 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -31,7 +31,8 @@ rbtree/1, mseg_clear_cache/1, erts_mmap/1, - cpool/1]). + cpool/1, + migration/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -43,7 +44,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [basic, coalesce, threads, realloc_copy, bucket_index, - bucket_mask, rbtree, mseg_clear_cache, erts_mmap, cpool]. + bucket_mask, rbtree, mseg_clear_cache, erts_mmap, cpool, migration]. groups() -> []. @@ -64,7 +65,7 @@ end_per_group(_GroupName, Config) -> init_per_testcase(Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?t:seconds(?DEFAULT_TIMETRAP_SECS)), - [{watchdog, Dog},{testcase, Case}|Config]. + [{watchdog, Dog}, {testcase, Case}, {debug,false} | Config]. end_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?config(watchdog, Config), @@ -112,6 +113,14 @@ cpool(suite) -> []; cpool(doc) -> []; cpool(Cfg) -> ?line drv_case(Cfg). +migration(Cfg) -> + case erlang:system_info(smp_support) of + true -> + drv_case(Cfg, concurrent, "+MZe true"); + false -> + {skipped, "No smp"} + end. + erts_mmap(Config) when is_list(Config) -> case {?t:os_type(), is_halfword_vm()} of {{unix, _}, false} -> @@ -176,18 +185,17 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) -> %% %% drv_case(Config) -> - drv_case(Config, ""). + drv_case(Config, one_shot, ""). -drv_case(Config, Command) when is_list(Config), - is_list(Command) -> +drv_case(Config, Mode, NodeOpts) when is_list(Config) -> case ?t:os_type() of {Family, _} when Family == unix; Family == win32 -> - ?line {ok, Node} = start_node(Config), + ?line {ok, Node} = start_node(Config, NodeOpts), ?line Self = self(), ?line Ref = make_ref(), ?line spawn_link(Node, fun () -> - Res = run_drv_case(Config, Command), + Res = run_drv_case(Config, Mode), Self ! {Ref, Res} end), ?line Result = receive {Ref, Rslt} -> Rslt end, @@ -199,49 +207,172 @@ drv_case(Config, Command) when is_list(Config), | io_lib:format("~p",[SkipOs])])} end. -run_drv_case(Config, Command) -> - ?line DataDir = ?config(data_dir,Config), - ?line CaseName = ?config(testcase,Config), - case erl_ddll:load_driver(DataDir, CaseName) of - ok -> ok; - {error, Error} -> - io:format("~s\n", [erl_ddll:format_error(Error)]), - ?line ?t:fail() +run_drv_case(Config, Mode) -> + DataDir = ?config(data_dir,Config), + CaseName = ?config(testcase,Config), + File = filename:join(DataDir, CaseName), + {ok,CaseName,Bin} = compile:file(File, [binary,return_errors]), + {module,CaseName} = erlang:load_module(CaseName,Bin), + print_stats(CaseName), + ok = CaseName:init(File), + + SlaveState = slave_init(CaseName), + case Mode of + one_shot -> + Result = one_shot(CaseName); + + concurrent -> + Result = concurrent(CaseName) end, - ?line Port = open_port({spawn, atom_to_list(CaseName)}, []), - ?line true = is_port(Port), - ?line Port ! {self(), {command, Command}}, - ?line Result = receive_drv_result(Port, CaseName), - ?line Port ! {self(), close}, - ?line receive - {Port, closed} -> - ok - end, - ?line ok = erl_ddll:unload_driver(CaseName), - ?line Result. - -receive_drv_result(Port, CaseName) -> - ?line receive - {print, Port, CaseName, Str} -> - ?line ?t:format("~s", [Str]), - ?line receive_drv_result(Port, CaseName); - {'EXIT', Port, Error} -> - ?line ?t:fail(Error); - {'EXIT', error, Error} -> - ?line ?t:fail(Error); - {failed, Port, CaseName, Comment} -> - ?line ?t:fail(Comment); - {skipped, Port, CaseName, Comment} -> - ?line {skipped, Comment}; - {succeeded, Port, CaseName, ""} -> - ?line succeeded; - {succeeded, Port, CaseName, Comment} -> - ?line {comment, Comment} - end. - -start_node(Config) -> - start_node(Config, []). + + wait_for_memory_deallocations(), + print_stats(CaseName), + + true = erlang:delete_module(CaseName), + slave_end(SlaveState), + Result. + +slave_init(migration) -> + A0 = case application:start(sasl) of + ok -> [sasl]; + _ -> [] + end, + case application:start(os_mon) of + ok -> [os_mon|A0]; + _ -> A0 + end; +slave_init(_) -> []. + +slave_end(Apps) -> + lists:foreach(fun (A) -> application:stop(A) end, Apps). + +wait_for_memory_deallocations() -> + try + erts_debug:set_internal_state(wait, deallocations) + catch + error:undef -> + erts_debug:set_internal_state(available_internal_state, true), + wait_for_memory_deallocations() + end. + +print_stats(migration) -> + {Btot,Ctot} = lists:foldl(fun({instance,Inr,Istats}, {Bacc,Cacc}) -> + {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats), + Btup = lists:keyfind(blocks, 1, MBCS), + Ctup = lists:keyfind(carriers, 1, MBCS), + io:format("{instance,~p,~p,~p}\n", [Inr, Btup, Ctup]), + {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup)}; + (_, Acc) -> Acc + end, + {{blocks,0,0,0},{carriers,0,0,0}}, + erlang:system_info({allocator,test_alloc})), + + io:format("Number of blocks : ~p\n", [Btot]), + io:format("Number of carriers: ~p\n", [Ctot]); + +print_stats(_) -> ok. + +tuple_add(T1, T2) -> + list_to_tuple(lists:zipwith(fun(E1,E2) when is_number(E1), is_number(E2) -> + E1 + E2; + (A,A) -> + A + end, + tuple_to_list(T1), tuple_to_list(T2))). + + +one_shot(CaseName) -> + State = CaseName:start({1, 0, erlang:system_info(build_type)}), + Result0 = CaseName:run(State), + false = (Result0 =:= continue), + Result1 = handle_result(State, Result0), + CaseName:stop(State), + Result1. + + +many_shot(CaseName, I, Mem) -> + State = CaseName:start({I, Mem, erlang:system_info(build_type)}), + Result1 = repeat_while(fun() -> + Result0 = CaseName:run(State), + handle_result(State, Result0) + end, + 10*1000, I), + CaseName:stop(State), + flush_log(), + Result1. + +concurrent(CaseName) -> + NSched = erlang:system_info(schedulers), + Mem = (free_memory() * 3) div 4, + PRs = lists:map(fun(I) -> spawn_opt(fun() -> + many_shot(CaseName, I, + Mem div NSched) + end, + [monitor, {scheduler,I}]) + end, + lists:seq(1, NSched)), + lists:foreach(fun({Pid,Ref}) -> + receive {'DOWN', Ref, process, Pid, Reason} -> + Reason + end + end, + PRs), + ok. + +repeat_while(Fun, Timeout, I) -> + TRef = erlang:start_timer(Timeout, self(), timeout), + R = repeat_while_loop(Fun, TRef, I), + erlang:cancel_timer(TRef, [{async,true},{info,false}]), + R. + +repeat_while_loop(Fun, TRef, I) -> + receive + {timeout, TRef, timeout} -> + io:format("~p: Timeout, enough is enough.",[I]), + succeeded + after 0 -> + %%io:format("~p calls fun\n", [self()]), + case Fun() of + continue -> repeat_while_loop(Fun, TRef, I); + R -> R + end + end. + +flush_log() -> + receive + {print, Str} -> + ?t:format("~s", [Str]), + flush_log() + after 0 -> + ok + end. + +handle_result(_State, Result0) -> + flush_log(), + case Result0 of + {'EXIT', Error} -> + ?line ?t:fail(Error); + {'EXIT', error, Error} -> + ?line ?t:fail(Error); + {failed, Comment} -> + ?line ?t:fail(Comment); + {skipped, Comment} -> + ?line {skipped, Comment}; + {succeeded, ""} -> + ?line succeeded; + {succeeded, Comment} -> + ?line {comment, Comment}; + continue -> + continue + end. + start_node(Config, Opts) when is_list(Config), is_list(Opts) -> + case ?config(debug,Config) of + true -> {ok, node()}; + _ -> start_node_1(Config, Opts) + end. + +start_node_1(Config, Opts) -> Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) ++ "-" @@ -252,6 +383,7 @@ start_node(Config, Opts) when is_list(Config), is_list(Opts) -> ++ integer_to_list(erlang:unique_integer([positive]))), ?t:start_node(Name, slave, [{args, Opts++" -pa "++Pa}]). +stop_node(Node) when Node =:= node() -> ok; stop_node(Node) -> ?t:stop_node(Node). @@ -261,3 +393,23 @@ is_halfword_vm() -> {4, 8} -> true; {WS, WS} -> false end. + +free_memory() -> + %% Free memory in MB. + try + SMD = memsup:get_system_memory_data(), + {value, {free_memory, Free}} = lists:keysearch(free_memory, 1, SMD), + TotFree = (Free + + case lists:keysearch(cached_memory, 1, SMD) of + {value, {cached_memory, Cached}} -> Cached; + false -> 0 + end + + case lists:keysearch(buffered_memory, 1, SMD) of + {value, {buffered_memory, Buffed}} -> Buffed; + false -> 0 + end), + TotFree div (1024*1024) + catch + error : undef -> + ?t:fail({"os_mon not built"}) + end. diff --git a/erts/emulator/test/alloc_SUITE_data/Makefile.src b/erts/emulator/test/alloc_SUITE_data/Makefile.src index a441fe946b..e31de54e1b 100644 --- a/erts/emulator/test/alloc_SUITE_data/Makefile.src +++ b/erts/emulator/test/alloc_SUITE_data/Makefile.src @@ -25,7 +25,8 @@ TEST_DRVS = basic@dll@ \ bucket_mask@dll@ \ rbtree@dll@ \ mseg_clear_cache@dll@ \ - cpool@dll@ + cpool@dll@ \ + migration@dll@ CC = @CC@ LD = @LD@ diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h index 1d6b2f4907..97ee58cdad 100644 --- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h +++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h @@ -20,9 +20,20 @@ #ifndef ALLOCATOR_TEST_H__ #define ALLOCATOR_TEST_H__ -typedef ErlDrvUInt Ulong; +#if SIZEOF_VOID_P == SIZEOF_INT +typedef unsigned int Ulong; +#elif SIZEOF_VOID_P == SIZEOF_LONG +typedef unsigned long Ulong; +#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG +typedef unsigned long long Ulong; +#else +# error No pointer sized integer type found ??? +#endif -#ifndef __WIN32__ +#ifdef __WIN32__ +typedef Ulong erts_alc_test_Fn(Ulong, Ulong, Ulong, Ulong); +# define erts_alc_test ((erts_alc_test_Fn*)WinDynNifCallbacks.erts_alc_test) +#else Ulong erts_alc_test(Ulong, Ulong, Ulong, Ulong); #endif @@ -85,6 +96,7 @@ typedef void* erts_cond; #define CPOOL_DELETE(A,B) ((Carrier_t *) ALC_TEST2(0x022, (A), (B))) #define CPOOL_IS_EMPTY(A) ((int) ALC_TEST1(0x023, (A))) #define CPOOL_IS_IN_POOL(A,B) ((int) ALC_TEST2(0x024, (A), (B))) +#define UMEM2BLK_TEST(P) ((Block_t*) ALC_TEST1(0x025, (P))) /* From erl_goodfit_alloc.c */ #define BKT_IX(A, S) ((Ulong) ALC_TEST2(0x100, (A), (S))) @@ -142,5 +154,9 @@ typedef void* erts_cond; #define THR_JOIN(T) ((void) ALC_TEST1(0xf11, (T))) #define THR_EXIT(R) ((void) ALC_TEST1(0xf12, (R))) #define IS_SMP_ENABLED ((int) ALC_TEST0(0xf13)) +#define ALLOC_TEST(S) ((void*) ALC_TEST1(0xf14, (S))) +#define FREE_TEST(P) ((void) ALC_TEST1(0xf15, (P))) +#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf16, (SZ), (CMBC), (DMBC))) +#define GET_TEST_MBC_SIZE() ((int) ALC_TEST0(0xf17)) #endif diff --git a/erts/emulator/test/alloc_SUITE_data/basic.c b/erts/emulator/test/alloc_SUITE_data/basic.c index 323a24a11f..debb3d7ebe 100644 --- a/erts/emulator/test/alloc_SUITE_data/basic.c +++ b/erts/emulator/test/alloc_SUITE_data/basic.c @@ -60,3 +60,6 @@ testcase_cleanup(TestCaseState_t *tcs) if (tcs->extra) STOP_ALC((Allctr_t *) tcs->extra); } + +ERL_NIF_INIT(basic, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/basic.erl b/erts/emulator/test/alloc_SUITE_data/basic.erl new file mode 100644 index 0000000000..a018fd5582 --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/basic.erl @@ -0,0 +1,10 @@ +-module(basic). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/bucket_index.c b/erts/emulator/test/alloc_SUITE_data/bucket_index.c index c13f229049..45cb53fbf7 100644 --- a/erts/emulator/test/alloc_SUITE_data/bucket_index.c +++ b/erts/emulator/test/alloc_SUITE_data/bucket_index.c @@ -113,3 +113,5 @@ test_it(TestCaseState_t *tcs, unsigned sbct) sbct ? sbct_buf : "default"); } +ERL_NIF_INIT(bucket_index, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/bucket_index.erl b/erts/emulator/test/alloc_SUITE_data/bucket_index.erl new file mode 100644 index 0000000000..c54f54e2f5 --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/bucket_index.erl @@ -0,0 +1,10 @@ +-module(bucket_index). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/bucket_mask.c b/erts/emulator/test/alloc_SUITE_data/bucket_mask.c index 8d6166771e..c94c265f4e 100644 --- a/erts/emulator/test/alloc_SUITE_data/bucket_mask.c +++ b/erts/emulator/test/alloc_SUITE_data/bucket_mask.c @@ -52,7 +52,7 @@ testcase_run(TestCaseState_t *tcs) typedef struct linked_block { struct linked_block* next; }Linked; - Linked* link; + Linked* link = NULL; Linked* fence_list; Linked* pad_list; void* tmp; @@ -183,3 +183,5 @@ testcase_run(TestCaseState_t *tcs) tcs->extra = NULL; } +ERL_NIF_INIT(bucket_mask, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/bucket_mask.erl b/erts/emulator/test/alloc_SUITE_data/bucket_mask.erl new file mode 100644 index 0000000000..589a50e1fa --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/bucket_mask.erl @@ -0,0 +1,10 @@ +-module(bucket_mask). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/coalesce.c b/erts/emulator/test/alloc_SUITE_data/coalesce.c index 0a5e0c5b0e..7791409a34 100644 --- a/erts/emulator/test/alloc_SUITE_data/coalesce.c +++ b/erts/emulator/test/alloc_SUITE_data/coalesce.c @@ -317,3 +317,6 @@ testcase_cleanup(TestCaseState_t *tcs) if (tcs->extra) STOP_ALC((Allctr_t *) tcs->extra); } + +ERL_NIF_INIT(coalesce, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/coalesce.erl b/erts/emulator/test/alloc_SUITE_data/coalesce.erl new file mode 100644 index 0000000000..453c726c4e --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/coalesce.erl @@ -0,0 +1,10 @@ +-module(coalesce). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/cpool.c b/erts/emulator/test/alloc_SUITE_data/cpool.c index 75c2bc13ae..73026cc758 100644 --- a/erts/emulator/test/alloc_SUITE_data/cpool.c +++ b/erts/emulator/test/alloc_SUITE_data/cpool.c @@ -86,13 +86,13 @@ thread_func(void *arg) for (i = 0; i < (TEST_NO_CARRIERS_PER_THREAD+TEST_CARRIERS_OFFSET); i++) { int d; if (i < TEST_NO_CARRIERS_PER_THREAD) { - CPOOL_INSERT(alloc, crr[i]); + (void) CPOOL_INSERT(alloc, crr[i]); if ((i & 0x7) == 0) FATAL_ASSERT(CPOOL_IS_IN_POOL(alloc, crr[i])); } d = i-TEST_CARRIERS_OFFSET; if (d >= 0) { - CPOOL_DELETE(alloc, crr[d]); + (void) CPOOL_DELETE(alloc, crr[d]); if ((d & 0x7) == 0) FATAL_ASSERT(!CPOOL_IS_IN_POOL(alloc, crr[d])); } @@ -129,7 +129,7 @@ testcase_run(TestCaseState_t *tcs) for (c = 0; c < TEST_NO_CARRIERS_PER_THREAD; c++) { Carrier_t *crr = (Carrier_t *) p; p += zcrr_sz; - ZERO_CRR_INIT(alloc, crr); + (void) ZERO_CRR_INIT(alloc, crr); threads[t].crr[c] = crr; } } @@ -156,3 +156,6 @@ testcase_run(TestCaseState_t *tcs) ASSERT(tcs, no_threads == TEST_NO_THREADS); } + +ERL_NIF_INIT(cpool, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/cpool.erl b/erts/emulator/test/alloc_SUITE_data/cpool.erl new file mode 100644 index 0000000000..89053471fa --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/cpool.erl @@ -0,0 +1,10 @@ +-module(cpool). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/migration.c b/erts/emulator/test/alloc_SUITE_data/migration.c new file mode 100644 index 0000000000..b006360043 --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/migration.c @@ -0,0 +1,343 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Test the carrier migration logic + */ + +#ifndef __WIN32__ +#include <sys/types.h> +#include <unistd.h> +#include <errno.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> +#include <string.h> +#include "testcase_driver.h" +#include "allocator_test.h" + +#define FATAL_ASSERT(A) \ + ((void) ((A) \ + ? 1 \ + : (fatal_assert_failed(#A, \ + (char *) __FILE__, \ + __LINE__), \ + 0))) + +static void +fatal_assert_failed(char* expr, char* file, int line) +{ + fflush(stdout); + fprintf(stderr, "%s:%d: Assertion failed: %s\n", + file, line, expr); + fflush(stderr); + abort(); +} + + +char * +testcase_name(void) +{ + return "migration"; +} + +/* Turns out random_r() is a nonstandard glibc extension. +#define HAVE_RANDOM_R +*/ +#ifdef HAVE_RANDOM_R + +typedef struct { struct random_data rnd; char rndbuf[32]; } MyRandState; + +static void myrand_init(MyRandState* mrs, unsigned int seed) +{ + int res; + memset(&mrs->rnd, 0, sizeof(mrs->rnd)); + res = initstate_r(seed, mrs->rndbuf, sizeof(mrs->rndbuf), &mrs->rnd); + FATAL_ASSERT(res == 0); +} + +static int myrand(MyRandState* mrs) +{ + int32_t x; + int res = random_r(&mrs->rnd, &x); + FATAL_ASSERT(res == 0); + return (int)x; +} + +#else /* !HAVE_RANDOM_R */ + +typedef unsigned int MyRandState; + +static void myrand_init(MyRandState* mrs, unsigned int seed) +{ + *mrs = seed; +} + +static int myrand(MyRandState* mrs) +{ + /* Taken from rand(3) man page. + * Modified to return a full 31-bit value by using low half of *mrs as well. + */ + *mrs = (*mrs) * 1103515245 + 12345; + return (int) (((*mrs >> 16) | (*mrs << 16)) & ~(1 << 31)); +} + +#endif /* !HAVE_RANDOM_R */ + +#define MAX_BLOCK_PER_THR 200 +#define BLOCKS_PER_MBC 10 +#define MAX_ROUNDS 10000 + +typedef struct MyBlock_ { + struct MyBlock_* next; + struct MyBlock_** prevp; +} MyBlock; + +typedef struct { + MyBlock* blockv[MAX_BLOCK_PER_THR]; + MyRandState rand_state; + enum { GROWING, SHRINKING, CLEANUP, DONE } phase; + int nblocks; + int goal_nblocks; + int round; + int nr_of_migrations; + int nr_of_carriers; + int max_blocks_in_mbc; + int block_size; + int max_nblocks; +} MigrationState; + +typedef struct { + ErlNifMutex* mtx; + int nblocks; + MyBlock* first; + MigrationState* employer; +} MyCrrInfo; + + +static int crr_info_offset = -1; +static void (*orig_create_mbc_fn)(Allctr_t *allctr, Carrier_t *carrier); +static void (*orig_destroying_mbc_fn)(Allctr_t *allctr, Carrier_t *carrier); + +static void my_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + MyCrrInfo* mci = (MyCrrInfo*) ((char*)carrier + crr_info_offset); + if (orig_create_mbc_fn) + orig_create_mbc_fn(allctr, carrier); + + mci->mtx = enif_mutex_create("alloc_SUITE.migration"); + mci->nblocks = 0; + mci->first = NULL; + mci->employer = NULL; +} + +static void my_destroying_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + MyCrrInfo* mci = (MyCrrInfo*) ((char*)carrier + crr_info_offset); + + FATAL_ASSERT(mci->nblocks == 0); + FATAL_ASSERT(mci->first == NULL); + enif_mutex_destroy(mci->mtx); + + if (orig_destroying_mbc_fn) + orig_destroying_mbc_fn(allctr, carrier); +} + +static int migration_init(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + void* creating_mbc_arg = (void*)my_creating_mbc; + void* destroying_mbc_arg = (void*)my_destroying_mbc; + + if (testcase_nif_init(env, priv_data, load_info)) + return -1; + + crr_info_offset = SET_TEST_MBC_USER_HEADER(sizeof(MyCrrInfo), + &creating_mbc_arg, + &destroying_mbc_arg); + FATAL_ASSERT(crr_info_offset >= 0); + orig_create_mbc_fn = creating_mbc_arg; + orig_destroying_mbc_fn = destroying_mbc_arg; + + return 0; +} + +static void add_block(MyBlock* p, MigrationState* state) +{ + MyCrrInfo* mci = (MyCrrInfo*)((char*)BLK_TO_MBC(UMEM2BLK_TEST(p)) + crr_info_offset); + + enif_mutex_lock(mci->mtx); + if (++mci->nblocks > state->max_blocks_in_mbc) + state->max_blocks_in_mbc = mci->nblocks; + p->next = mci->first; + p->prevp = &mci->first; + mci->first = p; + if (p->next) + p->next->prevp = &p->next; + if (mci->employer != state) { + if (!mci->employer) { + FATAL_ASSERT(mci->nblocks == 1); + state->nr_of_carriers++; + } + else { + state->nr_of_migrations++; + } + mci->employer = state; + } + enif_mutex_unlock(mci->mtx); +} + +static void remove_block(MyBlock* p) +{ + MyCrrInfo* mci = (MyCrrInfo*)((char*)BLK_TO_MBC(UMEM2BLK_TEST(p)) + crr_info_offset); + + enif_mutex_lock(mci->mtx); + mci->nblocks--; + if (p->next) + p->next->prevp = p->prevp; + *p->prevp = p->next; + enif_mutex_unlock(mci->mtx); +} + +static int rand_int(MigrationState* state, int low, int high) +{ + int x; + FATAL_ASSERT(high >= low); + x = myrand(&state->rand_state); + return low + (x % (high+1-low)); +} + + +static void do_cleanup(TestCaseState_t *tcs, MigrationState* state) +{ + if (state->nblocks == 0) { + state->phase = DONE; + testcase_printf(tcs, "%d: Done %d rounds", tcs->thr_nr, state->round); + testcase_printf(tcs, "%d: Cleanup all blocks", tcs->thr_nr); + testcase_printf(tcs, "%d: Empty carriers detected = %d", tcs->thr_nr, + state->nr_of_carriers); + testcase_printf(tcs, "%d: Migrations detected = %d", tcs->thr_nr, + state->nr_of_migrations); + testcase_printf(tcs, "%d: Max blocks in carrier = %d", tcs->thr_nr, + state->max_blocks_in_mbc); + } + else { + state->nblocks--; + if (state->blockv[state->nblocks]) { + remove_block(state->blockv[state->nblocks]); + FREE_TEST(state->blockv[state->nblocks]); + } + } +} + + +void +testcase_run(TestCaseState_t *tcs) +{ + MigrationState* state = (MigrationState*) tcs->extra; + + if (!tcs->extra) { + if (!IS_SMP_ENABLED) + testcase_skipped(tcs, "No SMP support"); + + tcs->extra = enif_alloc(sizeof(MigrationState)); + state = (MigrationState*) tcs->extra; + memset(state->blockv, 0, sizeof(state->blockv)); + myrand_init(&state->rand_state, tcs->thr_nr); + state->phase = GROWING; + state->nblocks = 0; + state->round = 0; + state->nr_of_migrations = 0; + state->nr_of_carriers = 0; + state->max_blocks_in_mbc = 0; + state->block_size = GET_TEST_MBC_SIZE() / (BLOCKS_PER_MBC+1); + if (MAX_BLOCK_PER_THR * state->block_size < tcs->free_mem) { + state->max_nblocks = MAX_BLOCK_PER_THR; + } else { + state->max_nblocks = tcs->free_mem / state->block_size; + } + state->goal_nblocks = rand_int(state, 1, state->max_nblocks); + } + + switch (state->phase) { + case GROWING: { + MyBlock* p; + FATAL_ASSERT(!state->blockv[state->nblocks]); + p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size)); + FATAL_ASSERT(p); + add_block(p, state); + state->blockv[state->nblocks] = p; + if (++state->nblocks >= state->goal_nblocks) { + /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/ + state->phase = SHRINKING; + state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1); + } + else + FATAL_ASSERT(!state->blockv[state->nblocks]); + break; + } + case SHRINKING: { + int ix = rand_int(state, 0, state->nblocks-1); + FATAL_ASSERT(state->blockv[ix]); + remove_block(state->blockv[ix]); + FREE_TEST(state->blockv[ix]); + state->blockv[ix] = state->blockv[--state->nblocks]; + state->blockv[state->nblocks] = NULL; + + if (state->nblocks <= state->goal_nblocks) { + /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/ + if (++state->round >= MAX_ROUNDS) { + state->phase = CLEANUP; + } else { + state->phase = GROWING; + state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks); + } + } + break; + } + case CLEANUP: + do_cleanup(tcs, state); + break; + + default: + FATAL_ASSERT(!"Invalid phase"); + } + + if (state->phase == DONE) { + } + else { + testcase_continue(tcs); + } +} + +void +testcase_cleanup(TestCaseState_t *tcs) +{ + MigrationState* state = (MigrationState*) tcs->extra; + + while (state->phase != DONE) + do_cleanup(tcs, state); + + enif_free(tcs->extra); + tcs->extra = NULL; +} + + +ERL_NIF_INIT(migration, testcase_nif_funcs, migration_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/migration.erl b/erts/emulator/test/alloc_SUITE_data/migration.erl new file mode 100644 index 0000000000..440a99becd --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/migration.erl @@ -0,0 +1,10 @@ +-module(migration). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/mseg_clear_cache.c b/erts/emulator/test/alloc_SUITE_data/mseg_clear_cache.c index 9c03f3a331..e5df3d647f 100644 --- a/erts/emulator/test/alloc_SUITE_data/mseg_clear_cache.c +++ b/erts/emulator/test/alloc_SUITE_data/mseg_clear_cache.c @@ -101,3 +101,6 @@ testcase_cleanup(TestCaseState_t *tcs) tcs->extra = NULL; } } + +ERL_NIF_INIT(mseg_clear_cache, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/mseg_clear_cache.erl b/erts/emulator/test/alloc_SUITE_data/mseg_clear_cache.erl new file mode 100644 index 0000000000..befd6c2e8e --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/mseg_clear_cache.erl @@ -0,0 +1,10 @@ +-module(mseg_clear_cache). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/rbtree.c b/erts/emulator/test/alloc_SUITE_data/rbtree.c index 8d4d5535a8..38bbbdf90c 100644 --- a/erts/emulator/test/alloc_SUITE_data/rbtree.c +++ b/erts/emulator/test/alloc_SUITE_data/rbtree.c @@ -20,7 +20,7 @@ #include "testcase_driver.h" #include "allocator_test.h" -#define NO_BLOCKS 100000 +int NO_BLOCKS; #define RIGHT_VISITED (1 << 0) #define LEFT_VISITED (1 << 1) @@ -265,9 +265,10 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) ASSERT(tcs, curr_blacks == 0); ASSERT(tcs, i == -1); + /* testcase_printf(tcs, "Red-Black Tree OK! Max depth = %d; " "Black depth = %d\n", max_i+1, blacks < 0 ? 0 : blacks); - + */ return res; } @@ -468,6 +469,12 @@ testcase_run(TestCaseState_t *tcs) Allctr_t *a; rbtree_test_data *td; + NO_BLOCKS = 100*1000; + if (enif_is_identical(tcs->build_type, + enif_make_atom(tcs->curr_env,"valgrind"))) { + NO_BLOCKS /= 10; + } + /* Best fit... */ testcase_printf(tcs, "Setup...\n"); @@ -577,3 +584,6 @@ testcase_run(TestCaseState_t *tcs) testcase_printf(tcs, "aoffcaobf test succeeded!\n"); } + +ERL_NIF_INIT(rbtree, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/rbtree.erl b/erts/emulator/test/alloc_SUITE_data/rbtree.erl new file mode 100644 index 0000000000..f5b7120ff2 --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/rbtree.erl @@ -0,0 +1,10 @@ +-module(rbtree). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/realloc_copy.c b/erts/emulator/test/alloc_SUITE_data/realloc_copy.c index e405f06225..c4147eb00d 100644 --- a/erts/emulator/test/alloc_SUITE_data/realloc_copy.c +++ b/erts/emulator/test/alloc_SUITE_data/realloc_copy.c @@ -278,3 +278,5 @@ testcase_cleanup(TestCaseState_t *tcs) STOP_ALC((Allctr_t *) tcs->extra); } +ERL_NIF_INIT(realloc_copy, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/realloc_copy.erl b/erts/emulator/test/alloc_SUITE_data/realloc_copy.erl new file mode 100644 index 0000000000..cc6617bf64 --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/realloc_copy.erl @@ -0,0 +1,10 @@ +-module(realloc_copy). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/alloc_SUITE_data/testcase_driver.c b/erts/emulator/test/alloc_SUITE_data/testcase_driver.c index bc674c56b7..7dcca544e5 100644 --- a/erts/emulator/test/alloc_SUITE_data/testcase_driver.c +++ b/erts/emulator/test/alloc_SUITE_data/testcase_driver.c @@ -23,141 +23,147 @@ #include <stdarg.h> #include <setjmp.h> #include <string.h> +#include <limits.h> #ifdef __WIN32__ -#undef HAVE_VSNPRINTF -#define HAVE_VSNPRINTF 1 -#define vsnprintf _vsnprintf +static void my_vsnprintf(char *outBuf, size_t size, const char *format, va_list ap) +{ + _vsnprintf(outBuf, size, format, ap); + outBuf[size-1] = 0; /* be sure string is terminated */ +} +#elif defined(HAVE_VSNPRINTF) +# define my_vsnprintf(B,S,F,A) (void)vsnprintf(B,S,F,A) +#else +# warning Using unsafe 'vsprintf' without buffer overflow protection +# define my_vsnprintf(B,S,F,A) (void)vsprintf(B,F,A) #endif -#ifndef HAVE_VSNPRINTF -#define HAVE_VSNPRINTF 0 -#endif +static void my_snprintf(char *outBuf, size_t size, const char *format, ...) +{ + va_list ap; + va_start(ap, format); + my_vsnprintf(outBuf, size, format, ap); + va_end(ap); +} #define COMMENT_BUF_SZ 4096 #define TESTCASE_FAILED 0 #define TESTCASE_SKIPPED 1 #define TESTCASE_SUCCEEDED 2 +#define TESTCASE_CONTINUE 3 typedef struct { TestCaseState_t visible; - ErlDrvPort port; - ErlDrvTermData port_id; int result; - jmp_buf done_jmp_buf; + jmp_buf* done_jmp_buf; char *comment; char comment_buf[COMMENT_BUF_SZ]; } InternalTestCaseState_t; -ErlDrvData testcase_drv_start(ErlDrvPort port, char *command); -void testcase_drv_stop(ErlDrvData drv_data); -void testcase_drv_run(ErlDrvData drv_data, char *buf, ErlDrvSizeT len); - -static ErlDrvEntry testcase_drv_entry = { - NULL, - testcase_drv_start, - testcase_drv_stop, - testcase_drv_run, - NULL, - NULL, - "testcase_drv", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - 0, - NULL, - NULL, - NULL +ERL_NIF_TERM testcase_nif_start(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM testcase_nif_stop(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM testcase_nif_run(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +ErlNifFunc testcase_nif_funcs[] = +{ + {"start", 1, testcase_nif_start}, + {"run", 1, testcase_nif_run}, + {"stop", 1, testcase_nif_stop} }; +static ErlNifResourceType* testcase_rt; +static ERL_NIF_TERM print_atom; -DRIVER_INIT(testcase_drv) +int testcase_nif_init(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { - testcase_drv_entry.driver_name = testcase_name(); - return &testcase_drv_entry; + testcase_rt = enif_open_resource_type(env, NULL, "testcase_rt", NULL, + ERL_NIF_RT_CREATE, NULL); + + print_atom = enif_make_atom(env, "print"); + return 0; } -ErlDrvData -testcase_drv_start(ErlDrvPort port, char *command) -{ +ERL_NIF_TERM +testcase_nif_start(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ /* (ThrNr, FreeMeg, BuildType) */ + ERL_NIF_TERM ret; InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) - driver_alloc(sizeof(InternalTestCaseState_t)); - if (!itcs) { - return ERL_DRV_ERROR_GENERAL; + enif_alloc_resource(testcase_rt, sizeof(InternalTestCaseState_t)); + int free_megabyte; + const int max_megabyte = INT_MAX / (1024*1024); + const ERL_NIF_TERM* tpl; + int tpl_arity; + + if (!itcs + || !enif_get_tuple(env, argv[0], &tpl_arity, &tpl) + || tpl_arity != 3 + || !enif_get_int(env, tpl[0], &itcs->visible.thr_nr) + || !enif_get_int(env, tpl[1], &free_megabyte)) { + enif_make_badarg(env); } - + itcs->visible.free_mem = (free_megabyte < max_megabyte ? + free_megabyte : max_megabyte) * (1024*1024); itcs->visible.testcase_name = testcase_name(); + itcs->visible.build_type = tpl[2]; itcs->visible.extra = NULL; - itcs->port = port; - itcs->port_id = driver_mk_port(port); itcs->result = TESTCASE_FAILED; itcs->comment = ""; - return (ErlDrvData) itcs; + ret = enif_make_resource(env, itcs); + enif_release_resource(itcs); + return ret; } -void -testcase_drv_stop(ErlDrvData drv_data) +ERL_NIF_TERM +testcase_nif_stop(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - testcase_cleanup((TestCaseState_t *) drv_data); - driver_free((void *) drv_data); + InternalTestCaseState_t *itcs; + if (!enif_get_resource(env, argv[0], testcase_rt, (void**)&itcs)) + return enif_make_badarg(env); + testcase_cleanup(&itcs->visible); + return enif_make_atom(env,"ok"); } -void -testcase_drv_run(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) +ERL_NIF_TERM +testcase_nif_run(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) drv_data; - ErlDrvTermData result_atom; - ErlDrvTermData msg[12]; + InternalTestCaseState_t *itcs; + const char* result_atom; + jmp_buf the_jmp_buf; + + if (!enif_get_resource(env, argv[0], testcase_rt, (void**)&itcs)) + return enif_make_badarg(env); - itcs->visible.command = buf; - itcs->visible.command_len = len; + itcs->visible.curr_env = env; - if (setjmp(itcs->done_jmp_buf) == 0) { - testcase_run((TestCaseState_t *) itcs); + /* For some unknown reason, first call to setjmp crashes on win64 + * when jmp_buf is allocated as part of the resource. But it works when + * allocated on stack. It used to work when this was a driver. + */ + itcs->done_jmp_buf = &the_jmp_buf; + + if (setjmp(the_jmp_buf) == 0) { + testcase_run(&itcs->visible); itcs->result = TESTCASE_SUCCEEDED; } switch (itcs->result) { - case TESTCASE_SUCCEEDED: - result_atom = driver_mk_atom("succeeded"); - break; - case TESTCASE_SKIPPED: - result_atom = driver_mk_atom("skipped"); - break; - case TESTCASE_FAILED: + case TESTCASE_CONTINUE: + return enif_make_atom(env, "continue"); + + case TESTCASE_SUCCEEDED: result_atom = "succeeded"; break; + case TESTCASE_SKIPPED: result_atom = "skipped"; break; + case TESTCASE_FAILED: result_atom = "failed"; break; default: - result_atom = driver_mk_atom("failed"); - break; + result_atom = "failed"; + my_snprintf(itcs->comment_buf, sizeof(itcs->comment_buf), + "Unexpected test result code %d.", itcs->result); + itcs->comment = itcs->comment_buf; } - msg[0] = ERL_DRV_ATOM; - msg[1] = (ErlDrvTermData) result_atom; - - msg[2] = ERL_DRV_PORT; - msg[3] = itcs->port_id; - - msg[4] = ERL_DRV_ATOM; - msg[5] = driver_mk_atom(itcs->visible.testcase_name); - - msg[6] = ERL_DRV_STRING; - msg[7] = (ErlDrvTermData) itcs->comment; - msg[8] = (ErlDrvTermData) strlen(itcs->comment); - - msg[9] = ERL_DRV_TUPLE; - msg[10] = (ErlDrvTermData) 4; - - erl_drv_output_term(itcs->port_id, msg, 11); + return enif_make_tuple2(env, enif_make_atom(env, result_atom), + enif_make_string(env, itcs->comment, ERL_NIF_LATIN1)); } int @@ -172,34 +178,22 @@ testcase_assertion_failed(TestCaseState_t *tcs, void testcase_printf(TestCaseState_t *tcs, char *frmt, ...) { - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - ErlDrvTermData msg[12]; + InternalTestCaseState_t* itcs = (InternalTestCaseState_t*)tcs; + ErlNifPid pid; + ErlNifEnv* msg_env = enif_alloc_env(); + ERL_NIF_TERM msg; va_list va; va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif + my_vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); va_end(va); - msg[0] = ERL_DRV_ATOM; - msg[1] = (ErlDrvTermData) driver_mk_atom("print"); + msg = enif_make_tuple2(msg_env, print_atom, + enif_make_string(msg_env, itcs->comment_buf, ERL_NIF_LATIN1)); - msg[2] = ERL_DRV_PORT; - msg[3] = itcs->port_id; + enif_send(itcs->visible.curr_env, enif_self(itcs->visible.curr_env, &pid), + msg_env, msg); - msg[4] = ERL_DRV_ATOM; - msg[5] = driver_mk_atom(itcs->visible.testcase_name); - - msg[6] = ERL_DRV_STRING; - msg[7] = (ErlDrvTermData) itcs->comment_buf; - msg[8] = (ErlDrvTermData) strlen(itcs->comment_buf); - - msg[9] = ERL_DRV_TUPLE; - msg[10] = (ErlDrvTermData) 4; - - erl_drv_output_term(itcs->port_id, msg, 11); + enif_free_env(msg_env); } @@ -208,17 +202,13 @@ void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...) InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; va_list va; va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif + my_vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); va_end(va); itcs->result = TESTCASE_SUCCEEDED; itcs->comment = itcs->comment_buf; - longjmp(itcs->done_jmp_buf, 1); + longjmp(*itcs->done_jmp_buf, 1); } void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...) @@ -226,17 +216,20 @@ void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...) InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; va_list va; va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif + my_vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); va_end(va); itcs->result = TESTCASE_SKIPPED; itcs->comment = itcs->comment_buf; - longjmp(itcs->done_jmp_buf, 1); + longjmp(*itcs->done_jmp_buf, 1); +} + +void testcase_continue(TestCaseState_t *tcs) +{ + InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; + itcs->result = TESTCASE_CONTINUE; + longjmp(*itcs->done_jmp_buf, 1); } void testcase_failed(TestCaseState_t *tcs, char *frmt, ...) @@ -246,37 +239,33 @@ void testcase_failed(TestCaseState_t *tcs, char *frmt, ...) size_t bufsz = sizeof(buf); va_list va; va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif + my_vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); va_end(va); itcs->result = TESTCASE_FAILED; itcs->comment = itcs->comment_buf; - if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0 + if (enif_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0 && strcmp("true", buf) == 0) { fprintf(stderr, "Testcase \"%s\" failed: %s\n", itcs->visible.testcase_name, itcs->comment); abort(); } - longjmp(itcs->done_jmp_buf, 1); + longjmp(*itcs->done_jmp_buf, 1); } void *testcase_alloc(size_t size) { - return driver_alloc(size); + return enif_alloc(size); } void *testcase_realloc(void *ptr, size_t size) { - return driver_realloc(ptr, size); + return enif_realloc(ptr, size); } void testcase_free(void *ptr) { - driver_free(ptr); + enif_free(ptr); } diff --git a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h index 5d17eaec64..f0ca91bd06 100644 --- a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h +++ b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h @@ -20,13 +20,15 @@ #ifndef TESTCASE_DRIVER_H__ #define TESTCASE_DRIVER_H__ -#include "erl_driver.h" +#include "erl_nif.h" #include <stdlib.h> typedef struct { + ErlNifEnv* curr_env; char *testcase_name; - char *command; - int command_len; + int thr_nr; + int free_mem; /* in bytes */ + ERL_NIF_TERM build_type; /* opt, debug, valgrind, ... */ void *extra; } TestCaseState_t; @@ -34,9 +36,11 @@ typedef struct { ((void) ((B) ? 1 : testcase_assertion_failed((TCS), __FILE__, __LINE__, #B))) +int testcase_nif_init(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info); void testcase_printf(TestCaseState_t *tcs, char *frmt, ...); void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...); void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...); +void testcase_continue(TestCaseState_t *tcs); void testcase_failed(TestCaseState_t *tcs, char *frmt, ...); int testcase_assertion_failed(TestCaseState_t *tcs, char *file, int line, char *assertion); @@ -45,8 +49,11 @@ void *testcase_realloc(void *ptr, size_t size); void testcase_free(void *ptr); +/* Implemented by testcase: */ char *testcase_name(void); void testcase_run(TestCaseState_t *tcs); void testcase_cleanup(TestCaseState_t *tcs); -#endif +extern ErlNifFunc testcase_nif_funcs[3]; + +#endif /* TESTCASE_DRIVER_H__ */ diff --git a/erts/emulator/test/alloc_SUITE_data/threads.c b/erts/emulator/test/alloc_SUITE_data/threads.c index edad24ee6b..a8a6a23695 100644 --- a/erts/emulator/test/alloc_SUITE_data/threads.c +++ b/erts/emulator/test/alloc_SUITE_data/threads.c @@ -86,7 +86,7 @@ static void fail(int t_no, char *frmt, ...) tc_failed = 1; - if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0 + if (enif_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0 && strcmp("true", buf) == 0) { fprintf(stderr, "Testcase \"%s\" failed: %s\n", testcase_name(), err_buf); @@ -187,7 +187,6 @@ testcase_run(TestCaseState_t *tcs) for(i = 1; i <= NO_OF_THREADS; i++) { char *alc; - int res; threads[i].arg.no_ops_per_bl = NO_OF_OPS_PER_BL; @@ -446,3 +445,6 @@ thread_func(void *arg) exit_thread(td->t_no, 1); return NULL; } + +ERL_NIF_INIT(threads, testcase_nif_funcs, testcase_nif_init, + NULL, NULL, NULL); diff --git a/erts/emulator/test/alloc_SUITE_data/threads.erl b/erts/emulator/test/alloc_SUITE_data/threads.erl new file mode 100644 index 0000000000..a7b4965f5e --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/threads.erl @@ -0,0 +1,10 @@ +-module(threads). + +-export([init/1, start/1, run/1, stop/1]). + +init(File) -> + ok = erlang:load_nif(File, 0). + +start(_) -> erlang:nif_error(not_loaded). +run(_) -> erlang:nif_error(not_loaded). +stop(_) -> erlang:nif_error(not_loaded). diff --git a/erts/emulator/test/distribution_SUITE_data/run.erl b/erts/emulator/test/distribution_SUITE_data/run.erl index f5169e160c..d5ed139369 100644 --- a/erts/emulator/test/distribution_SUITE_data/run.erl +++ b/erts/emulator/test/distribution_SUITE_data/run.erl @@ -30,16 +30,19 @@ from(H, [_ | T]) -> from(H, T); from(H, []) -> []. start() -> - net_kernel:start([fideridum,shortnames]), - {ok, Node} = slave:start(host(), heppel), - P = spawn(Node, a, b, []), - B1 = term_to_binary(P), - N1 = node(P), - ok = net_kernel:stop(), - N2 = node(P), - io:format("~w~n", [N1 == N2]), + Result = do_it(), + + %% Do GCs and node_and_dist_references + %% in an attempt to crash the VM (without OTP-13076 fix) + lists:foreach(fun(P) -> erlang:garbage_collect(P) end, + processes()), + erts_debug:set_internal_state(available_internal_state, true), + erts_debug:get_internal_state(node_and_dist_references), + + io:format("~w~n", [Result]), + if - N1 == N2 -> + Result -> init:stop(); true -> %% Make sure that the io:format/2 output is really written @@ -47,3 +50,29 @@ start() -> erlang:yield(), init:stop() end. + + +do_it() -> + {ok, _} = net_kernel:start([fideridum,shortnames]), + {ok, Node} = slave:start(host(), heppel), + P = spawn(Node, net_kernel, stop, []), + B1 = term_to_binary(P), + N1 = node(P), + ok = net_kernel:stop(), + N2 = node(P), + + %% OTP-13076 + %% Restart distribution with same node name as previous remote node + %% Repeat to wrap around creation + Result = lists:foldl(fun(_, Acc) -> + timer:sleep(2), % give net_kernel:stop() time to take effect :-( + {ok, _} = net_kernel:start([heppel,shortnames]), + N3 = node(P), + ok = net_kernel:stop(), + N4 = node(P), + Acc and (N3 =:= N1) and (N4 =:= N1) + end, + (N2 =:= N1), + lists:seq(1,3)), + + Result. diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl index 62a94e5281..6890c42b7a 100644 --- a/erts/emulator/test/map_SUITE.erl +++ b/erts/emulator/test/map_SUITE.erl @@ -58,6 +58,7 @@ %% erlang t_erlang_hash/1, t_map_encode_decode/1, + t_gc_rare_map_overflow/1, %% non specific BIF related t_bif_build_and_check/1, @@ -121,6 +122,7 @@ all() -> [ %% erlang t_erlang_hash, t_map_encode_decode, + t_gc_rare_map_overflow, t_map_size, t_is_map, %% non specific BIF related @@ -2991,6 +2993,100 @@ do_badmap_17(Config) -> id(I) -> I. +%% OTP-13146 +%% Provoke major GC with a lot of "fat" maps on external format in msg queue +%% causing heap fragments to be allocated. +t_gc_rare_map_overflow(Config) -> + Pa = filename:dirname(code:which(?MODULE)), + {ok, Node} = test_server:start_node(gc_rare_map_overflow, slave, [{args, "-pa \""++Pa++"\""}]), + Echo = spawn_link(Node, fun Loop() -> receive {From,Msg} -> From ! Msg + end, + Loop() + end), + FatMap = fatmap(34), + false = (flatmap =:= erts_internal:map_type(FatMap)), + + t_gc_rare_map_overflow_do(Echo, FatMap, fun() -> erlang:garbage_collect() end), + + % Repeat test for minor gc: + minor_collect(), % need this to make the next gc really be a minor + t_gc_rare_map_overflow_do(Echo, FatMap, fun() -> true = minor_collect() end), + + unlink(Echo), + test_server:stop_node(Node). + +t_gc_rare_map_overflow_do(Echo, FatMap, GcFun) -> + Master = self(), + true = receive M -> false after 0 -> true end, % assert empty msg queue + Echo ! {Master, token}, + repeat(1000, fun(_) -> Echo ! {Master, FatMap} end, void), + + timer:sleep(100), % Wait for maps to arrive in our msg queue + token = receive Tok -> Tok end, % and provoke move from outer to inner msg queue + + %% Do GC that will "overflow" and create heap frags due to all the fat maps + GcFun(), + + %% Now check that all maps in msg queueu are intact + %% Will crash emulator in OTP-18.1 + repeat(1000, fun(_) -> FatMap = receive FM -> FM end end, void), + ok. + +minor_collect() -> + minor_collect(minor_gcs()). + +minor_collect(Before) -> + After = minor_gcs(), + case After of + _ when After > Before -> true; + _ when After =:= Before -> minor_collect(Before); + 0 -> false + end. + +minor_gcs() -> + {garbage_collection, Info} = process_info(self(), garbage_collection), + {minor_gcs, GCS} = lists:keyfind(minor_gcs, 1, Info), + GCS. + +%% Generate a map with N (or N+1) keys that has an abnormal heap demand. +%% Done by finding keys that collide in the first 32-bit hash. +fatmap(N) -> + erts_debug:set_internal_state(available_internal_state, true), + Table = ets:new(void, [bag, private]), + + Seed0 = rand:seed_s(exsplus, {4711, 3141592, 2718281}), + Seed1 = fatmap_populate(Table, Seed0, (1 bsl 16)), + Keys = fatmap_generate(Table, Seed1, N, []), + ets:delete(Table), + maps:from_list([{K,K} || K <- Keys]). + +fatmap_populate(_, Seed, 0) -> Seed; +fatmap_populate(Table, Seed, N) -> + {I, NextSeed} = rand:uniform_s(1 bsl 48, Seed), + Hash = internal_hash(I), + ets:insert(Table, [{Hash, I}]), + fatmap_populate(Table, NextSeed, N-1). + + +fatmap_generate(_, _, N, Acc) when N =< 0 -> + Acc; +fatmap_generate(Table, Seed, N0, Acc0) -> + {I, NextSeed} = rand:uniform_s(1 bsl 48, Seed), + Hash = internal_hash(I), + case ets:member(Table, Hash) of + true -> + NewKeys = [I | ets:lookup_element(Table, Hash, 2)], + Acc1 = lists:usort(Acc0 ++ NewKeys), + N1 = N0 - (length(Acc1) - length(Acc0)), + fatmap_generate(Table, NextSeed, N1, Acc1); + false -> + fatmap_generate(Table, NextSeed, N0, Acc0) + end. + +internal_hash(Term) -> + erts_debug:get_internal_state({internal_hash, Term}). + + %% map external_format (fannerl). fannerl() -> <<131,116,0,0,0,28,100,0,13,108,101,97,114,110,105,110,103,95,114, diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 1e7c56dd8e..af1c198281 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -73,6 +73,7 @@ static const char plusM_au_allocs[]= { 'R', /* driver_alloc */ 'S', /* sl_alloc */ 'T', /* temp_alloc */ + 'Z', /* test_alloc */ '\0' }; diff --git a/lib/asn1/src/asn1rtt_ber.erl b/lib/asn1/src/asn1rtt_ber.erl index 39fa8aaf99..e50b14941c 100644 --- a/lib/asn1/src/asn1rtt_ber.erl +++ b/lib/asn1/src/asn1rtt_ber.erl @@ -739,6 +739,8 @@ encode_named_bit_string([H|_]=Bits, NamedBitList, TagIn) when is_atom(H) -> do_encode_named_bit_string(Bits, NamedBitList, TagIn); encode_named_bit_string([{bit,_}|_]=Bits, NamedBitList, TagIn) -> do_encode_named_bit_string(Bits, NamedBitList, TagIn); +encode_named_bit_string([], _NamedBitList, TagIn) -> + encode_unnamed_bit_string(<<>>, TagIn); encode_named_bit_string(Bits, _NamedBitList, TagIn) when is_bitstring(Bits) -> encode_unnamed_bit_string(Bits, TagIn). @@ -746,6 +748,8 @@ encode_named_bit_string(C, [H|_]=Bits, NamedBitList, TagIn) when is_atom(H) -> do_encode_named_bit_string(C, Bits, NamedBitList, TagIn); encode_named_bit_string(C, [{bit,_}|_]=Bits, NamedBitList, TagIn) -> do_encode_named_bit_string(C, Bits, NamedBitList, TagIn); +encode_named_bit_string(C, [], _NamedBitList, TagIn) -> + encode_unnamed_bit_string(C, <<>>, TagIn); encode_named_bit_string(C, Bits, _NamedBitList, TagIn) when is_bitstring(Bits) -> encode_unnamed_bit_string(C, Bits, TagIn). diff --git a/lib/asn1/test/testPrimStrings.erl b/lib/asn1/test/testPrimStrings.erl index fa778765b1..46793c6bff 100644 --- a/lib/asn1/test/testPrimStrings.erl +++ b/lib/asn1/test/testPrimStrings.erl @@ -123,6 +123,7 @@ bit_string(Rules, Opts) -> %% Bs2 ::= BIT STRING {su(0), mo(1), tu(2), we(3), th(4), fr(5), sa(6) } (SIZE (7)) %%========================================================== + roundtrip('Bs2', []), roundtrip('Bs2', [mo,tu,fr]), bs_roundtrip('Bs2', <<2#0110010:7>>, [mo,tu,fr]), bs_roundtrip('Bs2', <<2#0110011:7>>, [mo,tu,fr,sa]), @@ -131,6 +132,7 @@ bit_string(Rules, Opts) -> %% Bs3 ::= BIT STRING {su(0), mo(1), tu(2), we(3), th(4), fr(5), sa(6) } (SIZE (1..7)) %%========================================================== + roundtrip('Bs3', []), roundtrip('Bs3', [mo,tu,fr]), bs_roundtrip('Bs3', <<2#0110010:7>>, [mo,tu,fr]), bs_roundtrip('Bs3', <<2#0110010:7>>, [mo,tu,fr]), @@ -139,6 +141,13 @@ bit_string(Rules, Opts) -> bs_roundtrip('Bs3', <<2#11:2>>, [su,mo]), %%========================================================== + %% Bs4 ::= BIT STRING {su(0), mo(1), tu(2), we(3), th(4), fr(5), sa(6) } + %%========================================================== + + roundtrip('Bs4', []), + roundtrip('Bs4', [mo,tu,fr,sa]), + + %%========================================================== %% Bs7 ::= BIT STRING (SIZE (24)) %%========================================================== diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl index 2dd4fdac05..5239ec1ff8 100644 --- a/lib/common_test/src/ct_conn_log_h.erl +++ b/lib/common_test/src/ct_conn_log_h.erl @@ -104,18 +104,26 @@ terminate(_,#state{logs=Logs}) -> %%%----------------------------------------------------------------- %%% Writing reports write_report(_Time,#conn_log{header=false,module=ConnMod}=Info,Data,GL,State) -> - {LogType,Fd} = get_log(Info,GL,State), - io:format(Fd,"~n~ts",[format_data(ConnMod,LogType,Data)]); + case get_log(Info,GL,State) of + {silent,_} -> + ok; + {LogType,Fd} -> + io:format(Fd,"~n~ts",[format_data(ConnMod,LogType,Data)]) + end; write_report(Time,#conn_log{module=ConnMod}=Info,Data,GL,State) -> - {LogType,Fd} = get_log(Info,GL,State), - io:format(Fd,"~n~ts~ts~ts",[format_head(ConnMod,LogType,Time), - format_title(LogType,Info), - format_data(ConnMod,LogType,Data)]). + case get_log(Info,GL,State) of + {silent,_} -> + ok; + {LogType,Fd} -> + io:format(Fd,"~n~ts~ts~ts",[format_head(ConnMod,LogType,Time), + format_title(LogType,Info), + format_data(ConnMod,LogType,Data)]) + end. write_error(Time,#conn_log{module=ConnMod}=Info,Report,GL,State) -> case get_log(Info,GL,State) of - {html,_} -> + {LogType,_} when LogType==html; LogType==silent -> %% The error will anyway be written in the html log by the %% sasl error handler, so don't write it again. ok; diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 0de7bf03af..05977e5649 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -1272,6 +1272,14 @@ set_request_timer(T) -> {ok,TRef} = timer:send_after(T,{Ref,timeout}), {Ref,TRef}. +%%%----------------------------------------------------------------- +cancel_request_timer(undefined,undefined) -> + ok; +cancel_request_timer(Ref,TRef) -> + _ = timer:cancel(TRef), + receive {Ref,timeout} -> ok + after 0 -> ok + end. %%%----------------------------------------------------------------- client_hello(Options) when is_list(Options) -> @@ -1404,9 +1412,9 @@ handle_error(Reason, State) -> Pending -> %% Assuming the first request gets the %% first answer - P=#pending{tref=TRef,caller=Caller} = + P=#pending{tref=TRef,ref=Ref,caller=Caller} = lists:last(Pending), - _ = timer:cancel(TRef), + cancel_request_timer(Ref,TRef), Reason1 = {failed_to_parse_received_data,Reason}, ct_gen_conn:return(Caller,{error,Reason1}), lists:delete(P,Pending) @@ -1492,8 +1500,8 @@ decode({Tag,Attrs,_}=E, #state{connection=Connection,pending=Pending}=State) -> {error,Reason} -> {noreply,State#state{hello_status = {error,Reason}}} end; - #pending{tref=TRef,caller=Caller} -> - _ = timer:cancel(TRef), + #pending{tref=TRef,ref=Ref,caller=Caller} -> + cancel_request_timer(Ref,TRef), case decode_hello(E) of {ok,SessionId,Capabilities} -> ct_gen_conn:return(Caller,ok), @@ -1519,9 +1527,8 @@ decode({Tag,Attrs,_}=E, #state{connection=Connection,pending=Pending}=State) -> %% there is just one pending that matches (i.e. has %% undefined msg_id and op) case [P || P = #pending{msg_id=undefined,op=undefined} <- Pending] of - [#pending{tref=TRef, - caller=Caller}] -> - _ = timer:cancel(TRef), + [#pending{tref=TRef,ref=Ref,caller=Caller}] -> + cancel_request_timer(Ref,TRef), ct_gen_conn:return(Caller,E), {noreply,State#state{pending=[]}}; _ -> @@ -1542,8 +1549,8 @@ get_msg_id(Attrs) -> decode_rpc_reply(MsgId,{_,Attrs,Content0}=E,#state{pending=Pending} = State) -> case lists:keytake(MsgId,#pending.msg_id,Pending) of - {value, #pending{tref=TRef,op=Op,caller=Caller}, Pending1} -> - _ = timer:cancel(TRef), + {value, #pending{tref=TRef,ref=Ref,op=Op,caller=Caller}, Pending1} -> + cancel_request_timer(Ref,TRef), Content = forward_xmlns_attr(Attrs,Content0), {CallerReply,{ServerReply,State2}} = do_decode_rpc_reply(Op,Content,State#state{pending=Pending1}), @@ -1555,10 +1562,11 @@ decode_rpc_reply(MsgId,{_,Attrs,Content0}=E,#state{pending=Pending} = State) -> %% pending that matches (i.e. has undefined msg_id and op) case [P || P = #pending{msg_id=undefined,op=undefined} <- Pending] of [#pending{tref=TRef, + ref=Ref, msg_id=undefined, op=undefined, caller=Caller}] -> - _ = timer:cancel(TRef), + cancel_request_timer(Ref,TRef), ct_gen_conn:return(Caller,E), {noreply,State#state{pending=[]}}; _ -> diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index 64ebfbc463..aaa0723488 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -73,6 +73,7 @@ all() -> timeout_close_session, get, timeout_get, + flush_timeout_get, get_xpath, get_config, get_config_xpath, @@ -360,6 +361,28 @@ timeout_get(Config) -> ?ok = ct_netconfc:close_session(Client), ok. +%% Test OTP-13008 "ct_netconfc crash when receiving unknown timeout" +%% If the timer expires "at the same time" as the rpc reply is +%% received, the timeout message might already be sent when the timer +%% is cancelled. This test checks that the timeout message is flushed +%% from the message queue. If it isn't, the client crashes and the +%% session can not be closed afterwards. +%% Note that we can only hope that the test case triggers the problem +%% every now and then, as it is very timing dependent... +flush_timeout_get(Config) -> + DataDir = ?config(data_dir,Config), + {ok,Client} = open_success(DataDir), + Data = [{server,[{xmlns,"myns"}],[{name,[],["myserver"]}]}], + ?NS:expect_reply('get',{data,Data}), + timer:sleep(1000), + case ct_netconfc:get(Client,{server,[{xmlns,"myns"}],[]},1) of + {error,timeout} -> ok; % problem not triggered + {ok,Data} -> ok % problem possibly triggered + end, + ?NS:expect_do_reply('close-session',close,ok), + ?ok = ct_netconfc:close_session(Client), + ok. + get_xpath(Config) -> DataDir = ?config(data_dir,Config), {ok,Client} = open_success(DataDir), diff --git a/lib/erl_interface/src/connect/ei_resolve.c b/lib/erl_interface/src/connect/ei_resolve.c index 3f1be2b17d..6381b02393 100644 --- a/lib/erl_interface/src/connect/ei_resolve.c +++ b/lib/erl_interface/src/connect/ei_resolve.c @@ -601,6 +601,16 @@ struct hostent *ei_gethostbyaddr(const char *addr, int len, int type) return gethostbyaddr(addr, len, type); } +/* + * Imprecise way to select the actually available gethostbyname_r and + * gethostbyaddr_r. + * + * TODO: check this properly in configure.in + */ +#if (defined(__linux__) || (__FreeBSD_version >= 602000) || defined(__DragonFly__)) + #define HAVE_GETHOSTBYADDR_R_8 1 +#endif + struct hostent *ei_gethostbyaddr_r(const char *addr, int length, int type, @@ -616,7 +626,7 @@ struct hostent *ei_gethostbyaddr_r(const char *addr, #ifndef HAVE_GETHOSTBYNAME_R return my_gethostbyaddr_r(addr,length,type,hostp,buffer,buflen,h_errnop); #else -#if (defined(__GLIBC__) || (__FreeBSD_version >= 602000) || defined(__DragonFly__)) +#ifdef HAVE_GETHOSTBYADDR_R_8 struct hostent *result; gethostbyaddr_r(addr, length, type, hostp, buffer, buflen, &result, @@ -643,7 +653,7 @@ struct hostent *ei_gethostbyname_r(const char *name, #ifndef HAVE_GETHOSTBYNAME_R return my_gethostbyname_r(name,hostp,buffer,buflen,h_errnop); #else -#if (defined(__GLIBC__) || (__FreeBSD_version >= 602000) || defined(__DragonFly__) || defined(__ANDROID__)) +#ifdef HAVE_GETHOSTBYADDR_R_8 struct hostent *result; gethostbyname_r(name, hostp, buffer, buflen, &result, h_errnop); diff --git a/lib/eunit/include/eunit.hrl b/lib/eunit/include/eunit.hrl index 88e9d6c19b..8f678b0290 100644 --- a/lib/eunit/include/eunit.hrl +++ b/lib/eunit/include/eunit.hrl @@ -135,7 +135,6 @@ -define(_assertThrow(Term, Expr), ?_assertException(throw, Term, Expr)). -define(_assertNotException(Class, Term, Expr), ?_test(?assertNotException(Class, Term, Expr))). --define(_assertReceive(Guard, Expr), ?_test(?assertReceive(Guard, Expr))). %% Macros for running operating system commands. (Note that these %% require EUnit to be present at runtime, or at least eunit_lib.) diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index e6dcfee818..d1c52dcc78 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1827,11 +1827,13 @@ host_header(_, URI) -> tls_upgrade(#state{status = {ssl_tunnel, #request{settings = - #http_options{ssl = {_, TLSOptions} = SocketType}, - address = Address} = Request}, + #http_options{ssl = {_, TLSOptions0} = SocketType}, + address = {Host, _} = Address} = Request}, session = #session{socket = TCPSocket} = Session0, options = Options} = State) -> + TLSOptions = maybe_add_sni(Host, TLSOptions0), + case ssl:connect(TCPSocket, TLSOptions) of {ok, TLSSocket} -> ClientClose = httpc_request:is_client_closing(Request#request.headers), @@ -1862,6 +1864,15 @@ tls_upgrade(#state{status = {stop, normal, State#state{request = Request}} end. +maybe_add_sni(Host, Options) -> + case http_util:is_hostname(Host) andalso + not lists:keymember(server_name_indication, 1, Options) of + true -> + [{server_name_indication, Host} | Options]; + false -> + Options + end. + %% --------------------------------------------------------------------- %% Session wrappers %% --------------------------------------------------------------------- diff --git a/lib/inets/src/http_lib/http_chunk.erl b/lib/inets/src/http_lib/http_chunk.erl index 9699856bf8..7325f24809 100644 --- a/lib/inets/src/http_lib/http_chunk.erl +++ b/lib/inets/src/http_lib/http_chunk.erl @@ -25,7 +25,7 @@ -include("http_internal.hrl"). %% API --export([decode/3, encode/1, encode_last/0, handle_headers/2]). +-export([decode/3, encode/1, encode_last/0, encode_last/1, handle_headers/2]). %% Callback API - used for example if the chunkedbody is received a %% little at a time on a socket. -export([decode_size/1, ignore_extensions/1, decode_data/1, decode_trailer/1]). @@ -85,6 +85,11 @@ encode(Chunk) when is_list(Chunk)-> encode_last() -> <<$0, ?CR, ?LF, ?CR, ?LF >>. +encode_last([]) -> + encode_last(); +encode_last(Trailers0) -> + Trailers = list_to_binary(encode_trailers(Trailers0)), + <<$0, ?CR, ?LF, Trailers/binary>>. %%------------------------------------------------------------------------- %% handle_headers(HeaderRecord, ChunkedHeaders) -> NewHeaderRecord @@ -276,10 +281,18 @@ decode_trailer(<<?CR, ?LF, Rest/binary>>, Header, Headers, Body, BodyLength, Rem Body, BodyLength, RemainingSize, TotalMaxHeaderSize); decode_trailer(<<Octet, Rest/binary>>, Header, Headers, Body, BodyLength, RemainingSize, TotalMaxHeaderSize) -> - decode_trailer(Rest, [Octet | Header], Headers, - Body, BodyLength, RemainingSize - 1, TotalMaxHeaderSize). + decode_trailer(Rest, [Octet | Header], Headers, + Body, BodyLength, remaing_size(RemainingSize, 1), TotalMaxHeaderSize). remaing_size(nolimit, _) -> nolimit; remaing_size(Total, Consumed) -> Total - Consumed. + +encode_trailers(Trailers) -> + encode_trailers(Trailers, ""). + +encode_trailers([], Acc) -> + Acc ++ ?CRLF ++ ?CRLF; +encode_trailers([{Header, Value} | Rest], Acc) -> + encode_trailers(Rest, Header ++ ":" ++ Value ++ ?CRLF ++ Acc). diff --git a/lib/inets/src/http_lib/http_response.erl b/lib/inets/src/http_lib/http_response.erl index d13670700c..42e5dd263d 100644 --- a/lib/inets/src/http_lib/http_response.erl +++ b/lib/inets/src/http_lib/http_response.erl @@ -65,6 +65,8 @@ header_list(Headers) -> %%%======================================================================== fill_headers([], _, Headers) -> Headers; +fill_headers([[]], _, Headers) -> + Headers; fill_headers([[Ch|HeaderFold]|Tail], Folded, Headers) when Ch == $\t; Ch == $\s -> fill_headers(Tail, [HeaderFold|Folded], Headers); diff --git a/lib/inets/src/http_server/httpd_example.erl b/lib/inets/src/http_server/httpd_example.erl index d729affd6d..0222487a4b 100644 --- a/lib/inets/src/http_server/httpd_example.erl +++ b/lib/inets/src/http_server/httpd_example.erl @@ -24,7 +24,7 @@ -export([newformat/3]). %% These are used by the inets test-suite --export([delay/1]). +-export([delay/1, chunk_timeout/3]). print(String) -> @@ -142,3 +142,11 @@ i(F) -> i(F,[]). i(F,A) -> io:format(F ++ "~n",A). sleep(T) -> receive after T -> ok end. + +%% ------------------------------------------------------ + +chunk_timeout(SessionID, _, StrInt) -> + mod_esi:deliver(SessionID, "Tranfer-Encoding:chunked/html\r\n\r\n"), + mod_esi:deliver(SessionID, top("Test chunk encoding timeout")), + timer:sleep(20000), + mod_esi:deliver(SessionID, footer()). diff --git a/lib/inets/src/http_server/httpd_request_handler.erl b/lib/inets/src/http_server/httpd_request_handler.erl index 143d599edb..134576059d 100644 --- a/lib/inets/src/http_server/httpd_request_handler.erl +++ b/lib/inets/src/http_server/httpd_request_handler.erl @@ -630,21 +630,10 @@ decrease(N) when is_integer(N) -> decrease(N) -> N. -error_log(ReasonString, Info) -> +error_log(ReasonString, #mod{config_db = ConfigDB}) -> Error = lists:flatten( io_lib:format("Error reading request: ~s", [ReasonString])), - error_log(mod_log, Info, Error), - error_log(mod_disk_log, Info, Error). - -error_log(Mod, #mod{config_db = ConfigDB} = Info, String) -> - Modules = httpd_util:lookup(ConfigDB, modules, - [mod_get, mod_head, mod_log]), - case lists:member(Mod, Modules) of - true -> - Mod:error_log(Info, String); - _ -> - ok - end. + httpd_util:error_log(ConfigDB, Error). %%-------------------------------------------------------------------- diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index 71243f525a..c0b5f09faf 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -20,8 +20,8 @@ %% -module(httpd_response). -export([generate_and_send_response/1, send_status/3, send_header/3, - send_body/3, send_chunk/3, send_final_chunk/2, split_header/2, - is_disable_chunked_send/1, cache_headers/2]). + send_body/3, send_chunk/3, send_final_chunk/2, send_final_chunk/3, + split_header/2, is_disable_chunked_send/1, cache_headers/2]). -export([map_status_code/2]). -include_lib("inets/src/inets_app/inets_internal.hrl"). @@ -89,8 +89,7 @@ traverse_modules(ModData,[Module|Rest]) -> "~n Error: ~p" "~n Stack trace: ~p", [Module, T, E, ?STACK()])), - report_error(mod_log, ModData#mod.config_db, String), - report_error(mod_disk_log, ModData#mod.config_db, String), + httpd_util:error_log(ModData#mod.config_db, String), send_status(ModData, 500, none), done end. @@ -245,7 +244,6 @@ send_chunk(_, <<>>, _) -> ok; send_chunk(_, [], _) -> ok; - send_chunk(#mod{http_version = "HTTP/1.1", socket_type = Type, socket = Sock}, Response0, false) -> Response = http_chunk:encode(Response0), @@ -254,10 +252,13 @@ send_chunk(#mod{http_version = "HTTP/1.1", send_chunk(#mod{socket_type = Type, socket = Sock} = _ModData, Response, _) -> httpd_socket:deliver(Type, Sock, Response). +send_final_chunk(Mod, IsDisableChunkedSend) -> + send_final_chunk(Mod, [], IsDisableChunkedSend). + send_final_chunk(#mod{http_version = "HTTP/1.1", - socket_type = Type, socket = Sock}, false) -> - httpd_socket:deliver(Type, Sock, http_chunk:encode_last()); -send_final_chunk(#mod{socket_type = Type, socket = Sock}, _) -> + socket_type = Type, socket = Sock}, Trailers, false) -> + httpd_socket:deliver(Type, Sock, http_chunk:encode_last(Trailers)); +send_final_chunk(#mod{socket_type = Type, socket = Sock}, _, _) -> httpd_socket:close(Type, Sock). is_disable_chunked_send(Db) -> @@ -397,16 +398,6 @@ send_response_old(#mod{socket_type = Type, content_length(Body)-> integer_to_list(httpd_util:flatlength(Body)). -report_error(Mod, ConfigDB, Error) -> - Modules = httpd_util:lookup(ConfigDB, modules, - [mod_get, mod_head, mod_log]), - case lists:member(Mod, Modules) of - true -> - Mod:report_error(ConfigDB, Error); - _ -> - ok - end. - handle_headers([], NewHeaders) -> {ok, NewHeaders}; diff --git a/lib/inets/src/http_server/httpd_util.erl b/lib/inets/src/http_server/httpd_util.erl index 0387d71911..ab43f0b378 100644 --- a/lib/inets/src/http_server/httpd_util.erl +++ b/lib/inets/src/http_server/httpd_util.erl @@ -31,7 +31,7 @@ convert_netscapecookie_date/1, enable_debug/1, valid_options/3, modules_validate/1, module_validate/1, dir_validate/2, file_validate/2, mime_type_validate/1, - mime_types_validate/1, custom_date/0]). + mime_types_validate/1, custom_date/0, error_log/2]). -export([encode_hex/1, decode_hex/1]). -include_lib("kernel/include/file.hrl"). @@ -776,3 +776,17 @@ do_enable_debug([{Level,Modules}|Rest]) ok end, do_enable_debug(Rest). + +error_log(ConfigDb, Error) -> + error_log(mod_log, ConfigDb, Error), + error_log(mod_disk_log, ConfigDb, Error). + +error_log(Mod, ConfigDB, Error) -> + Modules = httpd_util:lookup(ConfigDB, modules, + [mod_get, mod_head, mod_log]), + case lists:member(Mod, Modules) of + true -> + Mod:report_error(ConfigDB, Error); + _ -> + ok + end. diff --git a/lib/inets/src/http_server/mod_esi.erl b/lib/inets/src/http_server/mod_esi.erl index b9a0797977..1923411449 100644 --- a/lib/inets/src/http_server/mod_esi.erl +++ b/lib/inets/src/http_server/mod_esi.erl @@ -376,7 +376,6 @@ erl_scheme_webpage_chunk(Mod, Func, Env, Input, ModData) -> end), Response = deliver_webpage_chunk(ModData, Pid), - process_flag(trap_exit,false), Response. @@ -418,7 +417,6 @@ deliver_webpage_chunk(#mod{config_db = Db} = ModData, Pid, Timeout) -> ?hdrv("deliver_webpage_chunk - timeout", []), send_headers(ModData, 504, [{"connection", "close"}]), httpd_socket:close(ModData#mod.socket_type, ModData#mod.socket), - process_flag(trap_exit,false), {proceed,[{response, {already_sent, 200, 0}} | ModData#mod.data]} end. @@ -446,7 +444,6 @@ send_headers(ModData, StatusCode, HTTPHeaders) -> ExtraHeaders ++ HTTPHeaders). handle_body(_, #mod{method = "HEAD"} = ModData, _, _, Size, _) -> - process_flag(trap_exit,false), {proceed, [{response, {already_sent, 200, Size}} | ModData#mod.data]}; handle_body(Pid, ModData, Body, Timeout, Size, IsDisableChunkedSend) -> @@ -454,34 +451,54 @@ handle_body(Pid, ModData, Body, Timeout, Size, IsDisableChunkedSend) -> httpd_response:send_chunk(ModData, Body, IsDisableChunkedSend), receive {esi_data, Data} when is_binary(Data) -> - ?hdrt("handle_body - received binary data (esi)", []), handle_body(Pid, ModData, Data, Timeout, Size + byte_size(Data), IsDisableChunkedSend); {esi_data, Data} -> - ?hdrt("handle_body - received data (esi)", []), handle_body(Pid, ModData, Data, Timeout, Size + length(Data), IsDisableChunkedSend); {ok, Data} -> - ?hdrt("handle_body - received data (ok)", []), handle_body(Pid, ModData, Data, Timeout, Size + length(Data), IsDisableChunkedSend); {'EXIT', Pid, normal} when is_pid(Pid) -> - ?hdrt("handle_body - exit:normal", []), httpd_response:send_final_chunk(ModData, IsDisableChunkedSend), {proceed, [{response, {already_sent, 200, Size}} | ModData#mod.data]}; {'EXIT', Pid, Reason} when is_pid(Pid) -> - ?hdrv("handle_body - exit", [{reason, Reason}]), - httpd_response:send_final_chunk(ModData, IsDisableChunkedSend), - exit({mod_esi_linked_process_died, Pid, Reason}) - + Error = lists:flatten(io_lib:format("mod_esi process failed with reason ~p", [Reason])), + httpd_util:error_log(ModData#mod.config_db, Error), + httpd_response:send_final_chunk(ModData, + [{"Warning", "199 inets server - body maybe incomplete, " + "internal server error"}], + IsDisableChunkedSend), + done after Timeout -> - ?hdrv("handle_body - timeout", []), - process_flag(trap_exit,false), - httpd_response:send_final_chunk(ModData, IsDisableChunkedSend), - exit({mod_esi_linked_process_timeout, Pid}) + kill_esi_delivery_process(Pid), + httpd_response:send_final_chunk(ModData, [{"Warning", "199 inets server - " + "body maybe incomplete, timed out"}], + IsDisableChunkedSend), + done end. +kill_esi_delivery_process(Pid) -> + exit(Pid, kill), + receive + {'EXIT', Pid, killed} -> + %% Clean message queue + receive + {esi_data, _} -> + ok + after 0 -> + ok + end, + receive + {ok, _} -> + ok + after 0 -> + ok + end + end. + + erl_script_timeout(Db) -> httpd_util:lookup(Db, erl_script_timeout, ?DEFAULT_ERL_TIMEOUT). diff --git a/lib/inets/test/httpd_1_1.erl b/lib/inets/test/httpd_1_1.erl index dd9d21bbfc..db6def9d17 100644 --- a/lib/inets/test/httpd_1_1.erl +++ b/lib/inets/test/httpd_1_1.erl @@ -24,7 +24,7 @@ -include_lib("kernel/include/file.hrl"). -export([host/4, chunked/4, expect/4, range/4, if_test/5, trace/4, - head/4, mod_cgi_chunked_encoding_test/5]). + head/4, mod_cgi_chunked_encoding_test/5, mod_esi_chunk_timeout/4]). %% -define(all_keys_lower_case,true). -ifndef(all_keys_lower_case). @@ -274,6 +274,15 @@ mod_cgi_chunked_encoding_test(Type, Port, Host, Node, [Request| Rest])-> [{statuscode, 200}]), mod_cgi_chunked_encoding_test(Type, Port, Host, Node, Rest). + +mod_esi_chunk_timeout(Type, Port, Host, Node) -> + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "GET /cgi-bin/erl/httpd_example/chunk_timeout?input=20000 HTTP/1.1\r\n" + "Host:"++ Host ++"\r\n" + "\r\n", + [{statuscode, 200}, + {header, "warning"}]). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 9bd6f3636c..1d8a603981 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -117,7 +117,7 @@ groups() -> {htaccess, [], [htaccess_1_1, htaccess_1_0, htaccess_0_9]}, {security, [], [security_1_1, security_1_0]}, %% Skip 0.9 as causes timing issus in test code {http_1_1, [], [host, chunked, expect, cgi, cgi_chunked_encoding_test, - trace, range, if_modified_since] ++ http_head() ++ http_get() ++ load()}, + trace, range, if_modified_since, mod_esi_chunk_timeout] ++ http_head() ++ http_get() ++ load()}, {http_1_0, [], [host, cgi, trace] ++ http_head() ++ http_get() ++ load()}, {http_0_9, [], http_head() ++ http_get() ++ load()} ]. @@ -757,6 +757,13 @@ esi(Config) when is_list(Config) -> Config, [{statuscode, 200}, {no_header, "cache-control"}]). %%------------------------------------------------------------------------- +mod_esi_chunk_timeout(Config) when is_list(Config) -> + ok = httpd_1_1:mod_esi_chunk_timeout(?config(type, Config), + ?config(port, Config), + ?config(host, Config), + ?config(node, Config)). + +%%------------------------------------------------------------------------- cgi() -> [{doc, "Test mod_cgi"}]. diff --git a/lib/inets/test/httpd_test_lib.erl b/lib/inets/test/httpd_test_lib.erl index a5b836f651..c58966ce10 100644 --- a/lib/inets/test/httpd_test_lib.erl +++ b/lib/inets/test/httpd_test_lib.erl @@ -235,11 +235,17 @@ validate(RequestStr, #state{status_line = {Version, StatusCode, _}, _ -> ok end, - do_validate(http_response:header_list(Headers), Options, N, P), - check_body(RequestStr, StatusCode, - Headers#http_response_h.'content-type', - list_to_integer(Headers#http_response_h.'content-length'), - Body). + HList = http_response:header_list(Headers), + do_validate(HList, Options, N, P), + case lists:keysearch("warning", 1, HList) of + {value, _} -> + ok; + _ -> + check_body(RequestStr, StatusCode, + Headers#http_response_h.'content-type', + list_to_integer(Headers#http_response_h.'content-length'), + Body) + end. %-------------------------------------------------------------------- %% Internal functions diff --git a/lib/kernel/src/file_io_server.erl b/lib/kernel/src/file_io_server.erl index bf8b9e2747..deb7b315b1 100644 --- a/lib/kernel/src/file_io_server.erl +++ b/lib/kernel/src/file_io_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2013. All Rights Reserved. +%% Copyright Ericsson AB 2000-2015. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -206,8 +206,8 @@ io_reply(From, ReplyAs, Reply) -> file_request({advise,Offset,Length,Advise}, #state{handle=Handle}=State) -> case ?PRIM_FILE:advise(Handle, Offset, Length, Advise) of - {error,_}=Reply -> - {stop,normal,Reply,State}; + {error,Reason}=Reply -> + {stop,Reason,Reply,State}; Reply -> {reply,Reply,State} end; @@ -215,62 +215,91 @@ file_request({allocate, Offset, Length}, #state{handle = Handle} = State) -> Reply = ?PRIM_FILE:allocate(Handle, Offset, Length), {reply, Reply, State}; +file_request({pread,At,Sz}, State) + when At =:= cur; + At =:= {cur,0} -> + case get_chars(Sz, latin1, State) of + {reply,Reply,NewState} + when is_list(Reply); + is_binary(Reply) -> + {reply,{ok,Reply},NewState}; + Other -> + Other + end; file_request({pread,At,Sz}, - #state{handle=Handle,buf=Buf,read_mode=ReadMode}=State) -> + #state{handle=Handle,buf=Buf}=State) -> case position(Handle, At, Buf) of - {ok,_Offs} -> - case ?PRIM_FILE:read(Handle, Sz) of - {ok,Bin} when ReadMode =:= list -> - std_reply({ok,binary_to_list(Bin)}, State); - Reply -> - std_reply(Reply, State) - end; - Reply -> - std_reply(Reply, State) + {error,_} = Reply -> + {error,Reply,State}; + _ -> + case get_chars(Sz, latin1, State#state{buf= <<>>}) of + {reply,Reply,NewState} + when is_list(Reply); + is_binary(Reply) -> + {reply,{ok,Reply},NewState}; + Other -> + Other + end end; +file_request({pwrite,At,Data}, + #state{buf= <<>>}=State) + when At =:= cur; + At =:= {cur,0} -> + put_chars(Data, latin1, State); file_request({pwrite,At,Data}, #state{handle=Handle,buf=Buf}=State) -> case position(Handle, At, Buf) of - {ok,_Offs} -> - std_reply(?PRIM_FILE:write(Handle, Data), State); - Reply -> - std_reply(Reply, State) + {error,_} = Reply -> + {error,Reply,State}; + _ -> + put_chars(Data, latin1, State) end; file_request(datasync, #state{handle=Handle}=State) -> case ?PRIM_FILE:datasync(Handle) of - {error,_}=Reply -> - {stop,normal,Reply,State}; + {error,Reason}=Reply -> + {stop,Reason,Reply,State}; Reply -> {reply,Reply,State} end; file_request(sync, #state{handle=Handle}=State) -> case ?PRIM_FILE:sync(Handle) of - {error,_}=Reply -> - {stop,normal,Reply,State}; + {error,Reason}=Reply -> + {stop,Reason,Reply,State}; Reply -> {reply,Reply,State} end; file_request(close, #state{handle=Handle}=State) -> - {stop,normal,?PRIM_FILE:close(Handle),State#state{buf= <<>>}}; + case ?PRIM_FILE:close(Handle) of + {error,Reason}=Reply -> + {stop,Reason,Reply,State#state{buf= <<>>}}; + Reply -> + {stop,normal,Reply,State#state{buf= <<>>}} + end; file_request({position,At}, #state{handle=Handle,buf=Buf}=State) -> - std_reply(position(Handle, At, Buf), State); + case position(Handle, At, Buf) of + {error,_} = Reply -> + {error,Reply,State}; + Reply -> + std_reply(Reply, State) + end; file_request(truncate, #state{handle=Handle}=State) -> case ?PRIM_FILE:truncate(Handle) of - {error,_Reason}=Reply -> - {stop,normal,Reply,State#state{buf= <<>>}}; + {error,Reason}=Reply -> + {stop,Reason,Reply,State#state{buf= <<>>}}; Reply -> - {reply,Reply,State} + std_reply(Reply, State) end; file_request(Unknown, #state{}=State) -> Reason = {request, Unknown}, {error,{error,Reason},State}. +%% Standard reply and clear buffer std_reply({error,_}=Reply, State) -> {error,Reply,State#state{buf= <<>>}}; std_reply(Reply, State) -> @@ -286,8 +315,8 @@ io_request({put_chars, Enc, Chars}, io_request({put_chars, Enc, Chars}, #state{handle=Handle,buf=Buf}=State) -> case position(Handle, cur, Buf) of - {error,_}=Reply -> - {stop,normal,Reply,State#state{buf= <<>>}}; + {error,Reason}=Reply -> + {stop,Reason,Reply,State}; _ -> put_chars(Chars, Enc, State#state{buf= <<>>}) end; @@ -368,23 +397,27 @@ io_request_loop([Request|Tail], %% I/O request put_chars %% put_chars(Chars, latin1, #state{handle=Handle, unic=latin1}=State) -> + NewState = State#state{buf = <<>>}, case ?PRIM_FILE:write(Handle, Chars) of - {error,_}=Reply -> - {stop,normal,Reply,State}; + {error,Reason}=Reply -> + {stop,Reason,Reply,NewState}; Reply -> - {reply,Reply,State} + {reply,Reply,NewState} end; put_chars(Chars, InEncoding, #state{handle=Handle, unic=OutEncoding}=State) -> + NewState = State#state{buf = <<>>}, case unicode:characters_to_binary(Chars,InEncoding,OutEncoding) of Bin when is_binary(Bin) -> case ?PRIM_FILE:write(Handle, Bin) of - {error,_}=Reply -> - {stop,normal,Reply,State}; + {error,Reason}=Reply -> + {stop,Reason,Reply,NewState}; Reply -> - {reply,Reply,State} + {reply,Reply,NewState} end; {error,_,_} -> - {stop,normal,{error,{no_translation, InEncoding, OutEncoding}},State} + {stop,no_translation, + {error,{no_translation, InEncoding, OutEncoding}}, + NewState} end. get_line(S, {<<>>, Cont}, OutEnc, @@ -884,11 +917,14 @@ cbv({utf32,little},_) -> %% Compensates ?PRIM_FILE:position/2 for the number of bytes %% we have buffered - -position(Handle, cur, Buf) -> - position(Handle, {cur, 0}, Buf); -position(Handle, {cur, Offs}, Buf) when is_binary(Buf) -> - ?PRIM_FILE:position(Handle, {cur, Offs-byte_size(Buf)}); -position(Handle, At, _Buf) -> - ?PRIM_FILE:position(Handle, At). - +position(Handle, At, Buf) -> + ?PRIM_FILE:position( + Handle, + case At of + cur -> + {cur, -byte_size(Buf)}; + {cur, Offs} -> + {cur, Offs-byte_size(Buf)}; + _ -> + At + end). diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 8856be31c2..09d9a45197 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ -export([cur_dir_0/1, cur_dir_1/1, make_del_dir/1, list_dir/1,list_dir_error/1, untranslatable_names/1, untranslatable_names_error/1, - pos1/1, pos2/1]). + pos1/1, pos2/1, pos3/1]). -export([close/1, consult1/1, path_consult/1, delete/1]). -export([ eval1/1, path_eval/1, script1/1, path_script/1, open1/1, @@ -80,6 +80,7 @@ -export([interleaved_read_write/1]). +-export([unicode/1]). -export([altname/1]). -export([large_file/1, large_write/1]). @@ -114,7 +115,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [altname, read_write_file, {group, dirs}, + [unicode, altname, read_write_file, {group, dirs}, {group, files}, delete, rename, names, {group, errors}, {group, compression}, {group, links}, copy, delayed_write, read_ahead, segment_read, segment_write, @@ -136,7 +137,7 @@ groups() -> [open1, old_modes, new_modes, path_open, close, access, read_write, pread_write, append, open_errors, exclusive]}, - {pos, [], [pos1, pos2]}, + {pos, [], [pos1, pos2, pos3]}, {file_info, [], [file_info_basic_file, file_info_basic_directory, file_info_bad, file_info_times, file_write_file_info]}, @@ -1370,6 +1371,27 @@ pos2(Config) when is_list(Config) -> ?line test_server:timetrap_cancel(Dog), ok. +pos3(suite) -> []; +pos3(doc) -> ["When it does not use raw mode, file:position had a bug."]; +pos3(Config) when is_list(Config) -> + ?line Dog = test_server:timetrap(test_server:seconds(5)), + ?line RootDir = ?config(data_dir, Config), + ?line Name = filename:join(RootDir, "realmen.html.gz"), + + ?line {ok, Fd} = ?FILE_MODULE:open(Name, [read, binary]), + ?line {ok, _} = ?FILE_MODULE:read(Fd, 5), + ?line {error, einval} = ?FILE_MODULE:position(Fd, {bof, -1}), + + %% Here ok had returned =( + ?line {error, einval} = ?FILE_MODULE:position(Fd, {cur, -10}), + %% That test is actually questionable since file:position/2 + %% is documented to leave the file position undefined after + %% it has returned an error. But on Posix systems the position + %% is guaranteed to be unchanged after an error return. On e.g + %% Windows there is nothing stated about this in the documentation. + + ?line test_server:timetrap_cancel(Dog), + ok. file_info_basic_file(suite) -> []; file_info_basic_file(doc) -> []; @@ -2761,6 +2783,40 @@ compress_async_crash_loop(N, Path, ExpectedData) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +unicode(Config) when is_list(Config) -> + Dir = ?config(priv_dir, Config), + Name = filename:join(Dir, "data-utf8.txt"), + Txt = lists:seq(128, 255), + D = unicode:characters_to_binary(Txt, latin1, latin1), + {ok,Fd1} = + ?FILE_MODULE:open(Name, [write,read,binary,{encoding,unicode}]), + ok = ?FILE_MODULE:truncate(Fd1), + ok = ?FILE_MODULE:write(Fd1, Txt), + {ok,0} = ?FILE_MODULE:position(Fd1, bof), + {ok,D} = ?FILE_MODULE:read(Fd1, 129), + {ok,0} = ?FILE_MODULE:position(Fd1, bof), + {ok,D1} = ?FILE_MODULE:read(Fd1, 64), + {ok,Pos} = ?FILE_MODULE:position(Fd1, cur), + {ok,D2} = ?FILE_MODULE:pread(Fd1, {cur,0}, 65), + D = <<D1/binary, D2/binary>>, + {ok,D1} = ?FILE_MODULE:pread(Fd1, bof, 64), + {ok,Pos} = ?FILE_MODULE:position(Fd1, Pos), + {ok,D2} = ?FILE_MODULE:read(Fd1, 64), + ok = ?FILE_MODULE:close(Fd1), + %% + RawD = unicode:characters_to_binary(Txt, latin1, unicode), + {ok,RawD} = ?FILE_MODULE:read_file(Name), + %% + {ok,Fd2} = ?FILE_MODULE:open(Name, [read,{encoding,unicode}]), + {ok,Txt} = ?FILE_MODULE:read(Fd2, 129), + {Txt1,Txt2} = lists:split(64, Txt), + {ok,Txt2} = ?FILE_MODULE:pread(Fd2, Pos, 65), + {ok,0} = ?FILE_MODULE:position(Fd2, bof), + {ok,Txt1} = ?FILE_MODULE:read(Fd2, 64), + ok = ?FILE_MODULE:close(Fd2). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + altname(doc) -> "Test the file:altname/1 function"; altname(suite) -> diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 012d7051eb..010b1b15c7 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,22 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.1.3</title> + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p> + SSH_MSG_KEX_DH_GEX_REQUEST_OLD implemented to make PuTTY + work with erl server.</p> + <p> + Own Id: OTP-13140</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 1e9acf4a99..18bced2d1d 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -85,6 +85,15 @@ <item><p><c>atom()</c> - Name of the Erlang module implementing the subsystem using the <c>ssh_channel</c> behavior, see <seealso marker="ssh_channel">ssh_channel(3)</seealso></p></item> + <tag><c>key_cb() =</c></tag> + <item> + <p><c>atom() | {atom(), list()}</c></p> + <p><c>atom()</c> - Name of the erlang module implementing the behaviours + <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> or + <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> as the + case maybe.</p> + <p><c>list()</c> - List of options that can be passed to the callback module.</p> + </item> <tag><c>channel_init_args() =</c></tag> <item><p><c>list()</c></p></item> @@ -272,11 +281,13 @@ kex is implicit but public_key is set explicitly.</p> password, if the password authentication method is attempted.</p> </item> - <tag><c><![CDATA[{key_cb, atom()}]]></c></tag> + <tag><c><![CDATA[{key_cb, key_cb()}]]></c></tag> <item> - <p>Module implementing the behaviour - <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso>. - Can be used to customize the handling of public keys. + <p>Module implementing the behaviour <seealso + marker="ssh_client_key_api">ssh_client_key_api</seealso>. Can be used to + customize the handling of public keys. If callback options are provided + along with the module name, they are made available to the callback + module via the options passed to it under the key 'key_cb_private'. </p> </item> <tag><c><![CDATA[{quiet_mode, atom() = boolean()}]]></c></tag> @@ -607,11 +618,13 @@ kex is implicit but public_key is set explicitly.</p> </p> </item> - <tag><c><![CDATA[{key_cb, atom()}]]></c></tag> + <tag><c><![CDATA[{key_cb, key_cb()}]]></c></tag> <item> - <p>Module implementing the behaviour - <seealso marker="ssh_server_key_api">ssh_server_key_api</seealso>. - Can be used to customize the handling of public keys. + <p>Module implementing the behaviour <seealso + marker="ssh_server_key_api">ssh_server_key_api</seealso>. Can be used to + customize the handling of public keys. If callback options are provided + along with the module name, they are made available to the callback + module via the options passed to it under the key 'key_cb_private'. </p> </item> diff --git a/lib/ssh/doc/src/ssh_app.xml b/lib/ssh/doc/src/ssh_app.xml index 29cbbd79a2..79dd1e210e 100644 --- a/lib/ssh/doc/src/ssh_app.xml +++ b/lib/ssh/doc/src/ssh_app.xml @@ -137,6 +137,19 @@ <p>Supported algorithms are:</p> <taglist> + <tag>Key exchange algorithms</tag> + <item> + <list type="bulleted"> + <item>ecdh-sha2-nistp256</item> + <item>ecdh-sha2-nistp384</item> + <item>ecdh-sha2-nistp521</item> + <item>diffie-hellman-group-exchange-sha1</item> + <item>diffie-hellman-group-exchange-sha256</item> + <item>diffie-hellman-group14-sha1</item> + <item>diffie-hellman-group1-sha1</item> + </list> + </item> + <tag>Public key algorithms</tag> <item> <list type="bulleted"> @@ -157,30 +170,26 @@ </list> </item> - <tag>Encryption algorithms</tag> + <tag>Encryption algorithms (ciphers)</tag> <item> <list type="bulleted"> + <item>[email protected] (AEAD_AES_128_GCM)</item> + <item>[email protected] (AEAD_AES_256_GCM)</item> <item>aes128-ctr</item> <item>aes192-ctr</item> <item>aes256-ctr</item> <item>aes128-cbc</item> <item>3des-cbc</item> </list> + <p>Following the internet de-facto standard, the cipher and mac algorithm AEAD_AES_128_GCM is selected when the + cipher [email protected] is negotiated. The cipher and mac algorithm AEAD_AES_256_GCM is selected when the + cipher [email protected] is negotiated. + </p> + <p>See the text at the description of <seealso marker="#rfc5647_note">the rfc 5647 further down</seealso> + for more information. + </p> </item> - - <tag>Key exchange algorithms</tag> - <item> - <list type="bulleted"> - <item>ecdh-sha2-nistp256</item> - <item>ecdh-sha2-nistp384</item> - <item>ecdh-sha2-nistp521</item> - <item>diffie-hellman-group-exchange-sha1</item> - <item>diffie-hellman-group-exchange-sha256</item> - <item>diffie-hellman-group14-sha1</item> - <item>diffie-hellman-group1-sha1</item> - </list> - </item> - + <tag>Compression algorithms</tag> <item> <list type="bulleted"> @@ -255,6 +264,30 @@ <p></p> </item> + <item><url href="https://tools.ietf.org/html/rfc5647">RFC 5647</url>, AES Galois Counter Mode for + the Secure Shell Transport Layer Protocol. + <p><marker id="rfc5647_note"/>There is an ambiguity in the synchronized selection of cipher and mac algorithm. + This is resolved by OpenSSH in the ciphers [email protected] and [email protected] which are implemented. + If the explicit ciphers and macs AEAD_AES_128_GCM or AEAD_AES_256_GCM are needed, + they could be enabled with the option preferred_algorithms. + <warning> + If the client or the server is not Erlang/OTP, it is the users responsibility to check that + other implementation has the same interpretation of AEAD_AES_*_GCM as the Erlang/OTP SSH before + enabling them. The aes*[email protected] variants are always safe to use since they lack the + ambiguity. + </warning> + </p> + <p>The second paragraph in section 5.1 is resolved as: + <list type="ordered"> + <item>If the negotiated cipher is AEAD_AES_128_GCM, the mac algorithm is set to AEAD_AES_128_GCM.</item> + <item>If the negotiated cipher is AEAD_AES_256_GCM, the mac algorithm is set to AEAD_AES_256_GCM.</item> + <item>If the mac algorithm is AEAD_AES_128_GCM, the cipher is set to AEAD_AES_128_GCM.</item> + <item>If the mac algorithm is AEAD_AES_256_GCM, the cipher is set to AEAD_AES_256_GCM.</item> + </list> + The first rule that matches when read in order from the top is applied + </p> + </item> + <item><url href="https://tools.ietf.org/html/rfc5656">RFC 5656</url>, Elliptic Curve Algorithm Integration in the Secure Shell Transport Layer. <p>Except @@ -266,6 +299,13 @@ </list> </p> </item> + + <item><url href="https://tools.ietf.org/html/rfc6668">RFC 6668</url>, SHA-2 Data Integrity Verification for + the Secure Shell (SSH) Transport Layer Protocol + <p>Comment: Defines hmac-sha2-256 and hmac-sha2-512 + </p> + </item> + </list> </section> diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index 5bde184070..1d29c95229 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -235,10 +235,27 @@ start_daemon(Host, Port, Options, Inet) -> {error, _Reason} = Error -> Error; {SocketOptions, SshOptions}-> - do_start_daemon(Host, Port,[{role, server} |SshOptions] , [Inet | SocketOptions]) + try + do_start_daemon(Host, Port,[{role, server} |SshOptions] , [Inet | SocketOptions]) + catch + throw:bad_fd -> {error,bad_fd}; + _C:_E -> {error,{cannot_start_daemon,_C,_E}} + end end. -do_start_daemon(Host, Port, Options, SocketOptions) -> +do_start_daemon(Host0, Port0, Options, SocketOptions) -> + {Host,Port} = try + case proplists:get_value(fd, SocketOptions) of + undefined -> + {Host0,Port0}; + Fd when Port0==0 -> + find_hostport(Fd); + _ -> + {Host0,Port0} + end + catch + _:_ -> throw(bad_fd) + end, Profile = proplists:get_value(profile, Options, ?DEFAULT_PROFILE), case ssh_system_sup:system_supervisor(Host, Port, Profile) of undefined -> @@ -272,6 +289,22 @@ do_start_daemon(Host, Port, Options, SocketOptions) -> end end. +find_hostport(Fd) -> + %% Using internal functions inet:open/8 and inet:close/0. + %% Don't try this at home unless you know what you are doing! + {ok,S} = inet:open(Fd, {0,0,0,0}, 0, [], tcp, inet, stream, inet_tcp), + {ok, HostPort} = inet:sockname(S), + ok = inet:close(S), + HostPort. + +%% find_port(Fd) -> +%% %% Hack.... +%% {ok,TmpSock} = gen_tcp:listen(0,[{fd,Fd}]), +%% {ok, {_,ThePort}} = inet:sockname(TmpSock), +%% gen_tcp:close(TmpSock), +%% ThePort. + + handle_options(Opts) -> try handle_option(algs_compatibility(proplists:unfold(Opts)), [], []) of {Inet, Ssh} -> @@ -336,8 +369,12 @@ handle_option([{user_passwords, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{pwdfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{key_cb, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); +handle_option([{key_cb, {Module, Options}} | Rest], SocketOptions, SshOptions) -> + handle_option(Rest, SocketOptions, [handle_ssh_option({key_cb, Module}), + handle_ssh_priv_option({key_cb_private, Options}) | + SshOptions]); +handle_option([{key_cb, Module} | Rest], SocketOptions, SshOptions) -> + handle_option([{key_cb, {Module, []}} | Rest], SocketOptions, SshOptions); handle_option([{keyboard_interact_fun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); %%Backwards compatibility @@ -511,6 +548,9 @@ handle_ssh_option({pwdfun, Value} = Opt) when is_function(Value,4) -> Opt; handle_ssh_option({key_cb, Value} = Opt) when is_atom(Value) -> Opt; +handle_ssh_option({key_cb, {CallbackMod, CallbackOptions}} = Opt) when is_atom(CallbackMod), + is_list(CallbackOptions) -> + Opt; handle_ssh_option({keyboard_interact_fun, Value} = Opt) when is_function(Value,3) -> Opt; handle_ssh_option({compression, Value} = Opt) when is_atom(Value) -> @@ -577,6 +617,9 @@ handle_ssh_option({profile, Value} = Opt) when is_atom(Value) -> handle_ssh_option(Opt) -> throw({error, {eoptions, Opt}}). +handle_ssh_priv_option({key_cb_private, Value} = Opt) when is_list(Value) -> + Opt. + handle_inet_option({active, _} = Opt) -> throw({error, {{eoptions, Opt}, "SSH has built in flow control, " "and active is handled internally, user is not allowed" diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 4ad936f742..8efc743b67 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -29,7 +29,6 @@ -define(SSH_DEFAULT_PORT, 22). -define(SSH_MAX_PACKET_SIZE, (256*1024)). --define(SSH_LENGHT_INDICATOR_SIZE, 4). -define(REKEY_TIMOUT, 3600000). -define(REKEY_DATA_TIMOUT, 60000). -define(DEFAULT_PROFILE, default). diff --git a/lib/ssh/src/ssh_acceptor.erl b/lib/ssh/src/ssh_acceptor.erl index c5ad1d7b6c..d94dedf1bf 100644 --- a/lib/ssh/src/ssh_acceptor.erl +++ b/lib/ssh/src/ssh_acceptor.erl @@ -56,7 +56,12 @@ acceptor_init(Parent, Port, Address, SockOpts, Opts, AcceptTimeout) -> error end. -do_socket_listen(Callback, Port, Opts) -> +do_socket_listen(Callback, Port0, Opts) -> + Port = + case proplists:get_value(fd, Opts) of + undefined -> Port0; + _ -> 0 + end, case Callback:listen(Port, Opts) of {error, nxdomain} -> Callback:listen(Port, lists:delete(inet6, Opts)); diff --git a/lib/ssh/src/ssh_connect.hrl b/lib/ssh/src/ssh_connect.hrl index 6db89c5d80..9f9f3de8fa 100644 --- a/lib/ssh/src/ssh_connect.hrl +++ b/lib/ssh/src/ssh_connect.hrl @@ -248,6 +248,9 @@ local_id, %% local channel id recv_window_size, + recv_window_pending = 0, %% Sum of window size updates that has not + %% yet been sent. This limits the number + %% of sent update msgs. recv_packet_size, recv_close = false, diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 8448218d91..516a09bf6a 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -433,6 +433,12 @@ key_exchange(#ssh_msg_kex_dh_gex_request{} = Msg, send_msg(GexGroup, State), {next_state, key_exchange_dh_gex_init, next_packet(State#state{ssh_params = Ssh})}; +key_exchange(#ssh_msg_kex_dh_gex_request_old{} = Msg, + #state{ssh_params = #ssh{role = server} = Ssh0} = State) -> + {ok, GexGroup, Ssh} = ssh_transport:handle_kex_dh_gex_request(Msg, Ssh0), + send_msg(GexGroup, State), + {next_state, key_exchange_dh_gex_init, next_packet(State#state{ssh_params = Ssh})}; + key_exchange(#ssh_msg_kex_dh_gex_group{} = Msg, #state{ssh_params = #ssh{role = client} = Ssh0} = State) -> {ok, KexGexInit, Ssh} = ssh_transport:handle_kex_dh_gex_group(Msg, Ssh0), @@ -731,13 +737,28 @@ handle_event({adjust_window, ChannelId, Bytes}, StateName, #connection{channel_cache = Cache}} = State0) -> State = case ssh_channel:cache_lookup(Cache, ChannelId) of - #channel{recv_window_size = WinSize, remote_id = Id} = Channel -> - ssh_channel:cache_update(Cache, Channel#channel{recv_window_size = - WinSize + Bytes}), - Msg = ssh_connection:channel_adjust_window_msg(Id, Bytes), + #channel{recv_window_size = WinSize, + recv_window_pending = Pending, + recv_packet_size = PktSize} = Channel + when (WinSize-Bytes) >= 2*PktSize -> + %% The peer can send at least two more *full* packet, no hurry. + ssh_channel:cache_update(Cache, + Channel#channel{recv_window_pending = Pending + Bytes}), + State0; + + #channel{recv_window_size = WinSize, + recv_window_pending = Pending, + remote_id = Id} = Channel -> + %% Now we have to update the window - we can't receive so many more pkts + ssh_channel:cache_update(Cache, + Channel#channel{recv_window_size = + WinSize + Bytes + Pending, + recv_window_pending = 0}), + Msg = ssh_connection:channel_adjust_window_msg(Id, Bytes + Pending), send_replies([{connection_reply, Msg}], State0); - undefined -> - State0 + + undefined -> + State0 end, {next_state, StateName, next_packet(State)}; @@ -970,57 +991,39 @@ handle_info({Protocol, Socket, Info}, hello, transport_protocol = Protocol} = State) -> event({info_line, Info}, hello, State); -handle_info({Protocol, Socket, Data}, Statename, +handle_info({Protocol, Socket, Data}, StateName, #state{socket = Socket, transport_protocol = Protocol, - ssh_params = #ssh{decrypt_block_size = BlockSize, - recv_mac_size = MacSize} = Ssh0, - decoded_data_buffer = <<>>, - encoded_data_buffer = EncData0} = State0) -> - - %% Implementations SHOULD decrypt the length after receiving the - %% first 8 (or cipher block size, whichever is larger) bytes of a - %% packet. (RFC 4253: Section 6 - Binary Packet Protocol) - case size(EncData0) + size(Data) >= erlang:max(8, BlockSize) of - true -> - {Ssh, SshPacketLen, DecData, EncData} = - - ssh_transport:decrypt_first_block(<<EncData0/binary, - Data/binary>>, Ssh0), - case SshPacketLen > ?SSH_MAX_PACKET_SIZE of - true -> - DisconnectMsg = - #ssh_msg_disconnect{code = - ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad packet length " - ++ integer_to_list(SshPacketLen), - language = "en"}, - handle_disconnect(DisconnectMsg, State0); - false -> - RemainingSshPacketLen = - (SshPacketLen + ?SSH_LENGHT_INDICATOR_SIZE) - - BlockSize + MacSize, - State = State0#state{ssh_params = Ssh}, - handle_ssh_packet_data(RemainingSshPacketLen, - DecData, EncData, Statename, - State) - end; - false -> - {next_state, Statename, - next_packet(State0#state{encoded_data_buffer = - <<EncData0/binary, Data/binary>>})} + ssh_params = Ssh0, + decoded_data_buffer = DecData0, + encoded_data_buffer = EncData0, + undecoded_packet_length = RemainingSshPacketLen0} = State0) -> + Encoded = <<EncData0/binary, Data/binary>>, + case ssh_transport:handle_packet_part(DecData0, Encoded, RemainingSshPacketLen0, Ssh0) of + {get_more, DecBytes, EncDataRest, RemainingSshPacketLen, Ssh1} -> + {next_state, StateName, + next_packet(State0#state{encoded_data_buffer = EncDataRest, + decoded_data_buffer = DecBytes, + undecoded_packet_length = RemainingSshPacketLen, + ssh_params = Ssh1})}; + {decoded, MsgBytes, EncDataRest, Ssh1} -> + generate_event(MsgBytes, StateName, + State0#state{ssh_params = Ssh1, + %% Important to be set for + %% next_packet +%%% FIXME: the following three seem to always be set in generate_event! + decoded_data_buffer = <<>>, + undecoded_packet_length = undefined, + encoded_data_buffer = EncDataRest}, + EncDataRest); + {bad_mac, Ssh1} -> + DisconnectMsg = + #ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, + description = "Bad mac", + language = ""}, + handle_disconnect(DisconnectMsg, State0#state{ssh_params=Ssh1}) end; - -handle_info({Protocol, Socket, Data}, Statename, - #state{socket = Socket, - transport_protocol = Protocol, - decoded_data_buffer = DecData, - encoded_data_buffer = EncData, - undecoded_packet_length = Len} = - State) when is_integer(Len) -> - handle_ssh_packet_data(Len, DecData, <<EncData/binary, Data/binary>>, - Statename, State); - + handle_info({CloseTag, _Socket}, _StateName, #state{transport_close_tag = CloseTag, ssh_params = #ssh{role = _Role, opts = _Opts}} = State) -> @@ -1631,57 +1634,6 @@ after_new_keys_events({connection_reply, _Data} = Reply, {StateName, State}) -> NewState = send_replies([Reply], State), {next_state, StateName, NewState}. -handle_ssh_packet_data(RemainingSshPacketLen, DecData, EncData, StateName, - State) -> - EncSize = size(EncData), - case RemainingSshPacketLen > EncSize of - true -> - {next_state, StateName, - next_packet(State#state{decoded_data_buffer = DecData, - encoded_data_buffer = EncData, - undecoded_packet_length = - RemainingSshPacketLen})}; - false -> - handle_ssh_packet(RemainingSshPacketLen, StateName, - State#state{decoded_data_buffer = DecData, - encoded_data_buffer = EncData}) - - end. - -handle_ssh_packet(Length, StateName, #state{decoded_data_buffer = DecData0, - encoded_data_buffer = EncData0, - ssh_params = Ssh0, - transport_protocol = _Protocol, - socket = _Socket} = State0) -> - try - {Ssh1, DecData, EncData, Mac} = - ssh_transport:unpack(EncData0, Length, Ssh0), - SshPacket = <<DecData0/binary, DecData/binary>>, - case ssh_transport:is_valid_mac(Mac, SshPacket, Ssh1) of - true -> - PacketData = ssh_transport:msg_data(SshPacket), - {Ssh1, Msg} = ssh_transport:decompress(Ssh1, PacketData), - generate_event(Msg, StateName, - State0#state{ssh_params = Ssh1, - %% Important to be set for - %% next_packet - decoded_data_buffer = <<>>}, - EncData); - false -> - DisconnectMsg = - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad mac", - language = "en"}, - handle_disconnect(DisconnectMsg, State0) - end - catch _:_ -> - Disconnect = - #ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad input", - language = "en"}, - handle_disconnect(Disconnect, State0) - end. - handle_disconnect(DisconnectMsg, State) -> handle_disconnect(own, DisconnectMsg, State). diff --git a/lib/ssh/src/ssh_sftpd.erl b/lib/ssh/src/ssh_sftpd.erl index a6549f1c73..819cba697e 100644 --- a/lib/ssh/src/ssh_sftpd.erl +++ b/lib/ssh/src/ssh_sftpd.erl @@ -30,6 +30,7 @@ -include("ssh.hrl"). -include("ssh_xfer.hrl"). +-include("ssh_connect.hrl"). %% For ?DEFAULT_PACKET_SIZE and ?DEFAULT_WINDOW_SIZE %%-------------------------------------------------------------------- %% External exports @@ -47,6 +48,7 @@ file_handler, % atom() - callback module file_state, % state for the file callback module max_files, % integer >= 0 max no files sent during READDIR + options, % from the subsystem declaration handles % list of open handles %% handle is either {<int>, directory, {Path, unread|eof}} or %% {<int>, file, {Path, IoDevice}} @@ -121,6 +123,7 @@ init(Options) -> MaxLength = proplists:get_value(max_files, Options, 0), Vsn = proplists:get_value(sftpd_vsn, Options, 5), {ok, State#state{cwd = CWD, root = Root, max_files = MaxLength, + options = Options, handles = [], pending = <<>>, xf = #ssh_xfer{vsn = Vsn, ext = []}}}. @@ -164,7 +167,9 @@ handle_ssh_msg({ssh_cm, _, {exit_status, ChannelId, Status}}, State) -> %% Description: Handles other messages %%-------------------------------------------------------------------- handle_msg({ssh_channel_up, ChannelId, ConnectionManager}, - #state{xf =Xf} = State) -> + #state{xf = Xf, + options = Options} = State) -> + maybe_increase_recv_window(ConnectionManager, ChannelId, Options), {ok, State#state{xf = Xf#ssh_xfer{cm = ConnectionManager, channel = ChannelId}}}. @@ -934,3 +939,18 @@ rename(Path, Path2, ReqId, State0) -> {Status, FS1} = FileMod:rename(Path, Path2, FS0), State1 = State0#state{file_state = FS1}, send_status(Status, ReqId, State1). + + +maybe_increase_recv_window(ConnectionManager, ChannelId, Options) -> + WantedRecvWindowSize = + proplists:get_value(recv_window_size, Options, 1000000), + NumPkts = WantedRecvWindowSize div ?DEFAULT_PACKET_SIZE, + Increment = NumPkts*?DEFAULT_PACKET_SIZE - ?DEFAULT_WINDOW_SIZE, + + if + Increment > 0 -> + ssh_connection:adjust_window(ConnectionManager, ChannelId, + Increment); + Increment =< 0 -> + do_nothing + end. diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 0c999b96cc..67a0d29bb8 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -31,10 +31,10 @@ -include("ssh.hrl"). -export([versions/2, hello_version_msg/1]). --export([next_seqnum/1, decrypt_first_block/2, decrypt_blocks/3, +-export([next_seqnum/1, supported_algorithms/0, supported_algorithms/1, default_algorithms/0, default_algorithms/1, - is_valid_mac/3, + handle_packet_part/4, handle_hello_version/1, key_exchange_init_msg/1, key_init/3, new_keys_message/1, @@ -45,9 +45,13 @@ handle_kex_ecdh_init/2, handle_kex_ecdh_reply/2, extract_public_key/1, - unpack/3, decompress/2, ssh_packet/2, pack/2, pack/3, msg_data/1, + ssh_packet/2, pack/2, sign/3, verify/4]). +%%% For test suites +-export([pack/3]). +-export([decompress/2, decrypt_blocks/3, is_valid_mac/3 ]). % FIXME: remove + %%%---------------------------------------------------------------------------- %%% %%% There is a difference between supported and default algorithms. The @@ -66,10 +70,15 @@ default_algorithms() -> [{K,default_algorithms(K)} || K <- algo_classes()]. algo_classes() -> [kex, public_key, cipher, mac, compression]. -%% default_algorithms(kex) -> % Example of how to disable an algorithm -%% supported_algorithms(kex, ['ecdh-sha2-nistp521']); + +default_algorithms(cipher) -> + supported_algorithms(cipher, same(['AEAD_AES_128_GCM', + 'AEAD_AES_256_GCM'])); +default_algorithms(mac) -> + supported_algorithms(mac, same(['AEAD_AES_128_GCM', + 'AEAD_AES_256_GCM'])); default_algorithms(Alg) -> - supported_algorithms(Alg). + supported_algorithms(Alg, []). supported_algorithms() -> [{K,supported_algorithms(K)} || K <- algo_classes()]. @@ -97,19 +106,25 @@ supported_algorithms(public_key) -> supported_algorithms(cipher) -> same( select_crypto_supported( - [{'aes256-ctr', [{ciphers,{aes_ctr,256}}]}, - {'aes192-ctr', [{ciphers,{aes_ctr,192}}]}, - {'aes128-ctr', [{ciphers,{aes_ctr,128}}]}, - {'aes128-cbc', [{ciphers,aes_cbc128}]}, - {'3des-cbc', [{ciphers,des3_cbc}]} + [{'aes256-ctr', [{ciphers,{aes_ctr,256}}]}, + {'aes192-ctr', [{ciphers,{aes_ctr,192}}]}, + {'aes128-ctr', [{ciphers,{aes_ctr,128}}]}, + {'aes128-cbc', [{ciphers,aes_cbc128}]}, + {'[email protected]', [{ciphers,{aes_gcm,128}}]}, + {'[email protected]', [{ciphers,{aes_gcm,256}}]}, + {'AEAD_AES_128_GCM', [{ciphers,{aes_gcm,128}}]}, + {'AEAD_AES_256_GCM', [{ciphers,{aes_gcm,256}}]}, + {'3des-cbc', [{ciphers,des3_cbc}]} ] )); supported_algorithms(mac) -> same( select_crypto_supported( - [{'hmac-sha2-256', [{hashs,sha256}]}, - {'hmac-sha2-512', [{hashs,sha512}]}, - {'hmac-sha1', [{hashs,sha}]} + [{'hmac-sha2-256', [{hashs,sha256}]}, + {'hmac-sha2-512', [{hashs,sha512}]}, + {'hmac-sha1', [{hashs,sha}]}, + {'AEAD_AES_128_GCM', [{ciphers,{aes_gcm,128}}]}, + {'AEAD_AES_256_GCM', [{ciphers,{aes_gcm,256}}]} ] )); supported_algorithms(compression) -> @@ -118,46 +133,6 @@ supported_algorithms(compression) -> 'zlib' ]). -%% Dialyzer complains when not called...supported_algorithms(Key, [{client2server,BL1},{server2client,BL2}]) -> -%% Dialyzer complains when not called... [{client2server,As1},{server2client,As2}] = supported_algorithms(Key), -%% Dialyzer complains when not called... [{client2server,As1--BL1},{server2client,As2--BL2}]; -%% Dialyzer complains when not called...supported_algorithms(Key, BlackList) -> -%% Dialyzer complains when not called... supported_algorithms(Key) -- BlackList. - -select_crypto_supported(L) -> - Sup = [{ec_curve,crypto_supported_curves()} | crypto:supports()], - [Name || {Name,CryptoRequires} <- L, - crypto_supported(CryptoRequires, Sup)]. - -crypto_supported_curves() -> - try crypto:ec_curves() - catch _:_ -> [] - end. - -crypto_supported(Conditions, Supported) -> - lists:all( fun({Tag,CryptoName}) when is_atom(CryptoName) -> - crypto_name_supported(Tag,CryptoName,Supported); - ({Tag,{Name=aes_ctr,Len}}) when is_integer(Len) -> - crypto_name_supported(Tag,Name,Supported) andalso - ctr_len_supported(Name,Len) - end, Conditions). - -crypto_name_supported(Tag, CryptoName, Supported) -> - lists:member(CryptoName, proplists:get_value(Tag,Supported,[])). - -ctr_len_supported(Name, Len) -> - try - crypto:stream_encrypt(crypto:stream_init(Name, <<0:Len>>, <<0:128>>), <<"">>) - of - {_,X} -> is_binary(X) - catch - _:_ -> false - end. - - -same(Algs) -> [{client2server,Algs}, {server2client,Algs}]. - - %%%---------------------------------------------------------------------------- versions(client, Options)-> Vsn = proplists:get_value(vsn, Options, ?DEFAULT_CLIENT_VERSION), @@ -196,12 +171,6 @@ hello_version_msg(Data) -> next_seqnum(SeqNum) -> (SeqNum + 1) band 16#ffffffff. -decrypt_first_block(Bin, #ssh{decrypt_block_size = BlockSize} = Ssh0) -> - <<EncBlock:BlockSize/binary, EncData/binary>> = Bin, - {Ssh, <<?UINT32(PacketLen), _/binary>> = DecData} = - decrypt(Ssh0, EncBlock), - {Ssh, PacketLen, DecData, EncData}. - decrypt_blocks(Bin, Length, Ssh0) -> <<EncBlocks:Length/binary, EncData/binary>> = Bin, {Ssh, DecData} = decrypt(Ssh0, EncBlocks), @@ -464,6 +433,40 @@ handle_kex_dh_gex_request(#ssh_msg_kex_dh_gex_request{min = Min0, language = ""}) end; +handle_kex_dh_gex_request(#ssh_msg_kex_dh_gex_request_old{n = NBits}, + Ssh0=#ssh{opts=Opts}) -> + %% server + %% + %% This message was in the draft-00 of rfc4419 + %% (https://tools.ietf.org/html/draft-ietf-secsh-dh-group-exchange-00) + %% In later drafts and the rfc is "is used for backward compatibility". + %% Unfortunatly the rfc does not specify how to treat the parameter n + %% if there is no group of that modulus length :( + %% The draft-00 however specifies that n is the "... number of bits + %% the subgroup should have at least". + %% Further, it says that "Servers and clients SHOULD support groups + %% with a modulus length of k bits, where 1024 <= k <= 8192." + %% + Min0 = NBits, + Max0 = 8192, + {Min, Max} = adjust_gex_min_max(Min0, Max0, Opts), + case public_key:dh_gex_group(Min, NBits, Max, + proplists:get_value(dh_gex_groups,Opts)) of + {ok, {_Sz, {G,P}}} -> + {Public, Private} = generate_key(dh, [P,G]), + {SshPacket, Ssh} = + ssh_packet(#ssh_msg_kex_dh_gex_group{p = P, g = G}, Ssh0), + {ok, SshPacket, + Ssh#ssh{keyex_key = {{Private, Public}, {G, P}}, + keyex_info = {-1, -1, NBits} % flag for kex_h hash calc + }}; + {error,_} -> + throw(#ssh_msg_disconnect{ + code = ?SSH_DISCONNECT_PROTOCOL_ERROR, + description = "No possible diffie-hellman-group-exchange group found", + language = ""}) + end; + handle_kex_dh_gex_request(_, _) -> throw({{error,bad_ssh_msg_kex_dh_gex_request}, #ssh_msg_disconnect{ @@ -757,8 +760,12 @@ known_host_key(#ssh{opts = Opts, key_cb = Mod, peer = Peer} = Ssh, %% The first algorithm in each list MUST be the preferred (guessed) %% algorithm. Each string MUST contain at least one algorithm name. select_algorithm(Role, Client, Server) -> - {Encrypt, Decrypt} = select_encrypt_decrypt(Role, Client, Server), - {SendMac, RecvMac} = select_send_recv_mac(Role, Client, Server), + {Encrypt0, Decrypt0} = select_encrypt_decrypt(Role, Client, Server), + {SendMac0, RecvMac0} = select_send_recv_mac(Role, Client, Server), + + {Encrypt, SendMac} = aead_gcm_simultan(Encrypt0, SendMac0), + {Decrypt, RecvMac} = aead_gcm_simultan(Decrypt0, RecvMac0), + {Compression, Decompression} = select_compression_decompression(Role, Client, Server), @@ -789,6 +796,38 @@ select_algorithm(Role, Client, Server) -> s_lng = S_Lng}, {ok, Alg}. + +%%% It is an agreed problem with RFC 5674 that if the selection is +%%% Cipher = AEAD_AES_x_GCM and +%%% Mac = AEAD_AES_y_GCM (where x =/= y) +%%% then it is undefined what length should be selected. +%%% +%%% If only one of the two lengths (128,256) is available, I claim that +%%% there is no such ambiguity. + +%%% From https://anongit.mindrot.org/openssh.git/plain/PROTOCOL +%%% (read Nov 20, 2015) +%%% 1.6 transport: AES-GCM +%%% +%%% OpenSSH supports the AES-GCM algorithm as specified in RFC 5647. +%%% Because of problems with the specification of the key exchange +%%% the behaviour of OpenSSH differs from the RFC as follows: +%%% +%%% AES-GCM is only negotiated as the cipher algorithms +%%% "[email protected]" or "[email protected]" and never as +%%% an MAC algorithm. Additionally, if AES-GCM is selected as the cipher +%%% the exchanged MAC algorithms are ignored and there doesn't have to be +%%% a matching MAC. + +aead_gcm_simultan('[email protected]', _) -> {'AEAD_AES_128_GCM', 'AEAD_AES_128_GCM'}; +aead_gcm_simultan('[email protected]', _) -> {'AEAD_AES_256_GCM', 'AEAD_AES_256_GCM'}; +aead_gcm_simultan('AEAD_AES_128_GCM', _) -> {'AEAD_AES_128_GCM', 'AEAD_AES_128_GCM'}; +aead_gcm_simultan('AEAD_AES_256_GCM', _) -> {'AEAD_AES_256_GCM', 'AEAD_AES_256_GCM'}; +aead_gcm_simultan(_, 'AEAD_AES_128_GCM') -> {'AEAD_AES_128_GCM', 'AEAD_AES_128_GCM'}; +aead_gcm_simultan(_, 'AEAD_AES_256_GCM') -> {'AEAD_AES_256_GCM', 'AEAD_AES_256_GCM'}; +aead_gcm_simultan(Cipher, Mac) -> {Cipher,Mac}. + + select_encrypt_decrypt(client, Client, Server) -> Encrypt = select(Client#ssh_msg_kexinit.encryption_algorithms_client_to_server, @@ -823,18 +862,18 @@ select_compression_decompression(client, Client, Server) -> Compression = select(Client#ssh_msg_kexinit.compression_algorithms_client_to_server, Server#ssh_msg_kexinit.compression_algorithms_client_to_server), - Decomprssion = + Decompression = select(Client#ssh_msg_kexinit.compression_algorithms_server_to_client, Server#ssh_msg_kexinit.compression_algorithms_server_to_client), - {Compression, Decomprssion}; + {Compression, Decompression}; select_compression_decompression(server, Client, Server) -> - Decomprssion = + Decompression = select(Client#ssh_msg_kexinit.compression_algorithms_client_to_server, Server#ssh_msg_kexinit.compression_algorithms_client_to_server), Compression = select(Client#ssh_msg_kexinit.compression_algorithms_server_to_client, Server#ssh_msg_kexinit.compression_algorithms_server_to_client), - {Compression, Decomprssion}. + {Compression, Decompression}. install_alg(SSH) -> SSH1 = alg_final(SSH), @@ -911,14 +950,39 @@ pack(Data, Ssh=#ssh{}) -> %%% Note: pack/3 is only to be called from tests that wants %%% to deliberetly send packets with wrong PacketLength! %%% Use pack/2 for all other purposes! -pack(Data0, #ssh{encrypt_block_size = BlockSize, - send_sequence = SeqNum, send_mac = MacAlg, - send_mac_key = MacKey, - random_length_padding = RandomLengthPadding} - = Ssh0, - PacketLenDeviationForTests) when is_binary(Data0) -> - {Ssh1, Data} = compress(Ssh0, Data0), - PL = (BlockSize - ((4 + 1 + size(Data)) rem BlockSize)) rem BlockSize, +pack(PlainText, + #ssh{send_sequence = SeqNum, + send_mac = MacAlg, + send_mac_key = MacKey, + encrypt = CryptoAlg} = Ssh0, PacketLenDeviationForTests) when is_binary(PlainText) -> + + {Ssh1, CompressedPlainText} = compress(Ssh0, PlainText), + {EcryptedPacket, MAC, Ssh3} = + case pkt_type(CryptoAlg) of + common -> + PaddingLen = padding_length(4+1+size(CompressedPlainText), Ssh0), + Padding = ssh_bits:random(PaddingLen), + PlainPacketLen = 1 + PaddingLen + size(CompressedPlainText) + PacketLenDeviationForTests, + PlainPacketData = <<?UINT32(PlainPacketLen),?BYTE(PaddingLen), CompressedPlainText/binary, Padding/binary>>, + {Ssh2, EcryptedPacket0} = encrypt(Ssh1, PlainPacketData), + MAC0 = mac(MacAlg, MacKey, SeqNum, PlainPacketData), + {EcryptedPacket0, MAC0, Ssh2}; + aead -> + PaddingLen = padding_length(1+size(CompressedPlainText), Ssh0), + Padding = ssh_bits:random(PaddingLen), + PlainPacketLen = 1 + PaddingLen + size(CompressedPlainText) + PacketLenDeviationForTests, + PlainPacketData = <<?BYTE(PaddingLen), CompressedPlainText/binary, Padding/binary>>, + {Ssh2, {EcryptedPacket0,MAC0}} = encrypt(Ssh1, {<<?UINT32(PlainPacketLen)>>,PlainPacketData}), + {<<?UINT32(PlainPacketLen),EcryptedPacket0/binary>>, MAC0, Ssh2} + end, + FinalPacket = [EcryptedPacket, MAC], + Ssh = Ssh3#ssh{send_sequence = (SeqNum+1) band 16#ffffffff}, + {FinalPacket, Ssh}. + + +padding_length(Size, #ssh{encrypt_block_size = BlockSize, + random_length_padding = RandomLengthPadding}) -> + PL = (BlockSize - (Size rem BlockSize)) rem BlockSize, MinPaddingLen = if PL < 4 -> PL + BlockSize; true -> PL end, @@ -927,45 +991,94 @@ pack(Data0, #ssh{encrypt_block_size = BlockSize, ExtraPaddingLen = try crypto:rand_uniform(0,MaxExtraBlocks)*PadBlockSize catch _:_ -> 0 end, - PaddingLen = MinPaddingLen + ExtraPaddingLen, - Padding = ssh_bits:random(PaddingLen), - PacketLen = 1 + PaddingLen + size(Data) + PacketLenDeviationForTests, - PacketData = <<?UINT32(PacketLen),?BYTE(PaddingLen), - Data/binary, Padding/binary>>, - {Ssh2, EncPacket} = encrypt(Ssh1, PacketData), - MAC = mac(MacAlg, MacKey, SeqNum, PacketData), - Packet = [EncPacket, MAC], - Ssh = Ssh2#ssh{send_sequence = (SeqNum+1) band 16#ffffffff}, - {Packet, Ssh}. - -unpack(EncodedSoFar, ReminingLenght, #ssh{recv_mac_size = MacSize} = Ssh0) -> - SshLength = ReminingLenght - MacSize, - {NoMac, Mac, Rest} = case MacSize of - 0 -> - <<NoMac0:SshLength/binary, - Rest0/binary>> = EncodedSoFar, - {NoMac0, <<>>, Rest0}; - _ -> - <<NoMac0:SshLength/binary, - Mac0:MacSize/binary, - Rest0/binary>> = EncodedSoFar, - {NoMac0, Mac0, Rest0} - end, - {Ssh1, DecData, <<>>} = - case SshLength of - 0 -> - {Ssh0, <<>>, <<>>}; - _ -> - decrypt_blocks(NoMac, SshLength, Ssh0) - end, - {Ssh1, DecData, Rest, Mac}. + MinPaddingLen + ExtraPaddingLen. + + + +handle_packet_part(<<>>, Encrypted0, undefined, #ssh{decrypt = CryptoAlg} = Ssh0) -> + %% New ssh packet + case get_length(pkt_type(CryptoAlg), Encrypted0, Ssh0) of + get_more -> + %% too short to get the length + {get_more, <<>>, Encrypted0, undefined, Ssh0}; -msg_data(PacketData) -> - <<Len:32, PaddingLen:8, _/binary>> = PacketData, - DataLen = Len - PaddingLen - 1, - <<_:32, _:8, Data:DataLen/binary, - _:PaddingLen/binary>> = PacketData, - Data. + {ok, PacketLen, _, _, _} when PacketLen > ?SSH_MAX_PACKET_SIZE -> + %% far too long message than expected + throw(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, + description = "Bad packet length " + ++ integer_to_list(PacketLen), + language = ""}); + + {ok, PacketLen, Decrypted, Encrypted1, + #ssh{recv_mac_size = MacSize} = Ssh1} -> + %% enough bytes so we got the length and can calculate how many + %% more bytes to expect for a full packet + TotalNeeded = (4 + PacketLen + MacSize), + handle_packet_part(Decrypted, Encrypted1, TotalNeeded, Ssh1) + end; + +handle_packet_part(DecryptedPfx, EncryptedBuffer, TotalNeeded, Ssh0) + when (size(DecryptedPfx)+size(EncryptedBuffer)) < TotalNeeded -> + %% need more bytes to finalize the packet + {get_more, DecryptedPfx, EncryptedBuffer, TotalNeeded, Ssh0}; + +handle_packet_part(DecryptedPfx, EncryptedBuffer, TotalNeeded, + #ssh{recv_mac_size = MacSize, + decrypt = CryptoAlg} = Ssh0) -> + %% enough bytes to decode the packet. + DecryptLen = TotalNeeded - size(DecryptedPfx) - MacSize, + <<EncryptedSfx:DecryptLen/binary, Mac:MacSize/binary, NextPacketBytes/binary>> = EncryptedBuffer, + case pkt_type(CryptoAlg) of + common -> + {Ssh1, DecryptedSfx} = decrypt(Ssh0, EncryptedSfx), + DecryptedPacket = <<DecryptedPfx/binary, DecryptedSfx/binary>>, + case is_valid_mac(Mac, DecryptedPacket, Ssh1) of + false -> + {bad_mac, Ssh1}; + true -> + {Ssh, DecompressedPayload} = decompress(Ssh1, payload(DecryptedPacket)), + {decoded, DecompressedPayload, NextPacketBytes, Ssh} + end; + aead -> + PacketLenBin = DecryptedPfx, + case decrypt(Ssh0, {PacketLenBin,EncryptedSfx,Mac}) of + {Ssh1, error} -> + {bad_mac, Ssh1}; + {Ssh1, DecryptedSfx} -> + DecryptedPacket = <<DecryptedPfx/binary, DecryptedSfx/binary>>, + {Ssh, DecompressedPayload} = decompress(Ssh1, payload(DecryptedPacket)), + {decoded, DecompressedPayload, NextPacketBytes, Ssh} + end + end. + + +get_length(common, EncryptedBuffer, #ssh{decrypt_block_size = BlockSize} = Ssh0) -> + case size(EncryptedBuffer) >= erlang:max(8, BlockSize) of + true -> + <<EncBlock:BlockSize/binary, EncryptedRest/binary>> = EncryptedBuffer, + {Ssh, + <<?UINT32(PacketLen),_/binary>> = Decrypted} = decrypt(Ssh0, EncBlock), + {ok, PacketLen, Decrypted, EncryptedRest, Ssh}; + false -> + get_more + end; +get_length(aead, EncryptedBuffer, Ssh) -> + case size(EncryptedBuffer) >= 4 of + true -> + <<?UINT32(PacketLen), EncryptedRest/binary>> = EncryptedBuffer, + {ok, PacketLen, <<?UINT32(PacketLen)>>, EncryptedRest, Ssh}; + false -> + get_more + end. + +pkt_type('AEAD_AES_128_GCM') -> aead; +pkt_type('AEAD_AES_256_GCM') -> aead; +pkt_type(_) -> common. + +payload(<<PacketLen:32, PaddingLen:8, PayloadAndPadding/binary>>) -> + PayloadLen = PacketLen - PaddingLen - 1, + <<Payload:PayloadLen/binary, _/binary>> = PayloadAndPadding, + Payload. sign(SigData, Hash, #'DSAPrivateKey'{} = Key) -> DerSignature = public_key:sign(SigData, Hash, Key), @@ -991,6 +1104,7 @@ verify(PlainText, Hash, Sig, {#'ECPoint'{},_} = Key) -> verify(PlainText, Hash, Sig, Key) -> public_key:verify(PlainText, Hash, Sig, Key). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% Encryption @@ -999,6 +1113,30 @@ verify(PlainText, Hash, Sig, Key) -> encrypt_init(#ssh{encrypt = none} = Ssh) -> {ok, Ssh}; +encrypt_init(#ssh{encrypt = 'AEAD_AES_128_GCM', role = client} = Ssh) -> + IV = hash(Ssh, "A", 12*8), + <<K:16/binary>> = hash(Ssh, "C", 128), + {ok, Ssh#ssh{encrypt_keys = K, + encrypt_block_size = 16, + encrypt_ctx = IV}}; +encrypt_init(#ssh{encrypt = 'AEAD_AES_128_GCM', role = server} = Ssh) -> + IV = hash(Ssh, "B", 12*8), + <<K:16/binary>> = hash(Ssh, "D", 128), + {ok, Ssh#ssh{encrypt_keys = K, + encrypt_block_size = 16, + encrypt_ctx = IV}}; +encrypt_init(#ssh{encrypt = 'AEAD_AES_256_GCM', role = client} = Ssh) -> + IV = hash(Ssh, "A", 12*8), + <<K:32/binary>> = hash(Ssh, "C", 256), + {ok, Ssh#ssh{encrypt_keys = K, + encrypt_block_size = 16, + encrypt_ctx = IV}}; +encrypt_init(#ssh{encrypt = 'AEAD_AES_256_GCM', role = server} = Ssh) -> + IV = hash(Ssh, "B", 12*8), + <<K:32/binary>> = hash(Ssh, "D", 256), + {ok, Ssh#ssh{encrypt_keys = K, + encrypt_block_size = 16, + encrypt_ctx = IV}}; encrypt_init(#ssh{encrypt = '3des-cbc', role = client} = Ssh) -> IV = hash(Ssh, "A", 64), <<K1:8/binary, K2:8/binary, K3:8/binary>> = hash(Ssh, "C", 192), @@ -1075,6 +1213,18 @@ encrypt_final(Ssh) -> encrypt(#ssh{encrypt = none} = Ssh, Data) -> {Ssh, Data}; +encrypt(#ssh{encrypt = 'AEAD_AES_128_GCM', + encrypt_keys = K, + encrypt_ctx = IV0} = Ssh, Data={_AAD,_Ptext}) -> + Enc = {_Ctext,_Ctag} = crypto:block_encrypt(aes_gcm, K, IV0, Data), + IV = next_gcm_iv(IV0), + {Ssh#ssh{encrypt_ctx = IV}, Enc}; +encrypt(#ssh{encrypt = 'AEAD_AES_256_GCM', + encrypt_keys = K, + encrypt_ctx = IV0} = Ssh, Data={_AAD,_Ptext}) -> + Enc = {_Ctext,_Ctag} = crypto:block_encrypt(aes_gcm, K, IV0, Data), + IV = next_gcm_iv(IV0), + {Ssh#ssh{encrypt_ctx = IV}, Enc}; encrypt(#ssh{encrypt = '3des-cbc', encrypt_keys = {K1,K2,K3}, encrypt_ctx = IV0} = Ssh, Data) -> @@ -1107,6 +1257,30 @@ encrypt(#ssh{encrypt = 'aes256-ctr', decrypt_init(#ssh{decrypt = none} = Ssh) -> {ok, Ssh}; +decrypt_init(#ssh{decrypt = 'AEAD_AES_128_GCM', role = client} = Ssh) -> + IV = hash(Ssh, "B", 12*8), + <<K:16/binary>> = hash(Ssh, "D", 128), + {ok, Ssh#ssh{decrypt_keys = K, + decrypt_block_size = 16, + decrypt_ctx = IV}}; +decrypt_init(#ssh{decrypt = 'AEAD_AES_128_GCM', role = server} = Ssh) -> + IV = hash(Ssh, "A", 12*8), + <<K:16/binary>> = hash(Ssh, "C", 128), + {ok, Ssh#ssh{decrypt_keys = K, + decrypt_block_size = 16, + decrypt_ctx = IV}}; +decrypt_init(#ssh{decrypt = 'AEAD_AES_256_GCM', role = client} = Ssh) -> + IV = hash(Ssh, "B", 12*8), + <<K:32/binary>> = hash(Ssh, "D", 256), + {ok, Ssh#ssh{decrypt_keys = K, + decrypt_block_size = 16, + decrypt_ctx = IV}}; +decrypt_init(#ssh{decrypt = 'AEAD_AES_256_GCM', role = server} = Ssh) -> + IV = hash(Ssh, "A", 12*8), + <<K:32/binary>> = hash(Ssh, "C", 256), + {ok, Ssh#ssh{decrypt_keys = K, + decrypt_block_size = 16, + decrypt_ctx = IV}}; decrypt_init(#ssh{decrypt = '3des-cbc', role = client} = Ssh) -> {IV, KD} = {hash(Ssh, "B", 64), hash(Ssh, "D", 192)}, @@ -1181,8 +1355,22 @@ decrypt_final(Ssh) -> decrypt_ctx = undefined, decrypt_block_size = 8}}. +decrypt(Ssh, <<>>) -> + {Ssh, <<>>}; decrypt(#ssh{decrypt = none} = Ssh, Data) -> {Ssh, Data}; +decrypt(#ssh{decrypt = 'AEAD_AES_128_GCM', + decrypt_keys = K, + decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) -> + Dec = crypto:block_decrypt(aes_gcm, K, IV0, Data), % Dec = PlainText | error + IV = next_gcm_iv(IV0), + {Ssh#ssh{decrypt_ctx = IV}, Dec}; +decrypt(#ssh{decrypt = 'AEAD_AES_256_GCM', + decrypt_keys = K, + decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) -> + Dec = crypto:block_decrypt(aes_gcm, K, IV0, Data), % Dec = PlainText | error + IV = next_gcm_iv(IV0), + {Ssh#ssh{decrypt_ctx = IV}, Dec}; decrypt(#ssh{decrypt = '3des-cbc', decrypt_keys = Keys, decrypt_ctx = IV0} = Ssh, Data) -> {K1, K2, K3} = Keys, @@ -1207,6 +1395,10 @@ decrypt(#ssh{decrypt = 'aes256-ctr', {State, Enc} = crypto:stream_decrypt(State0,Data), {Ssh#ssh{decrypt_ctx = State}, Enc}. + +next_gcm_iv(<<Fixed:32, InvCtr:64>>) -> <<Fixed:32, (InvCtr+1):64>>. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Compression %% @@ -1295,28 +1487,42 @@ decompress(#ssh{decompress = '[email protected]', decompress_ctx = Context, authe %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% send_mac_init(SSH) -> - case SSH#ssh.role of - client -> - KeySize =mac_key_size(SSH#ssh.send_mac), - Key = hash(SSH, "E", KeySize), - {ok, SSH#ssh { send_mac_key = Key }}; - server -> - KeySize = mac_key_size(SSH#ssh.send_mac), - Key = hash(SSH, "F", KeySize), - {ok, SSH#ssh { send_mac_key = Key }} + case pkt_type(SSH#ssh.send_mac) of + common -> + case SSH#ssh.role of + client -> + KeySize = mac_key_size(SSH#ssh.send_mac), + Key = hash(SSH, "E", KeySize), + {ok, SSH#ssh { send_mac_key = Key }}; + server -> + KeySize = mac_key_size(SSH#ssh.send_mac), + Key = hash(SSH, "F", KeySize), + {ok, SSH#ssh { send_mac_key = Key }} + end; + aead -> + %% Not applicable + {ok, SSH} end. send_mac_final(SSH) -> - {ok, SSH#ssh { send_mac = none, send_mac_key = undefined }}. + {ok, SSH#ssh {send_mac = none, + send_mac_key = undefined }}. + recv_mac_init(SSH) -> - case SSH#ssh.role of - client -> - Key = hash(SSH, "F", mac_key_size(SSH#ssh.recv_mac)), - {ok, SSH#ssh { recv_mac_key = Key }}; - server -> - Key = hash(SSH, "E", mac_key_size(SSH#ssh.recv_mac)), - {ok, SSH#ssh { recv_mac_key = Key }} + case pkt_type(SSH#ssh.recv_mac) of + common -> + case SSH#ssh.role of + client -> + Key = hash(SSH, "F", mac_key_size(SSH#ssh.recv_mac)), + {ok, SSH#ssh { recv_mac_key = Key }}; + server -> + Key = hash(SSH, "E", mac_key_size(SSH#ssh.recv_mac)), + {ok, SSH#ssh { recv_mac_key = Key }} + end; + aead -> + %% Not applicable + {ok, SSH} end. recv_mac_final(SSH) -> @@ -1399,8 +1605,11 @@ kex_h(SSH, Curve, Key, Q_c, Q_s, K) -> crypto:hash(sha(Curve), L). kex_h(SSH, Key, Min, NBits, Max, Prime, Gen, E, F, K) -> + KeyBin = public_key:ssh_encode(Key, ssh2_pubkey), L = if Min==-1; Max==-1 -> - KeyBin = public_key:ssh_encode(Key, ssh2_pubkey), + %% flag from 'ssh_msg_kex_dh_gex_request_old' + %% It was like this before that message was supported, + %% why? Ts = [string,string,binary,binary,binary, uint32, mpint,mpint,mpint,mpint,mpint], @@ -1409,7 +1618,6 @@ kex_h(SSH, Key, Min, NBits, Max, Prime, Gen, E, F, K) -> KeyBin, NBits, Prime, Gen, E,F,K], Ts); true -> - KeyBin = public_key:ssh_encode(Key, ssh2_pubkey), Ts = [string,string,binary,binary,binary, uint32,uint32,uint32, mpint,mpint,mpint,mpint,mpint], @@ -1447,6 +1655,8 @@ mac_digest_size('hmac-md5') -> 20; mac_digest_size('hmac-md5-96') -> 12; mac_digest_size('hmac-sha2-256') -> 32; mac_digest_size('hmac-sha2-512') -> 64; +mac_digest_size('AEAD_AES_128_GCM') -> 16; +mac_digest_size('AEAD_AES_256_GCM') -> 16; mac_digest_size(none) -> 0. peer_name({Host, _}) -> @@ -1476,6 +1686,68 @@ ecdh_curve('ecdh-sha2-nistp256') -> secp256r1; ecdh_curve('ecdh-sha2-nistp384') -> secp384r1; ecdh_curve('ecdh-sha2-nistp521') -> secp521r1. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% +%% Utils for default_algorithms/1 and supported_algorithms/1 +%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +supported_algorithms(Key, [{client2server,BL1},{server2client,BL2}]) -> + [{client2server,As1},{server2client,As2}] = supported_algorithms(Key), + [{client2server,As1--BL1},{server2client,As2--BL2}]; +supported_algorithms(Key, BlackList) -> + supported_algorithms(Key) -- BlackList. + + +select_crypto_supported(L) -> + Sup = [{ec_curve,crypto_supported_curves()} | crypto:supports()], + [Name || {Name,CryptoRequires} <- L, + crypto_supported(CryptoRequires, Sup)]. + +crypto_supported_curves() -> + try crypto:ec_curves() + catch _:_ -> [] + end. + +crypto_supported(Conditions, Supported) -> + lists:all( fun({Tag,CryptoName}) when is_atom(CryptoName) -> + crypto_name_supported(Tag,CryptoName,Supported); + ({Tag,{Name,Len}}) when is_integer(Len) -> + crypto_name_supported(Tag,Name,Supported) andalso + len_supported(Name,Len) + end, Conditions). + +crypto_name_supported(Tag, CryptoName, Supported) -> + lists:member(CryptoName, proplists:get_value(Tag,Supported,[])). + +len_supported(Name, Len) -> + try + case Name of + aes_ctr -> + {_, <<_/binary>>} = + %% Test encryption + crypto:stream_encrypt(crypto:stream_init(Name, <<0:Len>>, <<0:128>>), <<"">>); + aes_gcm -> + {<<_/binary>>, <<_/binary>>} = + crypto:block_encrypt(Name, + _Key = <<0:Len>>, + _IV = <<0:12/unsigned-unit:8>>, + {<<"AAD">>,"PT"}) + end + of + _ -> true + catch + _:_ -> false + end. + + +same(Algs) -> [{client2server,Algs}, {server2client,Algs}]. + + +%% default_algorithms(kex) -> % Example of how to disable an algorithm +%% supported_algorithms(kex, ['ecdh-sha2-nistp521']); + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% Other utils diff --git a/lib/ssh/test/Makefile b/lib/ssh/test/Makefile index 96c74c6c8a..781a876723 100644 --- a/lib/ssh/test/Makefile +++ b/lib/ssh/test/Makefile @@ -47,6 +47,8 @@ MODULES= \ ssh_to_openssh_SUITE \ ssh_upgrade_SUITE \ ssh_test_lib \ + ssh_key_cb \ + ssh_key_cb_options \ ssh_trpt_test_lib \ ssh_echo_server \ ssh_peername_sockname_server \ diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 400edb4d2c..d4cb03f2f2 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -36,6 +36,8 @@ cli/1, close/1, daemon_already_started/1, + daemon_opt_fd/1, + multi_daemon_opt_fd/1, double_close/1, exec/1, exec_compressed/1, @@ -52,8 +54,10 @@ send/1, shell/1, shell_no_unicode/1, - shell_unicode_string/1, - ssh_info_print/1 + shell_unicode_string/1, + ssh_info_print/1, + key_callback/1, + key_callback_options/1 ]). %%% Common test callbacks @@ -82,9 +86,12 @@ all() -> {group, ecdsa_sha2_nistp521_key}, {group, dsa_pass_key}, {group, rsa_pass_key}, + {group, key_cb}, {group, internal_error}, daemon_already_started, double_close, + daemon_opt_fd, + multi_daemon_opt_fd, packet_size_zero, ssh_info_print ]. @@ -97,6 +104,7 @@ groups() -> {ecdsa_sha2_nistp521_key, [], basic_tests()}, {dsa_pass_key, [], [pass_phrase]}, {rsa_pass_key, [], [pass_phrase]}, + {key_cb, [], [key_callback, key_callback_options]}, {internal_error, [], [internal_error]} ]. @@ -176,6 +184,11 @@ init_per_group(dsa_pass_key, Config) -> PrivDir = ?config(priv_dir, Config), ssh_test_lib:setup_dsa_pass_pharse(DataDir, PrivDir, "Password"), [{pass_phrase, {dsa_pass_phrase, "Password"}}| Config]; +init_per_group(key_cb, Config) -> + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + ssh_test_lib:setup_dsa(DataDir, PrivDir), + Config; init_per_group(internal_error, Config) -> DataDir = ?config(data_dir, Config), PrivDir = ?config(priv_dir, Config), @@ -243,6 +256,10 @@ end_per_group(rsa_pass_key, Config) -> PrivDir = ?config(priv_dir, Config), ssh_test_lib:clean_rsa(PrivDir), Config; +end_per_group(key_cb, Config) -> + PrivDir = ?config(priv_dir, Config), + ssh_test_lib:clean_dsa(PrivDir), + Config; end_per_group(internal_error, Config) -> PrivDir = ?config(priv_dir, Config), ssh_test_lib:clean_dsa(PrivDir), @@ -571,6 +588,56 @@ pass_phrase(Config) when is_list(Config) -> {ok, _ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), ssh:stop_daemon(Pid). +%%-------------------------------------------------------------------- +%%% Test that we can use key callback +key_callback(Config) when is_list(Config) -> + process_flag(trap_exit, true), + SystemDir = filename:join(?config(priv_dir, Config), system), + UserDir = ?config(priv_dir, Config), + NoPubKeyDir = filename:join(UserDir, "nopubkey"), + file:make_dir(NoPubKeyDir), + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {failfun, fun ssh_test_lib:failfun/2}]), + + ConnectOpts = [{silently_accept_hosts, true}, + {user_dir, NoPubKeyDir}, + {user_interaction, false}, + {key_cb, ssh_key_cb}], + + ConnectionRef = ssh_test_lib:connect(Host, Port, ConnectOpts), + + {ok, _ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), + ssh:stop_daemon(Pid). + + +%%-------------------------------------------------------------------- +%%% Test that we can use key callback with callback options +key_callback_options(Config) when is_list(Config) -> + process_flag(trap_exit, true), + SystemDir = filename:join(?config(priv_dir, Config), system), + UserDir = ?config(priv_dir, Config), + + NoPubKeyDir = filename:join(UserDir, "nopubkey"), + file:make_dir(NoPubKeyDir), + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {failfun, fun ssh_test_lib:failfun/2}]), + + {ok, PrivKey} = file:read_file(filename:join(UserDir, "id_dsa")), + + ConnectOpts = [{silently_accept_hosts, true}, + {user_dir, NoPubKeyDir}, + {user_interaction, false}, + {key_cb, {ssh_key_cb_options, [{priv_key, PrivKey}]}}], + + ConnectionRef = ssh_test_lib:connect(Host, Port, ConnectOpts), + + {ok, _ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), + ssh:stop_daemon(Pid). + %%-------------------------------------------------------------------- %%% Test that client does not hang if disconnects due to internal error @@ -705,6 +772,68 @@ double_close(Config) when is_list(Config) -> ok = ssh:close(CM). %%-------------------------------------------------------------------- +daemon_opt_fd(Config) -> + SystemDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + + {ok,S1} = gen_tcp:listen(0,[]), + {ok,Fd1} = prim_inet:getfd(S1), + + {ok,Pid1} = ssh:daemon(0, [{system_dir, SystemDir}, + {fd,Fd1}, + {user_dir, UserDir}, + {user_passwords, [{"vego", "morot"}]}, + {failfun, fun ssh_test_lib:failfun/2}]), + + {ok,{_Host1,Port1}} = inet:sockname(S1), + {ok, C1} = ssh:connect("localhost", Port1, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}]), + exit(C1, {shutdown, normal}), + ssh:stop_daemon(Pid1), + gen_tcp:close(S1). + + +%%-------------------------------------------------------------------- +multi_daemon_opt_fd(Config) -> + SystemDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + + Test = + fun() -> + {ok,S} = gen_tcp:listen(0,[]), + {ok,Fd} = prim_inet:getfd(S), + + {ok,Pid} = ssh:daemon(0, [{system_dir, SystemDir}, + {fd,Fd}, + {user_dir, UserDir}, + {user_passwords, [{"vego", "morot"}]}, + {failfun, fun ssh_test_lib:failfun/2}]), + + {ok,{_Host,Port}} = inet:sockname(S), + {ok, C} = ssh:connect("localhost", Port, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}]), + {S,Pid,C} + end, + + Tests = [Test(),Test(),Test(),Test(),Test(),Test()], + + [begin + gen_tcp:close(S), + ssh:stop_daemon(Pid), + exit(C, {shutdown, normal}) + end || {S,Pid,C} <- Tests]. + +%%-------------------------------------------------------------------- packet_size_zero(Config) -> SystemDir = ?config(data_dir, Config), PrivDir = ?config(priv_dir, Config), diff --git a/lib/ssh/test/ssh_key_cb.erl b/lib/ssh/test/ssh_key_cb.erl new file mode 100644 index 0000000000..388ec2ecc1 --- /dev/null +++ b/lib/ssh/test/ssh_key_cb.erl @@ -0,0 +1,45 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%%---------------------------------------------------------------------- + +%% Note: This module is used by ssh_basic_SUITE + +-module(ssh_key_cb). +-behaviour(ssh_client_key_api). +-compile(export_all). + +add_host_key(_, _, _) -> + ok. + +is_host_key(_, _, _, _) -> + true. + +user_key('ssh-dss', Opts) -> + UserDir = proplists:get_value(user_dir, Opts), + KeyFile = filename:join(filename:dirname(UserDir), "id_dsa"), + {ok, KeyBin} = file:read_file(KeyFile), + [Entry] = public_key:pem_decode(KeyBin), + Key = public_key:pem_entry_decode(Entry), + {ok, Key}; + +user_key(_Alg, _Opt) -> + {error, "Not Supported"}. diff --git a/lib/ssh/test/ssh_key_cb_options.erl b/lib/ssh/test/ssh_key_cb_options.erl new file mode 100644 index 0000000000..afccb34f0f --- /dev/null +++ b/lib/ssh/test/ssh_key_cb_options.erl @@ -0,0 +1,44 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%%---------------------------------------------------------------------- + +%% Note: This module is used by ssh_basic_SUITE + +-module(ssh_key_cb_options). +-behaviour(ssh_client_key_api). +-compile(export_all). + +add_host_key(_, _, _) -> + ok. + +is_host_key(_, _, _, _) -> + true. + +user_key('ssh-dss', Opts) -> + KeyCbOpts = proplists:get_value(key_cb_private, Opts), + KeyBin = proplists:get_value(priv_key, KeyCbOpts), + [Entry] = public_key:pem_decode(KeyBin), + Key = public_key:pem_entry_decode(Entry), + {ok, Key}; + +user_key(_Alg, _Opt) -> + {error, "Not Supported"}. diff --git a/lib/ssh/test/ssh_protocol_SUITE.erl b/lib/ssh/test/ssh_protocol_SUITE.erl index 3a7f47c2dd..4639904061 100644 --- a/lib/ssh/test/ssh_protocol_SUITE.erl +++ b/lib/ssh/test/ssh_protocol_SUITE.erl @@ -69,7 +69,9 @@ groups() -> gex_client_init_option_groups, gex_server_gex_limit, gex_client_init_option_groups_moduli_file, - gex_client_init_option_groups_file + gex_client_init_option_groups_file, + gex_client_old_request_exact, + gex_client_old_request_noexact ]}, {service_requests, [], [bad_service_name, bad_long_service_name, @@ -94,7 +96,9 @@ init_per_testcase(no_common_alg_server_disconnects, Config) -> init_per_testcase(TC, Config) when TC == gex_client_init_option_groups ; TC == gex_client_init_option_groups_moduli_file ; TC == gex_client_init_option_groups_file ; - TC == gex_server_gex_limit -> + TC == gex_server_gex_limit ; + TC == gex_client_old_request_exact ; + TC == gex_client_old_request_noexact -> Opts = case TC of gex_client_init_option_groups -> [{dh_gex_groups, [{2345, 3, 41}]}]; @@ -106,8 +110,10 @@ init_per_testcase(TC, Config) when TC == gex_client_init_option_groups ; DataDir = ?config(data_dir, Config), F = filename:join(DataDir, "dh_group_test.moduli"), [{dh_gex_groups, {ssh_moduli_file,F}}]; - gex_server_gex_limit -> - [{dh_gex_groups, [{ 500, 3, 18}, + _ when TC == gex_server_gex_limit ; + TC == gex_client_old_request_exact ; + TC == gex_client_old_request_noexact -> + [{dh_gex_groups, [{ 500, 3, 17}, {1000, 7, 91}, {3000, 5, 61}]}, {dh_gex_limits,{500,1500}} @@ -126,7 +132,9 @@ end_per_testcase(no_common_alg_server_disconnects, Config) -> end_per_testcase(TC, Config) when TC == gex_client_init_option_groups ; TC == gex_client_init_option_groups_moduli_file ; TC == gex_client_init_option_groups_file ; - TC == gex_server_gex_limit -> + TC == gex_server_gex_limit ; + TC == gex_client_old_request_exact ; + TC == gex_client_old_request_noexact -> stop_std_daemon(Config); end_per_testcase(_TestCase, Config) -> check_std_daemon_works(Config, ?LINE). @@ -381,6 +389,29 @@ do_gex_client_init(Config, {Min,N,Max}, {G,P}) -> ] ). +%%%-------------------------------------------------------------------- +gex_client_old_request_exact(Config) -> do_gex_client_init_old(Config, 500, {3,17}). +gex_client_old_request_noexact(Config) -> do_gex_client_init_old(Config, 800, {7,91}). + +do_gex_client_init_old(Config, N, {G,P}) -> + {ok,_} = + ssh_trpt_test_lib:exec( + [{set_options, [print_ops, print_seqnums, print_messages]}, + {connect, + server_host(Config),server_port(Config), + [{silently_accept_hosts, true}, + {user_dir, user_dir(Config)}, + {user_interaction, false}, + {preferred_algorithms,[{kex,['diffie-hellman-group-exchange-sha1']}]} + ]}, + receive_hello, + {send, hello}, + {send, ssh_msg_kexinit}, + {match, #ssh_msg_kexinit{_='_'}, receive_msg}, + {send, #ssh_msg_kex_dh_gex_request_old{n = N}}, + {match, #ssh_msg_kex_dh_gex_group{p=P, g=G, _='_'}, receive_msg} + ] + ). %%%-------------------------------------------------------------------- bad_service_name(Config) -> diff --git a/lib/ssh/test/ssh_renegotiate_SUITE.erl b/lib/ssh/test/ssh_renegotiate_SUITE.erl index ef631d54bd..227dfcddcd 100644 --- a/lib/ssh/test/ssh_renegotiate_SUITE.erl +++ b/lib/ssh/test/ssh_renegotiate_SUITE.erl @@ -32,9 +32,15 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> [rekey, rekey_limit, renegotiate1, renegotiate2]. +all() -> [{group,default_algs}, + {group,aes_gcm} + ]. -groups() -> []. +groups() -> [{default_algs, [], tests()}, + {aes_gcm, [], tests()} + ]. + +tests() -> [rekey, rekey_limit, renegotiate1, renegotiate2]. %%-------------------------------------------------------------------- init_per_suite(Config) -> @@ -50,6 +56,18 @@ end_per_suite(_Config) -> crypto:stop(). %%-------------------------------------------------------------------- +init_per_group(aes_gcm, Config) -> + [{preferred_algorithms, [{cipher,[{client2server,['[email protected]']}, + {server2client,['[email protected]']}]}]} + | Config]; +init_per_group(_, Config) -> + [{preferred_algorithms, ssh:default_algorithms()} | Config]. + + +end_per_group(_, Config) -> + Config. + +%%-------------------------------------------------------------------- init_per_testcase(_TestCase, Config) -> ssh:start(), Config. diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index 5816b708f2..424afc76fe 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -296,7 +296,7 @@ setup_dsa(DataDir, UserDir) -> file:make_dir(System), file:copy(filename:join(DataDir, "ssh_host_dsa_key"), filename:join(System, "ssh_host_dsa_key")), file:copy(filename:join(DataDir, "ssh_host_dsa_key.pub"), filename:join(System, "ssh_host_dsa_key.pub")), -ct:pal("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file:list_dir(DataDir), System, file:list_dir(System), UserDir, file:list_dir(UserDir)]), +ct:log("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file:list_dir(DataDir), System, file:list_dir(System), UserDir, file:list_dir(UserDir)]), setup_dsa_known_host(DataDir, UserDir), setup_dsa_auth_keys(DataDir, UserDir). @@ -306,7 +306,7 @@ setup_rsa(DataDir, UserDir) -> file:make_dir(System), file:copy(filename:join(DataDir, "ssh_host_rsa_key"), filename:join(System, "ssh_host_rsa_key")), file:copy(filename:join(DataDir, "ssh_host_rsa_key.pub"), filename:join(System, "ssh_host_rsa_key.pub")), -ct:pal("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file:list_dir(DataDir), System, file:list_dir(System), UserDir, file:list_dir(UserDir)]), +ct:log("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file:list_dir(DataDir), System, file:list_dir(System), UserDir, file:list_dir(UserDir)]), setup_rsa_known_host(DataDir, UserDir), setup_rsa_auth_keys(DataDir, UserDir). @@ -316,7 +316,7 @@ setup_ecdsa(Size, DataDir, UserDir) -> file:make_dir(System), file:copy(filename:join(DataDir, "ssh_host_ecdsa_key"++Size), filename:join(System, "ssh_host_ecdsa_key")), file:copy(filename:join(DataDir, "ssh_host_ecdsa_key"++Size++".pub"), filename:join(System, "ssh_host_ecdsa_key.pub")), -ct:pal("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file:list_dir(DataDir), System, file:list_dir(System), UserDir, file:list_dir(UserDir)]), +ct:log("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file:list_dir(DataDir), System, file:list_dir(System), UserDir, file:list_dir(UserDir)]), setup_ecdsa_known_host(Size, System, UserDir), setup_ecdsa_auth_keys(Size, UserDir, UserDir). @@ -502,7 +502,7 @@ default_algorithms(sshd, Host, Port) -> {user_interaction, false}]}])) catch _C:_E -> - ct:pal("***~p:~p: ~p:~p",[?MODULE,?LINE,_C,_E]), + ct:log("***~p:~p: ~p:~p",[?MODULE,?LINE,_C,_E]), [] end. @@ -522,7 +522,7 @@ default_algorithms(sshc, DaemonOptions) -> InitialState)) catch _C:_E -> - ct:pal("***~p:~p: ~p:~p",[?MODULE,?LINE,_C,_E]), + ct:log("***~p:~p: ~p:~p",[?MODULE,?LINE,_C,_E]), [] end} end), diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index d828bccd29..25b19133b1 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,4 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode SSH_VSN = 4.2 + APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl index b6e62a18c9..404ae93d20 100644 --- a/lib/ssl/src/inet_tls_dist.erl +++ b/lib/ssl/src/inet_tls_dist.erl @@ -30,7 +30,7 @@ childspecs() -> {ok, [{ssl_dist_sup,{ssl_dist_sup, start_link, []}, - permanent, 2000, worker, [ssl_dist_sup]}]}. + permanent, infinity, supervisor, [ssl_dist_sup]}]}. select(Node) -> case split_node(atom_to_list(Node), $@, []) of diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index f8afbdb41d..12a56df69f 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -1781,7 +1781,7 @@ handle_trusted_certs_db(#state{ssl_options = #ssl_options{cacertfile = <<>>, cac ok; handle_trusted_certs_db(#state{cert_db_ref = Ref, cert_db = CertDb, - ssl_options = #ssl_options{cacertfile = <<>>}}) -> + ssl_options = #ssl_options{cacertfile = <<>>}}) when CertDb =/= undefined -> %% Certs provided as DER directly can not be shared %% with other connections and it is safe to delete them when the connection ends. ssl_pkix_db:remove_trusted_certs(Ref, CertDb); diff --git a/lib/ssl/src/ssl_dist_sup.erl b/lib/ssl/src/ssl_dist_sup.erl index aa1fa57db8..435ad27a44 100644 --- a/lib/ssl/src/ssl_dist_sup.erl +++ b/lib/ssl/src/ssl_dist_sup.erl @@ -70,7 +70,7 @@ connection_manager_child_spec() -> Name = ssl_connection_dist, StartFunc = {tls_connection_sup, start_link_dist, []}, Restart = permanent, - Shutdown = 4000, + Shutdown = infinity, Modules = [tls_connection_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 3851b2bc6e..8c7ed9c0d1 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -78,6 +78,9 @@ -define(ALL_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2', dtlsv1]). -define(MIN_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2', dtlsv1]). +-define('24H_in_msec', 86400000). +-define('24H_in_sec', 86400). + -record(ssl_options, { protocol :: tls | dtls, versions :: [ssl_record:ssl_version()], %% ssl_record:atom_version() in API diff --git a/lib/ssl/src/ssl_manager.erl b/lib/ssl/src/ssl_manager.erl index 2e05ba5aa5..cc15678f23 100644 --- a/lib/ssl/src/ssl_manager.erl +++ b/lib/ssl/src/ssl_manager.erl @@ -57,8 +57,6 @@ clear_pem_cache }). --define('24H_in_msec', 86400000). --define('24H_in_sec', 86400). -define(GEN_UNIQUE_ID_MAX_TRIES, 10). -define(SESSION_VALIDATION_INTERVAL, 60000). -define(CLEAR_PEM_CACHE, 120000). diff --git a/lib/ssl/src/ssl_session.erl b/lib/ssl/src/ssl_session.erl index 0d6cc93a20..1849a05314 100644 --- a/lib/ssl/src/ssl_session.erl +++ b/lib/ssl/src/ssl_session.erl @@ -31,8 +31,6 @@ %% Internal application API -export([is_new/2, client_id/4, server_id/6, valid_session/2]). --define('24H_in_sec', 8640). - -type seconds() :: integer(). %%-------------------------------------------------------------------- diff --git a/lib/ssl/src/ssl_tls_dist_proxy.erl b/lib/ssl/src/ssl_tls_dist_proxy.erl index 273d3b5521..d384264b53 100644 --- a/lib/ssl/src/ssl_tls_dist_proxy.erl +++ b/lib/ssl/src/ssl_tls_dist_proxy.erl @@ -48,6 +48,47 @@ accept(Listen) -> connect(Ip, Port) -> gen_server:call(?MODULE, {connect, Ip, Port}, infinity). + +do_listen(Options) -> + {First,Last} = case application:get_env(kernel,inet_dist_listen_min) of + {ok,N} when is_integer(N) -> + case application:get_env(kernel, + inet_dist_listen_max) of + {ok,M} when is_integer(M) -> + {N,M}; + _ -> + {N,N} + end; + _ -> + {0,0} + end, + do_listen(First, Last, listen_options([{backlog,128}|Options])). + +do_listen(First,Last,_) when First > Last -> + {error,eaddrinuse}; +do_listen(First,Last,Options) -> + case gen_tcp:listen(First, Options) of + {error, eaddrinuse} -> + do_listen(First+1,Last,Options); + Other -> + Other + end. + +listen_options(Opts0) -> + Opts1 = + case application:get_env(kernel, inet_dist_use_interface) of + {ok, Ip} -> + [{ip, Ip} | Opts0]; + _ -> + Opts0 + end, + case application:get_env(kernel, inet_dist_listen_options) of + {ok,ListenOpts} -> + ListenOpts ++ Opts1; + _ -> + Opts1 + end. + %%==================================================================== %% gen_server callbacks %%==================================================================== @@ -62,13 +103,17 @@ init([]) -> handle_call({listen, Name}, _From, State) -> case gen_tcp:listen(0, [{active, false}, {packet,?PPRE}]) of {ok, Socket} -> - {ok, World} = gen_tcp:listen(0, [{active, false}, binary, {packet,?PPRE}]), + {ok, World} = do_listen([{active, false}, binary, {packet,?PPRE}, {reuseaddr, true}]), {ok, TcpAddress} = get_tcp_address(Socket), {ok, WorldTcpAddress} = get_tcp_address(World), {_,Port} = WorldTcpAddress#net_address.address, - {ok, Creation} = erl_epmd:register_node(Name, Port), - {reply, {ok, {Socket, TcpAddress, Creation}}, - State#state{listen={Socket, World}}}; + case erl_epmd:register_node(Name, Port) of + {ok, Creation} -> + {reply, {ok, {Socket, TcpAddress, Creation}}, + State#state{listen={Socket, World}}}; + {error, _} = Error -> + {reply, Error, State} + end; Error -> {reply, Error, State} end; @@ -134,6 +179,7 @@ accept_loop(Proxy, erts = Type, Listen, Extra) -> Extra ! {accept,self(),Socket,inet,proxy}, receive {_Kernel, controller, Pid} -> + inet:setopts(Socket, [nodelay()]), ok = gen_tcp:controlling_process(Socket, Pid), flush_old_controller(Pid, Socket), Pid ! {self(), controller}; @@ -167,7 +213,7 @@ accept_loop(Proxy, world = Type, Listen, Extra) -> accept_loop(Proxy, Type, Listen, Extra). try_connect(Port) -> - case gen_tcp:connect({127,0,0,1}, Port, [{active, false}, {packet,?PPRE}]) of + case gen_tcp:connect({127,0,0,1}, Port, [{active, false}, {packet,?PPRE}, nodelay()]) of R = {ok, _S} -> R; {error, _R} -> @@ -177,7 +223,7 @@ try_connect(Port) -> setup_proxy(Ip, Port, Parent) -> process_flag(trap_exit, true), Opts = get_ssl_options(client), - case ssl:connect(Ip, Port, [{active, true}, binary, {packet,?PPRE}] ++ Opts) of + case ssl:connect(Ip, Port, [{active, true}, binary, {packet,?PPRE}, nodelay()] ++ Opts) of {ok, World} -> {ok, ErtsL} = gen_tcp:listen(0, [{active, true}, {ip, {127,0,0,1}}, binary, {packet,?PPRE}]), {ok, #net_address{address={_,LPort}}} = get_tcp_address(ErtsL), @@ -193,25 +239,41 @@ setup_proxy(Ip, Port, Parent) -> Parent ! {self(), Err} end. + +%% we may not always want the nodelay behaviour +%% %% for performance reasons + +nodelay() -> + case application:get_env(kernel, dist_nodelay) of + undefined -> + {nodelay, true}; + {ok, true} -> + {nodelay, true}; + {ok, false} -> + {nodelay, false}; + _ -> + {nodelay, true} + end. + setup_connection(World, ErtsListen) -> process_flag(trap_exit, true), {ok, TcpAddress} = get_tcp_address(ErtsListen), {_Addr,Port} = TcpAddress#net_address.address, - {ok, Erts} = gen_tcp:connect({127,0,0,1}, Port, [{active, true}, binary, {packet,?PPRE}]), - ssl:setopts(World, [{active,true}, {packet,?PPRE}]), + {ok, Erts} = gen_tcp:connect({127,0,0,1}, Port, [{active, true}, binary, {packet,?PPRE}, nodelay()]), + ssl:setopts(World, [{active,true}, {packet,?PPRE}, nodelay()]), loop_conn_setup(World, Erts). loop_conn_setup(World, Erts) -> receive {ssl, World, Data = <<$a, _/binary>>} -> gen_tcp:send(Erts, Data), - ssl:setopts(World, [{packet,?PPOST}]), - inet:setopts(Erts, [{packet,?PPOST}]), + ssl:setopts(World, [{packet,?PPOST}, nodelay()]), + inet:setopts(Erts, [{packet,?PPOST}, nodelay()]), loop_conn(World, Erts); {tcp, Erts, Data = <<$a, _/binary>>} -> ssl:send(World, Data), - ssl:setopts(World, [{packet,?PPOST}]), - inet:setopts(Erts, [{packet,?PPOST}]), + ssl:setopts(World, [{packet,?PPOST}, nodelay()]), + inet:setopts(Erts, [{packet,?PPOST}, nodelay()]), loop_conn(World, Erts); {ssl, World, Data = <<_, _/binary>>} -> gen_tcp:send(Erts, Data), diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 6f6107de2c..f032c769e2 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -35,7 +35,6 @@ -include("tls_record.hrl"). -include("tls_handshake.hrl"). --define('24H_in_sec', 86400). -define(TIMEOUT, 20000). -define(EXPIRE, 10). -define(SLEEP, 500). diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 72d62b29a7..092015d3d8 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -40,7 +40,8 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- all() -> - [basic, payload, plain_options, plain_verify_options]. + [basic, payload, plain_options, plain_verify_options, nodelay_option, + listen_port_options, listen_options, use_interface]. groups() -> []. @@ -250,6 +251,146 @@ plain_verify_options(Config) when is_list(Config) -> stop_ssl_node(NH1), stop_ssl_node(NH2), success(Config). +%%-------------------------------------------------------------------- +nodelay_option() -> + [{doc,"Test specifying dist_nodelay option"}]. +nodelay_option(Config) -> + try + %% The default is 'true', so try setting it to 'false'. + application:set_env(kernel, dist_nodelay, false), + basic(Config) + after + application:unset_env(kernel, dist_nodelay) + end. + +listen_port_options() -> + [{doc, "Test specifying listening ports"}]. +listen_port_options(Config) when is_list(Config) -> + %% Start a node, and get the port number it's listening on. + NH1 = start_ssl_node(Config), + Node1 = NH1#node_handle.nodename, + Name1 = lists:takewhile(fun(C) -> C =/= $@ end, atom_to_list(Node1)), + {ok, NodesPorts} = apply_on_ssl_node(NH1, fun net_adm:names/0), + {Name1, Port1} = lists:keyfind(Name1, 1, NodesPorts), + + %% Now start a second node, configuring it to use the same port + %% number. + PortOpt1 = "-kernel inet_dist_listen_min " ++ integer_to_list(Port1) ++ + " inet_dist_listen_max " ++ integer_to_list(Port1), + + try start_ssl_node([{additional_dist_opts, PortOpt1} | Config]) of + #node_handle{} -> + %% If the node was able to start, it didn't take the port + %% option into account. + exit(unexpected_success) + catch + exit:{accept_failed, timeout} -> + %% The node failed to start, as expected. + ok + end, + + %% Try again, now specifying a high max port. + PortOpt2 = "-kernel inet_dist_listen_min " ++ integer_to_list(Port1) ++ + " inet_dist_listen_max 65535", + NH2 = start_ssl_node([{additional_dist_opts, PortOpt2} | Config]), + Node2 = NH2#node_handle.nodename, + Name2 = lists:takewhile(fun(C) -> C =/= $@ end, atom_to_list(Node2)), + {ok, NodesPorts2} = apply_on_ssl_node(NH2, fun net_adm:names/0), + {Name2, Port2} = lists:keyfind(Name2, 1, NodesPorts2), + + %% The new port should be higher: + if Port2 > Port1 -> + ok; + true -> + error({port, Port2, not_higher_than, Port1}) + end, + + stop_ssl_node(NH1), + stop_ssl_node(NH2), + success(Config). +%%-------------------------------------------------------------------- +listen_options() -> + [{doc, "Test inet_dist_listen_options"}]. +listen_options(Config) when is_list(Config) -> + Prio = 1, + case gen_udp:open(0, [{priority,Prio}]) of + {ok,Socket} -> + case inet:getopts(Socket, [priority]) of + {ok,[{priority,Prio}]} -> + ok = gen_udp:close(Socket), + do_listen_options(Prio, Config); + _ -> + ok = gen_udp:close(Socket), + {skip, + "Can not set priority "++integer_to_list(Prio)++ + " on socket"} + end; + {error,_} -> + {skip, "Can not set priority on socket"} + end. + +do_listen_options(Prio, Config) -> + PriorityString0 = "[{priority,"++integer_to_list(Prio)++"}]", + PriorityString = + case os:cmd("echo [{a,1}]") of + "[{a,1}]"++_ -> + PriorityString0; + _ -> + %% Some shells need quoting of [{}] + "'"++PriorityString0++"'" + end, + + Options = "-kernel inet_dist_listen_options " ++ PriorityString, + + NH1 = start_ssl_node([{additional_dist_opts, Options} | Config]), + NH2 = start_ssl_node([{additional_dist_opts, Options} | Config]), + Node2 = NH2#node_handle.nodename, + + pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), + + PrioritiesNode1 = + apply_on_ssl_node(NH1, fun get_socket_priorities/0), + PrioritiesNode2 = + apply_on_ssl_node(NH2, fun get_socket_priorities/0), + + Elevated1 = [P || P <- PrioritiesNode1, P =:= Prio], + ?t:format("Elevated1: ~p~n", [Elevated1]), + Elevated2 = [P || P <- PrioritiesNode2, P =:= Prio], + ?t:format("Elevated2: ~p~n", [Elevated2]), + [_|_] = Elevated1, + [_|_] = Elevated2, + + stop_ssl_node(NH1), + stop_ssl_node(NH2), + success(Config). +%%-------------------------------------------------------------------- +use_interface() -> + [{doc, "Test inet_dist_use_interface"}]. +use_interface(Config) when is_list(Config) -> + %% Force the node to listen only on the loopback interface. + IpString = "'{127,0,0,1}'", + Options = "-kernel inet_dist_use_interface " ++ IpString, + + %% Start a node, and get the port number it's listening on. + NH1 = start_ssl_node([{additional_dist_opts, Options} | Config]), + Node1 = NH1#node_handle.nodename, + Name = lists:takewhile(fun(C) -> C =/= $@ end, atom_to_list(Node1)), + {ok, NodesPorts} = apply_on_ssl_node(NH1, fun net_adm:names/0), + {Name, Port} = lists:keyfind(Name, 1, NodesPorts), + + %% Now find the socket listening on that port, and check its sockname. + Sockets = apply_on_ssl_node( + NH1, + fun() -> + [inet:sockname(P) || + P <- erlang:ports(), + {ok, Port} =:= (catch inet:port(P))] + end), + %% And check that it's actually listening on localhost. + [{ok,{{127,0,0,1},Port}}] = Sockets, + + stop_ssl_node(NH1), + success(Config). %%-------------------------------------------------------------------- %%% Internal functions ----------------------------------------------- @@ -264,6 +405,12 @@ tstsrvr_format(Fmt, ArgList) -> send_to_tstcntrl(Message) -> send_to_tstsrvr({message, Message}). +get_socket_priorities() -> + [Priority || + {ok,[{priority,Priority}]} <- + [inet:getopts(Port, [priority]) || + Port <- erlang:ports(), + element(2, erlang:port_info(Port, name)) =:= "tcp_inet"]]. %% %% test_server side api diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index c4cb5fdc80..5678e7eebe 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -293,6 +293,9 @@ format_error({variable_in_record_def,V}) -> %% --- binaries --- format_error({undefined_bittype,Type}) -> io_lib:format("bit type ~w undefined", [Type]); +format_error({bittype_mismatch,Val1,Val2,What}) -> + io_lib:format("conflict in ~s specification for bit field: '~p' and '~p'", + [What,Val1,Val2]); format_error(bittype_unit) -> "a bit unit size must not be specified unless a size is specified too"; format_error(illegal_bitsize) -> diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index f215a66812..ce1d9eb0ff 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -999,12 +999,7 @@ local_func(rl, [A], Bs0, _Shell, RT, Lf, Ef) -> {value,list_records(record_defs(RT, listify(Recs))),Bs}; local_func(rp, [A], Bs0, _Shell, RT, Lf, Ef) -> {[V],Bs} = expr_list([A], Bs0, Lf, Ef), - Cs = io_lib_pretty:print(V, ([{column, 1}, - {line_length, columns()}, - {depth, -1}, - {max_chars, ?CHAR_MAX}, - {record_print_fun, record_print_fun(RT)}] - ++ enc())), + Cs = pp(V, _Column=1, _Depth=-1, RT), io:requests([{put_chars, unicode, Cs}, nl]), {value,ok,Bs}; local_func(rr, [A], Bs0, _Shell, RT, Lf, Ef) -> @@ -1397,9 +1392,9 @@ get_history_and_results() -> {History, erlang:min(Results, History)}. pp(V, I, RT) -> - pp(V, I, RT, enc()). + pp(V, I, _Depth=?LINEMAX, RT). -pp(V, I, RT, Enc) -> +pp(V, I, D, RT) -> Strings = case application:get_env(stdlib, shell_strings) of {ok, false} -> @@ -1408,10 +1403,10 @@ pp(V, I, RT, Enc) -> true end, io_lib_pretty:print(V, ([{column, I}, {line_length, columns()}, - {depth, ?LINEMAX}, {max_chars, ?CHAR_MAX}, + {depth, D}, {max_chars, ?CHAR_MAX}, {strings, Strings}, {record_print_fun, record_print_fun(RT)}] - ++ Enc)). + ++ enc())). columns() -> case io:columns() of diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index 0424e2b967..5347ccaf1f 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -3604,7 +3604,10 @@ bin_syntax_errors(Config) -> t(<<X/unit:8>>) -> X; t(<<X:7/float>>) -> X; t(<< <<_:8>> >>) -> ok; - t(<<(x ! y):8/integer>>) -> ok. + t(<<(x ! y):8/integer>>) -> ok; + t(X) -> + {<<X/binary-integer>>,<<X/signed-unsigned-integer>>, + <<X/little-big>>,<<X/unit:4-unit:8>>}. ">>, [], {error,[{1,erl_lint,illegal_bitsize}, @@ -3613,7 +3616,12 @@ bin_syntax_errors(Config) -> {4,erl_lint,{undefined_bittype,bad_type}}, {5,erl_lint,bittype_unit}, {7,erl_lint,illegal_pattern}, - {8,erl_lint,illegal_pattern}], + {8,erl_lint,illegal_pattern}, + {10,erl_lint,{bittype_mismatch,integer,binary,"type"}}, + {10,erl_lint,{bittype_mismatch,unsigned,signed,"sign"}}, + {11,erl_lint,{bittype_mismatch,8,4,"unit"}}, + {11,erl_lint,{bittype_mismatch,big,little,"endianness"}} + ], [{6,erl_lint,{bad_bitsize,"float"}}]}} ], [] = run(Config, Ts), @@ -3920,22 +3928,35 @@ run_test2(Conf, Test, Warnings0) -> %% is no reason to produce an output file since we are only %% interested in the errors and warnings. - %% Print warnings, call erl_lint:format_error/1. + %% Print warnings, call erl_lint:format_error/1. (But note that + %% the compiler will ignore failing calls to erl_lint:format_error/1.) compile:file(File, [binary,report|Opts]), case compile:file(File, [binary|Opts]) of - {ok, _M, Code, Ws} when is_binary(Code) -> warnings(File, Ws); - {error, [{File,Es}], []} -> {errors, Es, []}; - {error, [{File,Es}], [{File,Ws}]} -> {error, Es, Ws}; - {error, [{File,Es1},{File,Es2}], []} -> {errors2, Es1, Es2} + {ok, _M, Code, Ws} when is_binary(Code) -> + warnings(File, Ws); + {error, [{File,Es}], []} -> + {errors, call_format_error(Es), []}; + {error, [{File,Es}], [{File,Ws}]} -> + {error, call_format_error(Es), call_format_error(Ws)}; + {error, [{File,Es1},{File,Es2}], []} -> + {errors2, Es1, Es2} end. warnings(File, Ws) -> case lists:append([W || {F, W} <- Ws, F =:= File]) of - [] -> []; - L -> {warnings, L} + [] -> + []; + L -> + {warnings, call_format_error(L)} end. +call_format_error(L) -> + %% Smoke test of format_error/1 to make sure that no crashes + %% slip through. + _ = [Mod:format_error(Term) || {_,Mod,Term} <- L], + L. + fail() -> io:format("failed~n"), ?t:fail(). diff --git a/otp_versions.table b/otp_versions.table index 09fe46cf57..9abffe87ed 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,4 @@ +OTP-18.1.5 : ssh-4.1.3 # asn1-4.0 common_test-1.11 compiler-6.0.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2 cosProperty-1.2 cosTime-1.2 cosTransactions-1.3 crypto-3.6.1 debugger-4.1.1 dialyzer-2.8.1 diameter-1.11 edoc-0.7.17 eldap-1.2 erl_docgen-0.4 erl_interface-3.8 erts-7.1 et-1.5.1 eunit-2.2.11 gs-1.6 hipe-3.13 ic-4.4 inets-6.0.3 jinterface-1.6 kernel-4.1 megaco-3.18 mnesia-4.13.2 observer-2.1 odbc-2.11.1 orber-3.8 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1 percept-0.8.11 public_key-1.0.1 reltool-0.7 runtime_tools-1.9.1 sasl-2.6 snmp-5.2 ssl-7.1 stdlib-2.6 syntax_tools-1.7 test_server-3.9 tools-2.8.1 typer-0.9.9 webtool-0.9 wx-1.5 xmerl-1.3.8 : OTP-18.1.4 : inets-6.0.3 # asn1-4.0 common_test-1.11 compiler-6.0.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2 cosProperty-1.2 cosTime-1.2 cosTransactions-1.3 crypto-3.6.1 debugger-4.1.1 dialyzer-2.8.1 diameter-1.11 edoc-0.7.17 eldap-1.2 erl_docgen-0.4 erl_interface-3.8 erts-7.1 et-1.5.1 eunit-2.2.11 gs-1.6 hipe-3.13 ic-4.4 jinterface-1.6 kernel-4.1 megaco-3.18 mnesia-4.13.2 observer-2.1 odbc-2.11.1 orber-3.8 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1 percept-0.8.11 public_key-1.0.1 reltool-0.7 runtime_tools-1.9.1 sasl-2.6 snmp-5.2 ssh-4.1.2 ssl-7.1 stdlib-2.6 syntax_tools-1.7 test_server-3.9 tools-2.8.1 typer-0.9.9 webtool-0.9 wx-1.5 xmerl-1.3.8 : OTP-18.1.3 : ssh-4.1.2 # asn1-4.0 common_test-1.11 compiler-6.0.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2 cosProperty-1.2 cosTime-1.2 cosTransactions-1.3 crypto-3.6.1 debugger-4.1.1 dialyzer-2.8.1 diameter-1.11 edoc-0.7.17 eldap-1.2 erl_docgen-0.4 erl_interface-3.8 erts-7.1 et-1.5.1 eunit-2.2.11 gs-1.6 hipe-3.13 ic-4.4 inets-6.0.2 jinterface-1.6 kernel-4.1 megaco-3.18 mnesia-4.13.2 observer-2.1 odbc-2.11.1 orber-3.8 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1 percept-0.8.11 public_key-1.0.1 reltool-0.7 runtime_tools-1.9.1 sasl-2.6 snmp-5.2 ssl-7.1 stdlib-2.6 syntax_tools-1.7 test_server-3.9 tools-2.8.1 typer-0.9.9 webtool-0.9 wx-1.5 xmerl-1.3.8 : OTP-18.1.2 : ssh-4.1.1 # asn1-4.0 common_test-1.11 compiler-6.0.1 cosEvent-2.2 cosEventDomain-1.2 cosFileTransfer-1.2 cosNotification-1.2 cosProperty-1.2 cosTime-1.2 cosTransactions-1.3 crypto-3.6.1 debugger-4.1.1 dialyzer-2.8.1 diameter-1.11 edoc-0.7.17 eldap-1.2 erl_docgen-0.4 erl_interface-3.8 erts-7.1 et-1.5.1 eunit-2.2.11 gs-1.6 hipe-3.13 ic-4.4 inets-6.0.2 jinterface-1.6 kernel-4.1 megaco-3.18 mnesia-4.13.2 observer-2.1 odbc-2.11.1 orber-3.8 os_mon-2.4 ose-1.1 otp_mibs-1.1 parsetools-2.1 percept-0.8.11 public_key-1.0.1 reltool-0.7 runtime_tools-1.9.1 sasl-2.6 snmp-5.2 ssl-7.1 stdlib-2.6 syntax_tools-1.7 test_server-3.9 tools-2.8.1 typer-0.9.9 webtool-0.9 wx-1.5 xmerl-1.3.8 : |