diff options
Diffstat (limited to 'erts/emulator/beam')
53 files changed, 8643 insertions, 2127 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index c097866c7e..ae3f30d82f 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -109,6 +109,7 @@ atom bag atom band atom big atom bif_return_trap +atom bif_timer_server atom binary atom binary_bin_to_list_trap atom binary_copy_trap @@ -144,9 +145,11 @@ atom catchlevel atom cd atom cdr atom cflags +atom CHANGE='CHANGE' atom characters_to_binary_int atom characters_to_list_int atom clear +atom clock_service atom close atom closed atom code @@ -156,6 +159,7 @@ atom compat_rel atom compile atom compressed atom config_h +atom convert_time_unit atom connect atom connected atom connection_closed @@ -237,7 +241,7 @@ atom first atom firstline atom flags atom flush -atom flush_monitor_message +atom flush_monitor_messages atom force atom format_cpu_topology atom free @@ -345,6 +349,8 @@ atom message_queue_len atom messages atom meta atom meta_match_spec +atom micro_seconds +atom milli_seconds atom min_heap_size atom min_bin_vheap_size atom minor_version @@ -355,12 +361,15 @@ atom monitored_by atom monitor atom monitor_nodes atom monitors +atom monotonic atom more atom multi_scheduling atom multiline +atom nano_seconds atom name atom named_table atom namelist +atom native atom native_addresses atom Neq='=/=' atom Neqeq='/=' @@ -451,6 +460,7 @@ atom ports atom port_count atom port_limit atom port_op +atom positive atom print atom priority atom private @@ -510,6 +520,7 @@ atom schedulers_online atom scheme atom scientific atom scope +atom seconds atom sensitive atom sequential_tracer atom sequential_trace_token @@ -555,6 +566,7 @@ atom term_to_binary_trap atom this atom thread_pool_size atom threads +atom time_offset atom timeout atom timeout_value atom Times='*' diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index e847403882..8fcdc72895 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -699,7 +699,7 @@ void** beam_ops; Fail; \ } -#define IsMap(Src, Fail) if (is_not_map(Src)) { Fail; } +#define IsMap(Src, Fail) if (!is_map(Src)) { Fail; } #define HasMapField(Src, Key, Fail) if (has_not_map_field(Src, Key)) { Fail; } @@ -2392,7 +2392,7 @@ void process_main(void) } OpCase(i_has_map_fields_fsI): { - map_t* mp; + flatmap_t* mp; Eterm map; Eterm field; Eterm *ks; @@ -2400,22 +2400,34 @@ void process_main(void) Uint sz,n; GetArg1(1, map); + n = (Uint)Arg(2); + fs = &Arg(3); /* pattern fields */ - /* this instruction assumes Arg1 is a map, - * i.e. that it follows a test is_map if needed. - */ + /* get term from field? */ + if (is_hashmap(map)) { + Uint32 hx; + while(n--) { + field = *fs++; + hx = hashmap_make_hash(field); + if (!erts_hashmap_get(hx,field,map)) { + SET_I((BeamInstr *) Arg(0)); + goto has_map_fields_fail; + } + } + goto has_map_fields_ok; + } + + ASSERT(is_flatmap(map)); - mp = (map_t *)map_val(map); - sz = map_get_size(mp); + mp = (flatmap_t *)flatmap_val(map); + sz = flatmap_get_size(mp); if (sz == 0) { SET_I((BeamInstr *) Arg(0)); goto has_map_fields_fail; } - ks = map_get_keys(mp); - n = (Uint)Arg(2); - fs = &Arg(3); /* pattern fields */ + ks = flatmap_get_keys(mp); ASSERT(n>0); @@ -2433,7 +2445,7 @@ void process_main(void) SET_I((BeamInstr *) Arg(0)); goto has_map_fields_fail; } - +has_map_fields_ok: I += 4 + Arg(2); has_map_fields_fail: ASSERT(VALID_INSTR(*I)); @@ -2460,12 +2472,8 @@ do { \ OpCase(i_get_map_elements_fsI): { Eterm map; - map_t *mp; - Eterm field; - Eterm *ks; - Eterm *vs; BeamInstr *fs; - Uint sz,n; + Uint sz, n; GetArg1(1, map); @@ -2473,36 +2481,56 @@ do { \ * i.e. that it follows a test is_map if needed. */ - mp = (map_t *)map_val(map); - sz = map_get_size(mp); - - if (sz == 0) { - SET_I((BeamInstr *) Arg(0)); - goto get_map_elements_fail; - } - n = (Uint)Arg(2) / 2; fs = &Arg(3); /* pattern fields and target registers */ - ks = map_get_keys(mp); - vs = map_get_values(mp); - while(sz) { - field = (Eterm)*fs; - if (EQ(field,*ks)) { - PUT_TERM_REG(*vs, fs[1]); - n--; + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Eterm *vs; + + mp = (flatmap_t *)flatmap_val(map); + sz = flatmap_get_size(mp); + + if (sz == 0) { + SET_I((BeamInstr *) Arg(0)); + goto get_map_elements_fail; + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + while(sz) { + if (EQ((Eterm)*fs,*ks)) { + PUT_TERM_REG(*vs, fs[1]); + n--; + fs += 2; + /* no more values to fetch, we are done */ + if (n == 0) break; + } + ks++; sz--; + vs++; + } + + if (n) { + SET_I((BeamInstr *) Arg(0)); + goto get_map_elements_fail; + } + } else { + const Eterm *v; + Uint32 hx; + ASSERT(is_hashmap(map)); + while(n--) { + hx = hashmap_make_hash((Eterm)*fs); + if ((v = erts_hashmap_get(hx,(Eterm)*fs, map)) == NULL) { + SET_I((BeamInstr *) Arg(0)); + goto get_map_elements_fail; + } + PUT_TERM_REG(*v, fs[1]); fs += 2; - /* no more values to fetch, we are done */ - if (n == 0) break; } - ks++; sz--; - vs++; } - if (n) { - SET_I((BeamInstr *) Arg(0)); - goto get_map_elements_fail; - } I += 4 + Arg(2); get_map_elements_fail: @@ -2801,6 +2829,7 @@ get_map_elements_fail: } PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); reg[0] = r(0); result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); @@ -6443,55 +6472,69 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) static int has_not_map_field(Eterm map, Eterm key) { - map_t* mp; - Eterm* keys; - Uint i; - Uint n; - - mp = (map_t *)map_val(map); - keys = map_get_keys(mp); - n = map_get_size(mp); - if (is_immed(key)) { - for (i = 0; i < n; i++) { - if (keys[i] == key) { - return 0; + Uint32 hx; + if (is_flatmap(map)) { + flatmap_t* mp; + Eterm* keys; + Uint i; + Uint n; + + mp = (flatmap_t *)flatmap_val(map); + keys = flatmap_get_keys(mp); + n = flatmap_get_size(mp); + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (keys[i] == key) { + return 0; + } } - } - } else { - for (i = 0; i < n; i++) { - if (EQ(keys[i], key)) { - return 0; + } else { + for (i = 0; i < n; i++) { + if (EQ(keys[i], key)) { + return 0; + } } } + return 1; } - return 1; + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + return erts_hashmap_get(hx,key,map) ? 0 : 1; } static Eterm get_map_element(Eterm map, Eterm key) { - map_t *mp; - Eterm* ks, *vs; - Uint i; - Uint n; - - mp = (map_t *)map_val(map); - ks = map_get_keys(mp); - vs = map_get_values(mp); - n = map_get_size(mp); - if (is_immed(key)) { - for (i = 0; i < n; i++) { - if (ks[i] == key) { - return vs[i]; + Uint32 hx; + const Eterm *vs; + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Uint i; + Uint n; + + mp = (flatmap_t *)flatmap_val(map); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (ks[i] == key) { + return vs[i]; + } } - } - } else { - for (i = 0; i < n; i++) { - if (EQ(ks[i], key)) { - return vs[i]; + } else { + for (i = 0; i < n; i++) { + if (EQ(ks[i], key)) { + return vs[i]; + } } } + return THE_NON_VALUE; } - return THE_NON_VALUE; + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + vs = erts_hashmap_get(hx,key,map); + return vs ? *vs : THE_NON_VALUE; } #define GET_TERM(term, dest) \ @@ -6524,7 +6567,30 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) Eterm *mhp,*thp; Eterm *E; BeamInstr *ptr; - map_t *mp; + flatmap_t *mp; + ErtsHeapFactory factory; + + ptr = &Arg(4); + + if (n > 2*MAP_SMALL_MAP_LIMIT) { + if (HeapWordsLeft(p) < n) { + erts_garbage_collect(p, n, reg, Arg(2)); + } + + mhp = p->htop; + thp = p->htop; + E = p->stop; + + for (i = 0; i < n/2; i++) { + GET_TERM(*ptr++, *mhp++); + GET_TERM(*ptr++, *mhp++); + } + + p->htop = mhp; + + factory.p = p; + return erts_hashmap_from_array(&factory, thp, n/2, 0); + } if (HeapWordsLeft(p) < need) { erts_garbage_collect(p, need, reg, Arg(2)); @@ -6533,12 +6599,11 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) thp = p->htop; mhp = thp + 1 + n/2; E = p->stop; - ptr = &Arg(4); keys = make_tuple(thp); *thp++ = make_arityval(n/2); - mp = (map_t *)mhp; mhp += MAP_HEADER_SIZE; - mp->thing_word = MAP_HEADER; + mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; mp->size = n/2; mp->keys = keys; @@ -6547,7 +6612,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) GET_TERM(*ptr++, *mhp++); } p->htop = mhp; - return make_map(mp); + return make_flatmap(mp); } static Eterm @@ -6557,7 +6622,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Uint num_old; Uint num_updates; Uint need; - map_t *old_mp, *mp; + flatmap_t *old_mp, *mp; Eterm res; Eterm* hp; Eterm* E; @@ -6567,12 +6632,44 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Eterm new_key; Eterm* kp; - if (is_not_map(map)) { - return THE_NON_VALUE; + new_p = &Arg(5); + num_updates = Arg(4) / 2; + + if (is_not_flatmap(map)) { + Uint32 hx; + Eterm val; + + /* apparently the compiler does not emit is_map instructions, + * bad compiler */ + + if (is_not_hashmap(map)) + return THE_NON_VALUE; + + res = map; + E = p->stop; + while(num_updates--) { + /* assoc can't fail */ + GET_TERM(new_p[0], new_key); + GET_TERM(new_p[1], val); + hx = hashmap_make_hash(new_key); + + res = erts_hashmap_insert(p, hx, new_key, val, res, 0); + if (p->mbuf) { + Uint live = Arg(3); + reg[live] = res; + erts_garbage_collect(p, 0, reg, live+1); + res = reg[live]; + } + + E = p->stop; + + new_p += 2; + } + return res; } - old_mp = (map_t *) map_val(map); - num_old = map_get_size(old_mp); + old_mp = (flatmap_t *) flatmap_val(map); + num_old = flatmap_get_size(old_mp); /* * If the old map is empty, create a new map. @@ -6587,14 +6684,13 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) * update list are new). */ - num_updates = Arg(4) / 2; - need = 2*(num_old+num_updates) + 1 + MAP_HEADER_SIZE; + need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ; if (HeapWordsLeft(p) < need) { Uint live = Arg(3); reg[live] = map; erts_garbage_collect(p, need, reg, live+1); map = reg[live]; - old_mp = (map_t *)map_val(map); + old_mp = (flatmap_t *)flatmap_val(map); } /* @@ -6625,16 +6721,15 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) kp = p->htop + 1; /* Point to first key */ hp = kp + num_old + num_updates; - res = make_map(hp); - mp = (map_t *)hp; - hp += MAP_HEADER_SIZE; - mp->thing_word = MAP_HEADER; + res = make_flatmap(hp); + mp = (flatmap_t *)hp; + hp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; mp->keys = make_tuple(kp-1); - old_vals = map_get_values(old_mp); - old_keys = map_get_keys(old_mp); + old_vals = flatmap_get_values(old_mp); + old_keys = flatmap_get_keys(old_mp); - new_p = &Arg(5); GET_TERM(*new_p, new_key); n = num_updates; @@ -6720,8 +6815,19 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) n = kp - p->htop - 1; /* Actual number of keys/values */ *p->htop = make_arityval(n); + p->htop = hp; mp->size = n; - p->htop = hp; + + /* The expensive case, need to build a hashmap */ + if (n > MAP_SMALL_MAP_LIMIT) { + res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n); + if (p->mbuf) { + Uint live = Arg(3); + reg[live] = res; + erts_garbage_collect(p, 0, reg, live+1); + res = reg[live]; + } + } return res; } @@ -6736,7 +6842,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Uint i; Uint num_old; Uint need; - map_t *old_mp, *mp; + flatmap_t *old_mp, *mp; Eterm res; Eterm* hp; Eterm* E; @@ -6745,12 +6851,48 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) BeamInstr* new_p; Eterm new_key; - if (is_not_map(map)) { - return THE_NON_VALUE; + new_p = &Arg(5); + n = Arg(4) / 2; /* Number of values to be updated */ + ASSERT(n > 0); + + if (is_not_flatmap(map)) { + Uint32 hx; + Eterm val; + + /* apparently the compiler does not emit is_map instructions, + * bad compiler */ + + if (is_not_hashmap(map)) + return THE_NON_VALUE; + + res = map; + E = p->stop; + while(n--) { + /* assoc can't fail */ + GET_TERM(new_p[0], new_key); + GET_TERM(new_p[1], val); + hx = hashmap_make_hash(new_key); + + res = erts_hashmap_insert(p, hx, new_key, val, res, 1); + if (is_non_value(res)) + return res; + + if (p->mbuf) { + Uint live = Arg(3); + reg[live] = res; + erts_garbage_collect(p, 0, reg, live+1); + res = reg[live]; + } + + E = p->stop; + + new_p += 2; + } + return res; } - old_mp = (map_t *) map_val(map); - num_old = map_get_size(old_mp); + old_mp = (flatmap_t *) flatmap_val(map); + num_old = flatmap_get_size(old_mp); /* * If the old map is empty, create a new map. @@ -6764,13 +6906,13 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) * Allocate the exact heap space needed. */ - need = num_old + MAP_HEADER_SIZE; + need = num_old + MAP_HEADER_FLATMAP_SZ; if (HeapWordsLeft(p) < need) { Uint live = Arg(3); reg[live] = map; erts_garbage_collect(p, need, reg, live+1); map = reg[live]; - old_mp = (map_t *)map_val(map); + old_mp = (flatmap_t *)flatmap_val(map); } /* @@ -6780,23 +6922,20 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) hp = p->htop; E = p->stop; - old_vals = map_get_values(old_mp); - old_keys = map_get_keys(old_mp); + old_vals = flatmap_get_values(old_mp); + old_keys = flatmap_get_keys(old_mp); - res = make_map(hp); - mp = (map_t *)hp; - hp += MAP_HEADER_SIZE; - mp->thing_word = MAP_HEADER; + res = make_flatmap(hp); + mp = (flatmap_t *)hp; + hp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; mp->size = num_old; mp->keys = old_mp->keys; /* Get array of key/value pairs to be updated */ - new_p = &Arg(5); GET_TERM(*new_p, new_key); /* Update all values */ - n = Arg(4) / 2; /* Number of values to be updated */ - ASSERT(n > 0); for (i = 0; i < num_old; i++) { if (!EQ(*old_keys, new_key)) { /* Not same keys */ diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index fce710f723..02689e5b19 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -3727,7 +3727,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, op->a[j+size] = Fail; #ifdef DEBUG - for (i = 0; i < size; i++) { + for (i = 0; i < size - 1; i++) { ASSERT(op->a[i+3].val <= op->a[i+4].val); } #endif diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h index 904564a96b..7fc3933f3d 100644 --- a/erts/emulator/beam/benchmark.h +++ b/erts/emulator/beam/benchmark.h @@ -141,10 +141,10 @@ extern unsigned long long major_gc; /* (Assuming Solaris) */ -#define BM_TIMER_T hrtime_t -#define BM_START_TIMER(t) system_clock = sys_gethrtime() +#define BM_TIMER_T ErtsMonotonicTime +#define BM_START_TIMER(t) system_clock = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) #define BM_STOP_TIMER(t) do { \ - BM_TIMER_T tmp = (sys_gethrtime() - system_clock) - timer_time; \ + BM_TIMER_T tmp = (ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) - system_clock) - timer_time; \ t##_time += (tmp > 0 ? tmp : 0); \ } while(0) diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 49996e7f0b..cc20ec7440 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -42,16 +42,22 @@ #define ERTS_PTAB_WANT_BIF_IMPL__ #include "erl_ptab.h" #include "erl_bits.h" +#include "erl_bif_unique.h" -static Export* flush_monitor_message_trap = NULL; +static Export* flush_monitor_messages_trap = NULL; static Export* set_cpu_topology_trap = NULL; static Export* await_proc_exit_trap = NULL; static Export* await_port_send_result_trap = NULL; Export* erts_format_cpu_topology_trap = NULL; static Export dsend_continue_trap_export; +Export *erts_convert_time_unit_trap = NULL; + static Export *await_sched_wall_time_mod_trap; static erts_smp_atomic32_t sched_wall_time; +static erts_smp_mtx_t ports_snapshot_mtx; +erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */ + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* @@ -393,7 +399,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) return res; } -static int demonitor(Process *c_p, Eterm ref) +static int demonitor(Process *c_p, Eterm ref, Eterm *multip) { ErtsMonitor *mon = NULL; /* The monitor entry to delete */ Process *rp; /* Local target process */ @@ -417,65 +423,73 @@ static int demonitor(Process *c_p, Eterm ref) goto done; } - if (mon->type != MON_ORIGIN) { - res = ERTS_DEMONITOR_BADARG; - goto done; - } - to = mon->pid; - - if (is_atom(to)) { - /* Monitoring a name at node to */ - ASSERT(is_node_name_atom(to)); - dep = erts_sysname_to_connected_dist_entry(to); - ASSERT(dep != erts_this_dist_entry); - if (dep) - deref_de = 1; - } else { - ASSERT(is_pid(to)); - dep = pid_dist_entry(to); - } - if (dep != erts_this_dist_entry) { - res = remote_demonitor(c_p, dep, ref, to); - /* remote_demonitor() unlocks link lock on c_p */ - unlock_link = 0; - } - else { /* Local monitor */ - if (deref_de) { - deref_de = 0; - erts_deref_dist_entry(dep); + switch (mon->type) { + case MON_TIME_OFFSET: + *multip = am_true; + erts_demonitor_time_offset(ref); + res = ERTS_DEMONITOR_TRUE; + break; + case MON_ORIGIN: + to = mon->pid; + *multip = am_false; + if (is_atom(to)) { + /* Monitoring a name at node to */ + ASSERT(is_node_name_atom(to)); + dep = erts_sysname_to_connected_dist_entry(to); + ASSERT(dep != erts_this_dist_entry); + if (dep) + deref_de = 1; + } else { + ASSERT(is_pid(to)); + dep = pid_dist_entry(to); } - dep = NULL; - rp = erts_pid2proc_opt(c_p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, - to, - ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); + if (dep != erts_this_dist_entry) { + res = remote_demonitor(c_p, dep, ref, to); + /* remote_demonitor() unlocks link lock on c_p */ + unlock_link = 0; + } + else { /* Local monitor */ + if (deref_de) { + deref_de = 0; + erts_deref_dist_entry(dep); + } + dep = NULL; + rp = erts_pid2proc_opt(c_p, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, + to, + ERTS_PROC_LOCK_LINK, + ERTS_P2P_FLG_ALLOW_OTHER_X); + mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); #ifndef ERTS_SMP - ASSERT(mon); + ASSERT(mon); #else - if (!mon) - res = ERTS_DEMONITOR_FALSE; - else + if (!mon) + res = ERTS_DEMONITOR_FALSE; + else #endif - { - res = ERTS_DEMONITOR_TRUE; - erts_destroy_monitor(mon); - } - if (rp) { - ErtsMonitor *rmon; - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - if (rp != c_p) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (rmon != NULL) - erts_destroy_monitor(rmon); - } - else { - ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p); - } + { + res = ERTS_DEMONITOR_TRUE; + erts_destroy_monitor(mon); + } + if (rp) { + ErtsMonitor *rmon; + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); + if (rp != c_p) + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + if (rmon != NULL) + erts_destroy_monitor(rmon); + } + else { + ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p); + } + } + break; + default: + res = ERTS_DEMONITOR_BADARG; + *multip = am_false; + break; } - done: if (unlock_link) @@ -492,7 +506,8 @@ static int demonitor(Process *c_p, Eterm ref) BIF_RETTYPE demonitor_1(BIF_ALIST_1) { - switch (demonitor(BIF_P, BIF_ARG_1)) { + Eterm multi; + switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { case ERTS_DEMONITOR_FALSE: case ERTS_DEMONITOR_TRUE: BIF_RET(am_true); @@ -510,6 +525,7 @@ BIF_RETTYPE demonitor_1(BIF_ALIST_1) BIF_RETTYPE demonitor_2(BIF_ALIST_2) { Eterm res = am_true; + Eterm multi = am_false; int info = 0; int flush = 0; Eterm list = BIF_ARG_2; @@ -532,13 +548,18 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2) if (is_not_nil(list)) goto badarg; - switch (demonitor(BIF_P, BIF_ARG_1)) { + switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { case ERTS_DEMONITOR_FALSE: if (info) res = am_false; - if (flush) - BIF_TRAP2(flush_monitor_message_trap, BIF_P, BIF_ARG_1, res); + if (flush) { + flush_messages: + BIF_TRAP3(flush_monitor_messages_trap, BIF_P, + BIF_ARG_1, multi, res); + } case ERTS_DEMONITOR_TRUE: + if (multi == am_true && flush) + goto flush_messages; BIF_RET(res); case ERTS_DEMONITOR_YIELD_TRUE: ERTS_BIF_YIELD_RETURN(BIF_P, am_true); @@ -597,14 +618,12 @@ erts_queue_monitor_message(Process *p, } static BIF_RETTYPE -local_pid_monitor(Process *p, Eterm target) +local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int bool) { BIF_RETTYPE ret; - Eterm mon_ref; Process *rp; ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK; - mon_ref = erts_make_ref(p); ERTS_BIF_PREP_RET(ret, mon_ref); if (target == p->common.id) { return ret; @@ -617,12 +636,18 @@ local_pid_monitor(Process *p, Eterm target) if (!rp) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; - erts_queue_monitor_message(p, &p_locks, - mon_ref, am_process, target, am_noproc); + if (bool) + ret = am_false; + else + erts_queue_monitor_message(p, &p_locks, + mon_ref, am_process, target, am_noproc); } else { ASSERT(rp != p); + if (bool) + ret = am_true; + erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL); erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL); @@ -746,13 +771,28 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) int deref_de = 0; /* Only process monitors are implemented */ - if (BIF_ARG_1 != am_process) { + switch (BIF_ARG_1) { + case am_time_offset: { + Eterm ref; + if (BIF_ARG_2 != am_clock_service) + goto error; + ref = erts_make_ref(BIF_P); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET, + ref, am_clock_service, NIL); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_monitor_time_offset(BIF_P->common.id, ref); + BIF_RET(ref); + } + case am_process: + break; + default: goto error; } if (is_internal_pid(target)) { local_pid: - ret = local_pid_monitor(BIF_P, target); + ret = local_pid_monitor(BIF_P, target, erts_make_ref(BIF_P), 0); } else if (is_external_pid(target)) { dep = external_pid_dist_entry(target); if (dep == erts_this_dist_entry) @@ -795,6 +835,25 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) return ret; } +BIF_RETTYPE erts_internal_monitor_process_2(BIF_ALIST_2) +{ + if (is_not_internal_pid(BIF_ARG_1)) { + if (is_external_pid(BIF_ARG_1) + && (external_pid_dist_entry(BIF_ARG_1) + == erts_this_dist_entry)) { + BIF_RET(am_false); + } + goto badarg; + } + + if (is_not_internal_ref(BIF_ARG_2)) + goto badarg; + + BIF_RET(local_pid_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, 1)); + +badarg: + BIF_ERROR(BIF_P, BADARG); +} /**********************************************************************/ /* this is a combination of the spawn and link BIFs */ @@ -3542,91 +3601,6 @@ BIF_RETTYPE self_0(BIF_ALIST_0) /**********************************************************************/ -/* - New representation of refs in R9, see erl_term.h - - In the first data word, only the usual 18 bits are used. Ordinarily, - in "long refs" all words are used (in other words, practically never - wrap around), but for compatibility with older nodes, "short refs" - exist. Short refs come into being by being converted from the old - external format for refs (tag REFERENCE_EXT). Short refs are - converted back to the old external format. - - When converting a long ref to the external format in the case of - preparing for sending to an older node, the ref is truncated by only - using the first word (with 18 significant bits), and using the old tag - REFERENCE_EXT. - - When comparing refs or different size, only the parts up to the length - of the shorter operand are used. This has the desirable effect that a - long ref sent to an old node and back will be treated as equal to - the original, although some of the bits have been lost. - - The hash value for a ref always considers only the first word, since - in the above scenario, the original and the copy should have the same - hash value. -*/ - -static Uint32 reference0; /* Initialized in erts_init_bif */ -static Uint32 reference1; -static Uint32 reference2; -static erts_smp_spinlock_t make_ref_lock; -static erts_smp_mtx_t ports_snapshot_mtx; -erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */ - -void -erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) -{ - erts_smp_spin_lock(&make_ref_lock); - - reference0++; - if (reference0 >= MAX_REFERENCE) { - reference0 = 0; - reference1++; - if (reference1 == 0) { - reference2++; - } - } - - ref[0] = reference0; - ref[1] = reference1; - ref[2] = reference2; - - erts_smp_spin_unlock(&make_ref_lock); -} - -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) -{ - Eterm* hp = buffer; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; - - erts_make_ref_in_array(ref); - write_ref_thing(hp, ref[0], ref[1], ref[2]); - return make_internal_ref(hp); -} - -Eterm erts_make_ref(Process *p) -{ - Eterm* hp; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - - hp = HAlloc(p, REF_THING_SIZE); - - erts_make_ref_in_array(ref); - write_ref_thing(hp, ref[0], ref[1], ref[2]); - - return make_internal_ref(hp); -} - -BIF_RETTYPE make_ref_0(BIF_ALIST_0) -{ - return erts_make_ref(BIF_P); -} - -/**********************************************************************/ - /* return the time of day */ BIF_RETTYPE time_0(BIF_ALIST_0) @@ -4607,6 +4581,28 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) break; } #endif + } else if (BIF_ARG_1 == am_time_offset + && ERTS_IS_ATOM_STR("finalize", BIF_ARG_2)) { + ErtsTimeOffsetState res; + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + res = erts_finalize_time_offset(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + switch (res) { + case ERTS_TIME_OFFSET_PRELIMINARY: { + DECL_AM(preliminary); + BIF_RET(AM_preliminary); + } + case ERTS_TIME_OFFSET_FINAL: { + DECL_AM(final); + BIF_RET(AM_final); + } + case ERTS_TIME_OFFSET_VOLATILE: { + DECL_AM(volatile); + BIF_RET(AM_volatile); + } + default: + ERTS_INTERNAL_ERROR("Unknown state"); + } } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) { int what; if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2)) @@ -4758,7 +4754,7 @@ BIF_RETTYPE bump_reductions_1(BIF_ALIST_1) } BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) { - int res = CMP_TERM(BIF_ARG_1,BIF_ARG_2); + Sint res = CMP_TERM(BIF_ARG_1,BIF_ARG_2); /* ensure -1, 0, 1 result */ if (res < 0) { @@ -4894,11 +4890,6 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, void erts_init_bif(void) { - reference0 = 0; - reference1 = 0; - reference2 = 0; - - erts_smp_spinlock_init(&make_ref_lock, "make_ref"); erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL); @@ -4919,9 +4910,13 @@ void erts_init_bif(void) am_erts_internal, am_dsend_continue_trap, 1, dsend_continue_trap_1); - flush_monitor_message_trap = erts_export_put(am_erlang, - am_flush_monitor_message, - 2); + flush_monitor_messages_trap = erts_export_put(am_erts_internal, + am_flush_monitor_messages, + 3); + + erts_convert_time_unit_trap = erts_export_put(am_erlang, + am_convert_time_unit, + 3); set_cpu_topology_trap = erts_export_put(am_erlang, am_set_cpu_topology, diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 837cb017ac..d461c3f479 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -21,6 +21,7 @@ #define __BIF_H__ extern Export* erts_format_cpu_topology_trap; +extern Export *erts_convert_time_unit_trap; #define BIF_RETTYPE Eterm diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 788c866e63..471f687101 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -92,6 +92,8 @@ bif erlang:loaded/0 bif erlang:localtime/0 bif erlang:localtime_to_universaltime/2 bif erlang:make_ref/0 +bif erlang:unique_integer/0 +bif erlang:unique_integer/1 bif erlang:md5/1 bif erlang:md5_init/0 bif erlang:md5_update/2 @@ -104,6 +106,13 @@ ubif erlang:node/1 ubif erlang:node/0 bif erlang:nodes/1 bif erlang:now/0 +bif erlang:monotonic_time/0 +bif erlang:monotonic_time/1 +bif erlang:system_time/0 +bif erlang:system_time/1 +bif erlang:time_offset/0 +bif erlang:time_offset/1 +bif erlang:timestamp/0 bif erlang:open_port/2 @@ -157,6 +166,17 @@ bif erts_internal:request_system_task/3 bif erts_internal:check_process_code/2 bif erts_internal:map_to_tuple_keys/1 +bif erts_internal:map_type/1 +bif erts_internal:map_hashmap_children/1 + +bif erts_internal:time_unit/0 + +bif erts_internal:get_bif_timer_servers/0 +bif erts_internal:create_bif_timer/0 +bif erts_internal:access_bif_timer/1 + +bif erts_internal:monitor_process/2 +bif erts_internal:is_system_process/1 # inet_db support bif erlang:port_set_data/2 @@ -200,11 +220,6 @@ bif math:sqrt/1 bif math:atan2/2 bif math:pow/2 -bif erlang:start_timer/3 -bif erlang:send_after/3 -bif erlang:cancel_timer/1 -bif erlang:read_timer/1 - bif erlang:make_tuple/2 bif erlang:append_element/2 bif erlang:make_tuple/3 @@ -348,6 +363,8 @@ bif os:getenv/0 bif os:getenv/1 bif os:getpid/0 bif os:timestamp/0 +bif os:system_time/0 +bif os:system_time/1 # # Bifs in the erl_ddll module (the module actually does not exist) @@ -614,6 +631,7 @@ bif erlang:fun_info_mfa/1 bif erlang:get_keys/0 bif ets:update_counter/4 +bif erts_debug:map_info/1 # # Obsolete diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index d1e46e3063..a4ea9c59ca 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1577,6 +1577,46 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) return make_big(hp); } +Eterm +erts_uint64_array_to_big(Uint **hpp, int neg, int len, Uint64 *array) +{ + Uint *headerp; + int i, pot_digits, digits; + + headerp = *hpp; + + pot_digits = digits = 0; + for (i = 0; i < len; i++) { +#if defined(ARCH_32) || HALFWORD_HEAP + Uint low_val = array[i] & ((Uint) 0xffffffff); + Uint high_val = (array[i] >> 32) & ((Uint) 0xffffffff); + BIG_DIGIT(headerp, pot_digits) = low_val; + pot_digits++; + if (low_val) + digits = pot_digits; + BIG_DIGIT(headerp, pot_digits) = high_val; + pot_digits++; + if (high_val) + digits = pot_digits; +#else + Uint val = array[i]; + BIG_DIGIT(headerp, pot_digits) = val; + pot_digits++; + if (val) + digits = pot_digits; +#endif + } + + if (neg) + *headerp = make_neg_bignum_header(digits); + else + *headerp = make_pos_bignum_header(digits); + + *hpp = headerp + 1 + digits; + + return make_big(headerp); +} + /* ** Convert a bignum to a double float */ diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index da31876d75..4e4611de16 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -104,6 +104,9 @@ typedef Uint dsize_t; /* Vector size type */ : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(Uint64)(X))) #define ERTS_UINT64_HEAP_SIZE(X) \ (IS_USMALL(0, (X)) ? 0 : ERTS_UINT64_BIG_HEAP_SIZE__((X))) +#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 2) +#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 2) +#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) (2*(LEN)+1) #else @@ -111,6 +114,9 @@ typedef Uint dsize_t; /* Vector size type */ (IS_SSMALL((X)) ? 0 : (1 + 1)) #define ERTS_UINT64_HEAP_SIZE(X) \ (IS_USMALL(0, (X)) ? 0 : (1 + 1)) +#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 1) +#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 1) +#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) ((LEN)+1) #endif @@ -156,6 +162,7 @@ int term_to_Uint(Eterm, Uint*); int term_to_UWord(Eterm, UWord*); int term_to_Sint(Eterm, Sint*); #if HAVE_INT64 +Eterm erts_uint64_array_to_big(Uint **, int, int, Uint64 *); int term_to_Uint64(Eterm, Uint64*); int term_to_Sint64(Eterm, Sint64*); #endif diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 4ede2c9d7d..e2fa572546 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -661,7 +661,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) { #ifdef ERTS_SMP ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */ - int bc; #endif int fd; size_t envsz; @@ -681,7 +680,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) /* Order all managed threads to block, this has to be done first to guarantee that this is the only thread to generate crash dump. */ - bc = erts_thr_progress_fatal_error_block(&tpd_buf); + erts_thr_progress_fatal_error_block(&tpd_buf); #ifdef ERTS_THR_HAVE_SIG_FUNCS /* diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 0010f6a440..4d12dae787 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -127,6 +127,52 @@ Uint size_object(Eterm obj) obj = *bptr; break; } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : + { + Uint n; + flatmap_t *mp; + mp = (flatmap_t*)flatmap_val_rel(obj,base); + ptr = (Eterm *)mp; + n = flatmap_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + ESTACK_PUSH(s, obj); + } + } + goto pop_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + { + Eterm *head; + Uint sz; + head = hashmap_val_rel(obj, base); + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + sz + header_arity(hdr); + head += 1 + header_arity(hdr); + + if (sz == 0) { + goto pop_next; + } + while(sz-- > 1) { + obj = head[sz]; + if (!IS_CONST(obj)) { + ESTACK_PUSH(s, obj); + } + } + obj = head[0]; + } + break; + default: + erl_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + break; case SUB_BINARY_SUBTAG: { Eterm real_bin; @@ -154,25 +200,7 @@ Uint size_object(Eterm obj) goto pop_next; } break; - case MAP_SUBTAG: - { - Uint n; - map_t *mp; - mp = (map_t*)map_val_rel(obj,base); - ptr = (Eterm *)mp; - n = map_get_size(mp) + 1; - sum += n + 2; - ptr += 2; /* hdr + size words */ - while (n--) { - obj = *ptr++; - if (!IS_CONST(obj)) { - ESTACK_PUSH(s, obj); - } - } - goto pop_next; - } - break; - case BIN_MATCHSTATE_SUBTAG: + case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "size_object: matchstate term not allowed"); default: @@ -340,15 +368,6 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } break; - case MAP_SUBTAG: - { - i = map_get_size(objp) + 3; - *argp = make_map_rel(htop, dst_base); - while (i--) { - *htop++ = *objp++; - } - } - break; case REFC_BINARY_SUBTAG: { ProcBin* pb; @@ -459,7 +478,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) { ExternalThing *etp = (ExternalThing *) htop; - i = thing_arityval(hdr) + 1; + i = thing_arityval(hdr) + 1; tp = htop; while (i--) { @@ -473,6 +492,28 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *argp = make_external_rel(tp, dst_base); } break; + case MAP_SUBTAG: + tp = htop; + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : + i = flatmap_get_size(objp) + 3; + *argp = make_flatmap_rel(htop, dst_base); + while (i--) { + *htop++ = *objp++; + } + break; + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + *htop++ = *objp++; + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr)); + while (i--) { *htop++ = *objp++; } + *argp = make_hashmap_rel(tp, dst_base); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + break; case BIN_MATCHSTATE_SUBTAG: erl_exit(ERTS_ABORT_EXIT, "copy_struct: matchstate term not allowed"); diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 61def65235..e2f8da38b9 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -269,6 +269,7 @@ type BUSY_CALLER_TAB SHORT_LIVED SYSTEM busy_caller_table type BUSY_CALLER SHORT_LIVED SYSTEM busy_caller type PROC_SYS_TSK SHORT_LIVED PROCESSES proc_sys_task type PROC_SYS_TSK_QS SHORT_LIVED PROCESSES proc_sys_task_queues +type NEW_TIME_OFFSET SHORT_LIVED SYSTEM new_time_offset +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -364,6 +365,7 @@ type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD_LOW SYSTEM port_data_heap +type BIF_TIMER_DATA LONG_LIVED_LOW SYSTEM bif_timer_data +else # "fullword" @@ -384,6 +386,7 @@ type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap +type BIF_TIMER_DATA LONG_LIVED SYSTEM bif_timer_data +endif diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index bd0d7c71cc..934904d58e 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -38,6 +38,7 @@ #include "big.h" #include "erl_binary.h" #include "erl_bits.h" +#include "erl_bif_unique.h" /* diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 56cd2ba04f..fc4f819f56 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -45,6 +45,7 @@ #include "big.h" #include "dist.h" #include "erl_version.h" +#include "erl_bif_unique.h" #include "dtrace-wrapper.h" #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index bbd8aa31d9..e7d84ebda1 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -459,23 +459,25 @@ Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live) Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live) { Eterm arg = reg[live]; - if (is_map(arg)) { - map_t *mp = (map_t*)map_val(arg); - Uint size = map_get_size(mp); - if (IS_USMALL(0, size)) { - return make_small(size); - } else { - Eterm* hp; - if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) { - erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live); - } - hp = p->htop; - p->htop += BIG_UINT_HEAP_SIZE; - return uint_to_big(size, hp); - } + Eterm* hp; + Uint size; + if (is_flatmap(arg)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(arg); + size = flatmap_get_size(mp); + } else if (is_hashmap(arg)) { + size = hashmap_size(arg); } else { BIF_ERROR(p, BADARG); } + if (IS_USMALL(0, size)) { + return make_small(size); + } + if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) { + erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live); + } + hp = p->htop; + p->htop += BIG_UINT_HEAP_SIZE; + return uint_to_big(size, hp); } Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index d750e34be3..b2658a1fd6 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -42,6 +42,7 @@ #include "erl_cpu_topology.h" #include "erl_async.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #define ERTS_PTAB_WANT_DEBUG_FUNCS__ #include "erl_ptab.h" #ifdef HIPE @@ -540,6 +541,7 @@ pi_locks(Eterm info) switch (info) { case am_status: case am_priority: + case am_trap_exit: return ERTS_PROC_LOCK_STATUS; case am_links: case am_monitors: @@ -592,7 +594,7 @@ static Eterm pi_args[] = { am_min_bin_vheap_size, am_current_location, am_current_stacktrace, -}; +}; #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm))) @@ -2102,6 +2104,48 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_opt); #endif BIF_RET(res); + } else if (BIF_ARG_1 == am_time_offset) { + switch (erts_time_offset_state()) { + case ERTS_TIME_OFFSET_PRELIMINARY: { + ERTS_DECL_AM(preliminary); + BIF_RET(AM_preliminary); + } + case ERTS_TIME_OFFSET_FINAL: { + ERTS_DECL_AM(final); + BIF_RET(AM_final); + } + case ERTS_TIME_OFFSET_VOLATILE: { + ERTS_DECL_AM(volatile); + BIF_RET(AM_volatile); + } + default: + ERTS_INTERNAL_ERROR("Invalid time offset state"); + } + } else if (ERTS_IS_ATOM_STR("os_monotonic_time_source", BIF_ARG_1)) { + BIF_RET(erts_monotonic_time_source(BIF_P)); + } else if (ERTS_IS_ATOM_STR("os_system_time_source", BIF_ARG_1)) { + BIF_RET(erts_system_time_source(BIF_P)); + } else if (ERTS_IS_ATOM_STR("time_correction", BIF_ARG_1)) { + BIF_RET(erts_has_time_correction() ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("start_time", BIF_ARG_1)) { + BIF_RET(erts_get_monotonic_start_time(BIF_P)); + } else if (ERTS_IS_ATOM_STR("time_warp_mode", BIF_ARG_1)) { + switch (erts_time_warp_mode()) { + case ERTS_NO_TIME_WARP_MODE: { + ERTS_DECL_AM(no_time_warp); + BIF_RET(AM_no_time_warp); + } + case ERTS_SINGLE_TIME_WARP_MODE: { + ERTS_DECL_AM(single_time_warp); + BIF_RET(AM_single_time_warp); + } + case ERTS_MULTI_TIME_WARP_MODE: { + ERTS_DECL_AM(multi_time_warp); + BIF_RET(AM_multi_time_warp); + } + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + } } else if (BIF_ARG_1 == am_allocated_areas) { res = erts_allocated_areas(NULL, NULL, BIF_P); BIF_RET(res); @@ -2703,9 +2747,11 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(make_small(erts_db_get_max_tabs())); } else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { - BIF_RET(erts_disable_tolerant_timeofday - ? am_disabled - : am_enabled); + if (erts_has_time_correction() + && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { + BIF_RET(am_enabled); + } + BIF_RET(am_disabled); } else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) { BIF_RET(erts_eager_check_io ? am_true : am_false); @@ -3403,6 +3449,29 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) { BIF_RET(erts_mmap_debug_info(BIF_P)); } + else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { + BIF_RET(erts_debug_get_unique_monotonic_integer_state(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("min_unique_monotonic_integer", BIF_ARG_1)) { + Sint64 value = erts_get_min_unique_monotonic_integer(); + if (IS_SSMALL(value)) + BIF_RET(make_small(value)); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(value); + Eterm *hp = HAlloc(BIF_P, hsz); + BIF_RET(erts_sint64_to_big(value, &hp)); + } + } + else if (ERTS_IS_ATOM_STR("min_unique_integer", BIF_ARG_1)) { + Sint64 value = erts_get_min_unique_integer(); + if (IS_SSMALL(value)) + BIF_RET(make_small(value)); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(value); + Eterm *hp = HAlloc(BIF_P, hsz); + BIF_RET(erts_sint64_to_big(value, &hp)); + } + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); @@ -3597,6 +3666,58 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); } + else if (ERTS_IS_ATOM_STR("internal_hash", tp[1])) { + Uint hash = (Uint) make_internal_hash(tp[2]); + Uint hsz = 0; + Eterm* hp; + erts_bld_uint(NULL, &hsz, hash); + hp = HAlloc(BIF_P,hsz); + return erts_bld_uint(&hp, NULL, hash); + } + else if (ERTS_IS_ATOM_STR("atom", tp[1])) { + Uint ix; + if (!term_to_Uint(tp[2], &ix)) + BIF_ERROR(BIF_P, BADARG); + while (ix >= atom_table_size()) { + char tmp[20]; + erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size()); + erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); + } + return make_atom(ix); + } + + break; + } + case 3: { + if (ERTS_IS_ATOM_STR("check_time_config", tp[1])) { + int res, time_correction; + ErtsTimeWarpMode time_warp_mode; + if (tp[2] == am_true) + time_correction = !0; + else if (tp[2] == am_false) + time_correction = 0; + else + break; + if (ERTS_IS_ATOM_STR("no_time_warp", tp[3])) + time_warp_mode = ERTS_NO_TIME_WARP_MODE; + else if (ERTS_IS_ATOM_STR("single_time_warp", tp[3])) + time_warp_mode = ERTS_SINGLE_TIME_WARP_MODE; + else if (ERTS_IS_ATOM_STR("multi_time_warp", tp[3])) + time_warp_mode = ERTS_MULTI_TIME_WARP_MODE; + else + break; + res = erts_check_time_adj_support(time_correction, + time_warp_mode); + BIF_RET(res ? am_true : am_false); + } + else if (ERTS_IS_ATOM_STR("make_unique_integer", tp[1])) { + Eterm res = erts_debug_make_unique_integer(BIF_P, + tp[2], + tp[3]); + if (is_non_value(res)) + break; + BIF_RET(res); + } break; } default: @@ -3606,6 +3727,24 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1) +{ + if (is_internal_pid(BIF_ARG_1)) { + Process *rp = erts_proc_lookup(BIF_ARG_1); + if (rp && (rp->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) + BIF_RET(am_true); + BIF_RET(am_false); + } + + if (is_external_pid(BIF_ARG_1) + && external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) { + BIF_RET(am_false); + } + + BIF_ERROR(BIF_P, BADARG); +} + + static erts_smp_atomic_t hipe_test_reschedule_flag; @@ -3907,6 +4046,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) #endif erl_exit(ERTS_DUMP_EXIT, "%T", BIF_ARG_2); } + else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { + int res = erts_debug_set_unique_monotonic_integer_state(BIF_ARG_2); + BIF_RET(res ? am_true : am_false); + } } BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index 03ac97283c..0bd8d20c34 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -27,6 +27,7 @@ #include "error.h" #include "big.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" /**************************************************************************** ** BIF Timer support @@ -480,7 +481,7 @@ setup_bif_timer(Uint32 xflags, tab_insert(btm); ASSERT(btm == tab_find(ref)); - btm->tm.active = 0; /* MUST be initalized */ + erts_init_timer(&btm->tm); erts_set_timer(&btm->tm, (ErlTimeoutProc) bif_timer_timeout, (ErlCancelProc) bif_timer_cleanup, @@ -489,8 +490,9 @@ setup_bif_timer(Uint32 xflags, return ref; } +BIF_RETTYPE old_send_after_3(BIF_ALIST_3); /* send_after(Time, Pid, Message) -> Ref */ -BIF_RETTYPE send_after_3(BIF_ALIST_3) +BIF_RETTYPE old_send_after_3(BIF_ALIST_3) { Eterm res; @@ -510,8 +512,9 @@ BIF_RETTYPE send_after_3(BIF_ALIST_3) } } +BIF_RETTYPE old_start_timer_3(BIF_ALIST_3); /* start_timer(Time, Pid, Message) -> Ref */ -BIF_RETTYPE start_timer_3(BIF_ALIST_3) +BIF_RETTYPE old_start_timer_3(BIF_ALIST_3) { Eterm res; @@ -531,8 +534,9 @@ BIF_RETTYPE start_timer_3(BIF_ALIST_3) } } +BIF_RETTYPE old_cancel_timer_1(BIF_ALIST_1); /* cancel_timer(Ref) -> false | RemainingTime */ -BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) +BIF_RETTYPE old_cancel_timer_1(BIF_ALIST_1) { Eterm res; ErtsBifTimer *btm; @@ -569,8 +573,9 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) BIF_RET(res); } +BIF_RETTYPE old_read_timer_1(BIF_ALIST_1); /* read_timer(Ref) -> false | RemainingTime */ -BIF_RETTYPE read_timer_1(BIF_ALIST_1) +BIF_RETTYPE old_read_timer_1(BIF_ALIST_1) { Eterm res; ErtsBifTimer *btm; @@ -652,7 +657,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks) erts_smp_btm_rwunlock(); } -void erts_bif_timer_init(void) +static void erts_old_bif_timer_init(void) { int i; no_bif_timers = 0; @@ -703,3 +708,146 @@ erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *), } } } + +typedef struct { + Uint ref_heap[REF_THING_SIZE]; + Eterm pid[1]; +} ErtsBifTimerServers; + +static ErtsBifTimerServers *bif_timer_servers; + +void erts_bif_timer_init(void) +{ + erts_old_bif_timer_init(); +} + +void +erts_bif_timer_start_servers(Eterm parent) +{ + Process *parent_proc; + Eterm *hp, btr_ref, arg_list_end; + ErlSpawnOpts so; + int i; + + bif_timer_servers = erts_alloc(ERTS_ALC_T_BIF_TIMER_DATA, + (sizeof(ErtsBifTimerServers) + + (sizeof(Eterm)*(erts_no_schedulers-1)))); + + so.flags = SPO_USE_ARGS|SPO_SYSTEM_PROC|SPO_PREFER_SCHED|SPO_OFF_HEAP_MSGS; + so.min_heap_size = H_MIN_SIZE; + so.min_vheap_size = BIN_VH_MIN_SIZE; + so.priority = PRIORITY_MAX; + so.max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); + + /* + * Parent is "init" and schedulers have not yet been started, so it + * *should* be alive and well... + */ + ASSERT(is_internal_pid(parent)); + parent_proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(parent)); + ASSERT(parent_proc); + ASSERT(parent_proc->common.id == parent); + ASSERT(!ERTS_PROC_IS_EXITING(parent_proc)); + + erts_smp_proc_lock(parent_proc, ERTS_PROC_LOCK_MAIN); + + hp = HAlloc(parent_proc, 2*erts_no_schedulers + 2 + REF_THING_SIZE); + + btr_ref = erts_make_ref_in_buffer(hp); + hp += REF_THING_SIZE; + + arg_list_end = CONS(hp, btr_ref, NIL); + hp += 2; + + for (i = 0; i < erts_no_schedulers; i++) { + int sched = i+1; + Eterm arg_list = CONS(hp, make_small(i+1), arg_list_end); + hp += 2; + + so.scheduler = sched; /* Preferred scheduler */ + + bif_timer_servers->pid[i] = erl_create_process(parent_proc, + am_erts_internal, + am_bif_timer_server, + arg_list, + &so); + } + + erts_smp_proc_unlock(parent_proc, ERTS_PROC_LOCK_MAIN); + + hp = internal_ref_val(btr_ref); + for (i = 0; i < REF_THING_SIZE; i++) + bif_timer_servers->ref_heap[i] = hp[i]; +} + +BIF_RETTYPE +erts_internal_get_bif_timer_servers_0(BIF_ALIST_0) +{ + int i; + Eterm *hp, res = NIL; + + hp = HAlloc(BIF_P, erts_no_schedulers*2); + for (i = erts_no_schedulers-1; i >= 0; i--) { + res = CONS(hp, bif_timer_servers->pid[i], res); + hp += 2; + } + BIF_RET(res); +} + +BIF_RETTYPE +erts_internal_access_bif_timer_1(BIF_ALIST_1) +{ + int ix; + Uint32 *rdp; + Eterm ref, pid, *hp, res; + + if (is_not_internal_ref(BIF_ARG_1)) { + if (is_not_ref(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + BIF_RET(am_undefined); + } + + rdp = internal_ref_numbers(BIF_ARG_1); + ix = (int) erts_get_ref_numbers_thr_id(rdp); + if (ix < 1 || erts_no_schedulers < ix) + BIF_RET(am_undefined); + + pid = bif_timer_servers->pid[ix-1]; + ASSERT(is_internal_pid(pid)); + + hp = HAlloc(BIF_P, 3 /* 2-tuple */ + REF_THING_SIZE); + for (ix = 0; ix < REF_THING_SIZE; ix++) + hp[ix] = bif_timer_servers->ref_heap[ix]; + ref = make_internal_ref(&hp[0]); + hp += REF_THING_SIZE; + + res = TUPLE2(hp, ref, pid); + BIF_RET(res); +} + +BIF_RETTYPE +erts_internal_create_bif_timer_0(BIF_ALIST_0) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(BIF_P); + Eterm *hp, btr_ref, t_ref, pid, res; + int ix; + + hp = HAlloc(BIF_P, 4 /* 3-tuple */ + 2*REF_THING_SIZE); + for (ix = 0; ix < REF_THING_SIZE; ix++) + hp[ix] = bif_timer_servers->ref_heap[ix]; + btr_ref = make_internal_ref(&hp[0]); + hp += REF_THING_SIZE; + + t_ref = erts_sched_make_ref_in_buffer(esdp, hp); + hp += REF_THING_SIZE; + + ASSERT(erts_get_ref_numbers_thr_id(internal_ref_numbers(t_ref)) + == (Uint32) esdp->no); + + pid = bif_timer_servers->pid[((int) esdp->no) - 1]; + + res = TUPLE3(hp, btr_ref, pid, t_ref); + + BIF_RET(res); +} diff --git a/erts/emulator/beam/erl_bif_timer.h b/erts/emulator/beam/erl_bif_timer.h index 1197c176f5..c2f5dfd3c3 100644 --- a/erts/emulator/beam/erl_bif_timer.h +++ b/erts/emulator/beam/erl_bif_timer.h @@ -33,4 +33,5 @@ void erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks); void erts_bif_timer_init(void); void erts_bif_timer_foreach(void (*func)(Eterm,Eterm,ErlHeapFragment *,void *), void *arg); +void erts_bif_timer_start_servers(Eterm); #endif diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index f5e582b1c5..ac57205c47 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -38,6 +38,7 @@ #include "beam_bp.h" #include "erl_binary.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c new file mode 100644 index 0000000000..57b0bab72f --- /dev/null +++ b/erts/emulator/beam/erl_bif_unique.c @@ -0,0 +1,556 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "erl_alloc.h" +#include "export.h" +#include "bif.h" +#include "erl_bif_unique.h" + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Reference * +\* */ + +static union { + erts_atomic64_t count; + char align__[ERTS_CACHE_LINE_SIZE]; +} global_reference erts_align_attribute(ERTS_CACHE_LINE_SIZE); + + +/* + * ref[0] indicate thread creating reference as follows: + * + * - ref[0] == 0 => Non-scheduler thread; + * - else; ref[0] <= erts_no_schedulers => + * ordinary scheduler with id == ref[0]; + * - else; ref[0] <= erts_no_schedulers + * + erts_no_dirty_cpu_schedulers => + * dirty cpu scheduler with id == 'ref[0] - erts_no_schedulers'; + * - else => + * dirty io scheduler with id == 'ref[0] + * - erts_no_schedulers + * - erts_no_dirty_cpu_schedulers' + */ + +#ifdef DEBUG +static Uint32 max_thr_id; +#endif + +static void +init_reference(void) +{ +#ifdef DEBUG + max_thr_id = (Uint32) erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + max_thr_id += (Uint32) erts_no_dirty_cpu_schedulers; + max_thr_id += (Uint32) erts_no_dirty_io_schedulers; +#endif +#endif + erts_atomic64_init_nob(&global_reference.count, 0); +} + +static ERTS_INLINE void +global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + Uint64 value; + + value = (Uint64) erts_atomic64_inc_read_mb(&global_reference.count); + + erts_set_ref_numbers(ref, thr_id, value); +} + +static ERTS_INLINE void +make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + erts_sched_make_ref_in_array(esdp, ref); + else + global_make_ref_in_array(0, ref); +} + +void +erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + make_ref_in_array(ref); +} + +Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) +{ + Eterm* hp = buffer; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + make_ref_in_array(ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + return make_internal_ref(hp); +} + +Eterm erts_make_ref(Process *c_p) +{ + Eterm* hp; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + + hp = HAlloc(c_p, REF_THING_SIZE); + + make_ref_in_array(ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + + return make_internal_ref(hp); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Unique Integer * +\* */ + +static struct { + union { + struct { + int left_shift; + int right_shift; + Uint64 mask; + Uint64 val0_max; + } o; + char align__[ERTS_CACHE_LINE_SIZE]; + } r; + union { + erts_atomic64_t val1; + char align__[ERTS_CACHE_LINE_SIZE]; + } w; +} unique_data erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static void +init_unique_integer(void) +{ + int bits; + unique_data.r.o.val0_max = (Uint64) erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + unique_data.r.o.val0_max += (Uint64) erts_no_dirty_cpu_schedulers; + unique_data.r.o.val0_max += (Uint64) erts_no_dirty_io_schedulers; +#endif + bits = erts_fit_in_bits_int64(unique_data.r.o.val0_max); + unique_data.r.o.left_shift = bits; + unique_data.r.o.right_shift = 64 - bits; + unique_data.r.o.mask = (((Uint64) 1) << bits) - 1; + erts_atomic64_init_nob(&unique_data.w.val1, -1); +} + +#define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2) + +static ERTS_INLINE Eterm +bld_unique_integer_term(Eterm **hpp, Uint *szp, + Uint64 val0, Uint64 val1, + int positive) +{ + Uint hsz; + Uint64 unique_val[2]; + + unique_val[0] = ((Uint64) val0); + unique_val[0] |= ((Uint64) val1) << unique_data.r.o.left_shift; + unique_val[1] = ((Uint64) val1) >> unique_data.r.o.right_shift; + unique_val[1] &= unique_data.r.o.mask; + + if (positive) { + unique_val[0]++; + if (unique_val[0] == 0) + unique_val[1]++; + } + else { + ASSERT(MIN_SMALL < 0); + if (unique_val[1] == 0 + && unique_val[0] < ((Uint64) -1*((Sint64) MIN_SMALL))) { + Sint64 s_unique_val = (Sint64) unique_val[0]; + s_unique_val += MIN_SMALL; + ASSERT(MIN_SMALL <= s_unique_val && s_unique_val < 0); + if (szp) + *szp = 0; + if (!hpp) + return THE_NON_VALUE; + return make_small((Sint) s_unique_val); + } + if (unique_val[0] < ((Uint64) -1*((Sint64) MIN_SMALL))) { + ASSERT(unique_val[1] != 0); + unique_val[1] -= 1; + } + unique_val[0] += MIN_SMALL; + } + + if (!unique_val[1]) { + if (unique_val[0] <= MAX_SMALL) { + if (szp) + *szp = 0; + if (!hpp) + return THE_NON_VALUE; + return make_small((Uint) unique_val[0]); + } + + if (szp) + *szp = ERTS_UINT64_HEAP_SIZE(unique_val[0]); + if (!hpp) + return THE_NON_VALUE; + return erts_uint64_to_big(unique_val[0], hpp); + } + else { + Eterm tmp, *tmp_hp, res; + DeclareTmpHeapNoproc(local_heap, 2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + UseTmpHeapNoproc(2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + tmp_hp = local_heap; + + tmp = erts_uint64_array_to_big(&tmp_hp, 0, 2, unique_val); + ASSERT(is_big(tmp)); + + hsz = big_arity(tmp) + 1; + + ASSERT(hsz <= ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + if (szp) + *szp = hsz; + + if (!hpp) + res = THE_NON_VALUE; + else { + int hix; + Eterm *hp = *hpp; + tmp_hp = big_val(tmp); + for (hix = 0; hix < hsz; hix++) + hp[hix] = tmp_hp[hix]; + + *hpp = hp + hsz; + res = make_big(hp); + } + + UnUseTmpHeapNoproc(2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + return res; + } +} + +static ERTS_INLINE Eterm unique_integer_bif(Process *c_p, int positive) +{ + ErtsSchedulerData *esdp; + Uint64 thr_id, unique; + Uint hsz; + Eterm *hp; + + esdp = ERTS_PROC_GET_SCHDATA(c_p); + thr_id = (Uint64) esdp->thr_id; + unique = esdp->unique++; + bld_unique_integer_term(NULL, &hsz, thr_id, unique, positive); + hp = hsz ? HAlloc(c_p, hsz) : NULL; + return bld_unique_integer_term(&hp, NULL, thr_id, unique, positive); +} + +Uint +erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +{ + Uint sz; + bld_unique_integer_term(NULL, &sz, val[0], val[1], 0); + return sz; +} + +Eterm +erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +{ + return bld_unique_integer_term(hpp, NULL, val[0], val[1], 0); +} + +void +erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) { + val[0] = (Uint64) esdp->thr_id; + val[1] = esdp->unique++; + } + else { + val[0] = (Uint64) 0; + val[1] = (Uint64) erts_atomic64_inc_read_nob(&unique_data.w.val1); + } +} + + +Sint64 +erts_get_min_unique_integer(void) +{ + return (Sint64) MIN_SMALL; +} + +/* --- Debug --- */ + +Eterm +erts_debug_make_unique_integer(Process *c_p, Eterm etval0, Eterm etval1) +{ + Uint64 val0, val1; + Uint hsz; + Eterm res, *hp, *end_hp; + + if (!term_to_Uint64(etval0, &val0)) + return THE_NON_VALUE; + + if (!term_to_Uint64(etval1, &val1)) + return THE_NON_VALUE; + + bld_unique_integer_term(NULL, &hsz, val0, val1, 0); + + hp = HAlloc(c_p, hsz); + end_hp = hp + hsz; + + res = bld_unique_integer_term(&hp, NULL, val0, val1, 0); + if (hp != end_hp) + ERTS_INTERNAL_ERROR("Heap allocation error"); + + return res; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Strict Monotonic Counter * +\* */ + +static struct { + union { + erts_atomic64_t value; + char align__[ERTS_CACHE_LINE_SIZE]; + } w; +} raw_unique_monotonic_integer erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#if defined(ARCH_32) || HALFWORD_HEAP +# define ERTS_UNIQUE_MONOTONIC_OFFSET ERTS_SINT64_MIN +#else +# define ERTS_UNIQUE_MONOTONIC_OFFSET MIN_SMALL +#endif + +static void +init_unique_monotonic_integer(void) +{ + erts_atomic64_init_nob(&raw_unique_monotonic_integer.w.value, + (erts_aint64_t) -1); +} + +static ERTS_INLINE Uint64 +get_raw_unique_monotonic_integer(void) +{ + return (Uint64) erts_atomic64_inc_read_mb(&raw_unique_monotonic_integer.w.value); +} + +static ERTS_INLINE Uint +get_unique_monotonic_integer_heap_size(Uint64 raw, int positive) +{ + if (positive) { + Uint64 value = raw+1; + return ERTS_UINT64_HEAP_SIZE(value); + } + else { + Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET; + if (IS_SSMALL(value)) + return 0; +#if defined(ARCH_32) || HALFWORD_HEAP + return ERTS_SINT64_HEAP_SIZE(value); +#else + return ERTS_UINT64_HEAP_SIZE((Uint64) value); +#endif + } +} + +static ERTS_INLINE Eterm +make_unique_monotonic_integer_value(Eterm *hp, Uint hsz, Uint64 raw, int positive) +{ + Eterm res; +#ifdef DEBUG + Eterm *end_hp = hp + hsz; +#endif + + if (positive) { + Uint64 value = raw+1; + res = hsz ? erts_uint64_to_big(value, &hp) : make_small(value); + } + else { + Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET; + if (hsz == 0) + res = make_small(value); + else { +#if defined(ARCH_32) || HALFWORD_HEAP + res = erts_sint64_to_big(value, &hp); +#else + res = erts_uint64_to_big((Uint64) value, &hp); +#endif + } + } + + ASSERT(end_hp == hp); + + return res; +} + +static ERTS_INLINE Eterm +unique_monotonic_integer_bif(Process *c_p, int positive) +{ + Uint64 raw; + Uint hsz; + Eterm *hp; + + raw = get_raw_unique_monotonic_integer(); + hsz = get_unique_monotonic_integer_heap_size(raw, positive); + hp = hsz ? HAlloc(c_p, hsz) : NULL; + return make_unique_monotonic_integer_value(hp, hsz, raw, positive); +} + +Sint64 +erts_raw_get_unique_monotonic_integer(void) +{ + return get_raw_unique_monotonic_integer(); +} + +Uint +erts_raw_unique_monotonic_integer_heap_size(Sint64 raw) +{ + return get_unique_monotonic_integer_heap_size(raw, 0); +} + +Eterm +erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw) +{ + Uint hsz = get_unique_monotonic_integer_heap_size(raw, 0); + Eterm res = make_unique_monotonic_integer_value(*hpp, hsz, raw, 0); + *hpp += hsz; + return res; +} + +Sint64 +erts_get_min_unique_monotonic_integer(void) +{ + return ERTS_UNIQUE_MONOTONIC_OFFSET; +} + +/* --- Debug --- */ + +int +erts_debug_set_unique_monotonic_integer_state(Eterm et_value) +{ + Sint64 value; + + if (!term_to_Sint64(et_value, &value)) { + Uint64 uvalue; + if (!term_to_Uint64(et_value, &uvalue)) + return 0; + value = (Sint64) uvalue; + } + + erts_atomic64_set_mb(&raw_unique_monotonic_integer.w.value, + (erts_aint64_t) value); + return 1; +} + +Eterm +erts_debug_get_unique_monotonic_integer_state(Process *c_p) +{ + Uint64 value; + Eterm hsz, *hp; + + value = (Uint64) erts_atomic64_read_mb(&raw_unique_monotonic_integer.w.value); + + if (IS_USMALL(0, value)) + return make_small(value); + hsz = ERTS_UINT64_HEAP_SIZE(value); + hp = HAlloc(c_p, hsz); + return erts_uint64_to_big(value, &hp); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Initilazation * +\* */ + +void +erts_bif_unique_init(void) +{ + init_reference(); + init_unique_monotonic_integer(); + init_unique_integer(); +} + +void +erts_sched_bif_unique_init(ErtsSchedulerData *esdp) +{ + esdp->unique = (Uint64) 0; + esdp->ref = (Uint64) 0; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * The BIFs * +\* */ + + +BIF_RETTYPE make_ref_0(BIF_ALIST_0) +{ + BIF_RETTYPE res; + Eterm* hp; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); + + hp = HAlloc(BIF_P, REF_THING_SIZE); + + res = erts_sched_make_ref_in_buffer(ERTS_PROC_GET_SCHDATA(BIF_P), hp); + + BIF_RET(res); +} + +BIF_RETTYPE unique_integer_0(BIF_ALIST_0) +{ + BIF_RET(unique_integer_bif(BIF_P, 0)); +} + +BIF_RETTYPE unique_integer_1(BIF_ALIST_1) +{ + Eterm modlist = BIF_ARG_1; + int monotonic = 0; + int positive = 0; + BIF_RETTYPE res; + + while (is_list(modlist)) { + Eterm *consp = list_val(modlist); + switch (CAR(consp)) { + case am_monotonic: + monotonic = 1; + break; + case am_positive: + positive = 1; + break; + default: + BIF_ERROR(BIF_P, BADARG); + } + modlist = CDR(consp); + } + + if (is_not_nil(modlist)) + BIF_ERROR(BIF_P, BADARG); + + if (monotonic) + res = unique_monotonic_integer_bif(BIF_P, positive); + else + res = unique_integer_bif(BIF_P, positive); + + BIF_RET(res); +} diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h new file mode 100644 index 0000000000..cd001172a1 --- /dev/null +++ b/erts/emulator/beam/erl_bif_unique.h @@ -0,0 +1,131 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERTS_BIF_UNIQUE_H__ +#define ERTS_BIF_UNIQUE_H__ + +#include "erl_process.h" +#include "big.h" + +void erts_bif_unique_init(void); +void erts_sched_bif_unique_init(ErtsSchedulerData *esdp); + +/* reference */ +Eterm erts_make_ref(Process *); +Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]); +void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); + +/* strict monotonic counter */ + +#define ERTS_MAX_UNIQUE_MONOTONIC_INTEGER_HEAP_SIZE ERTS_MAX_UINT64_HEAP_SIZE + +/* + * Note that a raw value is an intermediate value that + * not necessarily correspond to the end result. + */ +Sint64 erts_raw_get_unique_monotonic_integer(void); +Uint erts_raw_unique_monotonic_integer_heap_size(Sint64 raw); +Eterm erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw); + +Sint64 erts_get_min_unique_monotonic_integer(void); + +int erts_debug_set_unique_monotonic_integer_state(Eterm et_value); +Eterm erts_debug_get_unique_monotonic_integer_state(Process *c_p); + +/* unique integer */ +#define ERTS_UNIQUE_INT_RAW_VALUES 2 +#define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2) + +Uint erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); +Eterm erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); +void erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); +Sint64 erts_get_min_unique_integer(void); + +Eterm erts_debug_make_unique_integer(Process *c_p, + Eterm etval0, + Eterm etval1); + + +ERTS_GLB_INLINE void erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], + Uint32 thr_id, Uint64 value); +ERTS_GLB_INLINE Uint32 erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE Uint64 erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE void erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, + Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE Eterm erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp, + Eterm buffer[REF_THING_SIZE]); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], Uint32 thr_id, Uint64 value) +{ + /* + * We cannot use thread id in the first 18-bit word since + * the hash/phash/phash2 BIFs only hash on this word. If + * we did, we would get really poor hash values. Instead + * we have to shuffle the bits a bit. + */ + ASSERT(thr_id == (thr_id & ((Uint32) 0x3ffff))); + ref[0] = (Uint32) (value & ((Uint64) 0x3ffff)); + ref[1] = (((Uint32) (value & ((Uint64) 0xfffc0000))) + | (thr_id & ((Uint32) 0x3ffff))); + ref[2] = (Uint32) ((value >> 32) & ((Uint64) 0xffffffff)); +} + +ERTS_GLB_INLINE Uint32 +erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + return ref[1] & ((Uint32) 0x3ffff); +} + +ERTS_GLB_INLINE Uint64 +erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + return (((((Uint64) ref[2]) & ((Uint64) 0xffffffff)) << 32) + | (((Uint64) ref[1]) & ((Uint64) 0xfffc0000)) + | (((Uint64) ref[0]) & ((Uint64) 0x3ffff))); +} + +ERTS_GLB_INLINE void +erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, + Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + Uint64 value; + + ASSERT(esdp); + value = esdp->ref++; + erts_set_ref_numbers(ref, (Uint32) esdp->thr_id, value); +} + +ERTS_GLB_INLINE Eterm +erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp, + Eterm buffer[REF_THING_SIZE]) +{ + Eterm* hp = buffer; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + erts_sched_make_ref_in_array(esdp, ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + return make_internal_ref(hp); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_BIF_UNIQUE_H__ */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 8668a87ba1..045c8ae135 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -174,7 +174,7 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) /* optimised version of make_hash (normal case? atomic key) */ #define MAKE_HASH(term) \ ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \ - make_hash2(term)) % MAX_HASH) + make_internal_hash(term)) % MAX_HASH) #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 60da35da56..0bf562d937 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -231,7 +231,8 @@ typedef enum { matchConsA, /* Car is below Cdr */ matchConsB, /* Cdr is below Car (unusual) */ matchMkTuple, - matchMkMap, + matchMkFlatMap, + matchMkHashMap, matchCall0, matchCall1, matchCall2, @@ -1376,15 +1377,15 @@ restart: for (;;) { switch (t & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_BOXED: - if (is_map(t)) { - num_iters = map_get_size(map_val(t)); + if (is_flatmap(t)) { + num_iters = flatmap_get_size(flatmap_val(t)); if (!structure_checked) { DMC_PUSH(text, matchMap); DMC_PUSH(text, num_iters); } structure_checked = 0; for (i = 0; i < num_iters; ++i) { - Eterm key = map_get_keys(map_val(t))[i]; + Eterm key = flatmap_get_keys(flatmap_val(t))[i]; if (db_is_variable(key) >= 0) { if (context.err_info) { add_dmc_err(context.err_info, @@ -1404,7 +1405,7 @@ restart: DMC_PUSH(text, dmc_private_copy(&context, key)); { int old_stack = ++(context.stack_used); - Eterm value = map_get_values(map_val(t))[i]; + Eterm value = flatmap_get_values(flatmap_val(t))[i]; res = dmc_one_term(&context, &heap, &stack, &text, value); ASSERT(res != retFail); @@ -1424,6 +1425,63 @@ restart: } break; } + if (is_hashmap(t)) { + DECLARE_WSTACK(wstack); + Eterm *kv; + num_iters = hashmap_size(t); + if (!structure_checked) { + DMC_PUSH(text, matchMap); + DMC_PUSH(text, num_iters); + } + structure_checked = 0; + + hashmap_iterator_init(&wstack, t, 0); + + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + Eterm key = CAR(kv); + Eterm value = CDR(kv); + if (db_is_variable(key) >= 0) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Variable found in map key.", + -1, 0UL, dmcError); + } + DESTROY_WSTACK(wstack); + goto error; + } else if (key == am_Underscore) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Underscore found in map key.", + -1, 0UL, dmcError); + } + DESTROY_WSTACK(wstack); + goto error; + } + DMC_PUSH(text, matchKey); + DMC_PUSH(text, dmc_private_copy(&context, key)); + { + int old_stack = ++(context.stack_used); + res = dmc_one_term(&context, &heap, &stack, &text, + value); + ASSERT(res != retFail); + if (res == retRestart) { + DESTROY_WSTACK(wstack); + goto restart; + } + if (old_stack != context.stack_used) { + ASSERT(old_stack + 1 == context.stack_used); + DMC_PUSH(text, matchSwap); + } + if (context.stack_used > context.stack_need) { + context.stack_need = context.stack_used; + } + DMC_PUSH(text, matchPop); + --(context.stack_used); + } + } + DESTROY_WSTACK(wstack); + break; + } if (!is_tuple(t)) { goto simple_term; } @@ -1950,24 +2008,38 @@ restart: FAIL(); } n = *pc++; - if (map_get_size(map_val_rel(*ep, base)) < n) { - FAIL(); - } - ep = map_val_rel(*ep, base); + if (is_flatmap_rel(*ep,base)) { + if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) { + FAIL(); + } + } else { + ASSERT(is_hashmap_rel(*ep,base)); + if (hashmap_size_rel(*ep, base) < n) { + FAIL(); + } + } + ep = flatmap_val_rel(*ep, base); break; case matchPushM: if (!is_map_rel(*ep, base)) { FAIL(); } n = *pc++; - if (map_get_size(map_val_rel(*ep, base)) < n) { - FAIL(); - } - *sp++ = map_val_rel(*ep++, base); + if (is_flatmap_rel(*ep,base)) { + if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) { + FAIL(); + } + } else { + ASSERT(is_hashmap_rel(*ep,base)); + if (hashmap_size_rel(*ep, base) < n) { + FAIL(); + } + } + *sp++ = flatmap_val_rel(*ep++, base); break; case matchKey: t = (Eterm) *pc++; - tp = erts_maps_get_rel(t, make_map_rel(ep, base), base); + tp = erts_maps_get_rel(t, make_flatmap_rel(ep, base), base); if (!tp) { FAIL(); } @@ -2079,23 +2151,38 @@ restart: } *esp++ = t; break; - case matchMkMap: + case matchMkFlatMap: n = *pc++; - ehp = HAllocX(build_proc, 1 + MAP_HEADER_SIZE + n, HEAP_XTRA); + ehp = HAllocX(build_proc, 1 + MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA); t = *ehp++ = *--esp; { - map_t *m = (map_t *)ehp; - m->thing_word = MAP_HEADER; + flatmap_t *m = (flatmap_t *)ehp; + m->thing_word = MAP_HEADER_FLATMAP; m->size = n; m->keys = t; } - t = make_map(ehp); - ehp += MAP_HEADER_SIZE; + t = make_flatmap(ehp); + ehp += MAP_HEADER_FLATMAP_SZ; while (n--) { *ehp++ = *--esp; } *esp++ = t; break; + case matchMkHashMap: + n = *pc++; + esp -= 2*n; + ehp = HAllocX(build_proc, 2*n, HEAP_XTRA); + { + ErtsHeapFactory factory; + Uint ix; + factory.p = build_proc; + for (ix = 0; ix < 2*n; ix++){ + ehp[ix] = esp[ix]; + } + t = erts_hashmap_from_array(&factory, ehp, n, 0); + } + *esp++ = t; + break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; t = (*bif)(build_proc, bif_args); @@ -3286,10 +3373,10 @@ int db_has_variable(Eterm node) { while(arity--) { ESTACK_PUSH(s,*(++tuple)); } - } else if (is_map(node)) { - Eterm *values = map_get_values(map_val(node)); - int size = map_get_size(map_val(node)); - ESTACK_PUSH(s, ((map_t *) map_val(node))->keys); + } else if (is_flatmap(node)) { + Eterm *values = flatmap_get_values(flatmap_val(node)); + Uint size = flatmap_get_size(flatmap_val(node)); + ESTACK_PUSH(s, ((flatmap_t *) flatmap_val(node))->keys); while (size--) { ESTACK_PUSH(s, *(values++)); } @@ -3366,7 +3453,6 @@ static DMCRet dmc_one_term(DMCContext *context, Uint sz, sz2, sz3; Uint i, j; - switch (c & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_IMMED1: if ((n = db_is_variable(c)) >= 0) { /* variable */ @@ -3454,7 +3540,10 @@ static DMCRet dmc_one_term(DMCContext *context, DMC_PUSH(*stack, c); break; case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): - n = map_get_size(map_val(c)); + if (is_flatmap(c)) + n = flatmap_get_size(flatmap_val(c)); + else + n = hashmap_size(c); DMC_PUSH(*text, matchPushM); ++(context->stack_used); DMC_PUSH(*text, n); @@ -3745,30 +3834,87 @@ static DMCRet dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { - map_t *m = (map_t *)map_val(t); - Eterm *values = map_get_values(m); - int nelems = map_get_size(m); + int nelems; int constant_values; DMCRet ret; + if (is_flatmap(t)) { + flatmap_t *m = (flatmap_t *)flatmap_val(t); + Eterm *values = flatmap_get_values(m); - ret = dmc_array(context, heap, text, values, nelems, &constant_values); - if (ret != retOk) { - return ret; - } - if (constant_values) { - *constant = 1; + nelems = flatmap_get_size(m); + ret = dmc_array(context, heap, text, values, nelems, &constant_values); + + if (ret != retOk) { + return ret; + } + if (constant_values) { + *constant = 1; + return retOk; + } + DMC_PUSH(*text, matchPushC); + DMC_PUSH(*text, dmc_private_copy(context, m->keys)); + if (++context->stack_used > context->stack_need) { + context->stack_need = context->stack_used; + } + DMC_PUSH(*text, matchMkFlatMap); + DMC_PUSH(*text, nelems); + context->stack_used -= nelems; + *constant = 0; + return retOk; + } else { + DECLARE_WSTACK(wstack); + Eterm *kv; + int c; + + ASSERT(is_hashmap(t)); + + hashmap_iterator_init(&wstack, t, 1); + constant_values = 1; + nelems = hashmap_size(t); + + while ((kv=hashmap_iterator_prev(&wstack)) != NULL) { + if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (!c) + constant_values = 0; + } + + if (constant_values) { + *constant = 1; + DESTROY_WSTACK(wstack); + return retOk; + } + + *constant = 0; + + hashmap_iterator_init(&wstack, t, 1); + + while ((kv=hashmap_iterator_prev(&wstack)) != NULL) { + /* push key */ + if ((ret = dmc_expr(context, heap, text, CAR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (c) { + do_emit_constant(context, text, CAR(kv)); + } + /* push value */ + if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (c) { + do_emit_constant(context, text, CDR(kv)); + } + } + DMC_PUSH(*text, matchMkHashMap); + DMC_PUSH(*text, nelems); + context->stack_used -= nelems; + DESTROY_WSTACK(wstack); return retOk; } - DMC_PUSH(*text, matchPushC); - DMC_PUSH(*text, dmc_private_copy(context, m->keys)); - if (++context->stack_used > context->stack_need) { - context->stack_need = context->stack_used; - } - DMC_PUSH(*text, matchMkMap); - DMC_PUSH(*text, nelems); - context->stack_used -= nelems; - *constant = 0; - return retOk; } static DMCRet dmc_whole_expression(DMCContext *context, @@ -5462,11 +5608,17 @@ void db_match_dis(Binary *bp) ++t; erts_printf("MkTuple\t%beu\n", n); break; - case matchMkMap: + case matchMkFlatMap: + ++t; + n = *t; + ++t; + erts_printf("MkFlatMap\t%beu\n", n); + break; + case matchMkHashMap: ++t; n = *t; ++t; - erts_printf("MkMapA\t%beu\n", n); + erts_printf("MkHashMap\t%beu\n", n); break; case matchOr: ++t; diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index d1a7ee113b..4a116c0740 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -39,6 +39,7 @@ #include "hipe_mode_switch.h" #endif #include "dtrace-wrapper.h" +#include "erl_bif_unique.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -96,10 +97,10 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); -static Uint combined_message_size(Process* p); +static Uint combined_message_size(Process* p, int off_heap_msgs); static void remove_message_buffers(Process* p); -static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl); -static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl); +static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs); +static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs); static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj); static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size); static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size); @@ -402,7 +403,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) { Uint reclaimed_now = 0; int done = 0; + int off_heap_msgs; Uint ms1, s1, us1; + erts_aint32_t state; ErtsSchedulerData *esdp; #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); @@ -419,7 +422,8 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) trace_gc(p, am_gc_start); } - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); + state = erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); + off_heap_msgs = state & ERTS_PSFLG_OFF_HEAP_MSGS; if (erts_system_monitor_long_gc != 0) { get_now(&ms1, &s1, &us1); } @@ -445,11 +449,11 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) while (!done) { if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) { DTRACE2(gc_major_start, pidbuf, need); - done = major_collection(p, need, objv, nobj, &reclaimed_now); + done = major_collection(p, need, objv, nobj, &reclaimed_now, off_heap_msgs); DTRACE2(gc_major_end, pidbuf, reclaimed_now); } else { DTRACE2(gc_minor_start, pidbuf, need); - done = minor_collection(p, need, objv, nobj, &reclaimed_now); + done = minor_collection(p, need, objv, nobj, &reclaimed_now, off_heap_msgs); DTRACE2(gc_minor_end, pidbuf, reclaimed_now); } } @@ -832,7 +836,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, } static int -minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) +minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs) { Uint mature = HIGH_WATER(p) - HEAP_START(p); @@ -871,20 +875,22 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) Uint size_after; Uint need_after; Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); + Uint fragments = MBUF_SIZE(p) + combined_message_size(p, off_heap_msgs); Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0); do_minor(p, new_sz, objv, nobj); - /* - * Copy newly received message onto the end of the new heap. - */ - ErtsGcQuickSanityCheck(p); - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); - ErtsGcQuickSanityCheck(p); + if (!off_heap_msgs) { + /* + * Copy newly received message onto the end of the new heap. + */ + ErtsGcQuickSanityCheck(p); + for (msgp = p->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached) { + erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); + ErtsGcQuickSanityCheck(p); + } } } ErtsGcQuickSanityCheck(p); @@ -1210,7 +1216,7 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) */ static int -major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) +major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl, int off_heap_msgs) { Rootset rootset; Roots* roots; @@ -1223,8 +1229,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) Uint oh_size = (char *) OLD_HTOP(p) - oh; Uint n; Uint new_sz; - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - ErlMessage *msgp; + Uint fragments = MBUF_SIZE(p) + combined_message_size(p, off_heap_msgs); size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); @@ -1434,13 +1439,16 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ErtsGcQuickSanityCheck(p); - /* - * Copy newly received message onto the end of the new heap. - */ - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); - ErtsGcQuickSanityCheck(p); + if (!off_heap_msgs) { + ErlMessage *msgp; + /* + * Copy newly received message onto the end of the new heap. + */ + for (msgp = p->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached) { + erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); + ErtsGcQuickSanityCheck(p); + } } } @@ -1501,15 +1509,17 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int * mbuf list. */ static Uint -combined_message_size(Process* p) +combined_message_size(Process* p, int off_heap_msgs) { - Uint sz = 0; + Uint sz; ErlMessage *msgp; - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { + if (off_heap_msgs) + return 0; + + for (sz = 0, msgp = p->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached) sz += erts_msg_attached_data_size(msgp); - } } return sz; } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index bf0496c112..bd6dcc9078 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -55,7 +55,10 @@ do { \ nelts = header_arity(HDR); \ switch ((HDR) & _HEADER_SUBTAG_MASK) { \ case SUB_BINARY_SUBTAG: nelts++; break; \ - case MAP_SUBTAG: nelts+=map_get_size(PTR) + 1; break; \ + case MAP_SUBTAG: \ + if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1; \ + else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR)); \ + break; \ case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break; \ } \ gval = make_boxed(HTOP); \ @@ -63,7 +66,6 @@ do { \ *HTOP++ = HDR; \ *PTR++ = gval; \ while (nelts--) *HTOP++ = *PTR++; \ - \ } while(0) #define in_area(ptr,start,nbytes) \ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index fe065e196d..86d3416423 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -45,6 +45,7 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "erl_ptab.h" +#include "erl_bif_unique.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -134,7 +135,9 @@ static void erl_init(int ncpu, int legacy_proc_tab, int port_tab_sz, int port_tab_sz_ignore_files, - int legacy_port_tab); + int legacy_port_tab, + int time_correction, + ErtsTimeWarpMode time_warp_mode); static erts_atomic_t exiting; @@ -185,10 +188,6 @@ static int no_dirty_io_schedulers; Uint32 verbose; /* See erl_debug.h for information about verbose */ #endif -int erts_disable_tolerant_timeofday; /* Time correction can be disabled it is - * not and/or it is too slow. - */ - int erts_atom_table_size = ATOM_LIMIT; /* Maximum number of atoms */ int erts_pd_initial_size = 10; @@ -268,6 +267,19 @@ this_rel_num(void) return this_rel; } +static ERTS_INLINE void +set_default_time_adj(int *time_correction_p, ErtsTimeWarpMode *time_warp_mode_p) +{ + *time_correction_p = 1; + *time_warp_mode_p = ERTS_NO_TIME_WARP_MODE; + if (!erts_check_time_adj_support(*time_correction_p, + *time_warp_mode_p)) { + *time_correction_p = 0; + ASSERT(erts_check_time_adj_support(*time_correction_p, + *time_warp_mode_p)); + } +} + /* * Common error printout function, all error messages * that don't go to the error logger go through here. @@ -283,13 +295,22 @@ static int early_init(int *argc, char **argv); void erts_short_init(void) { - int ncpu = early_init(NULL, NULL); + + int ncpu; + int time_correction; + ErtsTimeWarpMode time_warp_mode; + + set_default_time_adj(&time_correction, + &time_warp_mode); + ncpu = early_init(NULL, NULL); erl_init(ncpu, ERTS_DEFAULT_MAX_PROCESSES, 0, ERTS_DEFAULT_MAX_PORTS, 0, - 0); + 0, + time_correction, + time_warp_mode); erts_initialized = 1; } @@ -299,12 +320,15 @@ erl_init(int ncpu, int legacy_proc_tab, int port_tab_sz, int port_tab_sz_ignore_files, - int legacy_port_tab) + int legacy_port_tab, + int time_correction, + ErtsTimeWarpMode time_warp_mode) { init_benchmarking(); + erts_bif_unique_init(); erts_init_monitors(); - erts_init_time(); + erts_init_time(time_correction, time_warp_mode); erts_init_sys_common_misc(); erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab); erts_init_scheduling(no_schedulers, @@ -315,6 +339,7 @@ erl_init(int ncpu, no_dirty_io_schedulers #endif ); + erts_late_init_time_sup(); erts_init_cpu_topology(); /* Must be after init_scheduling */ erts_init_gc(); /* Must be after init_scheduling */ erts_alloc_late_init(); @@ -364,12 +389,13 @@ erl_init(int ncpu, erl_nif_init(); } -static void +static Eterm erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) { int i; Eterm start_mod; Eterm args; + Eterm res; Eterm* hp; Process parent; ErlSpawnOpts so; @@ -399,10 +425,11 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** hp += 2; args = CONS(hp, env, args); - so.flags = 0; - (void) erl_create_process(&parent, start_mod, am_start, args, &so); + so.flags = SPO_SYSTEM_PROC; + res = erl_create_process(&parent, start_mod, am_start, args, &so); erts_smp_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN); erts_cleanup_empty_process(&parent); + return res; } Eterm @@ -508,9 +535,9 @@ void erts_usage(void) /* erts_fprintf(stderr, "-b func set the boot function (default boot)\n"); */ - erts_fprintf(stderr, "-c disable continuous date/time correction with\n"); - erts_fprintf(stderr, " respect to uptime\n"); - + erts_fprintf(stderr, "-c bool enable or disable time correction\n"); + erts_fprintf(stderr, "-C mode set time warp mode; valid modes are:\n"); + erts_fprintf(stderr, " no_time_warp|single_time_warp|multi_time_warp\n"); erts_fprintf(stderr, "-d don't write a crash dump for internally detected errors\n"); erts_fprintf(stderr, " (halt(String) will still produce a crash dump)\n"); erts_fprintf(stderr, "-fn[u|a|l] Control how filenames are interpreted\n"); @@ -682,7 +709,6 @@ early_init(int *argc, char **argv) /* erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; - erts_disable_tolerant_timeofday = 0; display_items = 200; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS; @@ -1188,7 +1214,11 @@ erl_start(int argc, char **argv) int port_tab_sz_ignore_files = 0; int legacy_proc_tab = 0; int legacy_port_tab = 0; + int time_correction; + ErtsTimeWarpMode time_warp_mode; + set_default_time_adj(&time_correction, + &time_warp_mode); envbufsz = sizeof(envbuf); if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -1906,9 +1936,55 @@ erl_start(int argc, char **argv) } break; } + case 'C': + arg = get_arg(argv[i]+2, argv[i+1], &i); + if (sys_strcmp(arg, "no_time_warp") == 0) + time_warp_mode = ERTS_NO_TIME_WARP_MODE; + else if (sys_strcmp(arg, "single_time_warp") == 0) + time_warp_mode = ERTS_SINGLE_TIME_WARP_MODE; + else if (sys_strcmp(arg, "multi_time_warp") == 0) + time_warp_mode = ERTS_MULTI_TIME_WARP_MODE; + else { + erts_fprintf(stderr, + "Invalid time warp mode: %s\n", arg); + erts_usage(); + } + break; case 'c': - if (argv[i][2] == 0) { /* -c: documented option */ - erts_disable_tolerant_timeofday = 1; + if (sys_strcmp(argv[i]+2, "false") == 0) + goto time_correction_false; + else if (sys_strcmp(argv[i]+2, "true") == 0) + goto time_correction_true; +#ifdef ERTS_OPCODE_COUNTER_SUPPORT + else if (argv[i][2] == 'i') { /* -ci: undcoumented option*/ + count_instructions = 1; + } +#endif + else if (argv[i][2] == '\0') { + if (i + 1 >= argc) + goto time_correction_false; + else { + if (sys_strcmp(argv[i+1], "false") == 0) { + (void) get_arg(argv[i]+2, argv[i+1], &i); + goto time_correction_false; + } + else if (sys_strcmp(argv[i+1], "true") == 0) { + (void) get_arg(argv[i]+2, argv[i+1], &i); + time_correction_true: + time_correction = 1; + break; + } + else { + time_correction_false: + time_correction = 0; + break; + } + } + } + else { + arg = get_arg(argv[i]+2, argv[i+1], &i); + erts_fprintf(stderr, "Invalid time correnction value: %s\n", arg); + erts_usage(); } break; case 'W': @@ -1955,6 +2031,30 @@ erl_start(int argc, char **argv) i++; } + if (!erts_check_time_adj_support(time_correction, time_warp_mode)) { + char *time_correction_str = time_correction ? "Enabled" : "Disabled"; + char *time_warp_str = "undefined"; + switch (time_warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + time_warp_str = "no"; + break; + case ERTS_SINGLE_TIME_WARP_MODE: + time_warp_str = "single"; + break; + case ERTS_MULTI_TIME_WARP_MODE: + time_warp_str = "multi"; + break; + default: + time_warp_str = "undefined"; + break; + } + erts_fprintf(stderr, "%s time correction with %s time warp mode " + "is not supported on this platform\n", + time_correction_str, + time_warp_str); + erts_usage(); + } + /* Output format on windows for sprintf defaults to three exponents. * We use two-exponent to mimic normal sprintf behaviour. */ @@ -1988,7 +2088,9 @@ erl_start(int argc, char **argv) legacy_proc_tab, port_tab_sz, port_tab_sz_ignore_files, - legacy_port_tab); + legacy_port_tab, + time_correction, + time_warp_mode); load_preloaded(); erts_end_staging_code_ix(); @@ -1996,7 +2098,11 @@ erl_start(int argc, char **argv) erts_initialized = 1; - erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv); + { + Eterm init = erl_first_process_otp("otp_ring0", NULL, 0, + boot_argc, boot_argv); + erts_bif_timer_start_servers(init); + } #ifdef ERTS_SMP erts_start_schedulers(); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index b105ece6f1..261460d054 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -140,7 +140,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "async_enq_mtx", NULL }, #ifdef ERTS_SMP { "atom_tab", NULL }, - { "make_ref", NULL }, { "misc_op_list_pre_alloc_lock", "address" }, { "message_pre_alloc_lock", "address" }, { "ptimer_pre_alloc_lock", "address", }, @@ -168,6 +167,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "timer_wheel", NULL }, { "system_block", NULL }, { "timeofday", NULL }, + { "get_time", NULL }, + { "get_corrected_time", NULL }, { "breakpoints", NULL }, { "pollsets_lock", NULL }, { "pix_lock", "address" }, @@ -184,10 +185,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "efile_drv dtrace mutex", NULL }, #endif { "mtrace_buf", NULL }, -#ifdef __WIN32__ #ifdef ERTS_SMP - { "sys_gethrtime", NULL }, -#endif + { "os_monotonic_time", NULL }, #endif { "erts_alloc_hard_debug", NULL }, { "hard_dbg_mseg", NULL }, diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index cf6996ea06..c6d8f4df95 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -104,17 +104,13 @@ static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) { } static void lcnt_time(erts_lcnt_time_t *time) { -#if 0 || defined(HAVE_GETHRTIME) - SysHrTime hr_time; - hr_time = sys_gethrtime(); - time->s = (unsigned long)(hr_time / 1000000000LL); - time->ns = (unsigned long)(hr_time - 1000000000LL*time->s); -#else - SysTimeval tv; - sys_gettimeofday(&tv); - time->s = tv.tv_sec; - time->ns = tv.tv_usec*1000LL; -#endif + /* + * erts_sys_hrtime() is the highest resolution + * we could find, it may or may not be monotonic... + */ + ErtsMonotonicTime mtime = erts_sys_hrtime(); + time->s = (unsigned long) (mtime / 1000000000LL); + time->ns = (unsigned long) (mtime - 1000000000LL*time->s); } static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) { diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index ffbb93da1b..09fadd7e9e 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -76,7 +76,7 @@ /* histogram */ #define ERTS_LCNT_HISTOGRAM_MAX_NS (((unsigned long)1LL << 28) - 1) -#if 0 || defined(HAVE_GETHRTIME) +#if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT) #define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (30) #define ERTS_LCNT_HISTOGRAM_RSHIFT (0) #else diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index b2a16eb5ed..ab40f47919 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -16,6 +16,9 @@ * * %CopyrightEnd% * + * hashmaps are an adaption of Rich Hickeys Persistent HashMaps + * which were an adaption of Phil Bagwells - Hash Array Mapped Tries + * * Author: Björn-Egil Dahlberg */ @@ -62,39 +65,78 @@ * - erts_internal:map_to_tuple_keys/1 */ +#ifndef DECL_AM +#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) +#endif + +/* for hashmap_from_list/1 */ +typedef struct { + Uint32 hx; + Uint32 skip; + Uint i; + Eterm val; +} hxnode_t; + + +static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB); +static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args); +static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB); +static Eterm hashmap_to_list(Process *p, Eterm map); +static Eterm hashmap_keys(Process *p, Eterm map); +static Eterm hashmap_values(Process *p, Eterm map); +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node); +static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size); +static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size); +static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys); +static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root); +static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root); +static Eterm hashmap_info(Process *p, Eterm node); +static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]); +static int hxnodecmp(hxnode_t* a, hxnode_t* b); +static int hxnodecmpkey(hxnode_t* a, hxnode_t* b); + /* erlang:map_size/1 * the corresponding instruction is implemented in: * beam/erl_bif_guard.c */ BIF_RETTYPE map_size_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_1)) { Eterm *hp; Uint hsz = 0; - map_t *mp = (map_t*)map_val(BIF_ARG_1); - Uint n = map_get_size(mp); + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + Uint n = flatmap_get_size(mp); erts_bld_uint(NULL, &hsz, n); hp = HAlloc(BIF_P, hsz); BIF_RET(erts_bld_uint(&hp, NULL, n)); + } else if (is_hashmap(BIF_ARG_1)) { + Eterm *head, *hp, res; + Uint size, hsz=0; + + head = hashmap_val(BIF_ARG_1); + size = head[1]; + (void) erts_bld_uint(NULL, &hsz, size); + hp = HAlloc(BIF_P, hsz); + res = erts_bld_uint(&hp, NULL, size); + BIF_RET(res); } BIF_ERROR(BIF_P, BADARG); } -/* maps:to_list/1 - */ +/* maps:to_list/1 */ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_1)) { Uint n; Eterm* hp; Eterm *ks,*vs, res, tup; - map_t *mp = (map_t*)map_val(BIF_ARG_1); + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); - ks = map_get_keys(mp); - vs = map_get_values(mp); - n = map_get_size(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); hp = HAlloc(BIF_P, (2 + 3) * n); res = NIL; @@ -104,6 +146,8 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { } BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + return hashmap_to_list(BIF_P, BIF_ARG_1); } BIF_ERROR(BIF_P, BADARG); @@ -120,34 +164,41 @@ erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base) erts_maps_get(Eterm key, Eterm map) #endif { - Eterm *ks, *vs; - map_t *mp; - Uint n, i; + Uint32 hx; + if (is_flatmap_rel(map, map_base)) { + Eterm *ks, *vs; + flatmap_t *mp; + Uint n, i; - mp = (map_t *)map_val_rel(map, map_base); - n = map_get_size(mp); + mp = (flatmap_t *)flatmap_val_rel(map, map_base); + n = flatmap_get_size(mp); - if (n == 0) { - return NULL; - } + if (n == 0) { + return NULL; + } - ks = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1; - vs = map_get_values(mp); + ks = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1; + vs = flatmap_get_values(mp); - if (is_immed(key)) { - for (i = 0; i < n; i++) { - if (ks[i] == key) { - return &vs[i]; - } - } - } + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (ks[i] == key) { + return &vs[i]; + } + } + } - for (i = 0; i < n; i++) { - if (eq_rel(ks[i], NULL, key, map_base)) { - return &vs[i]; - } + for (i = 0; i < n; i++) { + if (eq_rel(ks[i], map_base, key, NULL)) { + return &vs[i]; + } + } + return NULL; } - return NULL; + ASSERT(is_hashmap_rel(map, map_base)); + hx = hashmap_make_hash(key); + + return erts_hashmap_get_rel(hx, key, map, map_base); } BIF_RETTYPE maps_find_2(BIF_ALIST_2) { @@ -164,7 +215,6 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) { *hp++ = *value; BIF_RET(res); } - BIF_RET(am_error); } BIF_ERROR(BIF_P, BADARG); @@ -202,13 +252,8 @@ BIF_RETTYPE maps_get_2(BIF_ALIST_2) { */ BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) { - Eterm *kv, item = BIF_ARG_1; - Eterm *hp, *thp,*vs, *ks, keys, res; - map_t *mp; - Uint size = 0, unused_size = 0; - Sint c = 0; - Sint idx = 0; - + Eterm item = BIF_ARG_1, res, *kv; + Uint size = 0; if (is_list(item) || is_nil(item)) { /* Calculate size and check validity */ @@ -229,368 +274,1085 @@ BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) { if (is_not_nil(item)) goto error; - hp = HAlloc(BIF_P, 3 + 1 + (2 * size)); - thp = hp; + if (size > MAP_SMALL_MAP_LIMIT) { + BIF_RET(hashmap_from_validated_list(BIF_P, BIF_ARG_1, size)); + } else { + BIF_RET(flatmap_from_validated_list(BIF_P, BIF_ARG_1, size)); + } + } + +error: + + BIF_ERROR(BIF_P, BADARG); +} + +static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size) { + Eterm *kv, item = list; + Eterm *hp, *thp,*vs, *ks, keys, res; + flatmap_t *mp; + Uint unused_size = 0; + Sint c = 0; + Sint idx = 0; + + + hp = HAlloc(p, 3 + 1 + (2 * size)); + thp = hp; + keys = make_tuple(hp); + *hp++ = make_arityval(size); + ks = hp; + hp += size; + mp = (flatmap_t*)hp; + res = make_flatmap(mp); + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = size; /* set later, might shrink*/ + mp->keys = keys; + + if (size == 0) + return res; + + /* first entry */ + kv = tuple_val(CAR(list_val(item))); + ks[0] = kv[1]; + vs[0] = kv[2]; + size = 1; + item = CDR(list_val(item)); + + /* insert sort key/value pairs */ + while(is_list(item)) { + + kv = tuple_val(CAR(list_val(item))); + + /* compare ks backwards + * idx represent word index to be written (hole position). + * We cannot copy the elements when searching since we might + * have an equal key. So we search for just the index first =( + * + * It is perhaps faster to move the values in the first pass. + * Check for uniqueness during insert phase and then have a + * second phace compacting the map if duplicates are found + * during insert. .. or do someother sort .. shell-sort perhaps. + */ + + idx = size; + + while(idx > 0 && (c = CMP_TERM(kv[1],ks[idx-1])) < 0) { idx--; } + + if (c == 0) { + /* last compare was equal, + * i.e. we have to release memory + * and overwrite that key/value + */ + ks[idx-1] = kv[1]; + vs[idx-1] = kv[2]; + unused_size++; + } else { + Uint i = size; + while(i > idx) { + ks[i] = ks[i-1]; + vs[i] = vs[i-1]; + i--; + } + ks[idx] = kv[1]; + vs[idx] = kv[2]; + size++; + } + item = CDR(list_val(item)); + } + + if (unused_size) { + /* the key tuple is embedded in the heap + * write a bignum to clear it. + */ + /* release values as normal since they are on the top of the heap */ + + ks[size] = make_pos_bignum_header(unused_size - 1); + HRelease(p, vs + size + unused_size, vs + size); + } + + *thp = make_arityval(size); + mp->size = size; + return res; +} + +#define swizzle32(D,S) \ + do { \ + (D) = ((S) & 0x0000000f) << 28 | ((S) & 0x000000f0) << 20 \ + | ((S) & 0x00000f00) << 12 | ((S) & 0x0000f000) << 4 \ + | ((S) & 0x000f0000) >> 4 | ((S) & 0x00f00000) >> 12 \ + | ((S) & 0x0f000000) >> 20 | ((S) & 0xf0000000) >> 28; \ + } while(0) + +#define maskval(V,L) (((V) >> ((7 - (L))*4)) & 0xf) +#define cdepth(V1,V2) (hashmap_clz((V1) ^ (V2)) >> 2) + +static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) { + Eterm item = list; + Eterm *hp; + Eterm *kv, res; + Eterm tmp[2]; + Uint32 sw, hx; + Uint ix = 0; + hxnode_t *hxns; + ErtsHeapFactory factory; + + ASSERT(size > 0); + + hp = HAlloc(p, (2 * size)); + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, size * sizeof(hxnode_t)); + + while(is_list(item)) { + res = CAR(list_val(item)); + kv = tuple_val(res); + hx = hashmap_restore_hash(tmp,0,kv[1]); + swizzle32(sw,hx); + hxns[ix].hx = sw; + hxns[ix].val = CONS(hp, kv[1], kv[2]); hp += 2; + hxns[ix].skip = 1; /* will be reassigned in from_array */ + hxns[ix].i = ix; + ix++; + item = CDR(list_val(item)); + } + + factory.p = p; + res = hashmap_from_unsorted_array(&factory, hxns, size, 0); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + if (hashmap_size(res) <= MAP_SMALL_MAP_LIMIT) { + DECLARE_WSTACK(wstack); + Eterm *kv, *ks, *vs; + flatmap_t *mp; + Eterm keys; + Uint n = hashmap_size(res); + + /* build flat structure */ + hp = HAlloc(p, 3 + 1 + (2 * n)); keys = make_tuple(hp); - *hp++ = make_arityval(size); + *hp++ = make_arityval(n); ks = hp; - hp += size; - mp = (map_t*)hp; - res = make_map(mp); - hp += MAP_HEADER_SIZE; + hp += n; + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; vs = hp; - mp->thing_word = MAP_HEADER; - mp->size = size; /* set later, might shrink*/ + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; mp->keys = keys; - if (size == 0) - BIF_RET(res); + hashmap_iterator_init(&wstack, res, 0); - item = BIF_ARG_1; + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + *ks++ = CAR(kv); + *vs++ = CDR(kv); + } - /* first entry */ - kv = tuple_val(CAR(list_val(item))); - ks[0] = kv[1]; - vs[0] = kv[2]; - size = 1; - item = CDR(list_val(item)); + /* it cannot have multiple keys */ + erts_validate_and_sort_flatmap(mp); - /* insert sort key/value pairs */ - while(is_list(item)) { + DESTROY_WSTACK(wstack); + return make_flatmap(mp); + } - kv = tuple_val(CAR(list_val(item))); - - /* compare ks backwards - * idx represent word index to be written (hole position). - * We cannot copy the elements when searching since we might - * have an equal key. So we search for just the index first =( - * - * It is perhaps faster to move the values in the first pass. - * Check for uniqueness during insert phase and then have a - * second phace compacting the map if duplicates are found - * during insert. .. or do someother sort .. shell-sort perhaps. - */ + return res; +} + +Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n, + int reject_dupkeys) { + Uint32 sw, hx; + Uint ix; + hxnode_t *hxns; + Eterm res; + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t)); + + for (ix = 0; ix < n; ix++) { + hx = hashmap_make_hash(*leafs); + swizzle32(sw,hx); + hxns[ix].hx = sw; + hxns[ix].val = make_list(leafs); + hxns[ix].skip = 1; + hxns[ix].i = ix; + leafs += 2; + } - idx = size; + res = hashmap_from_unsorted_array(factory, hxns, n, reject_dupkeys); - while(idx > 0 && (c = CMP_TERM(kv[1],ks[idx-1])) < 0) { idx--; } + erts_free(ERTS_ALC_T_TMP, (void *) hxns); - if (c == 0) { - /* last compare was equal, - * i.e. we have to release memory - * and overwrite that key/value - */ - ks[idx-1] = kv[1]; - vs[idx-1] = kv[2]; - unused_size++; - } else { - Uint i = size; - while(i > idx) { - ks[i] = ks[i-1]; - vs[i] = vs[i-1]; - i--; + return res; +} + + +Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, + Eterm key, Eterm value) { + Uint32 sw, hx; + Uint i,sz; + hxnode_t *hxns; + ErtsHeapFactory factory; + Eterm *hp, res; + + sz = (key == THE_NON_VALUE) ? n : (n + 1); + ASSERT(sz > MAP_SMALL_MAP_LIMIT); + hp = HAlloc(p, 2 * sz); + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, sz * sizeof(hxnode_t)); + + for(i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; /* will be reassigned in from_array */ + hxns[i].i = i; + } + + if (key != THE_NON_VALUE) { + hx = hashmap_make_hash(key); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, key, value); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; + } + + factory.p = p; + res = hashmap_from_unsorted_array(&factory, hxns, sz, 0); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + return res; +} + +static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory, + hxnode_t *hxns, Uint n, + int reject_dupkeys) { + Uint jx = 0, ix = 0, lx, cx; + Eterm res; + + if (n == 0) { + Eterm *hp; + hp = erts_produce_heap(factory, 2, 0); + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(0); + hp[1] = 0; + + return make_hashmap(hp); + } + + /* sort and compact array (remove non-unique entries) */ + qsort(hxns, n, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp); + + ix = 0, cx = 0; + while(ix < n - 1) { + if (hxns[ix].hx == hxns[ix+1].hx) { + + /* find region of equal hash values */ + jx = ix + 1; + while(jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; } + /* find all correct keys from region + * (last in list but now hash sorted so we check highest id instead) */ + + /* resort with keys instead of hash value within region */ + + qsort(&hxns[ix], jx - ix, sizeof(hxnode_t), + (int (*)(const void *, const void *)) hxnodecmpkey); + + while(ix < jx) { + lx = ix; + while(ix < jx && EQ(CAR(list_val(hxns[ix].val)), CAR(list_val(hxns[lx].val)))) { + if (reject_dupkeys) + return THE_NON_VALUE; + + if (hxns[ix].i > hxns[lx].i) { + lx = ix; + } + ix++; } - ks[idx] = kv[1]; - vs[idx] = kv[2]; - size++; + hxns[cx].hx = hxns[lx].hx; + hxns[cx].val = hxns[lx].val; + cx++; } - item = CDR(list_val(item)); + ix = jx; + continue; + } + if (ix > cx) { + hxns[cx].hx = hxns[ix].hx; + hxns[cx].val = hxns[ix].val; } + cx++; + ix++; + } - if (unused_size) { - /* the key tuple is embedded in the heap - * write a bignum to clear it. - */ - /* release values as normal since they are on the top of the heap */ + if (ix < n) { + hxns[cx].hx = hxns[ix].hx; + hxns[cx].val = hxns[ix].val; + cx++; + } - ks[size] = make_pos_bignum_header(unused_size - 1); - HRelease(BIF_P, vs + size + unused_size, vs + size); - } + if (cx > 1) { + /* recursive decompose array */ + res = hashmap_from_sorted_unique_array(factory, hxns, cx, 0); + } else { + Eterm *hp; - *thp = make_arityval(size); - mp->size = size; - BIF_RET(res); + /* we only have one item, either because n was 1 or + * because we hade multiples of the same key. + * + * hash value has been swizzled, need to drag it down to get the + * correct slot. */ + + hp = erts_produce_heap(factory, HAMT_HEAD_BITMAP_SZ(1), 0); + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf)); + hp[1] = 1; + hp[2] = hxns[0].val; + res = make_hashmap(hp); } -error: + return res; +} - BIF_ERROR(BIF_P, BADARG); +static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory, + hxnode_t *hxns, Uint n, int lvl) { + Eterm res = NIL; + Uint i,ix,jx,elems; + Uint32 sw, hx; + Eterm val; + Eterm th[2]; + hxnode_t *tmp; + + ASSERT(lvl < 32); + ix = 0; + elems = 1; + while (ix < n - 1) { + if (hxns[ix].hx == hxns[ix+1].hx) { + jx = ix + 1; + while (jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; } + tmp = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, ((jx - ix)) * sizeof(hxnode_t)); + + for(i = 0; i < jx - ix; i++) { + val = hxns[i + ix].val; + hx = hashmap_restore_hash(th, lvl + 8, CAR(list_val(val))); + swizzle32(sw,hx); + tmp[i].hx = sw; + tmp[i].val = val; + tmp[i].i = i; + tmp[i].skip = 1; + } + + qsort(tmp, jx - ix, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp); + + hxns[ix].skip = jx - ix; + hxns[ix].val = hashmap_from_sorted_unique_array(factory, tmp, jx - ix, lvl + 8); + erts_free(ERTS_ALC_T_TMP, (void *) tmp); + ix = jx; + if (ix < n) { elems++; } + continue; + } + hxns[ix].skip = 1; + elems++; + ix++; + } + + res = hashmap_from_chunked_array(factory, hxns, elems, !lvl); + + ERTS_FACTORY_HOLE_CHECK(factory); + + return res; } -/* maps:is_key/2 - */ +#define HALLOC_EXTRA 200 +static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, + hxnode_t *hxns, Uint n, int is_root) { + Uint ix, d, dn, dc, slot, elems; + Uint32 v, vp, vn, hdr; + Uint bp, sz; + DECLARE_ESTACK(stack); + Eterm res = NIL, *hp = NULL, *nhp; -BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) { - if (is_map(BIF_ARG_2)) { - Eterm *ks, key; - map_t *mp; - Uint n,i; + ASSERT(n > 1); - mp = (map_t*)map_val(BIF_ARG_2); - key = BIF_ARG_1; - n = map_get_size(mp); - ks = map_get_keys(mp); + /* push initial nodes on the stack, + * this is the starting depth */ - if (n == 0) - BIF_RET(am_false); + ix = 0; + d = 0; + vp = hxns[ix].hx; + v = hxns[ix + hxns[ix].skip].hx; - if (is_immed(key)) { - for( i = 0; i < n; i++) { - if (ks[i] == key) { - BIF_RET(am_true); - } + ASSERT(vp > v); + slot = maskval(vp,d); + + while(slot == maskval(v,d)) { + ESTACK_PUSH(stack, 1 << slot); + d++; + slot = maskval(vp,d); + } + + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + dc = 7; + /* build collision nodes */ + while (dc > d) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(vp,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; + } + } + + ESTACK_PUSH2(stack,res,1 << slot); + + /* all of the other nodes .. */ + elems = n - 2; /* remove first and last elements */ + while(elems--) { + hdr = ESTACK_POP(stack); + ix = ix + hxns[ix].skip; + + /* determine if node or subtree should be built by looking + * at the next value. */ + + vn = hxns[ix + hxns[ix].skip].hx; + dn = cdepth(v,vn); + ASSERT(v > vn); + + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + int wat = (d > dn) ? d : dn; + dc = 7; + /* build collision nodes */ + while (dc > wat) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; } } - for( i = 0; i < n; i++) { - if (EQ(ks[i], key)) { - BIF_RET(am_true); + /* next depth is higher (implies collision) */ + if (d < dn) { + /* hdr is the popped one initially */ + while(d < dn) { + slot = maskval(v, d); + bp = 1 << slot; + ESTACK_PUSH(stack, hdr | bp); + d++; + hdr = 0; /* clear hdr for all other collisions */ } + + slot = maskval(v, d); + bp = 1 << slot; + /* no more collisions */ + ESTACK_PUSH2(stack,res,bp); + } else if (d == dn) { + /* no collisions at all */ + slot = maskval(v, d); + bp = 1 << slot; + ESTACK_PUSH2(stack,res,hdr | bp); + } else { + /* dn < n, we have a drop and we are done + * build nodes and subtree */ + while (dn != d) { + slot = maskval(v, d); + bp = 1 << slot; + /* OR bitposition before sz calculation to handle + * redundant collisions */ + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + *hp++ = res; sz--; + while (sz--) { *hp++ = ESTACK_POP(stack); } + ASSERT((hp - nhp) < 18); + res = make_hashmap(nhp); + + /* we need to pop the next hdr and push if we don't need it */ + + hdr = ESTACK_POP(stack); + d--; + } + ESTACK_PUSH2(stack,res,hdr); } - BIF_RET(am_false); + + vp = v; + v = vn; + d = dn; + ERTS_FACTORY_HOLE_CHECK(factory); + } + + /* v and vp are reused from above */ + dn = cdepth(vp,v); + ix = ix + hxns[ix].skip; + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + dc = 7; + /* build collision nodes */ + while (dc > dn) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; + } + } + + hdr = ESTACK_POP(stack); + /* pop remaining subtree if any */ + while (dn) { + slot = maskval(v, dn); + bp = 1 << slot; + /* OR bitposition before sz calculation to handle + * redundant collisions */ + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + *hp++ = res; sz--; + + while (sz--) { *hp++ = ESTACK_POP(stack); } + res = make_hashmap(nhp); + hdr = ESTACK_POP(stack); + dn--; + } + + /* and finally the root .. */ + + slot = maskval(v, dn); + bp = 1 << slot; + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, sz + /* hdr + item */ (is_root ? 2 : 1), 0); + nhp = hp; + + if (is_root) { + *hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr); + *hp++ = n; + } else { + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + } + + *hp++ = res; sz--; + while (sz--) { *hp++ = ESTACK_POP(stack); } + + res = make_hashmap(nhp); + + ASSERT(ESTACK_COUNT(stack) == 0); + DESTROY_ESTACK(stack); + ERTS_FACTORY_HOLE_CHECK(factory); + return res; +} +#undef HALLOC_EXTRA + +static int hxnodecmpkey(hxnode_t *a, hxnode_t *b) { + return CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val))); +} + +static int hxnodecmp(hxnode_t *a, hxnode_t *b) { + if (a->hx < b->hx) + return 1; + else if (a->hx == b->hx) + return 0; + else + return -1; +} + +/* maps:is_key/2 */ + +BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + BIF_RET(erts_maps_get(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_ERROR(BIF_P, BADARG); } -/* maps:keys/1 - */ +/* maps:keys/1 */ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_1)) { Eterm *hp, *ks, res = NIL; - map_t *mp; + flatmap_t *mp; Uint n; - mp = (map_t*)map_val(BIF_ARG_1); - n = map_get_size(mp); + mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + n = flatmap_get_size(mp); if (n == 0) BIF_RET(res); hp = HAlloc(BIF_P, (2 * n)); - ks = map_get_keys(mp); + ks = flatmap_get_keys(mp); while(n--) { res = CONS(hp, ks[n], res); hp += 2; } BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_keys(BIF_P, BIF_ARG_1)); } BIF_ERROR(BIF_P, BADARG); } -/* maps:merge/2 - */ +/* maps:merge/2 */ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) { - if (is_map(BIF_ARG_1) && is_map(BIF_ARG_2)) { - Eterm *hp,*thp; - Eterm tup; - Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2; - map_t *mp1,*mp2,*mp_new; - Uint n1,n2,i1,i2,need,unused_size=0; - int c = 0; - - mp1 = (map_t*)map_val(BIF_ARG_1); - mp2 = (map_t*)map_val(BIF_ARG_2); - n1 = map_get_size(mp1); - n2 = map_get_size(mp2); - - need = MAP_HEADER_SIZE + 1 + 2*(n1 + n2); - - hp = HAlloc(BIF_P, need); - thp = hp; - tup = make_tuple(thp); - ks = hp + 1; hp += 1 + n1 + n2; - mp_new = (map_t*)hp; hp += MAP_HEADER_SIZE; - vs = hp; hp += n1 + n2; - - mp_new->thing_word = MAP_HEADER; - mp_new->size = 0; - mp_new->keys = tup; - - i1 = 0; i2 = 0; - ks1 = map_get_keys(mp1); - vs1 = map_get_values(mp1); - ks2 = map_get_keys(mp2); - vs2 = map_get_values(mp2); - - while(i1 < n1 && i2 < n2) { - c = CMP_TERM(ks1[i1],ks2[i2]); - if ( c == 0) { - /* use righthand side arguments map value, - * but advance both maps */ - *ks++ = ks2[i2]; - *vs++ = vs2[i2]; - i1++, i2++, unused_size++; - } else if ( c < 0) { - *ks++ = ks1[i1]; - *vs++ = vs1[i1]; - i1++; - } else { - *ks++ = ks2[i2]; - *vs++ = vs2[i2]; - i2++; - } + if (is_flatmap(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_2)) { + BIF_RET(flatmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); + } else if (is_hashmap(BIF_ARG_2)) { + /* Will always become a tree */ + BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0)); } + } else if (is_hashmap(BIF_ARG_1)) { + if (is_hashmap(BIF_ARG_2)) { + BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); + } else if (is_flatmap(BIF_ARG_2)) { + /* Will always become a tree */ + BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1)); + } + } + BIF_ERROR(BIF_P, BADARG); +} - /* copy remaining */ - while (i1 < n1) { +static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { + Eterm *hp,*thp; + Eterm tup; + Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2; + flatmap_t *mp1,*mp2,*mp_new; + Uint n,n1,n2,i1,i2,need,unused_size=0; + Sint c = 0; + + mp1 = (flatmap_t*)flatmap_val(nodeA); + mp2 = (flatmap_t*)flatmap_val(nodeB); + n1 = flatmap_get_size(mp1); + n2 = flatmap_get_size(mp2); + + need = MAP_HEADER_FLATMAP_SZ + 1 + 2 * (n1 + n2); + + hp = HAlloc(p, need); + thp = hp; + tup = make_tuple(thp); + ks = hp + 1; hp += 1 + n1 + n2; + mp_new = (flatmap_t*)hp; hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; hp += n1 + n2; + + mp_new->thing_word = MAP_HEADER_FLATMAP; + mp_new->size = 0; + mp_new->keys = tup; + + i1 = 0; i2 = 0; + ks1 = flatmap_get_keys(mp1); + vs1 = flatmap_get_values(mp1); + ks2 = flatmap_get_keys(mp2); + vs2 = flatmap_get_values(mp2); + + while(i1 < n1 && i2 < n2) { + c = CMP_TERM(ks1[i1],ks2[i2]); + if (c == 0) { + /* use righthand side arguments map value, + * but advance both maps */ + *ks++ = ks2[i2]; + *vs++ = vs2[i2]; + i1++, i2++, unused_size++; + } else if (c < 0) { *ks++ = ks1[i1]; *vs++ = vs1[i1]; i1++; - } - - while (i2 < n2) { + } else { *ks++ = ks2[i2]; *vs++ = vs2[i2]; i2++; } + } - if (unused_size) { - /* the key tuple is embedded in the heap, write a bignum to clear it. - * - * release values as normal since they are on the top of the heap - * size = n1 + n1 - unused_size - */ + /* copy remaining */ + while (i1 < n1) { + *ks++ = ks1[i1]; + *vs++ = vs1[i1]; + i1++; + } - *ks = make_pos_bignum_header(unused_size - 1); - HRelease(BIF_P, vs + unused_size, vs); - } + while (i2 < n2) { + *ks++ = ks2[i2]; + *vs++ = vs2[i2]; + i2++; + } - mp_new->size = n1 + n2 - unused_size; - *thp = make_arityval(n1 + n2 - unused_size); + if (unused_size) { + /* the key tuple is embedded in the heap, write a bignum to clear it. + * + * release values as normal since they are on the top of the heap + * size = n1 + n1 - unused_size + */ - BIF_RET(make_map(mp_new)); + *ks = make_pos_bignum_header(unused_size - 1); + HRelease(p, vs + unused_size, vs); } - BIF_ERROR(BIF_P, BADARG); -} -/* maps:new/2 - */ -BIF_RETTYPE maps_new_0(BIF_ALIST_0) { - Eterm* hp; - Eterm tup; - map_t *mp; + n = n1 + n2 - unused_size; + *thp = make_arityval(n); + mp_new->size = n; - hp = HAlloc(BIF_P, (MAP_HEADER_SIZE + 1)); - tup = make_tuple(hp); - *hp++ = make_arityval(0); + /* Reshape map to a hashmap if the map exceeds the limit */ - mp = (map_t*)hp; - mp->thing_word = MAP_HEADER; - mp->size = 0; - mp->keys = tup; + if (n > MAP_SMALL_MAP_LIMIT) { + Uint32 hx,sw; + Uint i; + Eterm res; + hxnode_t *hxns; + ErtsHeapFactory factory; - BIF_RET(make_map(mp)); -} + ks = flatmap_get_keys(mp_new); + vs = flatmap_get_values(mp_new); -/* maps:put/3 - */ + hp = HAlloc(p, 2 * n); -Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { - Sint n,i; - Sint c = 0; - Eterm* hp, *shp; - Eterm *ks,*vs, res, tup; - map_t *mp = (map_t*)map_val(map); + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP,n * sizeof(hxnode_t)); - n = map_get_size(mp); + for (i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; + } - if (n == 0) { - hp = HAlloc(p, MAP_HEADER_SIZE + 1 + 2); - tup = make_tuple(hp); - *hp++ = make_arityval(1); - *hp++ = key; - res = make_map(hp); - *hp++ = MAP_HEADER; - *hp++ = 1; - *hp++ = tup; - *hp++ = value; + factory.p = p; + res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); return res; } - ks = map_get_keys(mp); - vs = map_get_values(mp); + return make_flatmap(mp_new); +} + +static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args) { + Eterm *ks, *vs, *hp, res; + flatmap_t *mp; + Uint n, i; + hxnode_t *hxns; + Uint32 sw, hx; + ErtsHeapFactory factory; + + /* convert flat to tree */ + + ASSERT(is_flatmap(flat)); + ASSERT(is_hashmap(tree)); + + mp = (flatmap_t*)flatmap_val(flat); + n = flatmap_get_size(mp); + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + hp = HAlloc(p, 2 * n); + + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t)); + + for (i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; + } + + factory.p = p; + res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + return swap_args ? hashmap_merge(p, tree, res) : hashmap_merge(p, res, tree); +} - /* only allocate for values, - * assume key-tuple will be intact +#define HALLOC_EXTRA 200 + +static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { +#define PSTACK_TYPE struct HashmapMergePStackType + struct HashmapMergePStackType { + Eterm *srcA, *srcB; + Uint32 abm, bbm, rbm; /* node bitmaps */ + int keepA; + int ix; + Eterm array[16]; + }; + PSTACK_DECLARE(s, 4); + struct HashmapMergePStackType* sp = PSTACK_PUSH(s); + Eterm *hp, *nhp; + Eterm hdrA, hdrB; + Eterm th[2]; + Uint32 ahx, bhx; + Uint size; /* total key-value counter */ + int keepA = 0; + unsigned lvl = 0; + Eterm res = THE_NON_VALUE; + + /* + * Strategy: Do depth-first traversal of both trees (at the same time) + * and merge each pair of nodes. */ - hp = HAlloc(p, MAP_HEADER_SIZE + n); - shp = hp; /* save hp, used if optimistic update fails */ - res = make_map(hp); - *hp++ = MAP_HEADER; - *hp++ = n; - *hp++ = mp->keys; - - if (is_immed(key)) { - for( i = 0; i < n; i ++) { - if (ks[i] == key) { - *hp++ = value; - vs++; - c = 1; + { + hashmap_head_t* a = (hashmap_head_t*) hashmap_val(nodeA); + hashmap_head_t* b = (hashmap_head_t*) hashmap_val(nodeB); + size = a->size + b->size; + } + +recurse: + + if (primary_tag(nodeA) == TAG_PRIMARY_BOXED && + primary_tag(nodeB) == TAG_PRIMARY_LIST) { + /* Avoid implementing this combination by switching places */ + Eterm tmp = nodeA; + nodeA = nodeB; + nodeB = tmp; + keepA = !keepA; + } + + switch (primary_tag(nodeA)) { + case TAG_PRIMARY_LIST: { + sp->srcA = list_val(nodeA); + switch (primary_tag(nodeB)) { + case TAG_PRIMARY_LIST: { /* LEAF + LEAF */ + sp->srcB = list_val(nodeB); + + if (EQ(CAR(sp->srcA), CAR(sp->srcB))) { + --size; + res = keepA ? nodeA : nodeB; } else { - *hp++ = *vs++; + ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA)); + bhx = hashmap_restore_hash(th, lvl, CAR(sp->srcB)); + sp->abm = 1 << hashmap_index(ahx); + sp->bbm = 1 << hashmap_index(bhx); + + sp->srcA = &nodeA; + sp->srcB = &nodeB; } + break; } - } else { - for( i = 0; i < n; i ++) { - if (EQ(ks[i], key)) { - *hp++ = value; - vs++; - c = 1; - } else { - *hp++ = *vs++; + case TAG_PRIMARY_BOXED: { /* LEAF + NODE */ + sp->srcB = boxed_val(nodeB); + ASSERT(is_header(*sp->srcB)); + hdrB = *sp->srcB++; + + ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA)); + sp->abm = 1 << hashmap_index(ahx); + sp->srcA = &nodeA; + switch(hdrB & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + sp->srcB++; + sp->bbm = 0xffff; + break; + + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; + case HAMT_SUBTAG_NODE_BITMAP: + sp->bbm = MAP_HEADER_VAL(hdrB); + break; + + default: + erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); + break; } + break; + } + default: + erl_exit(1, "bad primary tag %ld\r\n", nodeB); } + break; } + case TAG_PRIMARY_BOXED: { /* NODE + NODE */ + sp->srcA = boxed_val(nodeA); + hdrA = *sp->srcA++; + ASSERT(is_header(hdrA)); + switch (hdrA & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { + sp->srcA++; + ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED); + sp->abm = 0xffff; + sp->srcB = boxed_val(nodeB); + hdrB = *sp->srcB++; + ASSERT(is_header(hdrB)); + switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + sp->srcB++; + sp->bbm = 0xffff; + break; + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; + case HAMT_SUBTAG_NODE_BITMAP: + sp->bbm = MAP_HEADER_VAL(hdrB); + break; + default: + erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); + } + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++; + case HAMT_SUBTAG_NODE_BITMAP: { + ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED); + sp->abm = MAP_HEADER_VAL(hdrA); + sp->srcB = boxed_val(nodeB); + hdrB = *sp->srcB++; + ASSERT(is_header(hdrB)); + switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + sp->srcB++; + sp->bbm = 0xffff; + break; + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; + case HAMT_SUBTAG_NODE_BITMAP: + sp->bbm = MAP_HEADER_VAL(hdrB); + break; - if (c) - return res; + default: + erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); + } + break; + } + default: + erl_exit(1, "bad primary tag %ld\r\n", nodeA); + } + break; + } + default: + erl_exit(1, "bad primary tag %ld\r\n", nodeA); + } - /* need to make a new tuple, - * use old hp since it needs to be recreated anyway. - */ - tup = make_tuple(shp); - *shp++ = make_arityval(n+1); + for (;;) { + if (is_value(res)) { /* We have a complete (sub-)tree or leaf */ + if (lvl == 0) + break; + + /* Pop from stack and continue build parent node */ + lvl--; + sp = PSTACK_POP(s); + sp->array[sp->ix++] = res; + res = THE_NON_VALUE; + if (sp->rbm) { + sp->srcA++; + sp->srcB++; + keepA = sp->keepA; + } + } else { /* Start build a node */ + sp->ix = 0; + sp->rbm = sp->abm | sp->bbm; + ASSERT(!(sp->rbm == 0 && lvl > 0)); + } - hp = HAlloc(p, 3 + n + 1); - res = make_map(hp); - *hp++ = MAP_HEADER; - *hp++ = n + 1; - *hp++ = tup; + while (sp->rbm) { + Uint32 next = sp->rbm & (sp->rbm-1); + Uint32 bit = sp->rbm ^ next; + sp->rbm = next; + if (sp->abm & bit) { + if (sp->bbm & bit) { + /* Bit clash. Push and resolve by recursive merge */ + if (sp->rbm) { + sp->keepA = keepA; + } + nodeA = *sp->srcA; + nodeB = *sp->srcB; + lvl++; + sp = PSTACK_PUSH(s); + goto recurse; + } else { + sp->array[sp->ix++] = *sp->srcA++; + } + } else { + ASSERT(sp->bbm & bit); + sp->array[sp->ix++] = *sp->srcB++; + } + } - ks = map_get_keys(mp); - vs = map_get_values(mp); + ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm)); + if (lvl == 0) { + nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY + : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm)); + *hp++ = size; + } else { + nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm); + } + memcpy(hp, sp->array, sp->ix * sizeof(Eterm)); + res = make_boxed(nhp); + } + PSTACK_DESTROY(s); + return res; +} - ASSERT(n >= 0); +static int hash_cmp(Uint32 ha, Uint32 hb) +{ + int i; + for (i=0; i<8; i++) { + int cmp = (int)(ha & 0xF) - (int)(hb & 0xF); + if (cmp) + return cmp; + ha >>= 4; + hb >>= 4; + } + return 0; +} - /* copy map in order */ - while (n && ((c = CMP_TERM(*ks, key)) < 0)) { - *shp++ = *ks++; - *hp++ = *vs++; - n--; +int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp) +{ + Eterm th[2]; + unsigned lvl = 0; + + if (ap && bp) { + ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0); + for (;;) { + Uint32 ha = hashmap_restore_hash(th, lvl, CAR(ap)); + Uint32 hb = hashmap_restore_hash(th, lvl, CAR(bp)); + int cmp = hash_cmp(ha, hb); + if (cmp) + return cmp; + lvl += 8; + } } + return ap ? -1 : 1; +} + +/* maps:new/0 */ - *shp++ = key; - *hp++ = value; +BIF_RETTYPE maps_new_0(BIF_ALIST_0) { + Eterm* hp; + Eterm tup; + flatmap_t *mp; - ASSERT(n >= 0); + hp = HAlloc(BIF_P, (MAP_HEADER_FLATMAP_SZ + 1)); + tup = make_tuple(hp); + *hp++ = make_arityval(0); - while(n--) { - *shp++ = *ks++; - *hp++ = *vs++; - } - /* we have one word remaining - * this will work out fine once we get the size word - * in the header. - */ - *shp = make_pos_bignum_header(0); - return res; + mp = (flatmap_t*)hp; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = 0; + mp->keys = tup; + + BIF_RET(make_flatmap(mp)); } +/* maps:put/3 */ + BIF_RETTYPE maps_put_3(BIF_ALIST_3) { if (is_map(BIF_ARG_3)) { BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3)); @@ -598,81 +1360,87 @@ BIF_RETTYPE maps_put_3(BIF_ALIST_3) { BIF_ERROR(BIF_P, BADARG); } -/* maps:remove/3 - */ +/* maps:remove/3 */ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { - Sint n; - Uint need; - Eterm *hp_start; - Eterm *thp, *mhp; - Eterm *ks, *vs, tup; - map_t *mp = (map_t*)map_val(map); + Uint32 hx; + if (is_flatmap(map)) { + Sint n; + Uint need; + Eterm *hp_start; + Eterm *thp, *mhp; + Eterm *ks, *vs, tup; + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + + n = flatmap_get_size(mp); + + if (n == 0) { + *res = map; + return 1; + } - n = map_get_size(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); - if (n == 0) { - *res = map; - return 1; - } + /* Assume key exists. + * Release allocated if it didn't. + * Allocate key tuple first. + */ - ks = map_get_keys(mp); - vs = map_get_values(mp); + need = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */ + hp_start = HAlloc(p, need); + thp = hp_start; + mhp = thp + n; /* offset with tuple heap size */ - /* Assume key exists. - * Release allocated if it didn't. - * Allocate key tuple first. - */ + tup = make_tuple(thp); + *thp++ = make_arityval(n - 1); - need = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */ - hp_start = HAlloc(p, need); - thp = hp_start; - mhp = thp + n; /* offset with tuple heap size */ + *res = make_flatmap(mhp); + *mhp++ = MAP_HEADER_FLATMAP; + *mhp++ = n - 1; + *mhp++ = tup; - tup = make_tuple(thp); - *thp++ = make_arityval(n - 1); - - *res = make_map(mhp); - *mhp++ = MAP_HEADER; - *mhp++ = n - 1; - *mhp++ = tup; - - if (is_immed(key)) { - while (1) { - if (*ks == key) { - goto found_key; - } else if (--n) { - *mhp++ = *vs++; - *thp++ = *ks++; - } else - break; - } - } else { - while(1) { - if (EQ(*ks, key)) { - goto found_key; - } else if (--n) { - *mhp++ = *vs++; - *thp++ = *ks++; - } else - break; + if (is_immed(key)) { + while (1) { + if (*ks == key) { + goto found_key; + } else if (--n) { + *mhp++ = *vs++; + *thp++ = *ks++; + } else + break; + } + } else { + while(1) { + if (EQ(*ks, key)) { + goto found_key; + } else if (--n) { + *mhp++ = *vs++; + *thp++ = *ks++; + } else + break; + } } - } - /* Not found, remove allocated memory - * and return previous map. - */ - HRelease(p, hp_start + need, hp_start); + /* Not found, remove allocated memory + * and return previous map. + */ + HRelease(p, hp_start + need, hp_start); - *res = map; - return 1; + *res = map; + return 1; found_key: - /* Copy rest of keys and values */ - if (--n) { - sys_memcpy(mhp, vs+1, n*sizeof(Eterm)); - sys_memcpy(thp, ks+1, n*sizeof(Eterm)); + /* Copy rest of keys and values */ + if (--n) { + sys_memcpy(mhp, vs+1, n*sizeof(Eterm)); + sys_memcpy(thp, ks+1, n*sizeof(Eterm)); + } + return 1; } + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + *res = hashmap_delete(p, hx, key, map); return 1; } @@ -686,29 +1454,28 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) { BIF_ERROR(BIF_P, BADARG); } -/* maps:update/3 - */ - int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) { + Uint32 hx; + if (is_flatmap(map)) { Sint n,i; Eterm* hp,*shp; Eterm *ks,*vs; - map_t *mp = (map_t*)map_val(map); + flatmap_t *mp = (flatmap_t*)flatmap_val(map); - if ((n = map_get_size(mp)) == 0) { + if ((n = flatmap_get_size(mp)) == 0) { return 0; } - ks = map_get_keys(mp); - vs = map_get_values(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); /* only allocate for values, * assume key-tuple will be intact */ - hp = HAlloc(p, MAP_HEADER_SIZE + n); + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + n); shp = hp; - *hp++ = MAP_HEADER; + *hp++ = MAP_HEADER_FLATMAP; *hp++ = n; *hp++ = mp->keys; @@ -730,7 +1497,7 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) } } - HRelease(p, shp + MAP_HEADER_SIZE + n, shp); + HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); return 0; found_key: @@ -738,10 +1505,147 @@ found_key: vs++; if (++i < n) sys_memcpy(hp, vs, (n - i)*sizeof(Eterm)); - *res = make_map(shp); + *res = make_flatmap(shp); return 1; + } + + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + *res = erts_hashmap_insert(p, hx, key, value, map, 1); + if (is_value(*res)) + return 1; + + return 0; } +Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { + Uint32 hx; + Eterm res; + if (is_flatmap(map)) { + Sint n,i; + Sint c = 0; + Eterm* hp, *shp; + Eterm *ks, *vs, tup; + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + + n = flatmap_get_size(mp); + + if (n == 0) { + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + 1 + 2); + tup = make_tuple(hp); + *hp++ = make_arityval(1); + *hp++ = key; + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = 1; + *hp++ = tup; + *hp++ = value; + + return res; + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + /* only allocate for values, + * assume key-tuple will be intact + */ + + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + n); + shp = hp; /* save hp, used if optimistic update fails */ + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = n; + *hp++ = mp->keys; + + if (is_immed(key)) { + for( i = 0; i < n; i ++) { + if (ks[i] == key) { + *hp++ = value; + vs++; + c = 1; + } else { + *hp++ = *vs++; + } + } + } else { + for( i = 0; i < n; i ++) { + if (EQ(ks[i], key)) { + *hp++ = value; + vs++; + c = 1; + } else { + *hp++ = *vs++; + } + } + } + + if (c) + return res; + + /* the map will grow */ + + if (n >= MAP_SMALL_MAP_LIMIT) { + HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + res = erts_hashmap_from_ks_and_vs_extra(p,ks,vs,n,key,value); + + return res; + } + + /* still a small map. need to make a new tuple, + * use old hp since it needs to be recreated anyway. */ + + tup = make_tuple(shp); + *shp++ = make_arityval(n+1); + + hp = HAlloc(p, 3 + n + 1); + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = n + 1; + *hp++ = tup; + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + ASSERT(n >= 0); + + /* copy map in order */ + while (n && ((c = CMP_TERM(*ks, key)) < 0)) { + *shp++ = *ks++; + *hp++ = *vs++; + n--; + } + + *shp++ = key; + *hp++ = value; + + ASSERT(n >= 0); + + while(n--) { + *shp++ = *ks++; + *hp++ = *vs++; + } + /* we have one word remaining + * this will work out fine once we get the size word + * in the header. + */ + *shp = make_pos_bignum_header(0); + return res; + } + ASSERT(is_hashmap(map)); + + hx = hashmap_make_hash(key); + res = erts_hashmap_insert(p, hx, key, value, map, 0); + ASSERT(is_hashmap(res)); + + return res; +} + +/* maps:update/3 */ + BIF_RETTYPE maps_update_3(BIF_ALIST_3) { if (is_map(BIF_ARG_3)) { Eterm res; @@ -753,41 +1657,769 @@ BIF_RETTYPE maps_update_3(BIF_ALIST_3) { } -/* maps:values/1 - */ +/* maps:values/1 */ BIF_RETTYPE maps_values_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_1)) { Eterm *hp, *vs, res = NIL; - map_t *mp; + flatmap_t *mp; Uint n; - mp = (map_t*)map_val(BIF_ARG_1); - n = map_get_size(mp); + mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + n = flatmap_get_size(mp); if (n == 0) BIF_RET(res); hp = HAlloc(BIF_P, (2 * n)); - vs = map_get_values(mp); + vs = flatmap_get_values(mp); while(n--) { res = CONS(hp, vs[n], res); hp += 2; } BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_values(BIF_P, BIF_ARG_1)); } BIF_ERROR(BIF_P, BADARG); } -int erts_validate_and_sort_map(map_t* mp) +static Eterm hashmap_to_list(Process *p, Eterm node) { + DECLARE_WSTACK(stack); + Eterm *hp, *kv; + Eterm res = NIL; + + hp = HAlloc(p, hashmap_size(node) * (2 + 3)); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv)); + hp += 3; + res = CONS(hp, tup, res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { + Eterm hdr = *hashmap_val(node); + Uint sz; + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erl_exit(1, "bad header"); + } + + WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */ + (UWord)(!reverse ? 0 : sz+1), + (UWord)node); +} + +Eterm* hashmap_iterator_next(ErtsWStack* s) { + Eterm node, *ptr, hdr; + Uint32 sz; + Uint idx; + + for (;;) { + ASSERT(!WSTACK_ISEMPTY((*s))); + node = (Eterm) WSTACK_POP((*s)); + if (is_non_value(node)) { + return NULL; + } + idx = (Uint) WSTACK_POP((*s)); + for (;;) { + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + break; + default: + erl_exit(1, "bad header"); + } + + idx++; + + if (idx <= sz) { + WSTACK_PUSH2((*s), (UWord)idx, (UWord)node); + + if (is_list(ptr[idx])) { + return list_val(ptr[idx]); + } + ASSERT(is_boxed(ptr[idx])); + node = ptr[idx]; + idx = 0; + } + else + break; /* and pop parent node */ + } + } +} + +Eterm* hashmap_iterator_prev(ErtsWStack* s) { + Eterm node, *ptr, hdr; + Uint32 sz; + Uint idx; + + for (;;) { + ASSERT(!WSTACK_ISEMPTY((*s))); + node = (Eterm) WSTACK_POP((*s)); + if (is_non_value(node)) { + return NULL; + } + idx = (Uint) WSTACK_POP((*s)); + for (;;) { + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + break; + default: + erl_exit(1, "bad header"); + } + + if (idx > sz) + idx = sz; + else + idx--; + + if (idx >= 1) { + WSTACK_PUSH2((*s), (UWord)idx, (UWord)node); + + if (is_list(ptr[idx])) { + return list_val(ptr[idx]); + } + ASSERT(is_boxed(ptr[idx])); + node = ptr[idx]; + idx = 17; + } + else + break; /* and pop parent node */ + } + } +} + +const Eterm * +#if HALFWORD_HEAP +erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base) +#else +erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) +#endif { - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - Uint sz = map_get_size(mp); + Eterm *ptr, hdr, *res; + Uint ix, lvl = 0; + Uint32 hval,bp; + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + ASSERT(is_hashmap_header_head(hdr)); + ptr++; + + for (;;) { + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + if (hval != 0xffff) { + bp = 1 << ix; + if (!(bp & hval)) { + /* not occupied */ + res = NULL; + break; + } + ix = hashmap_bitcount(hval & (bp - 1)); + } + node = ptr[ix+1]; + + if (is_list(node)) { /* LEAF NODE [K|V] */ + ptr = list_val(node); + + res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL; + break; + } + + hx = hashmap_shift_hash(th,hx,lvl,key); + + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + ASSERT(!is_hashmap_header_head(hdr)); + } + + UnUseTmpHeapNoproc(2); + return res; +} + +Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, + Eterm map, int is_update) { + Uint size, upsz; + Eterm *hp, res = THE_NON_VALUE; + DECLARE_ESTACK(stack); + if (erts_hashmap_insert_down(hx, key, map, &size, &upsz, &stack, is_update)) { + hp = HAlloc(p, size); + res = erts_hashmap_insert_up(hp, key, value, &upsz, &stack); + } + + DESTROY_ESTACK(stack); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + ERTS_HOLE_CHECK(p); + + return res; +} + + +int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, + Uint *update_size, ErtsEStack *sp, int is_update) { + Eterm *ptr; + Eterm hdr, ckey; + Eterm th[2]; + Uint32 ix, cix, bp, hval, chx; + Uint slot, lvl = 0, clvl; + Uint size = 0, n = 0; + + *update_size = 1; + + for (;;) { + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */ + ptr = list_val(node); + ckey = CAR(ptr); + if (EQ(ckey, key)) { + *update_size = 0; + goto unroll; + } + if (is_update) { + return 0; + } + goto insert_subnodes; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = hashmap_index(hx); + hx = hashmap_shift_hash(th,hx,lvl,key); + size += HAMT_HEAD_ARRAY_SZ; + ESTACK_PUSH2(*sp, ix, node); + node = ptr[ix+2]; + break; + case HAMT_SUBTAG_NODE_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + + ESTACK_PUSH4(*sp, n, bp, slot, node); + + /* occupied */ + if (bp & hval) { + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+1]; + ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); + size += HAMT_NODE_BITMAP_SZ(n); + break; + } + /* not occupied */ + if (is_update) { + return 0; + } + size += HAMT_NODE_BITMAP_SZ(n+1); + goto unroll; + case HAMT_SUBTAG_HEAD_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + + ESTACK_PUSH4(*sp, n, bp, slot, node); + + /* occupied */ + if (bp & hval) { + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+2]; + ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); + size += HAMT_HEAD_BITMAP_SZ(n); + break; + } + /* not occupied */ + if (is_update) { + return 0; + } + size += HAMT_HEAD_BITMAP_SZ(n+1); + goto unroll; + default: + erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erl_exit(1, "bad primary tag %p\r\n", node); + break; + } + } +insert_subnodes: + clvl = lvl; + chx = hashmap_restore_hash(th,clvl,ckey); + size += HAMT_NODE_BITMAP_SZ(2); + ix = hashmap_index(hx); + cix = hashmap_index(chx); + + while (cix == ix) { + ESTACK_PUSH4(*sp, 0, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0)); + size += HAMT_NODE_BITMAP_SZ(1); + hx = hashmap_shift_hash(th,hx,lvl,key); + chx = hashmap_shift_hash(th,chx,clvl,ckey); + ix = hashmap_index(hx); + cix = hashmap_index(chx); + } + ESTACK_PUSH3(*sp, cix, ix, node); + +unroll: + *sz = size + /* res cons */ 2; + return 1; +} + +Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, + Uint *update_size, ErtsEStack *sp) { + Eterm node, *ptr, hdr; + Eterm res; + Eterm *nhp = NULL; + Uint32 ix, cix, bp, hval; + Uint slot, n; + /* Needed for halfword */ + DeclareTmpHeapNoproc(fake,1); + UseTmpHeapNoproc(1); + + res = CONS(hp, key, value); hp += 2; + + do { + node = ESTACK_POP(*sp); + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + ix = (Uint32) ESTACK_POP(*sp); + cix = (Uint32) ESTACK_POP(*sp); + + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP((1 << ix) | (1 << cix)); + if (ix < cix) { + *hp++ = res; + *hp++ = node; + } else { + *hp++ = node; + *hp++ = res; + } + res = make_hashmap(nhp); + break; + case TAG_PRIMARY_HEADER: + /* subnodes, fake it */ + *fake = node; + node = make_boxed(fake); + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + slot = (Uint) ESTACK_POP(*sp); + nhp = hp; + n = HAMT_HEAD_ARRAY_SZ - 2; + *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++; + *hp++ = (*ptr++) + *update_size; + while(n--) { *hp++ = *ptr++; } + nhp[slot+2] = res; + res = make_hashmap(nhp); + break; + case HAMT_SUBTAG_NODE_BITMAP: + slot = (Uint) ESTACK_POP(*sp); + bp = (Uint32) ESTACK_POP(*sp); + n = (Uint32) ESTACK_POP(*sp); + hval = MAP_HEADER_VAL(hdr); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval | bp); ptr++; + + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + if (hval & bp) { ptr++; n--; } + while(n--) { *hp++ = *ptr++; } + + res = make_hashmap(nhp); + break; + case HAMT_SUBTAG_HEAD_BITMAP: + slot = (Uint) ESTACK_POP(*sp); + bp = (Uint32) ESTACK_POP(*sp); + n = (Uint32) ESTACK_POP(*sp); + hval = MAP_HEADER_VAL(hdr); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval | bp); ptr++; + *hp++ = (*ptr++) + *update_size; + + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + if (hval & bp) { ptr++; n--; } + while(n--) { *hp++ = *ptr++; } + + if ((hval | bp) == 0xffff) { + *nhp = MAP_HEADER_HAMT_HEAD_ARRAY; + } + res = make_hashmap(nhp); + break; + default: + erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erl_exit(1, "bad primary tag %x\r\n", primary_tag(node)); + break; + } + + } while(!ESTACK_ISEMPTY(*sp)); + + UnUseTmpHeapNoproc(1); + return res; +} + +static Eterm hashmap_keys(Process* p, Eterm node) { + DECLARE_WSTACK(stack); + hashmap_head_t* root; + Eterm *hp, *kv; + Eterm res = NIL; + + root = (hashmap_head_t*) boxed_val(node); + hp = HAlloc(p, root->size * 2); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + res = CONS(hp, CAR(kv), res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +static Eterm hashmap_values(Process* p, Eterm node) { + DECLARE_WSTACK(stack); + hashmap_head_t* root; + Eterm *hp, *kv; + Eterm res = NIL; + + root = (hashmap_head_t*) boxed_val(node); + hp = HAlloc(p, root->size * 2); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + res = CONS(hp, CDR(kv), res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { + Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL; + Eterm th[2]; + Eterm *ptr; + Eterm hdr, res = map, node = map; + Uint32 ix, bp, hval; + Uint slot, lvl = 0; + Uint size = 0, n = 0; + DECLARE_ESTACK(stack); + + for (;;) { + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + if (EQ(CAR(list_val(node)), key)) { + goto unroll; + } + goto not_found; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = hashmap_index(hx); + hx = hashmap_shift_hash(th,hx,lvl,key); + size += HAMT_HEAD_ARRAY_SZ; + ESTACK_PUSH2(stack, ix, node); + node = ptr[ix+2]; + break; + case HAMT_SUBTAG_NODE_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + + ESTACK_PUSH4(stack, n, bp, slot, node); + + /* occupied */ + if (bp & hval) { + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+1]; + ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); + size += HAMT_NODE_BITMAP_SZ(n); + break; + } + /* not occupied */ + goto not_found; + case HAMT_SUBTAG_HEAD_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + + ESTACK_PUSH4(stack, n, bp, slot, node); + + /* occupied */ + if (bp & hval) { + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+2]; + ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); + size += HAMT_HEAD_BITMAP_SZ(n); + break; + } + /* not occupied */ + goto not_found; + default: + erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erl_exit(1, "bad primary tag %p\r\n", node); + break; + } + } + +unroll: + /* the size is bounded and atleast one less than the previous size */ + size -= 1; + n = hashmap_size(map) - 1; + + if (n <= MAP_SMALL_MAP_LIMIT) { + DECLARE_WSTACK(wstack); + Eterm *kv, *ks, *vs; + flatmap_t *mp; + Eterm keys; + + DESTROY_ESTACK(stack); + + /* build flat structure */ + hp = HAlloc(p, 3 + 1 + (2 * n)); + keys = make_tuple(hp); + *hp++ = make_arityval(n); + ks = hp; + hp += n; + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; + mp->keys = keys; + + hashmap_iterator_init(&wstack, map, 0); + + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + if (EQ(CAR(kv),key)) + continue; + *ks++ = CAR(kv); + *vs++ = CDR(kv); + } + + /* it cannot have multiple keys */ + erts_validate_and_sort_flatmap(mp); + + DESTROY_WSTACK(wstack); + return make_flatmap(mp); + } + + hp = HAlloc(p, size); + hp_end = hp + size; + res = THE_NON_VALUE; + + do { + node = ESTACK_POP(stack); + + /* all nodes are things */ + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = (Uint) ESTACK_POP(stack); + nhp = hp; + if (res == THE_NON_VALUE) { + n = 16; + n -= ix; + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(0xffff ^ (1 << ix)); ptr++; + *hp++ = (*ptr++) - 1; + while(ix--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } else { + n = 16; + *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++; + *hp++ = (*ptr++) - 1; + while(n--) { *hp++ = *ptr++; } + nhp[ix+2] = res; + res = make_hashmap(nhp); + } + break; + case HAMT_SUBTAG_NODE_BITMAP: + slot = (Uint) ESTACK_POP(stack); + bp = (Uint32) ESTACK_POP(stack); + n = (Uint32) ESTACK_POP(stack); + nhp = hp; + + /* bitmap change matrix + * res | none leaf bitmap + * ---------------------------- + * n=1 | remove remove keep + * n=2 | other keep keep + * n>2 | shrink keep keep + * + * other: (remember, n is 2) + * shrink if the other bitmap value is a bitmap node + * remove if the other bitmap value is a leaf + * + * remove: + * this bitmap node is removed, res is moved up in tree (could be none) + * this is a special case of shrink + * + * keep: + * the current path index is still used down in the tree, need to keep it + * copy as usual with the updated res + * + * shrink: + * the current path index is no longer used down in the tree, remove it (shrink) + */ + if (res == THE_NON_VALUE) { + if (n == 1) { + break; + } else if (n == 2) { + if (slot == 0) { + ix = 2; /* off by one 'cause hdr */ + } else { + ix = 1; /* off by one 'cause hdr */ + } + if (primary_tag(ptr[ix]) == TAG_PRIMARY_LIST) { + res = ptr[ix]; + } else { + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); + *hp++ = ptr[ix]; + res = make_hashmap(nhp); + } + } else { + /* n > 2 */ + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); ptr++; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } + } else if (primary_tag(res) == TAG_PRIMARY_LIST && n == 1) { + break; + } else { + /* res is bitmap or leaf && n > 1, keep */ + n -= slot; + *hp++ = *ptr++; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } + break; + case HAMT_SUBTAG_HEAD_BITMAP: + slot = (Uint) ESTACK_POP(stack); + bp = (Uint32) ESTACK_POP(stack); + n = (Uint32) ESTACK_POP(stack); + nhp = hp; + + if (res != THE_NON_VALUE) { + *hp++ = *ptr++; + *hp++ = (*ptr++) - 1; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + } else { + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval ^ bp); ptr++; + *hp++ = (*ptr++) - 1; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + } + res = make_hashmap(nhp); + break; + default: + erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + } while(!ESTACK_ISEMPTY(stack)); + HRelease(p, hp_end, hp); +not_found: + DESTROY_ESTACK(stack); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + ERTS_HOLE_CHECK(p); + return res; +} + + +int erts_validate_and_sort_flatmap(flatmap_t* mp) +{ + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + Uint sz = flatmap_get_size(mp); Uint ix,jx; Eterm tmp; - int c; + Sint c; /* sort */ @@ -811,6 +2443,55 @@ int erts_validate_and_sort_map(map_t* mp) return 1; } +/* Really rough estimate of sqrt(x) + * Guaranteed not to be less than sqrt(x) + */ +static int int_sqrt_ceiling(Uint x) +{ + int n; + + if (x <= 2) + return x; + + n = erts_fit_in_bits_uint(x-1); + if (n & 1) { + /* Calc: sqrt(2^n) = 2^(n/2) * sqrt(2) ~= 2^(n/2) * 3 / 2 */ + return (1 << (n/2 - 1)) * 3; + } + else { + /* Calc: sqrt(2^n) = 2^(n/2) */ + return 1 << (n / 2); + } +} + +Uint hashmap_over_estimated_heap_size(Uint k) +{ + /* k is nr of key-value pairs. + N(k) is expected nr of nodes in hamt. + + Observation: + For uniformly distributed hash values, average of N varies between + 0.3*k and 0.4*k (with a beautiful sine curve) + and standard deviation of N is about sqrt(k)/3. + + Assuming normal probability distribution, we overestimate nr of nodes + by 15 std.devs above the average, which gives a probability for overrun + less than 1.0e-49 (same magnitude as a git SHA1 collision). + */ + Uint max_nodes = 2*k/5 + (15/3)*int_sqrt_ceiling(k); + return (k*2 + /* leaf cons cells */ + k + /* leaf list terms */ + max_nodes*2); /* headers + parent boxed terms */ +} + + +BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) { + if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_info(BIF_P,BIF_ARG_1)); + } + BIF_ERROR(BIF_P, BADARG); +} + /* * erts_internal:map_to_tuple_keys/1 * @@ -818,9 +2499,220 @@ int erts_validate_and_sort_map(map_t* mp) */ BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { - map_t *mp = (map_t*)map_val(BIF_ARG_1); + if (is_flatmap(BIF_ARG_1)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); BIF_RET(mp->keys); } BIF_ERROR(BIF_P, BADARG); } + +/* + * erts_internal:map_type/1 + * + * Used in erts_debug:size/1 + */ + +BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) { + DECL_AM(hashmap); + DECL_AM(hashmap_node); + DECL_AM(flatmap); + if (is_flatmap(BIF_ARG_1)) { + BIF_RET(AM_flatmap); + } else if (is_hashmap(BIF_ARG_1)) { + Eterm hdr = *(boxed_val(BIF_ARG_1)); + ASSERT(is_header(hdr)); + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + BIF_RET(AM_hashmap); + case HAMT_SUBTAG_NODE_BITMAP: + BIF_RET(AM_hashmap_node); + default: + erl_exit(1, "bad header"); + } + } + BIF_ERROR(BIF_P, BADARG); +} + +/* + * erts_internal:map_hashmap_children/1 + * + * Used in erts_debug:size/1 + */ + +BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) { + if (is_hashmap(BIF_ARG_1)) { + Eterm node = BIF_ARG_1; + Eterm *ptr, hdr, *hp, res = NIL; + Uint sz = 0; + ptr = boxed_val(node); + hdr = *ptr; + + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ptr += 1; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ptr += 2; + break; + case HAMT_SUBTAG_HEAD_ARRAY: + sz = 16; + ptr += 2; + break; + default: + erl_exit(1, "bad header\r\n"); + break; + } + ASSERT(sz < 17); + hp = HAlloc(BIF_P, 2*sz); + while(sz--) { res = CONS(hp, *ptr++, res); hp += 2; } + BIF_RET(res); + } + BIF_ERROR(BIF_P, BADARG); +} + + +static Eterm hashmap_info(Process *p, Eterm node) { + Eterm *hp; + Eterm res = NIL, info = NIL; + Eterm *ptr, tup, hdr; + Uint sz; + DECL_AM(depth); + DECL_AM(leafs); + DECL_AM(bitmaps); + DECL_AM(arrays); + Uint nleaf=0, nbitmap=0, narray=0; + Uint bitmap_usage[16], leaf_usage[16]; + Uint lvl = 0, clvl; + DECLARE_ESTACK(stack); + + for (sz = 0; sz < 16; sz++) { + bitmap_usage[sz] = 0; + leaf_usage[sz] = 0; + } + + ptr = boxed_val(node); + ESTACK_PUSH(stack, 0); + ESTACK_PUSH(stack, node); + do { + node = ESTACK_POP(stack); + clvl = ESTACK_POP(stack); + if (lvl < clvl) + lvl = clvl; + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + nleaf++; + leaf_usage[clvl] += 1; + break; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_NODE_BITMAP: + nbitmap++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + bitmap_usage[sz-1] += 1; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+1]); + } + break; + case HAMT_SUBTAG_HEAD_BITMAP: + nbitmap++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + bitmap_usage[sz-1] += 1; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+2]); + } + break; + case HAMT_SUBTAG_HEAD_ARRAY: + narray++; + sz = 16; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+2]); + } + break; + default: + erl_exit(1, "bad header\r\n"); + break; + } + } + } while(!ESTACK_ISEMPTY(stack)); + + + /* size */ + sz = 0; + hashmap_bld_tuple_uint(NULL,&sz,16,leaf_usage); + hashmap_bld_tuple_uint(NULL,&sz,16,bitmap_usage); + + /* alloc */ + hp = HAlloc(p, 2+3 + 3*(2+4) + sz); + + info = hashmap_bld_tuple_uint(&hp,NULL,16,leaf_usage); + tup = TUPLE3(hp, AM_leafs, make_small(nleaf),info); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + info = hashmap_bld_tuple_uint(&hp,NULL,16,bitmap_usage); + tup = TUPLE3(hp, AM_bitmaps, make_small(nbitmap), info); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + tup = TUPLE3(hp, AM_arrays, make_small(narray),NIL); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + tup = TUPLE2(hp, AM_depth, make_small(lvl)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + DESTROY_ESTACK(stack); + ERTS_HOLE_CHECK(p); + return res; +} + +static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]) { + Eterm res = THE_NON_VALUE; + Eterm *ts = (Eterm *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm)); + Uint i; + + for (i = 0; i < n; i++) { + ts[i] = erts_bld_uint(hpp, szp, nums[i]); + } + res = erts_bld_tuplev(hpp, szp, n, ts); + erts_free(ERTS_ALC_T_TMP, (void *) ts); + return res; +} + + +/* implementation of builtin emulations */ + +#if !ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) +/* Count leading zeros emulation */ +Uint32 hashmap_clz(Uint32 x) { + Uint32 y; + int n = 32; + y = x >>16; if (y != 0) {n = n -16; x = y;} + y = x >> 8; if (y != 0) {n = n - 8; x = y;} + y = x >> 4; if (y != 0) {n = n - 4; x = y;} + y = x >> 2; if (y != 0) {n = n - 2; x = y;} + y = x >> 1; if (y != 0) return n - 2; + return n - x; +} + +const Uint32 SK5 = 0x55555555, SK3 = 0x33333333; +const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF; + +/* CTPOP emulation */ +Uint32 hashmap_bitcount(Uint32 x) { + x -= ((x >> 1 ) & SK5); + x = (x & SK3 ) + ((x >> 2 ) & SK3 ); + x = (x & SKF0) + ((x >> 4 ) & SKF0); + x += x >> 8; + return (x + (x >> 16)) & 0x3F; +} +#endif diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 2e02ca4677..2cc6768bfc 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -22,13 +22,23 @@ #define __ERL_MAP_H__ #include "sys.h" + +/* instrinsic wrappers */ +#if ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) +#define hashmap_clz(x) ((Uint32) __builtin_clz((unsigned int)(x))) +#define hashmap_bitcount(x) ((Uint32) __builtin_popcount((unsigned int) (x))) +#else +Uint32 hashmap_clz(Uint32 x); +Uint32 hashmap_bitcount(Uint32 x); +#endif + /* MAP */ -typedef struct map_s { +typedef struct flatmap_s { Eterm thing_word; Uint size; Eterm keys; /* tuple */ -} map_t; +} flatmap_t; /* map node * * ----------- @@ -42,39 +52,144 @@ typedef struct map_s { * ----------- */ +/* the head-node is a bitmap or array with an untagged size */ + + +#define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size) +#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE)) +#define hashmap_make_hash(Key) make_internal_hash(Key) + +#define hashmap_restore_hash(Heap,Lvl,Key) \ + (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7))) +#define hashmap_shift_hash(Heap,Hx,Lvl,Key) \ + (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key))) /* erl_term.h stuff */ -#define make_map(x) make_boxed((Eterm*)(x)) -#define make_map_rel(x, BASE) make_boxed_rel((Eterm*)(x),(BASE)) -#define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val((x)))) -#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE)) -#define is_not_map(x) (!is_map((x))) -#define is_map_header(x) (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP) -#define header_is_map(x) ((((x) & (_HEADER_SUBTAG_MASK)) == MAP_SUBTAG)) -#define map_val(x) (_unchecked_boxed_val((x))) -#define map_val_rel(RTERM, BASE) map_val(rterm2wterm(RTERM, BASE)) - -#define map_get_values(x) (((Eterm *)(x)) + 3) -#define map_get_keys(x) (((Eterm *)tuple_val(((map_t *)(x))->keys)) + 1) -#define map_get_size(x) (((map_t*)(x))->size) - -#define MAP_HEADER _make_header(1,_TAG_HEADER_MAP) -#define MAP_HEADER_SIZE (sizeof(map_t) / sizeof(Eterm)) - -Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map); -int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res); -int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res); -int erts_validate_and_sort_map(map_t* map); +#define flatmap_get_values(x) (((Eterm *)(x)) + 3) +#define flatmap_get_keys(x) (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1) +#define flatmap_get_size(x) (((flatmap_t*)(x))->size) + +#ifdef DEBUG +#define MAP_SMALL_MAP_LIMIT (3) +#else +#define MAP_SMALL_MAP_LIMIT (32) +#endif + +struct ErtsWStack_; +struct ErtsEStack_; + +Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map); +int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res); +int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res); + +Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, + Eterm node, int is_update); +int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, + Uint *upsz, struct ErtsEStack_ *sp, int is_update); +Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, + Uint *upsz, struct ErtsEStack_ *sp); + +int erts_validate_and_sort_flatmap(flatmap_t* map); +Uint hashmap_over_estimated_heap_size(Uint n); +void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse); +Eterm* hashmap_iterator_next(struct ErtsWStack_* s); +Eterm* hashmap_iterator_prev(struct ErtsWStack_* s); +int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp); +Eterm erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys); + +#define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \ + erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE); + +Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, + Eterm k, Eterm v); -#if HALFWORD_HEAP const Eterm * +#if HALFWORD_HEAP erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base); # define erts_maps_get(A, B) erts_maps_get_rel(A, B, NULL) #else -const Eterm * erts_maps_get(Eterm key, Eterm map); # define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B) #endif +const Eterm * +#if HALFWORD_HEAP +erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base); +# define erts_hashmap_get(Hx, K, M) erts_hashmap_get_rel(Hx, K, M, NULL) +#else +erts_hashmap_get(Uint32 hx, Eterm key, Eterm map); +# define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M) +#endif + +/* hamt nodes v2.0 + * + * node :: leaf | array | bitmap + * head + */ +typedef struct hashmap_head_s { + Eterm thing_word; + Uint size; + Eterm items[1]; +} hashmap_head_t; + +/* thing_word tagscheme + * Need two bits for map subtags + * + * Original HEADER representation: + * + * aaaaaaaaaaaaaaaa aaaaaaaaaatttt00 arity:26, tag:4 + * + * For maps we have: + * + * vvvvvvvvvvvvvvvv aaaaaaaamm111100 val:16, arity:8, mtype:2 + * + * unsure about trailing zeros + * + * map-tag: + * 00 - flat map tag (non-hamt) -> val:16 = #items + * 01 - map-node bitmap tag -> val:16 = bitmap + * 10 - map-head (array-node) -> val:16 = 0xffff + * 11 - map-head (bitmap-node) -> val:16 = bitmap + */ + +/* erl_map.h stuff */ + +#define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2))) + +#define MAKE_MAP_HEADER(Type,Arity,Val) \ + (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_MAP)) + +#define MAP_HEADER_FLATMAP \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_FLATMAP_HEAD,0x1,0x0) + +#define MAP_HEADER_HAMT_HEAD_ARRAY \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff) + +#define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp) + +#define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp) + +#define MAP_HEADER_FLATMAP_SZ (sizeof(flatmap_t) / sizeof(Eterm)) + +#define HAMT_NODE_ARRAY_SZ (17) +#define HAMT_HEAD_ARRAY_SZ (18) +#define HAMT_NODE_BITMAP_SZ(n) (1 + n) +#define HAMT_HEAD_BITMAP_SZ(n) (2 + n) + +/* 2 bits maps tag + 4 bits subtag + 2 ignore bits */ +#define _HEADER_MAP_SUBTAG_MASK (0xfc) +/* 1 bit map tag + 1 ignore bit + 4 bits subtag + 2 ignore bits */ +#define _HEADER_MAP_HASHMAP_HEAD_MASK (0xbc) + +#define HAMT_SUBTAG_NODE_BITMAP ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_ARRAY ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_BITMAP ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_FLATMAP ((MAP_HEADER_TAG_FLATMAP_HEAD << _HEADER_ARITY_OFFS) | MAP_SUBTAG) + +#define hashmap_index(hash) (((Uint32)hash) & 0xf) + + #endif diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 8870fac7d9..22cbae10d1 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -994,7 +994,7 @@ erts_send_message(Process* sender, #endif ); BM_SWAP_TIMER(send,system); - } else if (sender == receiver) { + } else if (sender == receiver && !(sender->flags & F_OFF_HEAP_MSGS)) { /* Drop message if receiver has a pending exit ... */ #ifdef ERTS_SMP ErtsProcLocks need_locks = (~(*receiver_locks) @@ -1146,3 +1146,15 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, } } +Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +{ + Eterm* res; + if (factory->p) { + res = HAllocX(factory->p, need, xtra); + } else { + res = factory->hp; + factory->hp += need; + } + return res; +} + diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 0f3bb8d281..8713941769 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -68,6 +68,21 @@ struct erl_heap_fragment { Eterm mem[1]; /* Data */ }; +typedef struct { + Process* p; + Eterm* hp; +} ErtsHeapFactory; + +Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); +#ifdef CHECK_FOR_HOLES +# define ERTS_FACTORY_HOLE_CHECK(f) do { \ + if ((f)->p) erts_check_for_holes((f)->p); \ + } while (0) +#else +# define ERTS_FACTORY_HOLE_CHECK(p) +#endif + + typedef struct erl_mesg { struct erl_mesg* next; /* Next message */ union { @@ -198,15 +213,25 @@ do { \ if ((M)->data.attached) { \ Uint need__ = erts_msg_attached_data_size((M)); \ if ((ST) - (HT) >= need__) { \ - Uint *htop__ = (HT); \ + Uint *htop__; \ + move__attached__msg__data____: \ + htop__ = (HT); \ erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\ ASSERT(htop__ - (HT) <= need__); \ (HT) = htop__; \ } \ else { \ + int off_heap_msgs__ = (int) (P)->flags & F_OFF_HEAP_MSGS; \ + if (!off_heap_msgs__) \ + need__ = 0; \ { SWPO ; } \ - (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ + (FC) -= erts_garbage_collect((P), need__, NULL, 0); \ { SWPI ; } \ + if (off_heap_msgs__) { \ + ASSERT((M)->data.attached); \ + ASSERT((ST) - (HT) >= need__); \ + goto move__attached__msg__data____; \ + } \ } \ ASSERT(!(M)->data.attached); \ } \ diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index fb11dbbd22..9972890db7 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -82,6 +82,7 @@ /* Type tags for monitors */ #define MON_ORIGIN 1 #define MON_TARGET 3 +#define MON_TIME_OFFSET 7 /* Type tags for links */ #define LINK_PID 1 /* ...Or port */ @@ -103,7 +104,7 @@ typedef struct erts_monitor_or_link { typedef struct erts_monitor { struct erts_monitor *left, *right; Sint16 balance; - Uint16 type; /* MON_ORIGIN | MON_TARGET */ + Uint16 type; /* MON_ORIGIN | MON_TARGET | MON_TIME_OFFSET */ Eterm ref; Eterm pid; /* In case of distributed named monitor, this is the nodename atom in MON_ORIGIN process, otherwise a pid or diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 198acfd128..660f446a52 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -36,6 +36,7 @@ #include "erl_thr_progress.h" #include "dtrace-wrapper.h" #include "erl_process.h" +#include "erl_bif_unique.h" #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP)) #define HAVE_USE_DTRACE 1 #endif @@ -1910,29 +1911,33 @@ int enif_is_map(ErlNifEnv* env, ERL_NIF_TERM term) int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size) { - if (is_map(term)) { - map_t *mp; - mp = (map_t*)map_val(term); - *size = map_get_size(mp); + if (is_flatmap(term)) { + flatmap_t *mp; + mp = (flatmap_t*)flatmap_val(term); + *size = flatmap_get_size(mp); return 1; } + else if (is_hashmap(term)) { + *size = hashmap_size(term); + return 1; + } return 0; } ERL_NIF_TERM enif_make_new_map(ErlNifEnv* env) { - Eterm* hp = alloc_heap(env,MAP_HEADER_SIZE+1); + Eterm* hp = alloc_heap(env,MAP_HEADER_FLATMAP_SZ+1); Eterm tup; - map_t *mp; + flatmap_t *mp; tup = make_tuple(hp); *hp++ = make_arityval(0); - mp = (map_t*)hp; - mp->thing_word = MAP_HEADER; + mp = (flatmap_t*)hp; + mp->thing_word = MAP_HEADER_FLATMAP; mp->size = 0; mp->keys = tup; - return make_map(mp); + return make_flatmap(mp); } int enif_make_map_put(ErlNifEnv* env, @@ -1941,7 +1946,7 @@ int enif_make_map_put(ErlNifEnv* env, Eterm value, Eterm *map_out) { - if (is_not_map(map_in)) { + if (!is_map(map_in)) { return 0; } flush_env(env); @@ -1956,7 +1961,7 @@ int enif_get_map_value(ErlNifEnv* env, Eterm *value) { const Eterm *ret; - if (is_not_map(map)) { + if (!is_map(map)) { return 0; } ret = erts_maps_get(key, map); @@ -1974,7 +1979,7 @@ int enif_make_map_update(ErlNifEnv* env, Eterm *map_out) { int res; - if (is_not_map(map_in)) { + if (!is_map(map_in)) { return 0; } @@ -1990,7 +1995,7 @@ int enif_make_map_remove(ErlNifEnv* env, Eterm *map_out) { int res; - if (is_not_map(map_in)) { + if (!is_map(map_in)) { return 0; } flush_env(env); @@ -2004,13 +2009,13 @@ int enif_map_iterator_create(ErlNifEnv *env, ErlNifMapIterator *iter, ErlNifMapIteratorEntry entry) { - if (is_map(map)) { - map_t *mp = (map_t*)map_val(map); + if (is_flatmap(map)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(map); size_t offset; switch (entry) { case ERL_NIF_MAP_ITERATOR_HEAD: offset = 0; break; - case ERL_NIF_MAP_ITERATOR_TAIL: offset = map_get_size(mp) - 1; break; + case ERL_NIF_MAP_ITERATOR_TAIL: offset = flatmap_get_size(mp) - 1; break; default: goto error; } @@ -2019,14 +2024,37 @@ int enif_map_iterator_create(ErlNifEnv *env, */ iter->map = map; - iter->ks = ((Eterm *)map_get_keys(mp)) + offset; - iter->vs = ((Eterm *)map_get_values(mp)) + offset; - iter->t_limit = map_get_size(mp) + 1; + iter->u.flat.ks = ((Eterm *)flatmap_get_keys(mp)) + offset; + iter->u.flat.vs = ((Eterm *)flatmap_get_values(mp)) + offset; + iter->size = flatmap_get_size(mp); iter->idx = offset + 1; return 1; } - + else if (is_hashmap(map)) { + iter->map = map; + iter->size = hashmap_size(map); + iter->u.hash.wstack = erts_alloc(ERTS_ALC_T_NIF, sizeof(ErtsDynamicWStack)); + WSTACK_INIT(iter->u.hash.wstack, ERTS_ALC_T_NIF); + + switch (entry) { + case ERL_NIF_MAP_ITERATOR_HEAD: + iter->idx = 1; + hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 0); + iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws); + break; + case ERL_NIF_MAP_ITERATOR_TAIL: + iter->idx = hashmap_size(map); + hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 1); + iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws); + break; + default: + goto error; + } + ASSERT(!!iter->u.hash.kv == (iter->idx >= 1 && + iter->idx <= iter->size)); + return 1; + } error: #ifdef DEBUG iter->map = THE_NON_VALUE; @@ -2036,48 +2064,97 @@ error: void enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter) { - /* not used */ + if (is_hashmap(iter->map)) { + WSTACK_DESTROY(iter->u.hash.wstack->ws); + erts_free(ERTS_ALC_T_NIF, iter->u.hash.wstack); + } + else + ASSERT(is_flatmap(iter->map)); + #ifdef DEBUG iter->map = THE_NON_VALUE; #endif - } int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter) { - ASSERT(iter && is_map(iter->map)); - ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1)); - return (iter->t_limit == 1 || iter->idx == iter->t_limit); + ASSERT(iter); + if (is_flatmap(iter->map)) { + ASSERT(iter->idx >= 0); + ASSERT(iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1); + return (iter->size == 0 || iter->idx > iter->size); + } + else { + ASSERT(is_hashmap(iter->map)); + return iter->idx > iter->size; + } } int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter) { - ASSERT(iter && is_map(iter->map)); - ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1)); - return (iter->t_limit == 1 || iter->idx == 0); + ASSERT(iter); + if (is_flatmap(iter->map)) { + ASSERT(iter->idx >= 0); + ASSERT(iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1); + return (iter->size == 0 || iter->idx == 0); + } + else { + ASSERT(is_hashmap(iter->map)); + return iter->idx == 0; + } } int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter) { - ASSERT(iter && is_map(iter->map)); - if (iter->idx < iter->t_limit) { - iter->idx++; - iter->ks++; - iter->vs++; + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx <= iter->size) { + iter->idx++; + iter->u.flat.ks++; + iter->u.flat.vs++; + } + return (iter->idx <= iter->size); + } + else { + ASSERT(is_hashmap(iter->map)); + + if (iter->idx <= hashmap_size(iter->map)) { + if (iter->idx < 1) { + hashmap_iterator_init(&iter->u.hash.wstack->ws, iter->map, 0); + } + iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws); + iter->idx++; + ASSERT(!!iter->u.hash.kv == (iter->idx <= iter->size)); + } + return iter->idx <= iter->size; } - return (iter->idx != iter->t_limit); } int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter) { - ASSERT(iter && is_map(iter->map)); - if (iter->idx > 0) { - iter->idx--; - iter->ks--; - iter->vs--; + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx > 0) { + iter->idx--; + iter->u.flat.ks--; + iter->u.flat.vs--; + } + return iter->idx > 0; + } + else { + ASSERT(is_hashmap(iter->map)); + + if (iter->idx > 0) { + if (iter->idx > iter->size) { + hashmap_iterator_init(&iter->u.hash.wstack->ws, iter->map, 1); + } + iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws); + iter->idx--; + ASSERT(!!iter->u.hash.kv == (iter->idx > 0)); + } + return iter->idx > 0; } - return (iter->idx > 0); } int enif_map_iterator_get_pair(ErlNifEnv *env, @@ -2085,15 +2162,25 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, Eterm *key, Eterm *value) { - ASSERT(iter && is_map(iter->map)); - if (iter->idx > 0 && iter->idx < iter->t_limit) { - ASSERT(iter->ks >= map_get_keys(map_val(iter->map)) && - iter->ks < (map_get_keys(map_val(iter->map)) + map_get_size(map_val(iter->map)))); - ASSERT(iter->vs >= map_get_values(map_val(iter->map)) && - iter->vs < (map_get_values(map_val(iter->map)) + map_get_size(map_val(iter->map)))); - *key = *(iter->ks); - *value = *(iter->vs); - return 1; + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx > 0 && iter->idx <= iter->size) { + ASSERT(iter->u.flat.ks >= flatmap_get_keys(flatmap_val(iter->map)) && + iter->u.flat.ks < (flatmap_get_keys(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map)))); + ASSERT(iter->u.flat.vs >= flatmap_get_values(flatmap_val(iter->map)) && + iter->u.flat.vs < (flatmap_get_values(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map)))); + *key = *(iter->u.flat.ks); + *value = *(iter->u.flat.vs); + return 1; + } + } + else { + ASSERT(is_hashmap(iter->map)); + if (iter->idx > 0 && iter->idx <= iter->size) { + *key = CAR(iter->u.hash.kv); + *value = CDR(iter->u.hash.kv); + return 1; + } } return 0; } diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 849024453c..9b2b90c82d 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -201,10 +201,18 @@ typedef enum typedef struct /* All fields all internal and may change */ { ERL_NIF_TERM map; - ERL_NIF_UINT t_limit; + ERL_NIF_UINT size; ERL_NIF_UINT idx; - ERL_NIF_TERM *ks; - ERL_NIF_TERM *vs; + union { + struct { + ERL_NIF_TERM *ks; + ERL_NIF_TERM *vs; + }flat; + struct { + struct ErtsDynamicWStack_* wstack; + ERL_NIF_TERM* kv; + }hash; + }u; void* __spare__[2]; /* for future additions to be ABI compatible (same struct size) */ } ErlNifMapIterator; diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index c982dc2080..e0dfbd31b8 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -247,6 +247,17 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) #define PRT_PATCH_FUN_SIZE ((Eterm) 7) #define PRT_LAST_ARRAY_ELEMENT ((Eterm) 8) /* Note! Must be last... */ +#if 0 +static char *format_binary(Uint16 x, char *b) { + int z; + b[16] = '\0'; + for (z = 0; z < 16; z++) { + b[15-z] = ((x>>z) & 0x1) ? '1' : '0'; + } + return b; +} +#endif + static int print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, Eterm* obj_base) /* ignored if !HALFWORD_HEAP */ @@ -554,28 +565,74 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } break; case MAP_DEF: - { - Uint n; - Eterm *ks, *vs; - map_t *mp = (map_t *)map_val(wobj); - n = map_get_size(mp); - ks = map_get_keys(mp); - vs = map_get_values(mp); - - PRINT_CHAR(res, fn, arg, '#'); - PRINT_CHAR(res, fn, arg, '{'); - WSTACK_PUSH(s, PRT_CLOSE_TUPLE); - if (n > 0) { - n--; - WSTACK_PUSH5(s, vs[n], PRT_TERM, PRT_ASSOC, ks[n], PRT_TERM); - while (n--) { - WSTACK_PUSH6(s, PRT_COMMA, vs[n], PRT_TERM, PRT_ASSOC, - ks[n], PRT_TERM); - } - } - } - break; - default: + if (is_flatmap(wobj)) { + Uint n; + Eterm *ks, *vs; + flatmap_t *mp = (flatmap_t *)flatmap_val(wobj); + n = flatmap_get_size(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + PRINT_CHAR(res, fn, arg, '#'); + PRINT_CHAR(res, fn, arg, '{'); + WSTACK_PUSH(s, PRT_CLOSE_TUPLE); + if (n > 0) { + n--; + WSTACK_PUSH5(s, vs[n], PRT_TERM, PRT_ASSOC, ks[n], PRT_TERM); + while (n--) { + WSTACK_PUSH6(s, PRT_COMMA, vs[n], PRT_TERM, PRT_ASSOC, + ks[n], PRT_TERM); + } + } + } else { + Uint n, mapval; + Eterm *head; + head = hashmap_val(wobj); + mapval = MAP_HEADER_VAL(*head); + switch (MAP_HEADER_TYPE(*head)) { + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY: + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP: + PRINT_STRING(res, fn, arg, "#<"); + PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval); + PRINT_STRING(res, fn, arg, ">{"); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + n = hashmap_bitcount(mapval); + ASSERT(n < 17); + head += 2; + if (n > 0) { + n--; + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + while (n--) { + WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + } + } + break; + case MAP_HEADER_TAG_HAMT_NODE_BITMAP: + n = hashmap_bitcount(mapval); + head++; + PRINT_CHAR(res, fn, arg, '<'); + PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval); + PRINT_STRING(res, fn, arg, ">{"); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + ASSERT(n < 17); + if (n > 0) { + n--; + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + while (n--) { + WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + } + } + break; + } + } + break; + default: PRINT_STRING(res, fn, arg, "<unknown:"); PRINT_POINTER(res, fn, arg, wobj); PRINT_CHAR(res, fn, arg, '>'); @@ -584,11 +641,11 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } L_done: - DESTROY_WSTACK(s); return res; } + int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, ErlPfEterm* term_base) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index ba09ee57c2..f74a2ee54c 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -43,6 +43,7 @@ #include "erl_async.h" #include "dtrace-wrapper.h" #include "erl_ptab.h" +#include "erl_bif_unique.h" #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) @@ -701,8 +702,8 @@ init_sched_wall_time(ErtsSchedWallTime *swtp) static ERTS_INLINE Uint64 sched_wall_time_ts(void) { -#ifdef HAVE_GETHRTIME - return (Uint64) sys_gethrtime(); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return (Uint64) erts_os_monotonic_time(); #else Uint64 res; SysTimeval tv; @@ -2185,7 +2186,7 @@ aux_work_timeout_late_init(void) { aux_work_tmo->initialized = 1; if (erts_atomic32_read_nob(&aux_work_tmo->refc)) { - aux_work_tmo->timer.data.active = 0; + erts_init_timer(&aux_work_tmo->timer.data); erts_set_timer(&aux_work_tmo->timer.data, aux_work_timeout, NULL, @@ -2218,7 +2219,6 @@ aux_work_timeout(void *unused) if (refc != 1 || 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) { /* Setup next timeout... */ - aux_work_tmo->timer.data.active = 0; erts_set_timer(&aux_work_tmo->timer.data, aux_work_timeout, NULL, @@ -2237,7 +2237,7 @@ setup_aux_work_timer(void) else #endif { - aux_work_tmo->timer.data.active = 0; + erts_init_timer(&aux_work_tmo->timer.data); erts_set_timer(&aux_work_tmo->timer.data, aux_work_timeout, NULL, @@ -2638,6 +2638,13 @@ thr_prgr_fin_wait(void *vssi) static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); +void +erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time) +{ + /* TODO only poke when needed (based on timeout_time) */ + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1)); +} + static void * aux_thread(void *unused) { @@ -2646,6 +2653,11 @@ aux_thread(void *unused) erts_aint32_t aux_work; ErtsThrPrgrCallbacks callbacks; int thr_prgr_active = 1; + ErtsTimerWheel *timer_wheel = erts_default_timer_wheel; + ErtsNextTimeoutRef nxt_tmo_ref = erts_get_next_timeout_reference(timer_wheel); + + if (!timer_wheel) + ERTS_INTERNAL_ERROR("Missing aux timer wheel"); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -2669,6 +2681,7 @@ aux_thread(void *unused) sched_prep_spin_wait(ssi); while (1) { + ErtsMonotonicTime current_time; erts_aint32_t flgs; aux_work = erts_atomic32_read_acqb(&ssi->aux_work); @@ -2680,28 +2693,56 @@ aux_thread(void *unused) erts_thr_progress_leader_update(NULL); } - if (!aux_work) { - if (thr_prgr_active) - erts_thr_progress_active(NULL, thr_prgr_active = 0); - erts_thr_progress_prepare_wait(NULL); + if (aux_work) { + current_time = erts_get_monotonic_time(); + if (current_time >= erts_next_timeout_time(nxt_tmo_ref)) { + if (!thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 1); + erts_bump_timers(timer_wheel, current_time); + } + } + else { + ErtsMonotonicTime timeout_time; + timeout_time = erts_check_next_timeout_time(timer_wheel, + ERTS_SEC_TO_MONOTONIC(10*60)); + current_time = erts_get_monotonic_time(); + if (current_time >= timeout_time) { + if (!thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 1); + } + else { + if (thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 0); + erts_thr_progress_prepare_wait(NULL); - ERTS_SCHED_FAIR_YIELD(); + ERTS_SCHED_FAIR_YIELD(); - flgs = sched_spin_wait(ssi, 0); + flgs = sched_spin_wait(ssi, 0); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + current_time = erts_get_monotonic_time(); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + res = erts_tse_twait(ssi->event, timeout); + current_time = erts_get_monotonic_time(); + } while (res == EINTR); + } } + erts_thr_progress_finalize_wait(NULL); } - erts_thr_progress_finalize_wait(NULL); + if (current_time >= timeout_time) + erts_bump_timers(timer_wheel, current_time); } flgs = sched_prep_spin_wait(ssi); @@ -2768,6 +2809,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_wall_time_change(esdp, thr_prgr_active); while (1) { + ErtsMonotonicTime current_time; aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work) { @@ -2781,34 +2823,65 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_leader_update(esdp); } - if (aux_work) + if (aux_work) { flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + current_time = erts_get_monotonic_time(); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } else { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + ErtsMonotonicTime timeout_time; + timeout_time = erts_check_next_timeout_time(esdp->timer_wheel, + ERTS_SEC_TO_MONOTONIC(10*60)); + current_time = erts_get_monotonic_time(); + if (current_time >= timeout_time) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); } - erts_thr_progress_prepare_wait(esdp); } + else { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } - ERTS_SCHED_FAIR_YIELD(); + ERTS_SCHED_FAIR_YIELD(); - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + current_time = erts_get_monotonic_time(); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + res = erts_tse_twait(ssi->event, timeout); + current_time = erts_get_monotonic_time(); + } while (res == EINTR); + } } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + erts_thr_progress_finalize_wait(esdp); } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) - erts_thr_progress_finalize_wait(esdp); + if (current_time >= timeout_time) + erts_bump_timers(esdp->timer_wheel, current_time); } if (!(flgs & ERTS_SSI_FLG_WAITING)) { @@ -2841,7 +2914,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) else #endif { - erts_aint_t dt; erts_smp_atomic32_set_relb(&function_calls, 0); *fcalls = 0; @@ -2866,6 +2938,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto sys_aux_work; while (spincount-- > 0) { + ErtsMonotonicTime current_time; sys_poll_aux_work: @@ -2875,8 +2948,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(1); /* Might give us something to do */ - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + current_time = erts_get_monotonic_time(); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); sys_aux_work: #ifndef ERTS_SMP @@ -2991,8 +3065,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erl_sys_schedule(0); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + { + ErtsMonotonicTime current_time = erts_get_monotonic_time(); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); + } #ifndef ERTS_SMP if (rq->len == 0 && !rq->misc.start) @@ -5262,6 +5339,10 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, #else esdp->no = (Uint) num; #endif + + esdp->timer_wheel = erts_default_timer_wheel; + esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel); + esdp->ssi = ssi; esdp->current_process = NULL; esdp->current_port = NULL; @@ -5274,6 +5355,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->run_queue = runq; esdp->run_queue->scheduler = esdp; + esdp->thr_id = (Uint32) num; + erts_sched_bif_unique_init(esdp); + if (daww_ptr) { init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr); #ifdef ERTS_SMP @@ -5834,6 +5918,13 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces int check_emigration_need; #endif +#ifdef ERTS_SMP + if ((p->static_flags & ERTS_STC_FLG_PREFER_SCHED) + && p->preferred_run_queue != RUNQ_READ_RQ(&p->run_queue)) { + RUNQ_SET_RQ(&p->run_queue, p->preferred_run_queue); + } +#endif + a = state; while (1) { @@ -5872,6 +5963,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces free_proxy_proc(proxy); erts_smp_runq_lock(c_rq); + return 0; #ifdef ERTS_DIRTY_SCHEDULERS @@ -6584,7 +6676,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) int res; do { - res = erts_tse_wait(ssi->event); + res = erts_tse_twait(ssi->event, -1); } while (res == EINTR); } } @@ -6747,6 +6839,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_mtx_unlock(&schdlr_sspnd.mtx); while (1) { + ErtsMonotonicTime current_time; erts_aint32_t qmask; erts_aint32_t flgs; @@ -6771,30 +6864,64 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - if (!aux_work) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + if (aux_work) { + current_time = erts_get_monotonic_time(); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); } - erts_thr_progress_prepare_wait(esdp); - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); + } + else { + ErtsMonotonicTime timeout_time; + timeout_time = erts_check_next_timeout_time(esdp->timer_wheel, + ERTS_SEC_TO_MONOTONIC(60*60)); + current_time = erts_get_monotonic_time(); + + if (current_time >= timeout_time) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + } + else { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) { - int res; - - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + current_time = erts_get_monotonic_time(); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + res = erts_tse_twait(ssi->event, timeout); + current_time = erts_get_monotonic_time(); + } while (res == EINTR); + } } + erts_thr_progress_finalize_wait(esdp); } - erts_thr_progress_finalize_wait(esdp); + + if (current_time >= timeout_time) + erts_bump_timers(esdp->timer_wheel, current_time); } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING @@ -7613,6 +7740,9 @@ sched_thread_func(void *vesdp) ErtsThrPrgrCallbacks callbacks; ErtsSchedulerData *esdp = vesdp; Uint no = esdp->no; + + esdp->timer_wheel = erts_create_timer_wheel((int) no); + esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel); #ifdef ERTS_SMP ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); callbacks.arg = (void *) esdp->ssi; @@ -7715,6 +7845,8 @@ sched_dirty_cpu_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + esdp->thr_id += erts_no_schedulers; + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -7776,6 +7908,8 @@ sched_dirty_io_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers; + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -8878,7 +9012,6 @@ Process *schedule(Process *p, int calls) { Process *proxy_p = NULL; ErtsRunQueue *rq; - erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; int fcalls; @@ -9008,11 +9141,13 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - dt = erts_do_time_read_and_reset(); - if (dt) { - erts_smp_runq_unlock(rq); - erts_bump_timer(dt); - erts_smp_runq_lock(rq); + { + ErtsMonotonicTime current_time = erts_get_monotonic_time(); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + erts_smp_runq_unlock(rq); + erts_bump_timers(esdp->timer_wheel, current_time); + erts_smp_runq_lock(rq); + } } BM_STOP_TIMER(system); @@ -9158,6 +9293,7 @@ Process *schedule(Process *p, int calls) else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && (fcalls > input_reductions && prepare_for_sys_schedule(esdp, !0))) { + ErtsMonotonicTime current_time; /* * Schedule system-level activities. */ @@ -9170,8 +9306,10 @@ Process *schedule(Process *p, int calls) #endif erts_smp_runq_unlock(rq); erl_sys_schedule(1); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + + current_time = erts_get_monotonic_time(); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); #ifdef ERTS_SMP erts_smp_runq_lock(rq); @@ -10476,7 +10614,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). int ix = so->scheduler-1; ASSERT(0 <= ix && ix < erts_no_run_queues); rq = ERTS_RUNQ_IX(ix); - state |= ERTS_PSFLG_BOUND; + if (!(so->flags & SPO_PREFER_SCHED)) { + /* Unsupported feature... */ + state |= ERTS_PSFLG_BOUND; + } } prio = (erts_aint32_t) so->priority; } @@ -10484,6 +10625,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET) | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET)); + if (so->flags & SPO_OFF_HEAP_MSGS) + state |= ERTS_PSFLG_OFF_HEAP_MSGS; + if (!rq) rq = erts_get_runq_proc(parent); @@ -10507,11 +10651,25 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). heap_need = arg_size; p->flags = erts_default_process_flags; + if (so->flags & SPO_OFF_HEAP_MSGS) + p->flags |= F_OFF_HEAP_MSGS; +#ifdef ERTS_SMP + p->preferred_run_queue = NULL; +#endif + p->static_flags = 0; + if (so->flags & SPO_SYSTEM_PROC) + p->static_flags |= ERTS_STC_FLG_SYSTEM_PROC; if (so->flags & SPO_USE_ARGS) { p->min_heap_size = so->min_heap_size; p->min_vheap_size = so->min_vheap_size; p->max_gen_gcs = so->max_gen_gcs; + if (so->flags & SPO_PREFER_SCHED) { +#ifdef ERTS_SMP + p->preferred_run_queue = rq; +#endif + p->static_flags |= ERTS_STC_FLG_PREFER_SCHED; + } } else { p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; @@ -10589,7 +10747,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef ERTS_SMP p->common.u.alive.ptimer = NULL; #else - sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer)); + erts_init_timer(&p->common.u.alive.tm); #endif p->common.u.alive.reg = NULL; @@ -10782,7 +10940,7 @@ void erts_init_empty_process(Process *p) #ifdef ERTS_SMP p->common.u.alive.ptimer = NULL; #else - memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer)); + erts_init_timer(&p->common.u.alive.tm); #endif p->next = NULL; p->off_heap.first = NULL; @@ -10832,6 +10990,8 @@ void erts_init_empty_process(Process *p) p->parent = NIL; p->approx_started = 0; + p->static_flags = 0; + p->common.u.alive.started_interval = 0; #ifdef HIPE @@ -10857,6 +11017,7 @@ void erts_init_empty_process(Process *p) p->pending_suspenders = NULL; p->pending_exit.reason = THE_NON_VALUE; p->pending_exit.bp = NULL; + p->preferred_run_queue = NULL; erts_proc_lock_init(p); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0)); @@ -11482,7 +11643,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ErtsMonitor *rmon; Process *rp; - if (mon->type == MON_ORIGIN) { + switch (mon->type) { + case MON_ORIGIN: /* We are monitoring someone else, we need to demonitor that one.. */ if (is_atom(mon->pid)) { /* remote by name */ ASSERT(is_node_name_atom(mon->pid)); @@ -11545,7 +11707,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } } - } else { /* type == MON_TARGET */ + break; + case MON_TARGET: ASSERT(mon->type == MON_TARGET); ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid)); if (is_internal_port(mon->pid)) { @@ -11604,6 +11767,12 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } } + break; + case MON_TIME_OFFSET: + erts_demonitor_time_offset(mon->ref); + break; + default: + ERTS_INTERNAL_ERROR("Invalid monitor type"); } done: /* As the monitors are previously removed from the process, @@ -11762,6 +11931,9 @@ erts_do_exit_process(Process* p, Eterm reason) } #endif + if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC) + erl_exit(1, "System process %T terminated: %T\n", p->common.id, reason); + #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index d12ac792af..743711cc3b 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -348,7 +348,7 @@ typedef struct { } ErtsRunQueueInfo; -#ifdef HAVE_GETHRTIME +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT # undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT # define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1 #endif @@ -564,6 +564,8 @@ struct ErtsSchedulerData_ { Eterm* x_reg_array; /* X registers */ FloatDef* f_reg_array; /* Floating point registers. */ + ErtsTimerWheel *timer_wheel; + ErtsNextTimeoutRef next_tmo_ref; #ifdef ERTS_SMP ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ @@ -590,6 +592,10 @@ struct ErtsSchedulerData_ { ErtsAuxWorkData aux_work_data; ErtsAtomCacheMap atom_cache_map; + Uint32 thr_id; + Uint64 unique; + Uint64 ref; + ErtsSchedAllocData alloc_data; Uint64 reductions; @@ -936,6 +942,8 @@ struct process { Eterm parent; /* Pid of process that created this process. */ erts_approx_time_t approx_started; /* Time when started. */ + Uint32 static_flags; /* Flags that do *not* change */ + /* This is the place, where all fields that differs between memory * architectures, have gone to. */ @@ -967,6 +975,7 @@ struct process { ErtsSchedulerData *scheduler_data; Eterm suspendee; ErtsPendingSuspend *pending_suspenders; + ErtsRunQueue *preferred_run_queue; erts_smp_atomic_t run_queue; #ifdef HIPE struct hipe_process_state_smp hipe_smp; @@ -1076,14 +1085,15 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLG_RUNNING_SYS ERTS_PSFLG_BIT(15) #define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16) #define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17) +#define ERTS_PSFLG_OFF_HEAP_MSGS ERTS_PSFLG_BIT(18) #ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(18) -#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(19) -#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q ERTS_PSFLG_BIT(20) -#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q ERTS_PSFLG_BIT(21) -#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22) +#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(19) +#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(20) +#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q ERTS_PSFLG_BIT(21) +#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q ERTS_PSFLG_BIT(22) +#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 23) #else -#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18) +#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 19) #endif #define ERTS_PSFLGS_IN_PRQ_MASK (ERTS_PSFLG_IN_PRQ_MAX \ @@ -1098,6 +1108,12 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \ (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) +/* + * Static flags that do not change after process creation. + */ +#define ERTS_STC_FLG_SYSTEM_PROC (((Uint32) 1) << 0) +#define ERTS_STC_FLG_PREFER_SCHED (((Uint32) 1) << 1) + /* The sequential tracing token is a tuple of size 5: * * {Flags, Label, Serial, Sender} @@ -1125,6 +1141,9 @@ void erts_check_for_holes(Process* p); #define SPO_LINK 1 #define SPO_USE_ARGS 2 #define SPO_MONITOR 4 +#define SPO_OFF_HEAP_MSGS 8 +#define SPO_SYSTEM_PROC 16 +#define SPO_PREFER_SCHED 32 /* * The following struct contains options for a process to be spawned. @@ -1212,6 +1231,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ #define F_DISABLE_GC (1 << 11) /* Disable GC */ +#define F_OFF_HEAP_MSGS (1 << 12) /* process trace_flags */ #define F_SENSITIVE (1 << 0) @@ -2228,6 +2248,8 @@ extern int erts_disable_proc_not_running_opt; void erts_smp_notify_inc_runq(ErtsRunQueue *runq); +void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time); + #ifdef ERTS_SMP void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index 28cbe7004f..bc04d7b78e 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -86,11 +86,12 @@ unsigned tag_val_def(Wterm x) case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE): return EXTERNAL_PID_DEF; case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): return EXTERNAL_PORT_DEF; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): return EXTERNAL_REF_DEF; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): return MAP_DEF; case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; - case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): return MAP_DEF; } + break; } case TAG_PRIMARY_IMMED1: { diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 37014ccf94..095aa54ddd 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -147,21 +147,21 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #define MAP_SUBTAG (0xF << _TAG_PRIMARY_SIZE) /* MAP */ -#define _TAG_HEADER_ARITYVAL (TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG) -#define _TAG_HEADER_FUN (TAG_PRIMARY_HEADER|FUN_SUBTAG) -#define _TAG_HEADER_POS_BIG (TAG_PRIMARY_HEADER|POS_BIG_SUBTAG) -#define _TAG_HEADER_NEG_BIG (TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG) -#define _TAG_HEADER_FLOAT (TAG_PRIMARY_HEADER|FLOAT_SUBTAG) -#define _TAG_HEADER_EXPORT (TAG_PRIMARY_HEADER|EXPORT_SUBTAG) -#define _TAG_HEADER_REF (TAG_PRIMARY_HEADER|REF_SUBTAG) -#define _TAG_HEADER_REFC_BIN (TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG) -#define _TAG_HEADER_HEAP_BIN (TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG) -#define _TAG_HEADER_SUB_BIN (TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG) -#define _TAG_HEADER_EXTERNAL_PID (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG) -#define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG) -#define _TAG_HEADER_EXTERNAL_REF (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG) +#define _TAG_HEADER_ARITYVAL (TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG) +#define _TAG_HEADER_FUN (TAG_PRIMARY_HEADER|FUN_SUBTAG) +#define _TAG_HEADER_POS_BIG (TAG_PRIMARY_HEADER|POS_BIG_SUBTAG) +#define _TAG_HEADER_NEG_BIG (TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG) +#define _TAG_HEADER_FLOAT (TAG_PRIMARY_HEADER|FLOAT_SUBTAG) +#define _TAG_HEADER_EXPORT (TAG_PRIMARY_HEADER|EXPORT_SUBTAG) +#define _TAG_HEADER_REF (TAG_PRIMARY_HEADER|REF_SUBTAG) +#define _TAG_HEADER_REFC_BIN (TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG) +#define _TAG_HEADER_HEAP_BIN (TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG) +#define _TAG_HEADER_SUB_BIN (TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG) +#define _TAG_HEADER_EXTERNAL_PID (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG) +#define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG) +#define _TAG_HEADER_EXTERNAL_REF (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG) #define _TAG_HEADER_BIN_MATCHSTATE (TAG_PRIMARY_HEADER|BIN_MATCHSTATE_SUBTAG) -#define _TAG_HEADER_MAP (TAG_PRIMARY_HEADER|MAP_SUBTAG) +#define _TAG_HEADER_MAP (TAG_PRIMARY_HEADER|MAP_SUBTAG) #define _TAG_HEADER_MASK 0x3F @@ -296,9 +296,11 @@ _ET_DECLARE_CHECKED(Uint,atom_val,Eterm) #define atom_val(x) _ET_APPLY(atom_val,(x)) /* header (arityval or thing) access methods */ -#define _make_header(sz,tag) ((Uint)(((sz) << _HEADER_ARITY_OFFS) + (tag))) +#define _make_header(sz,tag) ((Uint)(((Uint)(sz) << _HEADER_ARITY_OFFS) + (tag))) #define is_header(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER) -#define _unchecked_header_arity(x) ((x) >> _HEADER_ARITY_OFFS) +//#define _unchecked_header_arity(x) ((x) >> _HEADER_ARITY_OFFS) +#define _unchecked_header_arity(x) \ + (is_map_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS)) _ET_DECLARE_CHECKED(Uint,header_arity,Eterm) #define header_arity(x) _ET_APPLY(header_arity,(x)) @@ -361,6 +363,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) ((((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_REFC_BIN) || \ (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HEAP_BIN) || \ (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_SUB_BIN)) + #define make_binary(x) make_boxed((Eterm*)(x)) #define is_binary(x) (is_boxed((x)) && is_binary_header(*boxed_val((x)))) #define is_not_binary(x) (!is_binary((x))) @@ -990,6 +993,43 @@ _ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm) _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #define external_ref_node(x) _ET_APPLY(external_ref_node,(x)) +/* maps */ + +#define MAP_HEADER_TAG_SZ (2) +#define MAP_HEADER_ARITY_SZ (8) +#define MAP_HEADER_VAL_SZ (16) + +#define MAP_HEADER_TAG_FLATMAP_HEAD (0x0) +#define MAP_HEADER_TAG_HAMT_NODE_BITMAP (0x1) +#define MAP_HEADER_TAG_HAMT_HEAD_ARRAY (0x2) +#define MAP_HEADER_TAG_HAMT_HEAD_BITMAP (0x3) + +#define MAP_HEADER_TYPE(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS)) & (0x3)) +#define MAP_HEADER_ARITY(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ)) & (0xff)) +#define MAP_HEADER_VAL(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff)) + +#define make_hashmap(x) make_boxed((Eterm*)(x)) +#define make_hashmap_rel make_boxed_rel +#define is_hashmap(x) (is_boxed((x)) && is_hashmap_header(*boxed_val((x)))) +#define is_not_hashmap(x) (!is_hashmap(x)) +#define is_hashmap_rel(RTERM,BASE) is_hashmap(rterm2wterm(RTERM,BASE)) +#define is_hashmap_header(x) (((x) & (_HEADER_MAP_HASHMAP_HEAD_MASK)) == HAMT_SUBTAG_HEAD_ARRAY) +#define hashmap_val(x) _unchecked_boxed_val((x)) +#define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE)) + +#define make_flatmap(x) make_boxed((Eterm*)(x)) +#define make_flatmap_rel(x, BASE) make_boxed_rel((Eterm*)(x),(BASE)) +#define is_flatmap(x) (is_boxed((x)) && is_flatmap_header(*boxed_val((x)))) +#define is_flatmap_rel(RTERM,BASE) is_flatmap(rterm2wterm(RTERM,BASE)) +#define is_not_flatmap(x) (!is_flatmap((x))) +#define is_flatmap_header(x) (((x) & (_HEADER_MAP_SUBTAG_MASK)) == HAMT_SUBTAG_HEAD_FLATMAP) +#define flatmap_val(x) (_unchecked_boxed_val((x))) +#define flatmap_val_rel(RTERM, BASE) flatmap_val(rterm2wterm(RTERM, BASE)) + +#define is_map_header(x) (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP) +#define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val(x))) +#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE)) + /* number tests */ #define is_integer(x) (is_small(x) || is_big(x)) @@ -1096,6 +1136,8 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) #define BIG_DEF 0xf #define SMALL_DEF 0x10 +#define FIRST_VACANT_TAG_DEF 0x11 + #if ET_DEBUG extern unsigned tag_val_def_debug(Wterm, const char*, unsigned); #define tag_val_def(x) tag_val_def_debug((x),__FILE__,__LINE__) diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index c2365c5cf7..4c9b00d2ee 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -1359,18 +1359,16 @@ void erts_thr_progress_fatal_error_wait(SWord timeout) { erts_aint32_t bc; SWord time_left = timeout; - SysTimeval to; + ErtsMonotonicTime timeout_time; /* * Counting poll intervals may give us a too long timeout - * if cpu is busy. If we got tolerant time of day we use it - * to prevent this. + * if cpu is busy. We use timeout time to try to prevent + * this. In case we havn't got time correction this may + * however fail too... */ - if (!erts_disable_tolerant_timeofday) { - erts_get_timeval(&to); - to.tv_sec += timeout / 1000; - to.tv_sec += timeout % 1000; - } + timeout_time = erts_get_monotonic_time(); + timeout_time += ERTS_MSEC_TO_MONOTONIC((ErtsMonotonicTime) timeout); while (1) { if (erts_milli_sleep(ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL) == 0) @@ -1380,14 +1378,8 @@ erts_thr_progress_fatal_error_wait(SWord timeout) { break; /* Succefully blocked all managed threads */ if (time_left <= 0) break; /* Timeout */ - if (!erts_disable_tolerant_timeofday) { - SysTimeval now; - erts_get_timeval(&now); - if (now.tv_sec > to.tv_sec) - break; /* Timeout */ - if (now.tv_sec == to.tv_sec && now.tv_usec >= to.tv_usec) - break; /* Timeout */ - } + if (timeout_time <= erts_get_monotonic_time()) + break; /* Timeout */ } } diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 1fd800d524..dc20ac207f 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -652,6 +652,8 @@ ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); +ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo); +ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo); ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); ERTS_GLB_INLINE int erts_thr_get_main_status(void); @@ -3490,6 +3492,27 @@ ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) #endif } +ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo) +{ +#ifdef USE_THREADS + return ethr_event_twait(&((ethr_ts_event *) ep)->event, + (ethr_sint64_t) tmo); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo) +{ +#ifdef USE_THREADS + return ethr_event_stwait(&((ethr_ts_event *) ep)->event, + spincount, + (ethr_sint64_t) tmo); +#else + return ENOTSUP; +#endif +} + ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) { #ifdef USE_THREADS diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 7ed1a395ad..cb7764addc 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -20,11 +20,16 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ -#define ERTS_SHORT_TIME_T_MAX ERTS_AINT32_T_MAX -#define ERTS_SHORT_TIME_T_MIN ERTS_AINT32_T_MIN -typedef erts_aint32_t erts_short_time_t; +#if defined(DEBUG) || 0 +#define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B) +#else +#define ERTS_TIME_ASSERT(B) ((void) 1) +#endif + +typedef struct ErtsTimerWheel_ ErtsTimerWheel; +typedef erts_atomic64_t * ErtsNextTimeoutRef; +extern ErtsTimerWheel *erts_default_timer_wheel; -extern erts_smp_atomic32_t do_time; /* set at clock interrupt */ extern SysTimeval erts_first_emu_time; /* @@ -34,8 +39,8 @@ typedef struct erl_timer { struct erl_timer* next; /* next entry tiw slot or chain */ struct erl_timer* prev; /* prev entry tiw slot or chain */ Uint slot; /* slot in timer wheel */ - Uint count; /* number of loops remaining */ - int active; /* 1=activated, 0=deactivated */ + erts_smp_atomic_t wheel; + ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ /* called when timeout */ void (*timeout)(void*); /* called when cancel (may be NULL) */ @@ -62,7 +67,6 @@ union ErtsSmpPTimer_ { ErtsSmpPTimer *next; }; - void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, Eterm id, ErlTimeoutProc timeout_func, @@ -70,36 +74,42 @@ void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer); #endif +void erts_monitor_time_offset(Eterm id, Eterm ref); +int erts_demonitor_time_offset(Eterm ref); + +void erts_late_init_time_sup(void); + /* timer-wheel api */ -void erts_init_time(void); +ErtsTimerWheel *erts_create_timer_wheel(int); +ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *); +void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode); void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); void erts_cancel_timer(ErlTimer*); -void erts_bump_timer(erts_short_time_t); -Uint erts_timer_wheel_memory_size(void); Uint erts_time_left(ErlTimer *); -erts_short_time_t erts_next_time(void); +void erts_bump_timers(ErtsTimerWheel *, ErtsMonotonicTime); +Uint erts_timer_wheel_memory_size(void); #ifdef DEBUG void erts_p_slpq(void); #endif -ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void); -ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t); +ErtsMonotonicTime erts_check_next_timeout_time(ErtsTimerWheel *, + ErtsMonotonicTime); + +ERTS_GLB_INLINE void erts_init_timer(ErlTimer *p); +ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void) +ERTS_GLB_INLINE void erts_init_timer(ErlTimer *p) { - erts_short_time_t time = erts_smp_atomic32_xchg_acqb(&do_time, 0); - if (time < 0) - erl_exit(ERTS_ABORT_EXIT, "Internal time management error\n"); - return time; + erts_smp_atomic_init_nob(&p->wheel, (erts_aint_t) NULL); } -ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed) +ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref) { - erts_smp_atomic32_add_relb(&do_time, elapsed); + return (ErtsMonotonicTime) erts_atomic64_read_acqb((erts_atomic64_t *) nxt_tmo_ref); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ @@ -121,25 +131,220 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); typedef UWord erts_approx_time_t; erts_approx_time_t erts_get_approx_time(void); -void erts_get_timeval(SysTimeval *tv); -erts_time_t erts_get_time(void); +int erts_has_time_correction(void); +int erts_check_time_adj_support(int time_correction, + ErtsTimeWarpMode time_warp_mode); + +ErtsTimeWarpMode erts_time_warp_mode(void); -ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); +typedef enum { + ERTS_TIME_OFFSET_PRELIMINARY, + ERTS_TIME_OFFSET_FINAL, + ERTS_TIME_OFFSET_VOLATILE +} ErtsTimeOffsetState; + +ErtsTimeOffsetState erts_time_offset_state(void); +ErtsTimeOffsetState erts_finalize_time_offset(void); +struct process; +Eterm erts_get_monotonic_start_time(struct process *c_p); +Eterm erts_monotonic_time_source(struct process*c_p); +Eterm erts_system_time_source(struct process*c_p); + +#ifdef SYS_CLOCK_RESOLUTION +#define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000)) +#else +#define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution) +#endif + +struct erts_time_sup_read_only__ { + ErtsMonotonicTime monotonic_time_unit; +#ifndef SYS_CLOCK_RESOLUTION + ErtsMonotonicTime clktck_resolution; +#endif +}; + +typedef struct { + union { + struct erts_time_sup_read_only__ o; + char align__[(((sizeof(struct erts_time_sup_read_only__) - 1) + / ASSUMED_CACHE_LINE_SIZE) + 1) + * ASSUMED_CACHE_LINE_SIZE]; + } r; +} ErtsTimeSupData; + +extern ErtsTimeSupData erts_time_sup__; + +ERTS_GLB_INLINE Uint64 +erts_time_unit_conversion(Uint64 value, + Uint32 from_time_unit, + Uint32 to_time_unit); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE int -erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) +ERTS_GLB_INLINE Uint64 +erts_time_unit_conversion(Uint64 value, + Uint32 from_time_unit, + Uint32 to_time_unit) { - if (t1p->tv_sec == t2p->tv_sec) { - if (t1p->tv_usec < t2p->tv_usec) - return -1; - else if (t1p->tv_usec > t2p->tv_usec) - return 1; - return 0; - } - return t1p->tv_sec < t2p->tv_sec ? -1 : 1; + Uint64 high, low, result; + if (value <= ~((Uint64) 0)/to_time_unit) + return (value*to_time_unit)/from_time_unit; + + low = value & ((Uint64) 0xffffffff); + high = (value >> 32) & ((Uint64) 0xffffffff); + + low *= to_time_unit; + high *= to_time_unit; + + high += (low >> 32) & ((Uint64) 0xffffffff); + low &= ((Uint64) 0xffffffff); + + result = high % from_time_unit; + high /= from_time_unit; + high <<= 32; + + result <<= 32; + result += low; + result /= from_time_unit; + result += high; + + return result; } -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + +/* + * If the monotonic time unit is a compile time constant, + * it is assumed (and need) to be a power of 10. + */ + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT < 1000*1000 +# error Compile time time unit needs to be at least 1000000 +#endif + +#define ERTS_MONOTONIC_TIME_UNIT \ + ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT) + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000*1000 +/* Nano-second time unit */ + +#define ERTS_MONOTONIC_TO_SEC__(NSEC) ((NSEC) / (1000*1000*1000)) +#define ERTS_MONOTONIC_TO_MSEC__(NSEC) ((NSEC) / (1000*1000)) +#define ERTS_MONOTONIC_TO_USEC__(NSEC) ((NSEC) / 1000) +#define ERTS_MONOTONIC_TO_NSEC__(NSEC) (NSEC) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000*1000)) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*(1000*1000)) +#define ERTS_USEC_TO_MONOTONIC__(USEC) (((ErtsMonotonicTime) (USEC))*1000) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) ((ErtsMonotonicTime) (NSEC)) + +#elif ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000 +/* Micro-second time unit */ + +#define ERTS_MONOTONIC_TO_SEC__(USEC) ((USEC) / (1000*1000)) +#define ERTS_MONOTONIC_TO_MSEC__(USEC) ((USEC) / 1000) +#define ERTS_MONOTONIC_TO_USEC__(USEC) (USEC) +#define ERTS_MONOTONIC_TO_NSEC__(USEC) ((USEC)*1000) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000)) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*1000) +#define ERTS_USEC_TO_MONOTONIC__(USEC) ((ErtsMonotonicTime) (USEC)) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/1000) + +#else +#error Missing implementation for monotonic time unit +#endif + +#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \ + ((MON) / (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION)) +#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \ + ((TCKS) * (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION)) + +#else /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */ + +#define ERTS_MONOTONIC_TIME_UNIT (erts_time_sup__.r.o.monotonic_time_unit) + +#define ERTS_CONV_FROM_MON_UNIT___(M, TO) \ + ((ErtsMonotonicTime) \ + erts_time_unit_conversion((Uint64) (M), \ + (Uint32) ERTS_MONOTONIC_TIME_UNIT, \ + (Uint32) (TO))) + +#define ERTS_CONV_TO_MON_UNIT___(M, FROM) \ + ((ErtsMonotonicTime) \ + erts_time_unit_conversion((Uint64) (M), \ + (Uint32) (FROM), \ + (Uint32) ERTS_MONOTONIC_TIME_UNIT)) \ + +#define ERTS_MONOTONIC_TO_SEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1) +#define ERTS_MONOTONIC_TO_MSEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000) +#define ERTS_MONOTONIC_TO_USEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000) +#define ERTS_MONOTONIC_TO_NSEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000*1000) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) \ + ERTS_CONV_TO_MON_UNIT___((SEC), 1) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) \ + ERTS_CONV_TO_MON_UNIT___((MSEC), 1000) +#define ERTS_USEC_TO_MONOTONIC__(USEC) \ + ERTS_CONV_TO_MON_UNIT___((USEC), 1000*1000) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) \ + ERTS_CONV_TO_MON_UNIT___((NSEC), 1000*1000*1000) + +#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \ + ERTS_CONV_FROM_MON_UNIT___((MON), ERTS_CLKTCK_RESOLUTION) +#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \ + ERTS_CONV_TO_MON_UNIT___((TCKS), ERTS_CLKTCK_RESOLUTION) + +#endif /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */ + +#define ERTS_MSEC_TO_CLKTCKS__(MON) \ + ((MON) * (ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_CLKTCKS_TO_MSEC__(TCKS) \ + ((TCKS) / (ERTS_CLKTCK_RESOLUTION/1000)) + +#define ERTS_MONOTONIC_TO_SEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_SEC__((X))) +#define ERTS_MONOTONIC_TO_MSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_MSEC__((X))) +#define ERTS_MONOTONIC_TO_USEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_USEC__((X))) +#define ERTS_MONOTONIC_TO_NSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_NSEC__((X))) +#define ERTS_SEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_SEC_TO_MONOTONIC__((X))) +#define ERTS_MSEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MSEC_TO_MONOTONIC__((X))) +#define ERTS_USEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_USEC_TO_MONOTONIC__((X))) +#define ERTS_NSEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_NSEC_TO_MONOTONIC__((X))) + +#define ERTS_MONOTONIC_TO_CLKTCKS(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_CLKTCKS__((X))) +#define ERTS_CLKTCKS_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_CLKTCKS_TO_MONOTONIC__((X))) + +#define ERTS_MSEC_TO_CLKTCKS(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MSEC_TO_CLKTCKS__((X))) +#define ERTS_CLKTCKS_TO_MSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_CLKTCKS_TO_MSEC__((X))) + #endif /* ERL_TIME_H__ */ diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 3272a5326d..bbdedcc128 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2015. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -18,60 +18,10 @@ */ /* -** Support routines for the timer wheel -** -** This code contains two strategies for dealing with -** date/time changes in the system. -** If the system has some kind of high resolution timer (HAVE_GETHRTIME), -** the high resolution timer is used to correct the time-of-day and the -** timeouts, the base source is the hrtimer, but at certain intervals the -** OS time-of-day is checked and if it is not within certain bounds, the -** delivered time gets slowly adjusted for each call until -** it corresponds to the system time (built-in adjtime...). -** The call gethrtime() is detected by autoconf on Unix, but other -** platforms may define it in erl_*_sys.h and implement -** their own high resolution timer. The high resolution timer -** strategy is (probably) best on all systems where the timer have -** a resolution higher or equal to gettimeofday (or what's implemented -** is sys_gettimeofday()). The actual resolution is the interesting thing, -** not the unit's thats used (i.e. on VxWorks, nanoseconds can be -** retrieved in terms of units, but the actual resolution is the same as -** for the clock ticks). -** If the systems best timer routine is kernel ticks returned from -** sys_times(), and the actual resolution of sys_gettimeofday() is -** better (like most unixes that does not have any realtime extensions), -** another strategy is used. The tolerant gettimeofday() corrects -** the value with respect to uptime (sys_times() return value) and checks -** for correction both when delivering timeticks and delivering nowtime. -** this strategy is slower, but accurate on systems without better timer -** routines. The kernel tick resolution is not enough to implement -** a gethrtime routine. On Linux and other non solaris unix-boxes the second -** strategy is used, on all other platforms we use the first. -** -** The following is expected (from sys.[ch] and erl_*_sys.h): -** -** 64 bit integers. So it is, and so it will be. -** -** sys_init_time(), will return the clock resolution in MS and -** that's about it. More could be added of course -** If the clock-rate is constant (i.e. 1 ms) one can define -** SYS_CLOCK_RESOLUTION (to 1), -** which makes erts_deliver_time/erts_time_remaining a bit faster. -** -** if HAVE_GETHRTIME is defined: -** sys_gethrtime() will return a SysHrTime (long long) representing -** nanoseconds, sys_init_hrtime() will do any initialization. -** else -** a long (64bit) integer type called Sint64 should be defined. -** -** sys_times() will return clock_ticks since start and -** fill in a SysTimes structure (struct tms). Instead of CLK_TCK, -** SYS_CLK_TCK is used to determine the resolution of kernel ticks. -** -** sys_gettimeofday() will take a SysTimeval (a struct timeval) as parameter -** and fill it in as gettimeofday(X,NULL). -** -*/ + * Support routines for the time + */ + +/* #define ERTS_TIME_CORRECTION_PRINT */ #ifdef HAVE_CONFIG_H # include "config.h" @@ -80,384 +30,1107 @@ #include "sys.h" #include "erl_vm.h" #include "global.h" - + static erts_smp_mtx_t erts_timeofday_mtx; - -static SysTimeval inittv; /* Used everywhere, the initial time-of-day */ +static erts_smp_mtx_t erts_get_time_mtx; static SysTimes t_start; /* Used in elapsed_time_both */ -static SysTimeval gtv; /* Used in wall_clock_elapsed_time_both */ -static SysTimeval then; /* Used in get_now */ -static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */ -SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */ +static ErtsMonotonicTime prev_wall_clock_elapsed; /* Used in wall_clock_elapsed_time_both */ +static ErtsMonotonicTime previous_now; /* Used in get_now */ -union { - erts_smp_atomic_t time; - char align[ERTS_CACHE_LINE_SIZE]; -} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE); +static ErtsMonitor *time_offset_monitors = NULL; +static Uint no_time_offset_monitors = 0; + +#ifdef DEBUG +static int time_sup_initialized = 0; +#endif + +#define ERTS_MONOTONIC_TIME_KILO \ + ((ErtsMonotonicTime) 1000) +#define ERTS_MONOTONIC_TIME_MEGA \ + (ERTS_MONOTONIC_TIME_KILO*ERTS_MONOTONIC_TIME_KILO) +#define ERTS_MONOTONIC_TIME_GIGA \ + (ERTS_MONOTONIC_TIME_MEGA*ERTS_MONOTONIC_TIME_KILO) +#define ERTS_MONOTONIC_TIME_TERA \ + (ERTS_MONOTONIC_TIME_GIGA*ERTS_MONOTONIC_TIME_KILO) static void -init_approx_time(void) +schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset); + +/* + * NOTE! ERTS_MONOTONIC_TIME_START *need* to be a multiple + * of ERTS_MONOTONIC_TIME_UNIT. + */ + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + +#ifdef ARCH_32 +/* + * Want to use a big-num of arity 2 as long as possible (584 years + * in the nano-second time unit case). + */ +#define ERTS_MONOTONIC_TIME_START \ + (((((((ErtsMonotonicTime) 1) << 32)-1) \ + / ERTS_MONOTONIC_TIME_UNIT) \ + * ERTS_MONOTONIC_TIME_UNIT) \ + + ERTS_MONOTONIC_TIME_UNIT) + +#else /* ARCH_64 */ + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 10*1000*1000 + +/* + * Using micro second time unit or lower. Start at zero since + * time will remain an immediate for a very long time anyway + * (1827 years in the 10 micro second case)... + */ +#define ERTS_MONOTONIC_TIME_START ((ErtsMonotonicTime) 0) + +#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */ + +/* + * Want to use an immediate as long as possible (36 years in the + * nano-second time unit case). +*/ +#define ERTS_MONOTONIC_TIME_START \ + ((((ErtsMonotonicTime) MIN_SMALL) \ + / ERTS_MONOTONIC_TIME_UNIT) \ + * ERTS_MONOTONIC_TIME_UNIT) + +#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */ + +#endif /* ARCH_64 */ + +#define ERTS_MONOTONIC_OFFSET_NATIVE \ + (ERTS_MONOTONIC_TIME_START - ERTS_MONOTONIC_TIME_UNIT) +#define ERTS_MONOTONIC_OFFSET_NSEC \ + ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_USEC \ + ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_MSEC \ + ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_SEC \ + ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_OFFSET_NATIVE) + +#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */ + +/* + * Initialized in erts_init_time_sup()... + */ + +#define ERTS_MONOTONIC_TIME_START (time_sup.r.o.start) +#define ERTS_MONOTONIC_OFFSET_NATIVE (time_sup.r.o.start_offset.native) +#define ERTS_MONOTONIC_OFFSET_NSEC (time_sup.r.o.start_offset.nsec) +#define ERTS_MONOTONIC_OFFSET_USEC (time_sup.r.o.start_offset.usec) +#define ERTS_MONOTONIC_OFFSET_MSEC (time_sup.r.o.start_offset.msec) +#define ERTS_MONOTONIC_OFFSET_SEC (time_sup.r.o.start_offset.sec) + +#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */ + +struct time_sup_read_only__ { + ErtsMonotonicTime (*get_time)(void); + int correction; + ErtsTimeWarpMode warp_mode; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + ErtsMonotonicTime moffset; + int os_monotonic_time_disable; + char *os_monotonic_time_func; + char *os_monotonic_time_clock_id; + int os_monotonic_time_locked; + Uint64 os_monotonic_time_resolution; + Uint64 os_monotonic_time_extended; +#endif + char *os_system_time_func; + char *os_system_time_clock_id; + int os_system_time_locked; + Uint64 os_system_time_resolution; + Uint64 os_system_time_extended; +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + ErtsMonotonicTime start; + struct { + ErtsMonotonicTime native; + ErtsMonotonicTime nsec; + ErtsMonotonicTime usec; + ErtsMonotonicTime msec; + ErtsMonotonicTime sec; + } start_offset; +#endif + struct { + ErtsMonotonicTime large_diff; + ErtsMonotonicTime small_diff; + } adj; +}; + +typedef struct { +#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC + ErtsMonotonicTime drift; /* Correction for os monotonic drift */ +#endif + ErtsMonotonicTime error; /* Correction for error between system times */ +} ErtsMonotonicCorrection; + +typedef struct { + ErtsMonotonicTime erl_mtime; + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrection correction; +} ErtsMonotonicCorrectionInstance; + +#define ERTS_DRIFT_INTERVALS 5 +typedef struct { + struct { + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } diff; + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } time; + } intervals[ERTS_DRIFT_INTERVALS]; + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } acc; + int ix; + int dirty_counter; +} ErtsMonotonicDriftData; + +typedef struct { + ErtsMonotonicCorrectionInstance prev; + ErtsMonotonicCorrectionInstance curr; +#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC + ErtsMonotonicDriftData drift; +#endif + ErtsMonotonicTime last_check; + int short_check_interval; +} ErtsMonotonicCorrectionData; + +struct time_sup_infrequently_changed__ { +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + struct { + erts_smp_rwmtx_t rwmtx; + ErlTimer timer; + ErtsMonotonicCorrectionData cdata; + } parmon; + ErtsMonotonicTime minit; +#endif + int finalized_offset; + ErtsSystemTime sinit; + ErtsMonotonicTime not_corrected_moffset; + erts_atomic64_t offset; +}; + +struct time_sup_frequently_changed__ { + ErtsMonotonicTime last_not_corrected_time; +}; + +static struct { + union { + struct time_sup_read_only__ o; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_read_only__))]; + } r; + union { + struct time_sup_infrequently_changed__ c; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_infrequently_changed__))]; + } inf; + union { + struct time_sup_frequently_changed__ c; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_frequently_changed__))]; + } f; +} time_sup erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +ErtsTimeSupData erts_time_sup__ erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +/* + * erts_get_approx_time() returns an *approximate* time + * in seconds. NOTE that this time may jump backwards!!! + */ +erts_approx_time_t +erts_get_approx_time(void) { - erts_smp_atomic_init_nob(&approx.time, 0); + ErtsSystemTime stime = erts_os_system_time(); + return (erts_approx_time_t) ERTS_MONOTONIC_TO_SEC(stime); } -static ERTS_INLINE erts_approx_time_t -get_approx_time(void) +static ERTS_INLINE void +init_time_offset(ErtsMonotonicTime offset) { - return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time); + erts_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset); } static ERTS_INLINE void -update_approx_time(SysTimeval *tv) +set_time_offset(ErtsMonotonicTime offset) { - erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec; - erts_approx_time_t old_secs = get_approx_time(); - if (old_secs != new_secs) - erts_smp_atomic_set_nob(&approx.time, new_secs); + erts_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset); } -/* - * erts_get_approx_time() returns an *approximate* time - * in seconds. NOTE that this time may jump backwards!!! - */ -erts_approx_time_t -erts_get_approx_time(void) +static ERTS_INLINE ErtsMonotonicTime +get_time_offset(void) { - return get_approx_time(); + return (ErtsMonotonicTime) erts_atomic64_read_acqb(&time_sup.inf.c.offset); } -#ifdef HAVE_GETHRTIME -int erts_disable_tolerant_timeofday; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT -static SysHrTime hr_init_time, hr_last_correction_check, - hr_correction, hr_last_time; +/* + * Time correction adjustments made due to + * error between Erlang system time and OS + * system time: + * - Large adjustment ~1% + * - Small adjustment ~0.05% + */ +#define ERTS_TCORR_ERR_UNIT 2048 +#define ERTS_TCORR_ERR_LARGE_ADJ 20 +#define ERTS_TCORR_ERR_SMALL_ADJ 1 + +#define ERTS_INIT_SHORT_INTERVAL_COUNTER 10 +#define ERTS_LONG_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(60) +#define ERTS_SHORT_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(15) + +#define ERTS_TIME_DRIFT_MAX_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(50) +#define ERTS_TIME_DRIFT_MIN_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(5) -static void init_tolerant_timeofday(void) +static ERTS_INLINE ErtsMonotonicTime +calc_corrected_erl_mtime(ErtsMonotonicTime os_mtime, + ErtsMonotonicCorrectionInstance *cip, + ErtsMonotonicTime *os_mdiff_p) { - /* Should be in sys.c */ -#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_CONF) - if (sysconf(_SC_NPROCESSORS_CONF) > 1) { - char b[1024]; - int maj,min,build; - os_flavor(b,1024); - os_version(&maj,&min,&build); - if (!strcmp(b,"sunos") && maj <= 5 && min <= 7) { - erts_disable_tolerant_timeofday = 1; - } - } + ErtsMonotonicTime erl_mtime, diff = os_mtime - cip->os_mtime; + ERTS_TIME_ASSERT(diff >= 0); +#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC + diff += (cip->correction.drift*diff)/ERTS_MONOTONIC_TIME_UNIT; #endif - hr_init_time = sys_gethrtime(); - hr_last_correction_check = hr_last_time = hr_init_time; - hr_correction = 0; + erl_mtime = cip->erl_mtime; + erl_mtime += diff; + erl_mtime += cip->correction.error*(diff/ERTS_TCORR_ERR_UNIT); + if (os_mdiff_p) + *os_mdiff_p = diff; + return erl_mtime; } -static void get_tolerant_timeofday(SysTimeval *tv) +static ErtsMonotonicTime get_corrected_time(void) { - SysHrTime diff_time, curr; + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrectionData cdata; + ErtsMonotonicCorrectionInstance *cip; + + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); - if (erts_disable_tolerant_timeofday) { - sys_gettimeofday(tv); - return; + os_mtime = erts_os_monotonic_time(); + + cdata = time_sup.inf.c.parmon.cdata; + + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + + if (os_mtime >= cdata.curr.os_mtime) + cip = &cdata.curr; + else { + if (os_mtime < cdata.prev.os_mtime) + erl_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + cip = &cdata.prev; } - *tv = inittv; - diff_time = ((curr = sys_gethrtime()) + hr_correction - hr_init_time) / 1000; - if (curr < hr_init_time) { - erl_exit(1,"Unexpected behaviour from operating system high " - "resolution timer"); + return calc_corrected_erl_mtime(os_mtime, cip, NULL); +} + +#ifdef ERTS_TIME_CORRECTION_PRINT + +static ERTS_INLINE void +print_correction(int change, + ErtsMonotonicTime sdiff, + ErtsMonotonicTime old_ecorr, + ErtsMonotonicTime old_dcorr, + ErtsMonotonicTime new_ecorr, + ErtsMonotonicTime new_dcorr, + Uint tmo) +{ + ErtsMonotonicTime usec_sdiff; + if (sdiff < 0) + usec_sdiff = -1*ERTS_MONOTONIC_TO_USEC(-1*sdiff); + else + usec_sdiff = ERTS_MONOTONIC_TO_USEC(sdiff); + + if (!change) + fprintf(stderr, + "sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppb] : " + "tmo = %lld msec\r\n", + (long long) usec_sdiff, + (long long) (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT, + (long long) (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + (long long) tmo); + else + fprintf(stderr, + "sdiff = %lld usec : [ec=%lld ppm, dc=%lld ppb] " + "-> [ec=%lld ppm, dc=%lld ppb] : tmo = %lld msec\r\n", + (long long) usec_sdiff, + (long long) (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT, + (long long) (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + (long long) (1000000*new_ecorr) / ERTS_TCORR_ERR_UNIT, + (long long) (1000000000*new_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + (long long) tmo); + +} + +#endif + +static void +check_time_correction(void *unused) +{ +#ifndef ERTS_TIME_CORRECTION_PRINT +# define ERTS_PRINT_CORRECTION +#else +# ifdef ERTS_HAVE_CORRECTED_OS_MONOTONIC +# define ERTS_PRINT_CORRECTION \ + print_correction(set_new_correction, \ + sdiff, \ + cip->correction.error, \ + 0, \ + new_correction.error, \ + 0, \ + timeout) +# else +# define ERTS_PRINT_CORRECTION \ + print_correction(set_new_correction, \ + sdiff, \ + cip->correction.error, \ + cip->correction.drift, \ + new_correction.error, \ + new_correction.drift, \ + timeout) +# endif +#endif + ErtsMonotonicCorrectionData cdata; + ErtsMonotonicCorrection new_correction; + ErtsMonotonicCorrectionInstance *cip; + ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime, + erl_stime, time_offset; + Uint timeout; + int set_new_correction, begin_short_intervals = 0; + + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + + ASSERT(time_sup.inf.c.finalized_offset); + + erts_os_times(&os_mtime, &os_stime); + + cdata = time_sup.inf.c.parmon.cdata; + + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + + if (os_mtime < cdata.curr.os_mtime) + erl_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + cip = &cdata.curr; + + erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, &mdiff); + time_offset = get_time_offset(); + erl_stime = erl_mtime + time_offset; + + sdiff = erl_stime - os_stime; + + new_correction = cip->correction; + + if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE + && (sdiff < -2*time_sup.r.o.adj.small_diff + || 2*time_sup.r.o.adj.small_diff < sdiff)) { + /* System time diff exeeded limits; change time offset... */ + time_offset -= sdiff; + sdiff = 0; + set_time_offset(time_offset); + schedule_send_time_offset_changed_notifications(time_offset); + begin_short_intervals = 1; + if (cdata.curr.correction.error == 0) + set_new_correction = 0; + else { + set_new_correction = 1; + new_correction.error = 0; + } + } + else if (cdata.curr.correction.error == 0) { + if (sdiff < -time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff < -time_sup.r.o.adj.large_diff) + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ; + } + else if (sdiff > time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff > time_sup.r.o.adj.large_diff) + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ; + } + else { + set_new_correction = 0; + } + } + else if (cdata.curr.correction.error > 0) { + if (sdiff < 0) { + if (cdata.curr.correction.error == ERTS_TCORR_ERR_LARGE_ADJ + || -time_sup.r.o.adj.large_diff <= sdiff) + set_new_correction = 0; + else { + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + set_new_correction = 1; + } + } + else if (sdiff > time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff > time_sup.r.o.adj.large_diff) + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ; + } + else { + set_new_correction = 1; + new_correction.error = 0; + } + } + else /* if (cdata.curr.correction.error < 0) */ { + if (0 < sdiff) { + if (cdata.curr.correction.error == -ERTS_TCORR_ERR_LARGE_ADJ + || sdiff <= time_sup.r.o.adj.large_diff) + set_new_correction = 0; + else { + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + set_new_correction = 1; + } + set_new_correction = 0; + } + else if (sdiff < -time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff < -time_sup.r.o.adj.large_diff) + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ; + } + else { + set_new_correction = 1; + new_correction.error = 0; + } } - if ((curr - hr_last_correction_check) / 1000 > 1000000) { - /* Check the correction need */ - SysHrTime tv_diff, diffdiff; - SysTimeval tmp; - int done = 0; - - sys_gettimeofday(&tmp); - tv_diff = ((SysHrTime) tmp.tv_sec) * 1000000 + tmp.tv_usec; - tv_diff -= ((SysHrTime) inittv.tv_sec) * 1000000 + inittv.tv_usec; - diffdiff = diff_time - tv_diff; - if (diffdiff > 10000) { - SysHrTime corr = (curr - hr_last_time) / 100; - if (corr / 1000 >= diffdiff) { - ++done; - hr_correction -= ((SysHrTime)diffdiff) * 1000; - } else { - hr_correction -= corr; +#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC + { + ErtsMonotonicDriftData *ddp = &time_sup.inf.c.parmon.cdata.drift; + int ix = ddp->ix; + ErtsMonotonicTime mtime_diff, old_os_mtime; + + old_os_mtime = ddp->intervals[ix].time.mon; + mtime_diff = os_mtime - old_os_mtime; + + if (mtime_diff >= ERTS_SEC_TO_MONOTONIC(10)) { + ErtsMonotonicTime drift_adj, drift_adj_diff, old_os_stime, + stime_diff, mtime_acc, stime_acc, avg_drift_adj; + + old_os_stime = ddp->intervals[ix].time.sys; + + mtime_acc = ddp->acc.mon; + stime_acc = ddp->acc.sys; + + avg_drift_adj = (((stime_acc - mtime_acc)*ERTS_MONOTONIC_TIME_UNIT) + / mtime_acc); + + mtime_diff = os_mtime - old_os_mtime; + stime_diff = os_stime - old_os_stime; + drift_adj = (((stime_diff - mtime_diff)*ERTS_MONOTONIC_TIME_UNIT) + / mtime_diff); + + ix++; + if (ix >= ERTS_DRIFT_INTERVALS) + ix = 0; + mtime_acc -= ddp->intervals[ix].diff.mon; + mtime_acc += mtime_diff; + stime_acc -= ddp->intervals[ix].diff.sys; + stime_acc += stime_diff; + + ddp->intervals[ix].diff.mon = mtime_diff; + ddp->intervals[ix].diff.sys = stime_diff; + ddp->intervals[ix].time.mon = os_mtime; + ddp->intervals[ix].time.sys = os_stime; + + ddp->ix = ix; + ddp->acc.mon = mtime_acc; + ddp->acc.sys = stime_acc; + + drift_adj_diff = avg_drift_adj - drift_adj; + if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF + || ERTS_TIME_DRIFT_MAX_ADJ_DIFF < drift_adj_diff) { + ddp->dirty_counter = ERTS_DRIFT_INTERVALS; + begin_short_intervals = 1; } - diff_time = (curr + hr_correction - hr_init_time) / 1000; - } else if (diffdiff < -10000) { - SysHrTime corr = (curr - hr_last_time) / 100; - if (corr / 1000 >= -diffdiff) { - ++done; - hr_correction -= ((SysHrTime)diffdiff) * 1000; - } else { - hr_correction += corr; + else { + if (ddp->dirty_counter <= 0) { + drift_adj = ((stime_acc - mtime_acc) + *ERTS_MONOTONIC_TIME_UNIT) / mtime_acc; + } + if (ddp->dirty_counter >= 0) { + if (ddp->dirty_counter == 0) { + /* Force set new drift correction... */ + set_new_correction = 1; + } + ddp->dirty_counter--; + } + drift_adj_diff = drift_adj - new_correction.drift; + if (drift_adj_diff) { + if (drift_adj_diff > ERTS_TIME_DRIFT_MAX_ADJ_DIFF) + drift_adj_diff = ERTS_TIME_DRIFT_MAX_ADJ_DIFF; + else if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF) + drift_adj_diff = -ERTS_TIME_DRIFT_MAX_ADJ_DIFF; + new_correction.drift += drift_adj_diff; + + if (drift_adj_diff < -ERTS_TIME_DRIFT_MIN_ADJ_DIFF + || ERTS_TIME_DRIFT_MIN_ADJ_DIFF < drift_adj_diff) { + set_new_correction = 1; + } + } } - diff_time = (curr + hr_correction - hr_init_time) / 1000; - } else { - ++done; } - if (done) { - hr_last_correction_check = curr; + } +#endif + + begin_short_intervals |= set_new_correction; + + if (begin_short_intervals) { + time_sup.inf.c.parmon.cdata.short_check_interval + = ERTS_INIT_SHORT_INTERVAL_COUNTER; + } + else if ((os_mtime - time_sup.inf.c.parmon.cdata.last_check + >= ERTS_SHORT_TIME_CORRECTION_CHECK - ERTS_MONOTONIC_TIME_UNIT) + && time_sup.inf.c.parmon.cdata.short_check_interval > 0) { + time_sup.inf.c.parmon.cdata.short_check_interval--; + } + time_sup.inf.c.parmon.cdata.last_check = os_mtime; + + if (new_correction.error == 0) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK); + else { + ErtsMonotonicTime ecorr = new_correction.error; + if (sdiff < 0) + sdiff = -1*sdiff; + if (ecorr < 0) + ecorr = -1*ecorr; + if (sdiff > ecorr*(ERTS_LONG_TIME_CORRECTION_CHECK/ERTS_TCORR_ERR_UNIT)) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK); + else { + timeout = ERTS_MONOTONIC_TO_MSEC((ERTS_TCORR_ERR_UNIT*sdiff)/ecorr); + if (timeout < 10) + timeout = 10; } } - tv->tv_sec += (int) (diff_time / ((SysHrTime) 1000000)); - tv->tv_usec += (int) (diff_time % ((SysHrTime) 1000000)); - if (tv->tv_usec >= 1000000) { - tv->tv_usec -= 1000000; - tv->tv_sec += 1; + + if (timeout > ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK) + && time_sup.inf.c.parmon.cdata.short_check_interval) { + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK); + } + + ERTS_PRINT_CORRECTION; + + if (set_new_correction) { + erts_smp_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx); + + os_mtime = erts_os_monotonic_time(); + + /* Save previous correction instance */ + time_sup.inf.c.parmon.cdata.prev = *cip; + + /* + * Current correction instance begin when + * OS monotonic time has increased one unit. + */ + os_mtime++; + + /* + * Erlang monotonic time corresponding to + * next OS monotonic time using previous + * correction. + */ + erl_mtime = calc_corrected_erl_mtime(os_mtime, cip, NULL); + + /* + * Save new current correction instance. + */ + time_sup.inf.c.parmon.cdata.curr.erl_mtime = erl_mtime; + time_sup.inf.c.parmon.cdata.curr.os_mtime = os_mtime; + time_sup.inf.c.parmon.cdata.curr.correction = new_correction; + + erts_smp_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx); } - hr_last_time = curr; + + erts_set_timer(&time_sup.inf.c.parmon.timer, + check_time_correction, + NULL, + NULL, + timeout); + +#undef ERTS_PRINT_CORRECTION } -#define correction (hr_correction/1000000) +#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC -#else /* !HAVE_GETHRTIME */ -#if !defined(CORRECT_USING_TIMES) -#define init_tolerant_timeofday() -#define get_tolerant_timeofday(tvp) sys_gettimeofday(tvp) -#else +static void +init_check_time_correction(void *unused) +{ + ErtsMonotonicDriftData *ddp; + ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff, + stime_diff; + int ix; + + ddp = &time_sup.inf.c.parmon.cdata.drift; + ix = ddp->ix; + old_mtime = ddp->intervals[0].time.mon; + old_stime = ddp->intervals[0].time.sys; + + erts_os_times(&mtime, &stime); + + mtime_diff = mtime - old_mtime; + stime_diff = stime - old_stime; + if (100*stime_diff < 80*mtime_diff || 120*mtime_diff < 100*stime_diff ) { + /* Had a system time leap... pretend no drift... */ + stime_diff = mtime_diff; + } + + /* + * We use old time values in order to trigger + * a drift adjustment, and repeat this interval + * in all slots... + */ + for (ix = 0; ix < ERTS_DRIFT_INTERVALS; ix++) { + ddp->intervals[ix].diff.mon = mtime_diff; + ddp->intervals[ix].diff.sys = stime_diff; + ddp->intervals[ix].time.mon = old_mtime; + ddp->intervals[ix].time.sys = old_stime; + } -typedef Sint64 Milli; - -static clock_t init_ct; -static Sint64 ct_wrap; -static Milli init_tv_m; -static Milli correction_supress; -static Milli last_ct_diff; -static Milli last_cc; -static clock_t last_ct; - -/* sys_times() might need to be wrapped and the values shifted (right) - a bit to cope with newer linux (2.5.*) kernels, this has to be taken care - of dynamically to start with, a special version that uses - the times() return value as a high resolution timer can be made - to fully utilize the faster ticks, like on windows, but for now, we'll - settle with this silly workaround */ -#ifdef ERTS_WRAP_SYS_TIMES -#define KERNEL_TICKS() (sys_times_wrap() & \ - ((1UL << ((sizeof(clock_t) * 8) - 1)) - 1)) -#else -SysTimes dummy_tms; + ddp->acc.sys = stime_diff*ERTS_DRIFT_INTERVALS; + ddp->acc.mon = mtime_diff*ERTS_DRIFT_INTERVALS; + ddp->ix = 0; + ddp->dirty_counter = ERTS_DRIFT_INTERVALS; -#define KERNEL_TICKS() (sys_times(&dummy_tms) & \ - ((1UL << ((sizeof(clock_t) * 8) - 1)) - 1)) + check_time_correction(NULL); +} #endif -static void init_tolerant_timeofday(void) +static ErtsMonotonicTime +finalize_corrected_time_offset(ErtsSystemTime *stimep) { - last_ct = init_ct = KERNEL_TICKS(); - last_cc = 0; - init_tv_m = (((Milli) inittv.tv_sec) * 1000) + - (inittv.tv_usec / 1000); - ct_wrap = 0; - correction_supress = 0; -} + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrectionData cdata; + ErtsMonotonicCorrectionInstance *cip; + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + + erts_os_times(&os_mtime, stimep); + + cdata = time_sup.inf.c.parmon.cdata; + + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + + if (os_mtime < cdata.curr.os_mtime) + erl_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + cip = &cdata.curr; + + return calc_corrected_erl_mtime(os_mtime, cip, NULL); +} -static void get_tolerant_timeofday(SysTimeval *tvp) +static void +late_init_time_correction(void) { - clock_t current_ct; - SysTimeval current_tv; - Milli ct_diff; - Milli tv_diff; - Milli current_correction; - Milli act_correction; /* long shown to be too small */ - Milli max_adjust; - - if (erts_disable_tolerant_timeofday) { - sys_gettimeofday(tvp); - return; - } + if (time_sup.inf.c.finalized_offset) { -#ifdef ERTS_WRAP_SYS_TIMES -#define TICK_MS (1000 / SYS_CLK_TCK_WRAP) + erts_init_timer(&time_sup.inf.c.parmon.timer); + erts_set_timer(&time_sup.inf.c.parmon.timer, +#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC + init_check_time_correction, #else -#define TICK_MS (1000 / SYS_CLK_TCK) + check_time_correction, #endif - current_ct = KERNEL_TICKS(); - sys_gettimeofday(¤t_tv); - - /* I dont know if uptime can move some units backwards - on some systems, but I allow for small backward - jumps to avoid such problems if they exist...*/ - if (last_ct > 100 && current_ct < (last_ct - 100)) { - ct_wrap += ((Sint64) 1) << ((sizeof(clock_t) * 8) - 1); + NULL, + NULL, + ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK)); } - last_ct = current_ct; - ct_diff = ((ct_wrap + current_ct) - init_ct) * TICK_MS; +} - /* - * We will adjust the time in milliseconds and we allow for 1% - * adjustments, but if this function is called more often then every 100 - * millisecond (which is obviously possible), we will never adjust, so - * we accumulate small times by setting last_ct_diff iff max_adjust > 0 - */ - if ((max_adjust = (ct_diff - last_ct_diff)/100) > 0) - last_ct_diff = ct_diff; +#endif /* ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT */ - tv_diff = ((((Milli) current_tv.tv_sec) * 1000) + - (current_tv.tv_usec / 1000)) - init_tv_m; +static ErtsMonotonicTime get_not_corrected_time(void) +{ + ErtsMonotonicTime stime, mtime; - current_correction = ((ct_diff - tv_diff) / TICK_MS) * TICK_MS; /* trunc */ + erts_smp_mtx_lock(&erts_get_time_mtx); - /* - * We allow the current_correction value to wobble a little, as it - * suffers from the low resolution of the kernel ticks. - * if it hasn't changed more than one tick in either direction, - * we will keep the old value. - */ - if ((last_cc > current_correction + TICK_MS) || - (last_cc < current_correction - TICK_MS)) { - last_cc = current_correction; - } else { - current_correction = last_cc; - } - - /* - * As time goes, we try to get the actual correction to 0, - * that is, make erlangs time correspond to the systems dito. - * The act correction is what we seem to need (current_correction) - * minus the correction suppression. The correction supression - * will change slowly (max 1% of elapsed time) but in millisecond steps. - */ - act_correction = current_correction - correction_supress; - if (max_adjust > 0) { - /* - * Here we slowly adjust erlangs time to correspond with the - * system time by changing the correction_supress variable. - * It can change max_adjust milliseconds which is 1% of elapsed time - */ - if (act_correction > 0) { - if (current_correction - correction_supress > max_adjust) { - correction_supress += max_adjust; - } else { - correction_supress = current_correction; - } - act_correction = current_correction - correction_supress; - } else if (act_correction < 0) { - if (correction_supress - current_correction > max_adjust) { - correction_supress -= max_adjust; - } else { - correction_supress = current_correction; + stime = erts_os_system_time(); + + mtime = stime - time_sup.inf.c.not_corrected_moffset; + + if (mtime >= time_sup.f.c.last_not_corrected_time) + time_sup.f.c.last_not_corrected_time = mtime; + else { + mtime = time_sup.f.c.last_not_corrected_time; + + if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE) { + ErtsMonotonicTime new_offset = stime - mtime; + new_offset = ERTS_MONOTONIC_TO_USEC(new_offset); + new_offset = ERTS_USEC_TO_MONOTONIC(new_offset); + if (time_sup.inf.c.not_corrected_moffset != new_offset) { + time_sup.inf.c.not_corrected_moffset = new_offset; + set_time_offset(new_offset); + schedule_send_time_offset_changed_notifications(new_offset); } - act_correction = current_correction - correction_supress; } + } - /* - * The actual correction will correct the timeval so that system - * time warps gets smothed down. - */ - current_tv.tv_sec += act_correction / 1000; - current_tv.tv_usec += (act_correction % 1000) * 1000; - - if (current_tv.tv_usec >= 1000000) { - ++current_tv.tv_sec ; - current_tv.tv_usec -= 1000000; - } else if (current_tv.tv_usec < 0) { - --current_tv.tv_sec; - current_tv.tv_usec += 1000000; - } - *tvp = current_tv; -#undef TICK_MS + + ASSERT(stime == mtime + time_sup.inf.c.not_corrected_moffset); + + erts_smp_mtx_unlock(&erts_get_time_mtx); + + return mtime; } -#endif /* CORRECT_USING_TIMES */ -#endif /* !HAVE_GETHRTIME */ +int erts_check_time_adj_support(int time_correction, + ErtsTimeWarpMode time_warp_mode) +{ + if (!time_correction) + return 1; + + /* User wants time correction */ -/* -** Why this? Well, most platforms have a constant clock resolution of 1, -** we dont want the deliver_time/time_remaining routines to waste -** time dividing and multiplying by/with a variable that's always one. -** so the return value of sys_init_time is ignored on those platforms. -*/ - -#ifndef SYS_CLOCK_RESOLUTION -static int clock_resolution; -#define CLOCK_RESOLUTION clock_resolution +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return !time_sup.r.o.os_monotonic_time_disable; #else -#define CLOCK_RESOLUTION SYS_CLOCK_RESOLUTION + return 0; #endif +} -/* -** The clock resolution should really be the resolution of the -** time function in use, which on most platforms -** is 1. On VxWorks the resolution should be -** the number of ticks per second (or 1, which would work nicely to). -** -** Setting lower resolutions is mostly interesting when timers are used -** instead of something like select. -*/ - -static SysTimeval last_delivered; - -static void init_erts_deliver_time(const SysTimeval *inittv) +int +erts_has_time_correction(void) { - /* We set the initial values for deliver_time here */ - last_delivered = *inittv; - last_delivered.tv_usec = 1000 * (last_delivered.tv_usec / 1000); - /* ms resolution */ + return time_sup.r.o.correction; } -static void do_erts_deliver_time(const SysTimeval *current) +void erts_init_sys_time_sup(void) { - SysTimeval cur_time; - erts_time_t elapsed; - - /* calculate and deliver appropriate number of ticks */ - cur_time = *current; - cur_time.tv_usec = 1000 * (cur_time.tv_usec / 1000); /* ms resolution */ - elapsed = (1000 * (cur_time.tv_sec - last_delivered.tv_sec) + - (cur_time.tv_usec - last_delivered.tv_usec) / 1000) / - CLOCK_RESOLUTION; + ErtsSysInitTimeResult sys_init_time_res + = ERTS_SYS_INIT_TIME_RESULT_INITER; - /* Sometimes the time jump backwards, - resulting in a negative elapsed time. We compensate for - this by simply pretend as if the time stood still. :) */ + sys_init_time(&sys_init_time_res); - if (elapsed > 0) { + erts_time_sup__.r.o.monotonic_time_unit + = sys_init_time_res.os_monotonic_time_unit; - ASSERT(elapsed < ((erts_time_t) ERTS_SHORT_TIME_T_MAX)); +#ifndef SYS_CLOCK_RESOLUTION + erts_time_sup__.r.o.clktck_resolution + = sys_init_time_res.sys_clock_resolution; + erts_time_sup__.r.o.clktck_resolution *= 1000; +#endif - erts_do_time_add((erts_short_time_t) elapsed); - last_delivered = cur_time; - } +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + time_sup.r.o.os_monotonic_time_disable + = !sys_init_time_res.have_os_monotonic_time; + time_sup.r.o.os_monotonic_time_func + = sys_init_time_res.os_monotonic_time_info.func; + time_sup.r.o.os_monotonic_time_clock_id + = sys_init_time_res.os_monotonic_time_info.clock_id; + time_sup.r.o.os_monotonic_time_locked + = sys_init_time_res.os_monotonic_time_info.locked_use; + time_sup.r.o.os_monotonic_time_resolution + = sys_init_time_res.os_monotonic_time_info.resolution; + time_sup.r.o.os_monotonic_time_extended + = sys_init_time_res.os_monotonic_time_info.extended; +#endif + time_sup.r.o.os_system_time_func + = sys_init_time_res.os_system_time_info.func; + time_sup.r.o.os_system_time_clock_id + = sys_init_time_res.os_system_time_info.clock_id; + time_sup.r.o.os_system_time_locked + = sys_init_time_res.os_system_time_info.locked_use; + time_sup.r.o.os_system_time_resolution + = sys_init_time_res.os_system_time_info.resolution; } int -erts_init_time_sup(void) +erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) { + ErtsMonotonicTime resolution; +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + ErtsMonotonicTime abs_native_offset, native_offset; +#endif + + ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX); + erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); + erts_smp_mtx_init(&erts_get_time_mtx, "get_time"); - init_approx_time(); + time_sup.r.o.correction = time_correction; + time_sup.r.o.warp_mode = time_warp_mode; + + if (time_warp_mode == ERTS_SINGLE_TIME_WARP_MODE) + time_sup.inf.c.finalized_offset = 0; + else + time_sup.inf.c.finalized_offset = ~0; + +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + +#ifdef ARCH_32 + time_sup.r.o.start = ((((ErtsMonotonicTime) 1) << 32)-1); + time_sup.r.o.start /= ERTS_MONOTONIC_TIME_UNIT; + time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT; + time_sup.r.o.start += ERTS_MONOTONIC_TIME_UNIT; + native_offset = time_sup.r.o.start - ERTS_MONOTONIC_TIME_UNIT; + native_offset = native_offset; +#else /* ARCH_64 */ + if (ERTS_MONOTONIC_TIME_UNIT <= 10*1000*1000) { + time_sup.r.o.start = 0; + native_offset = -ERTS_MONOTONIC_TIME_UNIT; + abs_native_offset = ERTS_MONOTONIC_TIME_UNIT; + } + else { + time_sup.r.o.start = ((ErtsMonotonicTime) MIN_SMALL); + time_sup.r.o.start /= ERTS_MONOTONIC_TIME_UNIT; + time_sup.r.o.start *= ERTS_MONOTONIC_TIME_UNIT; + native_offset = time_sup.r.o.start - ERTS_MONOTONIC_TIME_UNIT; + abs_native_offset = -1*native_offset; + } +#endif - last_emu_time.tv_sec = 0; - last_emu_time.tv_usec = 0; + time_sup.r.o.start_offset.native = (time_sup.r.o.start + - ERTS_MONOTONIC_TIME_UNIT); + time_sup.r.o.start_offset.nsec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000*1000*1000); + time_sup.r.o.start_offset.usec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000*1000); + time_sup.r.o.start_offset.msec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000); + time_sup.r.o.start_offset.sec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1); + if (native_offset < 0) { + time_sup.r.o.start_offset.nsec *= -1; + time_sup.r.o.start_offset.usec *= -1; + time_sup.r.o.start_offset.msec *= -1; + time_sup.r.o.start_offset.sec *= -1; + } -#ifndef SYS_CLOCK_RESOLUTION - clock_resolution = sys_init_time(); -#else - (void) sys_init_time(); #endif - sys_gettimeofday(&inittv); + + resolution = time_sup.r.o.os_system_time_resolution; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (resolution > time_sup.r.o.os_monotonic_time_resolution) + resolution = time_sup.r.o.os_monotonic_time_resolution; +#endif + + time_sup.r.o.adj.large_diff = erts_time_sup__.r.o.monotonic_time_unit; + time_sup.r.o.adj.large_diff *= 50; + time_sup.r.o.adj.large_diff /= resolution; + if (time_sup.r.o.adj.large_diff < ERTS_USEC_TO_MONOTONIC(500)) + time_sup.r.o.adj.large_diff = ERTS_USEC_TO_MONOTONIC(500); + time_sup.r.o.adj.small_diff = time_sup.r.o.adj.large_diff/10; + +#ifdef ERTS_TIME_CORRECTION_PRINT + fprintf(stderr, "start = %lld\n\r", (long long) ERTS_MONOTONIC_TIME_START); + fprintf(stderr, "native offset = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_NATIVE); + fprintf(stderr, "nsec offset = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_NSEC); + fprintf(stderr, "usec offset = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_USEC); + fprintf(stderr, "msec offset = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_MSEC); + fprintf(stderr, "sec offset = %lld\n\r", (long long) ERTS_MONOTONIC_OFFSET_SEC); + fprintf(stderr, "large diff = %lld usec\r\n", + (long long) ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.large_diff)); + fprintf(stderr, "small diff = %lld usec\r\n", + (long long) ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.small_diff)); +#endif + + if (ERTS_MONOTONIC_TIME_UNIT < ERTS_CLKTCK_RESOLUTION) + ERTS_INTERNAL_ERROR("Too small monotonic time time unit"); + +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + time_sup.r.o.correction = 0; +#else + if (time_sup.r.o.os_monotonic_time_disable) + time_sup.r.o.correction = 0; + + if (time_sup.r.o.correction) { + ErtsMonotonicCorrectionData *cdatap; + erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + ErtsMonotonicTime offset; + erts_os_times(&time_sup.inf.c.minit, + &time_sup.inf.c.sinit); + time_sup.r.o.moffset = -1*time_sup.inf.c.minit; + offset = time_sup.inf.c.sinit; + offset -= ERTS_MONOTONIC_TIME_UNIT; + init_time_offset(offset); + + rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx, + &rwmtx_opts, "get_corrected_time"); + + cdatap = &time_sup.inf.c.parmon.cdata; -#ifdef HAVE_GETHRTIME - sys_init_hrtime(); +#ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC + cdatap->drift.intervals[0].time.sys = time_sup.inf.c.sinit; + cdatap->drift.intervals[0].time.mon = time_sup.inf.c.minit; + cdatap->curr.correction.drift = 0; #endif - init_tolerant_timeofday(); + cdatap->curr.correction.error = 0; + cdatap->curr.erl_mtime = ERTS_MONOTONIC_TIME_UNIT; + cdatap->curr.os_mtime = time_sup.inf.c.minit; + cdatap->last_check = time_sup.inf.c.minit; + cdatap->short_check_interval = ERTS_INIT_SHORT_INTERVAL_COUNTER; + cdatap->prev = cdatap->curr; + + time_sup.r.o.get_time = get_corrected_time; + } + else +#endif + { + ErtsMonotonicTime stime, offset; + time_sup.r.o.get_time = get_not_corrected_time; + stime = time_sup.inf.c.sinit = erts_os_system_time(); + offset = stime - ERTS_MONOTONIC_TIME_UNIT; + time_sup.inf.c.not_corrected_moffset = offset; + init_time_offset(offset); + time_sup.f.c.last_not_corrected_time = 0; + } - init_erts_deliver_time(&inittv); - gtv = inittv; - then.tv_sec = then.tv_usec = 0; + prev_wall_clock_elapsed = 0; - erts_deliver_time(); + previous_now = ERTS_MONOTONIC_TO_USEC(get_time_offset()); - return CLOCK_RESOLUTION; +#ifdef DEBUG + time_sup_initialized = 1; +#endif + + return ERTS_CLKTCK_RESOLUTION/1000; } + +void +erts_late_init_time_sup(void) +{ +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + /* Timer wheel must be initialized */ + if (time_sup.r.o.get_time == get_corrected_time) + late_init_time_correction(); +#endif + erts_late_sys_init_time(); +} + +ErtsTimeWarpMode erts_time_warp_mode(void) +{ + return time_sup.r.o.warp_mode; +} + +ErtsTimeOffsetState erts_time_offset_state(void) +{ + switch (time_sup.r.o.warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_FINAL; + case ERTS_SINGLE_TIME_WARP_MODE: + if (time_sup.inf.c.finalized_offset) + return ERTS_TIME_OFFSET_FINAL; + return ERTS_TIME_OFFSET_PRELIMINARY; + case ERTS_MULTI_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_VOLATILE; + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + return ERTS_TIME_OFFSET_VOLATILE; + } +} + +/* + * erts_finalize_time_offset() will only change time offset + * the first time it is called when the emulator has been + * started in "single time warp" mode. Returns previous + * state: + * * ERTS_TIME_OFFSET_PRELIMINARY - Finalization performed + * * ERTS_TIME_OFFSET_FINAL - Already finialized; nothing changed + * * ERTS_TIME_OFFSET_VOLATILE - Not supported, either in + * * no correction mode (or multi time warp mode; not yet implemented). + */ + +ErtsTimeOffsetState +erts_finalize_time_offset(void) +{ + switch (time_sup.r.o.warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_FINAL; + case ERTS_MULTI_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_VOLATILE; + case ERTS_SINGLE_TIME_WARP_MODE: { + ErtsTimeOffsetState res = ERTS_TIME_OFFSET_FINAL; + + erts_smp_mtx_lock(&erts_get_time_mtx); + + if (!time_sup.inf.c.finalized_offset) { + ErtsMonotonicTime mtime, new_offset; + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (!time_sup.r.o.correction) +#endif + { + ErtsMonotonicTime stime = erts_os_system_time(); + + mtime = stime - time_sup.inf.c.not_corrected_moffset; + + if (mtime >= time_sup.f.c.last_not_corrected_time) { + time_sup.f.c.last_not_corrected_time = mtime; + new_offset = time_sup.inf.c.not_corrected_moffset; + } + else { + mtime = time_sup.f.c.last_not_corrected_time; + + ASSERT(time_sup.inf.c.not_corrected_moffset != stime - mtime); + new_offset = stime - mtime; + time_sup.inf.c.not_corrected_moffset = new_offset; + } + + } +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + else { + ErtsSystemTime stime; + mtime = finalize_corrected_time_offset(&stime); + new_offset = stime - mtime; + } +#endif + new_offset = ERTS_MONOTONIC_TO_USEC(new_offset); + new_offset = ERTS_USEC_TO_MONOTONIC(new_offset); + + set_time_offset(new_offset); + schedule_send_time_offset_changed_notifications(new_offset); + + time_sup.inf.c.finalized_offset = ~0; + res = ERTS_TIME_OFFSET_PRELIMINARY; + } + + erts_smp_mtx_unlock(&erts_get_time_mtx); + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (res == ERTS_TIME_OFFSET_PRELIMINARY + && time_sup.r.o.get_time == get_corrected_time) { + late_init_time_correction(); + } +#endif + + return res; + } + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + return ERTS_TIME_OFFSET_VOLATILE; + } +} + /* info functions */ void @@ -498,23 +1171,16 @@ elapsed_time_both(UWord *ms_user, UWord *ms_sys, void wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff) { - UWord prev_total; - SysTimeval tv; + ErtsMonotonicTime now, elapsed; erts_smp_mtx_lock(&erts_timeofday_mtx); - get_tolerant_timeofday(&tv); + now = time_sup.r.o.get_time(); - *ms_total = 1000 * (tv.tv_sec - inittv.tv_sec) + - (tv.tv_usec - inittv.tv_usec) / 1000; - - prev_total = 1000 * (gtv.tv_sec - inittv.tv_sec) + - (gtv.tv_usec - inittv.tv_usec) / 1000; - *ms_diff = *ms_total - prev_total; - gtv = tv; - - /* must sync the machine's idea of time here */ - do_erts_deliver_time(&tv); + elapsed = ERTS_MONOTONIC_TO_MSEC(now); + *ms_total = (UWord) elapsed; + *ms_diff = (UWord) (elapsed - prev_wall_clock_elapsed); + prev_wall_clock_elapsed = elapsed; erts_smp_mtx_unlock(&erts_timeofday_mtx); } @@ -890,146 +1556,555 @@ univ_to_local(Sint *year, Sint *month, Sint *day, return 0; } - /* get a timestamp */ void get_now(Uint* megasec, Uint* sec, Uint* microsec) { - SysTimeval now; + ErtsMonotonicTime now_megasec, now_sec, now, mtime, time_offset; + mtime = time_sup.r.o.get_time(); + time_offset = get_time_offset(); + now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset); + erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&now); - do_erts_deliver_time(&now); - - /* Make sure time is later than last */ - if (then.tv_sec > now.tv_sec || - (then.tv_sec == now.tv_sec && then.tv_usec >= now.tv_usec)) { - now = then; - now.tv_usec++; - } - /* Check for carry from above + general reasonability */ - if (now.tv_usec >= 1000000) { - now.tv_usec = 0; - now.tv_sec++; - } - then = now; + + /* Make sure now time is later than last time */ + if (now <= previous_now) + now = previous_now + 1; + + previous_now = now; erts_smp_mtx_unlock(&erts_timeofday_mtx); - - *megasec = (Uint) (now.tv_sec / 1000000); - *sec = (Uint) (now.tv_sec % 1000000); - *microsec = (Uint) (now.tv_usec); - update_approx_time(&now); + now_megasec = now / ERTS_MONOTONIC_TIME_TERA; + now_sec = now / ERTS_MONOTONIC_TIME_MEGA; + *megasec = (Uint) now_megasec; + *sec = (Uint) (now_sec - now_megasec*ERTS_MONOTONIC_TIME_MEGA); + *microsec = (Uint) (now - now_sec*ERTS_MONOTONIC_TIME_MEGA); + + ASSERT(((ErtsMonotonicTime) *megasec)*ERTS_MONOTONIC_TIME_TERA + + ((ErtsMonotonicTime) *sec)*ERTS_MONOTONIC_TIME_MEGA + + ((ErtsMonotonicTime) *microsec) == now); +} + +ErtsMonotonicTime +erts_get_monotonic_time(void) +{ + return time_sup.r.o.get_time(); } void get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) { - SysTimeval now; - - sys_gettimeofday(&now); - - *megasec = (Uint) (now.tv_sec / 1000000); - *sec = (Uint) (now.tv_sec % 1000000); - *microsec = (Uint) (now.tv_usec); + ErtsSystemTime stime = erts_os_system_time(); + ErtsSystemTime ms, s, us; + + us = ERTS_MONOTONIC_TO_USEC(stime); + s = us / (1000*1000); + ms = s / (1000*1000); - update_approx_time(&now); + *megasec = (Uint) ms; + *sec = (Uint) (s - ms*(1000*1000)); + *microsec = (Uint) (us - s*(1000*1000)); } +#ifdef HAVE_ERTS_NOW_CPU +void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { + SysCpuTime t; + SysTimespec tp; + + sys_get_proc_cputime(t, tp); + *microsec = (Uint)(tp.tv_nsec / 1000); + t = (tp.tv_sec / 1000000); + *megasec = (Uint)(t % 1000000); + *sec = (Uint)(tp.tv_sec % 1000000); +} +#endif -/* deliver elapsed *ticks* to the machine - takes a pointer - to a struct timeval representing current time (to save - a gettimeofday() where possible) or NULL */ +#include "big.h" -void erts_deliver_time(void) { - SysTimeval now; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&now); - do_erts_deliver_time(&now); - - erts_smp_mtx_unlock(&erts_timeofday_mtx); +void +erts_monitor_time_offset(Eterm id, Eterm ref) +{ + erts_smp_mtx_lock(&erts_get_time_mtx); + erts_add_monitor(&time_offset_monitors, MON_TIME_OFFSET, ref, id, NIL); + no_time_offset_monitors++; + erts_smp_mtx_unlock(&erts_get_time_mtx); +} - update_approx_time(&now); +int +erts_demonitor_time_offset(Eterm ref) +{ + int res; + ErtsMonitor *mon; + ASSERT(is_internal_ref(ref)); + erts_smp_mtx_lock(&erts_get_time_mtx); + mon = erts_remove_monitor(&time_offset_monitors, ref); + if (!mon) + res = 0; + else { + ASSERT(no_time_offset_monitors > 0); + no_time_offset_monitors--; + res = 1; + } + erts_smp_mtx_unlock(&erts_get_time_mtx); + if (res) + erts_destroy_monitor(mon); + return res; } -/* get *real* time (not ticks) remaining until next timeout - if there - isn't one, give a "long" time, that is guaranteed - to not cause overflow when we report elapsed time later on */ +typedef struct { + Eterm pid; + Eterm ref; + Eterm heap[REF_THING_SIZE]; +} ErtsTimeOffsetMonitorInfo; -void erts_time_remaining(SysTimeval *rem_time) +typedef struct { + Uint ix; + ErtsTimeOffsetMonitorInfo *to_mon_info; +} ErtsTimeOffsetMonitorContext; + +static void +save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt) { - erts_time_t ticks; - SysTimeval cur_time; - erts_time_t elapsed; - - /* erts_next_time() returns no of ticks to next timeout or -1 if none */ - - ticks = (erts_time_t) erts_next_time(); - if (ticks == (erts_time_t) -1) { - /* timer queue empty */ - /* this will cause at most 100000000 ticks */ - rem_time->tv_sec = 100000; - rem_time->tv_usec = 0; - } else { - /* next timeout after ticks ticks */ - ticks *= CLOCK_RESOLUTION; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&cur_time); - cur_time.tv_usec = 1000 * - (cur_time.tv_usec / 1000);/* ms resolution*/ - elapsed = 1000 * (cur_time.tv_sec - last_delivered.tv_sec) + - (cur_time.tv_usec - last_delivered.tv_usec) / 1000; - - erts_smp_mtx_unlock(&erts_timeofday_mtx); + ErtsTimeOffsetMonitorContext *cntxt; + Eterm *from_hp, *to_hp; + Uint mix; + int hix; + + cntxt = (ErtsTimeOffsetMonitorContext *) vcntxt; + mix = (cntxt->ix)++; + cntxt->to_mon_info[mix].pid = mon->pid; + to_hp = &cntxt->to_mon_info[mix].heap[0]; + + ASSERT(is_internal_ref(mon->ref)); + from_hp = internal_ref_val(mon->ref); + ASSERT(thing_arityval(*from_hp) + 1 == REF_THING_SIZE); + + for (hix = 0; hix < REF_THING_SIZE; hix++) + to_hp[hix] = from_hp[hix]; + + cntxt->to_mon_info[mix].ref + = make_internal_ref(&cntxt->to_mon_info[mix].heap[0]); + +} + +static void +send_time_offset_changed_notifications(void *new_offsetp) +{ + ErtsMonotonicTime new_offset; + ErtsTimeOffsetMonitorInfo *to_mon_info = NULL; /* Shut up faulty warning */ + Uint no_monitors; + char *tmp = NULL; + +#ifdef ARCH_64 + new_offset = (ErtsMonotonicTime) new_offsetp; +#else + new_offset = *((ErtsMonotonicTime *) new_offsetp); + erts_free(ERTS_ALC_T_NEW_TIME_OFFSET, new_offsetp); +#endif + new_offset -= ERTS_MONOTONIC_OFFSET_NATIVE; + + erts_smp_mtx_lock(&erts_get_time_mtx); + + no_monitors = no_time_offset_monitors; + if (no_monitors) { + ErtsTimeOffsetMonitorContext cntxt; + Uint alloc_sz; - if (ticks <= elapsed) { /* Ooops, better hurry */ - rem_time->tv_sec = rem_time->tv_usec = 0; - return; + /* Monitor info array size */ + alloc_sz = no_monitors*sizeof(ErtsTimeOffsetMonitorInfo); + /* + template max size */ + alloc_sz += 6*sizeof(Eterm); /* 5-tuple */ + alloc_sz += ERTS_MAX_SINT64_HEAP_SIZE*sizeof(Eterm); /* max offset size */ + tmp = erts_alloc(ERTS_ALC_T_TMP, alloc_sz); + + to_mon_info = (ErtsTimeOffsetMonitorInfo *) tmp; + cntxt.ix = 0; + cntxt.to_mon_info = to_mon_info; + + erts_doforall_monitors(time_offset_monitors, + save_time_offset_monitor, + &cntxt); + + ASSERT(cntxt.ix == no_monitors); + } + + erts_smp_mtx_unlock(&erts_get_time_mtx); + + if (no_monitors) { + Eterm *hp, *patch_refp, new_offset_term, message_template; + Uint mix, hsz; + + /* Make message template */ + + hp = (Eterm *) (tmp + no_monitors*sizeof(ErtsTimeOffsetMonitorInfo)); + + hsz = 6; /* 5-tuple */ + hsz += REF_THING_SIZE; + hsz += ERTS_SINT64_HEAP_SIZE(new_offset); + + if (IS_SSMALL(new_offset)) + new_offset_term = make_small(new_offset); + else + new_offset_term = erts_sint64_to_big(new_offset, &hp); + message_template = TUPLE5(hp, + am_CHANGE, + THE_NON_VALUE, /* Patch point for ref */ + am_time_offset, + am_clock_service, + new_offset_term); + patch_refp = &hp[2]; + + ASSERT(*patch_refp == THE_NON_VALUE); + + for (mix = 0; mix < no_monitors; mix++) { + Process *rp = erts_proc_lookup(to_mon_info[mix].pid); + if (rp) { + Eterm ref = to_mon_info[mix].ref; + ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) { + ErlHeapFragment *bp; + ErlOffHeap *ohp; + Eterm message; + + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + *patch_refp = ref; + ASSERT(hsz == size_object(message_template)); + message = copy_struct(message_template, hsz, &hp, ohp); + erts_queue_message(rp, &rp_locks, bp, message, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); + } + erts_smp_proc_unlock(rp, rp_locks); + } } - rem_time->tv_sec = (ticks - elapsed) / 1000; - rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000); + + erts_free(ERTS_ALC_T_TMP, tmp); } } -void erts_get_timeval(SysTimeval *tv) +static void +schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset) { - erts_smp_mtx_lock(&erts_timeofday_mtx); - get_tolerant_timeofday(tv); - erts_smp_mtx_unlock(&erts_timeofday_mtx); - update_approx_time(tv); +#ifdef ARCH_64 + void *new_offsetp = (void *) new_offset; + ASSERT(sizeof(void *) == sizeof(ErtsMonotonicTime)); +#else + void *new_offsetp = erts_alloc(ERTS_ALC_T_NEW_TIME_OFFSET, + sizeof(ErtsMonotonicTime)); + *((ErtsMonotonicTime *) new_offsetp) = new_offset; +#endif + erts_schedule_misc_aux_work(1, + send_time_offset_changed_notifications, + new_offsetp); } -erts_time_t -erts_get_time(void) +static ERTS_INLINE Eterm +make_time_val(Process *c_p, ErtsMonotonicTime time_val) { - SysTimeval sys_tv; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&sys_tv); - - erts_smp_mtx_unlock(&erts_timeofday_mtx); + Sint64 val = (Sint64) time_val; + Eterm *hp; + Uint sz; - update_approx_time(&sys_tv); + if (IS_SSMALL(val)) + return make_small(val); - return sys_tv.tv_sec; + sz = ERTS_SINT64_HEAP_SIZE(val); + hp = HAlloc(c_p, sz); + return erts_sint64_to_big(val, &hp); } -#ifdef HAVE_ERTS_NOW_CPU -void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { - SysCpuTime t; - SysTimespec tp; +Eterm +erts_get_monotonic_start_time(struct process *c_p) +{ + return make_time_val(c_p, ERTS_MONOTONIC_TIME_START); +} - sys_get_proc_cputime(t, tp); - *microsec = (Uint)(tp.tv_nsec / 1000); - t = (tp.tv_sec / 1000000); - *megasec = (Uint)(t % 1000000); - *sec = (Uint)(tp.tv_sec % 1000000); +static Eterm +bld_monotonic_time_source(Uint **hpp, Uint *szp, Sint64 os_mtime) +{ +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return NIL; +#else + int i = 0; + Eterm k[6]; + Eterm v[6]; + + if (time_sup.r.o.os_monotonic_time_disable) + return NIL; + + k[i] = erts_bld_atom(hpp, szp, "function"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_monotonic_time_func); + + if (time_sup.r.o.os_monotonic_time_clock_id) { + k[i] = erts_bld_atom(hpp, szp, "clock_id"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_monotonic_time_clock_id); + } + + k[i] = erts_bld_atom(hpp, szp, "resolution"); + v[i++] = erts_bld_uint64(hpp, szp, + time_sup.r.o.os_monotonic_time_resolution); + + k[i] = erts_bld_atom(hpp, szp, "extended"); + v[i++] = time_sup.r.o.os_monotonic_time_extended ? am_yes : am_no; + + k[i] = erts_bld_atom(hpp, szp, "parallel"); + v[i++] = time_sup.r.o.os_monotonic_time_locked ? am_no : am_yes; + + k[i] = erts_bld_atom(hpp, szp, "time"); + v[i++] = erts_bld_sint64(hpp, szp, os_mtime); + + return erts_bld_2tup_list(hpp, szp, (Sint) i, k, v); +#endif +} + +Eterm +erts_monotonic_time_source(struct process *c_p) +{ + Uint hsz = 0; + Eterm *hp = NULL; + Sint64 os_mtime = 0; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (!time_sup.r.o.os_monotonic_time_disable) + os_mtime = (Sint64) erts_os_monotonic_time(); +#endif + + bld_monotonic_time_source(NULL, &hsz, os_mtime); + if (hsz) + hp = HAlloc(c_p, hsz); + return bld_monotonic_time_source(&hp, NULL, os_mtime); +} + +static Eterm +bld_system_time_source(Uint **hpp, Uint *szp, Sint64 os_stime) +{ + int i = 0; + Eterm k[5]; + Eterm v[5]; + + k[i] = erts_bld_atom(hpp, szp, "function"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_system_time_func); + + if (time_sup.r.o.os_system_time_clock_id) { + k[i] = erts_bld_atom(hpp, szp, "clock_id"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_system_time_clock_id); + } + + k[i] = erts_bld_atom(hpp, szp, "resolution"); + v[i++] = erts_bld_uint64(hpp, szp, + time_sup.r.o.os_system_time_resolution); + + k[i] = erts_bld_atom(hpp, szp, "parallel"); + v[i++] = am_yes; + + k[i] = erts_bld_atom(hpp, szp, "time"); + v[i++] = erts_bld_sint64(hpp, szp, os_stime); + + return erts_bld_2tup_list(hpp, szp, (Sint) i, k, v); } + +Eterm +erts_system_time_source(struct process *c_p) +{ + Uint hsz = 0; + Eterm *hp = NULL; + Sint64 os_stime = (Sint64) erts_os_system_time(); + + bld_system_time_source(NULL, &hsz, os_stime); + if (hsz) + hp = HAlloc(c_p, hsz); + return bld_system_time_source(&hp, NULL, os_stime); +} + + +#include "bif.h" + +static ERTS_INLINE Eterm +time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonotonicTime muloff) +{ + ErtsMonotonicTime result; + BIF_RETTYPE ret; + + if (val < 0) + goto trap_to_erlang_code; + + /* Convert to common user specified time units */ + switch (term) { + case am_seconds: + case make_small(1): + result = ERTS_MONOTONIC_TO_SEC(val) + muloff*ERTS_MONOTONIC_OFFSET_SEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + case am_milli_seconds: + case make_small(1000): + result = ERTS_MONOTONIC_TO_MSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_MSEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + case am_micro_seconds: + case make_small(1000*1000): + result = ERTS_MONOTONIC_TO_USEC(val) + muloff*ERTS_MONOTONIC_OFFSET_USEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; +#ifdef ARCH_64 + case am_nano_seconds: + case make_small(1000*1000*1000): + result = ERTS_MONOTONIC_TO_NSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_NSEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; +#endif + default: { + Eterm value, native_res; +#ifndef ARCH_64 + Sint user_res; + if (term == am_nano_seconds) + goto to_nano_seconds; + if (term_to_Sint(term, &user_res)) { + if (user_res == 1000*1000*1000) { + to_nano_seconds: + result = (ERTS_MONOTONIC_TO_NSEC(val) + + muloff*ERTS_MONOTONIC_OFFSET_NSEC); + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + } + if (user_res <= 0) + goto badarg; + } +#else + if (is_small(term)) { + if (signed_val(term) <= 0) + goto badarg; + } #endif + else if (is_big(term)) { + if (big_sign(term)) + goto badarg; + } + else { + badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + break; + } + + trap_to_erlang_code: + /* Do it in erlang code instead; pass along values to use... */ + value = make_time_val(c_p, val + muloff*ERTS_MONOTONIC_OFFSET_NATIVE); + native_res = make_time_val(c_p, ERTS_MONOTONIC_TIME_UNIT); + + ERTS_BIF_PREP_TRAP3(ret, erts_convert_time_unit_trap, c_p, + value, native_res, term); + + break; + } + } + + return ret; +} + +/* Built in functions */ + +BIF_RETTYPE monotonic_time_0(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime = time_sup.r.o.get_time(); + mtime += ERTS_MONOTONIC_OFFSET_NATIVE; + BIF_RET(make_time_val(BIF_P, mtime)); +} + +BIF_RETTYPE monotonic_time_1(BIF_ALIST_1) +{ + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, time_sup.r.o.get_time(), 1)); +} + +BIF_RETTYPE system_time_0(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime, offset; + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + BIF_RET(make_time_val(BIF_P, mtime + offset)); +} + +BIF_RETTYPE system_time_1(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime, offset; + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime + offset, 0)); +} + +BIF_RETTYPE erts_internal_time_unit_0(BIF_ALIST_0) +{ + BIF_RET(make_time_val(BIF_P, ERTS_MONOTONIC_TIME_UNIT)); +} + +BIF_RETTYPE time_offset_0(BIF_ALIST_0) +{ + ErtsMonotonicTime time_offset = get_time_offset(); + time_offset -= ERTS_MONOTONIC_OFFSET_NATIVE; + BIF_RET(make_time_val(BIF_P, time_offset)); +} + +BIF_RETTYPE time_offset_1(BIF_ALIST_1) +{ + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, get_time_offset(), -1)); +} + + +BIF_RETTYPE timestamp_0(BIF_ALIST_0) +{ + Eterm *hp, res; + ErtsMonotonicTime stime, mtime, all_sec, offset; + Uint mega_sec, sec, micro_sec; + + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + stime = ERTS_MONOTONIC_TO_USEC(mtime + offset); + all_sec = stime / ERTS_MONOTONIC_TIME_MEGA; + mega_sec = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA); + sec = (Uint) (all_sec - (((ErtsMonotonicTime) mega_sec) + * ERTS_MONOTONIC_TIME_MEGA)); + micro_sec = (Uint) (stime - all_sec*ERTS_MONOTONIC_TIME_MEGA); + + ASSERT(((ErtsMonotonicTime) mega_sec)*ERTS_MONOTONIC_TIME_TERA + + ((ErtsMonotonicTime) sec)*ERTS_MONOTONIC_TIME_MEGA + + micro_sec == stime); + + /* + * Mega seconds is the only value that potentially + * ever could be a bignum. However, that wont happen + * during at least the next 4 million years... + * + * (System time will also have wrapped in the + * 64-bit integer before we get there...) + */ + + ASSERT(IS_USMALL(0, mega_sec)); + ASSERT(IS_USMALL(0, sec)); + ASSERT(IS_USMALL(0, micro_sec)); + + hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, + make_small(mega_sec), + make_small(sec), + make_small(micro_sec)); + BIF_RET(res); +} + +BIF_RETTYPE os_system_time_0(BIF_ALIST_0) +{ + ErtsSystemTime stime = erts_os_system_time(); + BIF_RET(make_time_val(BIF_P, stime)); +} + +BIF_RETTYPE os_system_time_1(BIF_ALIST_0) +{ + ErtsSystemTime stime = erts_os_system_time(); + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, stime, 0)); +} + diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index c32f8fd61c..7cb8972e29 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -113,12 +113,14 @@ void erts_silence_warn_unused_result(long unused); int erts_fit_in_bits_int64(Sint64); int erts_fit_in_bits_int32(Sint32); +int erts_fit_in_bits_uint(Uint); int erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 make_broken_hash(Eterm); Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); +Uint32 make_internal_hash(Eterm); void erts_save_emu_args(int argc, char **argv); Eterm erts_get_emu_args(struct process *c_p); diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 6e9216bef3..3a9fb1e07b 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -117,9 +117,9 @@ #if defined(DEBUG) || defined(CHECK_FOR_HOLES) #if HALFWORD_HEAP -# define ERTS_HOLE_MARKER (0xaf5e78ccU) +# define ERTS_HOLE_MARKER (0xdeadbeef) #else -# define ERTS_HOLE_MARKER (((0xaf5e78ccUL << 24) << 8) | 0xaf5e78ccUL) +# define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef) #endif #endif diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index e5fb2d3ec1..2117dbec62 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1182,7 +1182,8 @@ typedef struct { Eterm* hp_end; int remaining_n; char* remaining_bytes; - Eterm* maps_head; + Eterm* maps_list; + struct dec_term_hamt_placeholder* hamt_list; } B2TDecodeContext; typedef struct { @@ -1508,7 +1509,8 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size); ctx->u.dc.hp = ctx->u.dc.hp_start; ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; - ctx->u.dc.maps_head = NULL; + ctx->u.dc.maps_list = NULL; + ctx->u.dc.hamt_list = NULL; ctx->state = B2TDecode; /*fall through*/ case B2TDecode: @@ -2304,7 +2306,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_PATCH_FUN_SIZE ((Eterm) 2) #define ENC_BIN_COPY ((Eterm) 3) #define ENC_MAP_PAIR ((Eterm) 4) -#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 5) +#define ENC_HASHMAP_NODE ((Eterm) 5) +#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 6) static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, @@ -2413,6 +2416,13 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, WSTACK_PUSH2(s, ENC_TERM, *vptr); break; } + case ENC_HASHMAP_NODE: + if (is_list(obj)) { /* leaf node [K|V] */ + ptr = list_val(obj); + WSTACK_PUSH2(s, ENC_TERM, CDR(ptr)); + obj = CAR(ptr); + } + break; case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { @@ -2594,23 +2604,54 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; case MAP_DEF: - { - map_t *mp = (map_t*)map_val(obj); - Uint size = map_get_size(mp); + if (is_flatmap(obj)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint size = flatmap_get_size(mp); *ep++ = MAP_EXT; put_int32(size, ep); ep += 4; if (size > 0) { - Eterm *kptr = map_get_keys(mp); - Eterm *vptr = map_get_values(mp); + Eterm *kptr = flatmap_get_keys(mp); + Eterm *vptr = flatmap_get_values(mp); WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr, ENC_MAP_PAIR, size); } + } else { + Eterm hdr; + Uint node_sz; + ptr = boxed_val(obj); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + *ep++ = MAP_EXT; + ptr++; + put_int32(*ptr, ep); ep += 4; + node_sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + *ep++ = MAP_EXT; + ptr++; + put_int32(*ptr, ep); ep += 4; + /*fall through*/ + case HAMT_SUBTAG_NODE_BITMAP: + node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(node_sz < 17); + break; + default: + erl_exit(1, "bad header\r\n"); + } + + ptr++; + WSTACK_RESERVE(s, node_sz*2); + while(node_sz--) { + WSTACK_FAST_PUSH(s, ENC_HASHMAP_NODE); + WSTACK_FAST_PUSH(s, *ptr++); + } } break; - case FLOAT_DEF: GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { @@ -2892,9 +2933,19 @@ undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) #endif /* DEBUG */ } +struct dec_term_hamt_placeholder +{ + struct dec_term_hamt_placeholder* next; + Eterm* objp; /* write result here */ + Uint size; /* nr of leafs */ + Eterm leafs[1]; +}; + +#define DEC_TERM_HAMT_PLACEHOLDER_SIZE \ + (offsetof(struct dec_term_hamt_placeholder, leafs) / sizeof(Eterm)) /* Decode term from external format into *objp. -** On failure return NULL and (R13B04) *hpp will be unchanged. +** On failure return NULL and *hpp will be unchanged. */ static byte* dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, @@ -2904,7 +2955,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ - Eterm *maps_head; /* for validation of maps */ + Eterm *maps_list; /* for preprocessing of small maps */ + struct dec_term_hamt_placeholder* hamt_list; /* for preprocessing of big maps */ Eterm* next; SWord reds; @@ -2914,7 +2966,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, next = ctx->u.dc.next; ep = ctx->u.dc.ep; hpp = &ctx->u.dc.hp; - maps_head = ctx->u.dc.maps_head; + maps_list = ctx->u.dc.maps_list; + hamt_list = ctx->u.dc.hamt_list; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -2995,7 +3048,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; - maps_head = NULL; + maps_list = NULL; + hamt_list = NULL; } hp = *hpp; @@ -3399,6 +3453,7 @@ dec_term_atom_common: pb->size = n; pb->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3450,6 +3505,7 @@ dec_term_atom_common: pb->size = n; pb->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3531,46 +3587,67 @@ dec_term_atom_common: break; case MAP_EXT: { - map_t *mp; Uint32 size,n; Eterm *kptr,*vptr; Eterm keys; size = get_int32(ep); ep += 4; - keys = make_tuple(hp); - *hp++ = make_arityval(size); - hp += size; - kptr = hp - 1; - - mp = (map_t*)hp; - hp += MAP_HEADER_SIZE; - hp += size; - vptr = hp - 1; - - /* kptr, last word for keys - * vptr, last word for values - */ - - /* - * Use thing_word to link through decoded maps. - * The list of maps is for later validation. - */ - - mp->thing_word = (Eterm) COMPRESS_POINTER(maps_head); - maps_head = (Eterm *) mp; - - mp->size = size; - mp->keys = keys; - *objp = make_map(mp); - - for (n = size; n; n--) { - *vptr = (Eterm) COMPRESS_POINTER(next); - *kptr = (Eterm) COMPRESS_POINTER(vptr); - next = kptr; - vptr--; - kptr--; - } + if (size <= MAP_SMALL_MAP_LIMIT) { + flatmap_t *mp; + + keys = make_tuple(hp); + *hp++ = make_arityval(size); + hp += size; + kptr = hp - 1; + + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + hp += size; + vptr = hp - 1; + + /* kptr, last word for keys + * vptr, last word for values + */ + + /* + * Use thing_word to link through decoded maps. + * The list of maps is for later validation. + */ + + mp->thing_word = (Eterm) COMPRESS_POINTER(maps_list); + maps_list = (Eterm *) mp; + + mp->size = size; + mp->keys = keys; + *objp = make_flatmap(mp); + + for (n = size; n; n--) { + *vptr = (Eterm) COMPRESS_POINTER(next); + *kptr = (Eterm) COMPRESS_POINTER(vptr); + next = kptr; + vptr--; + kptr--; + } + } + else { /* Make hamt */ + struct dec_term_hamt_placeholder* holder = + (struct dec_term_hamt_placeholder*) hp; + + holder->next = hamt_list; + hamt_list = holder; + holder->objp = objp; + holder->size = size; + + hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE; + + for (n = size; n; n--) { + CDR(hp) = (Eterm) COMPRESS_POINTER(next); + CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp)); + next = &CAR(hp); + hp += 2; + } + } } break; case NEW_FUN_EXT: @@ -3748,6 +3825,7 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; *objp = make_binary(pb); break; @@ -3765,6 +3843,7 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; sub = (ErlSubBin*)hp; @@ -3791,7 +3870,8 @@ dec_term_atom_common: ctx->u.dc.ep = ep; ctx->u.dc.next = next; ctx->u.dc.hp = hp; - ctx->u.dc.maps_head = maps_head; + ctx->u.dc.maps_list = maps_list; + ctx->u.dc.hamt_list = hamt_list; ctx->reds = 0; return NULL; } @@ -3806,12 +3886,40 @@ dec_term_atom_common: * - done here for when we know it is complete. */ - while (maps_head) { - next = (Eterm *)(EXPAND_POINTER(*maps_head)); - *maps_head = MAP_HEADER; - if (!erts_validate_and_sort_map((map_t*)maps_head)) + while (maps_list) { + next = (Eterm *)(EXPAND_POINTER(*maps_list)); + *maps_list = MAP_HEADER_FLATMAP; + if (!erts_validate_and_sort_flatmap((flatmap_t*)maps_list)) goto error; - maps_head = next; + maps_list = next; + } + + /* Iterate through all the hamts and build tree nodes. + */ + if (hamt_list) { + ErtsHeapFactory factory; + + factory.p = NULL; + factory.hp = hp; + /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ + + do { + struct dec_term_hamt_placeholder* hamt = hamt_list; + *hamt->objp = erts_hashmap_from_array(&factory, + hamt->leafs, + hamt->size, + 1); + if (is_non_value(*hamt->objp)) + goto error; + + hamt_list = hamt->next; + + /* Yes, we waste a couple of heap words per hamt + for the temporary placeholder */ + *(Eterm*)hamt = make_pos_bignum_header(DEC_TERM_HAMT_PLACEHOLDER_SIZE-1); + } while (hamt_list); + + hp = factory.hp; } if (ctx) { @@ -3996,8 +4104,9 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, } for (i = 1; i <= arity; ++i) { if (is_list(ptr[i])) { - if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { + if ((m = is_string(ptr[i])) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; + continue; } else { result += 5; } @@ -4008,43 +4117,99 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, } break; case MAP_DEF: - { - map_t *mp = (map_t*)map_val(obj); - Uint size = map_get_size(mp); + if (is_flatmap(obj)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint size = flatmap_get_size(mp); Uint i; Eterm *ptr; result += 1 + 4; /* tag + 4 bytes size */ /* push values first */ - ptr = map_get_values(mp); - i = size; - while(i--) { + ptr = flatmap_get_values(mp); + for (i = size; i; i--, ptr++) { if (is_list(*ptr)) { if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; + continue; } else { result += 5; } } ESTACK_PUSH(s,*ptr); - ++ptr; } - ptr = map_get_keys(mp); - i = size; - while(i--) { + ptr = flatmap_get_keys(mp); + for (i = size; i; i--, ptr++) { if (is_list(*ptr)) { if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; + continue; } else { result += 5; } } ESTACK_PUSH(s,*ptr); - ++ptr; } goto outer_loop; + } else { + Eterm *ptr; + Eterm hdr; + Uint node_sz; + ptr = boxed_val(obj); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + node_sz = 16; + result += 1 + 4; /* tag + 4 bytes size */ + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + result += 1 + 4; /* tag + 4 bytes size */ + /*fall through*/ + case HAMT_SUBTAG_NODE_BITMAP: + node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(node_sz < 17); + break; + default: + erl_exit(1, "bad header\r\n"); + } + + ptr++; + ESTACK_RESERVE(s, node_sz*2); + while(node_sz--) { + if (is_list(*ptr)) { + Eterm* leaf = list_val(*ptr); + if (is_not_list(CAR(leaf))) { + ESTACK_FAST_PUSH(s, CAR(leaf)); + } + else { + if ((m = is_string(CAR(leaf))) && (m < MAX_STRING_LEN)) { + result += m + 2 + 1; + } else { + result += 5; + ESTACK_FAST_PUSH(s, CAR(leaf)); + } + } + if (is_not_list(CDR(leaf))) { + ESTACK_FAST_PUSH(s, CDR(leaf)); + } + else { + if ((m = is_string(CDR(leaf))) && (m < MAX_STRING_LEN)) { + result += m + 2 + 1; + } else { + result += 5; + ESTACK_FAST_PUSH(s, CDR(leaf)); + } + } + } + else { + ESTACK_FAST_PUSH(s, *ptr); + } + ptr++; + } } break; case FLOAT_DEF: @@ -4149,6 +4314,8 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, return 0; } + + static Sint decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) { @@ -4342,7 +4509,11 @@ init_done: n = get_int32(ep); ep += 4; ADDTERMS(2*n); - heap_size += 3 + n + 1 + n; + if (n <= MAP_SMALL_MAP_LIMIT) { + heap_size += 3 + n + 1 + n; + } else { + heap_size += hashmap_over_estimated_heap_size(n); + } break; case STRING_EXT: CHKSIZE(2); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 5330f389e0..634fe533d0 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -348,8 +348,6 @@ extern Uint display_items; /* no of items to display in traces etc */ extern int erts_backtrace_depth; extern erts_smp_atomic32_t erts_max_gen_gcs; -extern int erts_disable_tolerant_timeofday; - extern int bif_reductions; /* reductions + fcalls (when doing call_bif) */ extern int stackdump_on_exit; @@ -372,16 +370,17 @@ extern int stackdump_on_exit; * DESTROY_ESTACK(Stack) */ -typedef struct { +typedef struct ErtsEStack_ { Eterm* start; Eterm* sp; Eterm* end; + Eterm* edefault; ErtsAlcType_t alloc_type; }ErtsEStack; #define DEF_ESTACK_SIZE (16) -void erl_grow_estack(ErtsEStack*, Eterm* def_stack); +void erl_grow_estack(ErtsEStack*, Uint need); #define ESTK_CONCAT(a,b) a##b #define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack) @@ -391,22 +390,23 @@ void erl_grow_estack(ErtsEStack*, Eterm* def_stack); ESTK_DEF_STACK(s), /* start */ \ ESTK_DEF_STACK(s), /* sp */ \ ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ESTK_DEF_STACK(s), /* default */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } #define ESTACK_CHANGE_ALLOCATOR(s,t) \ do { \ - if (s.start != ESTK_DEF_STACK(s)) { \ + if ((s).start != ESTK_DEF_STACK(s)) { \ erl_exit(1, "Internal error - trying to change allocator " \ "type of active estack\n"); \ } \ - s.alloc_type = (t); \ + (s).alloc_type = (t); \ } while (0) #define DESTROY_ESTACK(s) \ do { \ - if (s.start != ESTK_DEF_STACK(s)) { \ - erts_free(s.alloc_type, s.start); \ + if ((s).start != ESTK_DEF_STACK(s)) { \ + erts_free((s).alloc_type, (s).start); \ } \ } while(0) @@ -417,16 +417,17 @@ do { \ */ #define ESTACK_SAVE(s,dst)\ do {\ - if (s.start == ESTK_DEF_STACK(s)) {\ + if ((s).start == ESTK_DEF_STACK(s)) {\ UWord _wsz = ESTACK_COUNT(s);\ - (dst)->start = erts_alloc(s.alloc_type,\ + (dst)->start = erts_alloc((s).alloc_type,\ DEF_ESTACK_SIZE * sizeof(Eterm));\ - memcpy((dst)->start, s.start,_wsz*sizeof(Eterm));\ + memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\ (dst)->sp = (dst)->start + _wsz;\ (dst)->end = (dst)->start + DEF_ESTACK_SIZE;\ - (dst)->alloc_type = s.alloc_type;\ + (dst)->edefault = NULL;\ + (dst)->alloc_type = (s).alloc_type;\ } else\ - *(dst) = s;\ + *(dst) = (s);\ } while (0) #define DESTROY_SAVED_ESTACK(estack)\ @@ -445,83 +446,114 @@ do {\ */ #define ESTACK_RESTORE(s, src) \ do { \ - ASSERT(s.start == ESTK_DEF_STACK(s)); \ - s = *(src); /* struct copy */ \ + ASSERT((s).start == ESTK_DEF_STACK(s)); \ + (s) = *(src); /* struct copy */ \ (src)->start = NULL; \ - ASSERT(s.sp >= s.start); \ - ASSERT(s.sp <= s.end); \ + ASSERT((s).sp >= (s).start); \ + ASSERT((s).sp <= (s).end); \ } while (0) -#define ESTACK_IS_STATIC(s) (s.start == ESTK_DEF_STACK(s))) +#define ESTACK_IS_STATIC(s) ((s).start == ESTK_DEF_STACK(s)) -#define ESTACK_PUSH(s, x) \ -do { \ - if (s.sp == s.end) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ - } \ - *s.sp++ = (x); \ +#define ESTACK_PUSH(s, x) \ +do { \ + if ((s).sp == (s).end) { \ + erl_grow_estack(&(s), 1); \ + } \ + *(s).sp++ = (x); \ } while(0) #define ESTACK_PUSH2(s, x, y) \ do { \ - if (s.sp > s.end - 2) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + if ((s).sp > (s).end - 2) { \ + erl_grow_estack(&(s), 2); \ } \ - *s.sp++ = (x); \ - *s.sp++ = (y); \ + *(s).sp++ = (x); \ + *(s).sp++ = (y); \ } while(0) #define ESTACK_PUSH3(s, x, y, z) \ do { \ - if (s.sp > s.end - 3) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + if ((s).sp > (s).end - 3) { \ + erl_grow_estack(&s, 3); \ } \ - *s.sp++ = (x); \ - *s.sp++ = (y); \ - *s.sp++ = (z); \ + *(s).sp++ = (x); \ + *(s).sp++ = (y); \ + *(s).sp++ = (z); \ } while(0) #define ESTACK_PUSH4(s, E1, E2, E3, E4) \ do { \ - if (s.sp > s.end - 4) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + if ((s).sp > (s).end - 4) { \ + erl_grow_estack(&s, 4); \ } \ - *s.sp++ = (E1); \ - *s.sp++ = (E2); \ - *s.sp++ = (E3); \ - *s.sp++ = (E4); \ + *(s).sp++ = (E1); \ + *(s).sp++ = (E2); \ + *(s).sp++ = (E3); \ + *(s).sp++ = (E4); \ } while(0) -#define ESTACK_COUNT(s) (s.sp - s.start) -#define ESTACK_ISEMPTY(s) (s.sp == s.start) -#define ESTACK_POP(s) (*(--s.sp)) +#define ESTACK_RESERVE(s, push_cnt) \ +do { \ + if ((s).sp > (s).end - (push_cnt)) { \ + erl_grow_estack(&(s), (push_cnt)); \ + } \ +} while(0) + +/* Must be preceded by ESTACK_RESERVE */ +#define ESTACK_FAST_PUSH(s, x) \ +do { \ + ASSERT((s).sp < (s).end); \ + *s.sp++ = (x); \ +} while(0) + +#define ESTACK_COUNT(s) ((s).sp - (s).start) +#define ESTACK_ISEMPTY(s) ((s).sp == (s).start) +#define ESTACK_POP(s) (*(--(s).sp)) /* * WSTACK: same as ESTACK but with UWord instead of Eterm */ -typedef struct { +typedef struct ErtsWStack_ { UWord* wstart; UWord* wsp; UWord* wend; + UWord* wdefault; ErtsAlcType_t alloc_type; }ErtsWStack; #define DEF_WSTACK_SIZE (16) -void erl_grow_wstack(ErtsWStack*, UWord* def_stack); +void erl_grow_wstack(ErtsWStack*, Uint need); #define WSTK_CONCAT(a,b) a##b #define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack) -#define DECLARE_WSTACK(s) \ +#define WSTACK_DECLARE(s) \ UWord WSTK_DEF_STACK(s)[DEF_WSTACK_SIZE]; \ ErtsWStack s = { \ WSTK_DEF_STACK(s), /* wstart */ \ WSTK_DEF_STACK(s), /* wsp */ \ WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + WSTK_DEF_STACK(s), /* wdflt */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } +#define DECLARE_WSTACK WSTACK_DECLARE + +typedef struct ErtsDynamicWStack_ { + UWord default_stack[DEF_WSTACK_SIZE]; + ErtsWStack ws; +}ErtsDynamicWStack; + +#define WSTACK_INIT(dwsp, ALC_TYPE) \ +do { \ + (dwsp)->ws.wstart = (dwsp)->default_stack; \ + (dwsp)->ws.wsp = (dwsp)->default_stack; \ + (dwsp)->ws.wend = (dwsp)->default_stack + DEF_WSTACK_SIZE;\ + (dwsp)->ws.wdefault = (dwsp)->default_stack; \ + (dwsp)->ws.alloc_type = ALC_TYPE; \ +} while (0) #define WSTACK_CHANGE_ALLOCATOR(s,t) \ do { \ @@ -532,13 +564,20 @@ do { \ s.alloc_type = (t); \ } while (0) -#define DESTROY_WSTACK(s) \ +#define WSTACK_DESTROY(s) \ do { \ - if (s.wstart != WSTK_DEF_STACK(s)) { \ + if (s.wstart != s.wdefault) { \ erts_free(s.alloc_type, s.wstart); \ } \ } while(0) +#define DESTROY_WSTACK WSTACK_DESTROY +#define WSTACK_DEBUG(s) \ + do { \ + fprintf(stderr, "wstack size = %ld\r\n", s.wsp - s.wstart); \ + fprintf(stderr, "wstack wstart = %p\r\n", s.wstart); \ + fprintf(stderr, "wstack wsp = %p\r\n", s.wsp); \ + } while(0) /* * Do not free the stack after this, it may have pointers into what @@ -553,6 +592,7 @@ do {\ memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\ (dst)->wsp = (dst)->wstart + _wsz;\ (dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\ + (dst)->wdefault = NULL;\ (dst)->alloc_type = s.alloc_type;\ } else\ *(dst) = s;\ @@ -581,12 +621,12 @@ do { \ ASSERT(s.wsp <= s.wend); \ } while (0) -#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s))) +#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s)) #define WSTACK_PUSH(s, x) \ do { \ if (s.wsp == s.wend) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + erl_grow_wstack(&s, 1); \ } \ *s.wsp++ = (x); \ } while(0) @@ -594,7 +634,7 @@ do { \ #define WSTACK_PUSH2(s, x, y) \ do { \ if (s.wsp > s.wend - 2) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + erl_grow_wstack(&s, 2); \ } \ *s.wsp++ = (x); \ *s.wsp++ = (y); \ @@ -602,8 +642,8 @@ do { \ #define WSTACK_PUSH3(s, x, y, z) \ do { \ - if (s.wsp > s.wend - 3) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + if (s.wsp > s.wend - 3) { \ + erl_grow_wstack(&s, 3); \ } \ *s.wsp++ = (x); \ *s.wsp++ = (y); \ @@ -612,8 +652,8 @@ do { \ #define WSTACK_PUSH4(s, A1, A2, A3, A4) \ do { \ - if (s.wsp > s.wend - 4) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + if (s.wsp > s.wend - 4) { \ + erl_grow_wstack(&s, 4); \ } \ *s.wsp++ = (A1); \ *s.wsp++ = (A2); \ @@ -624,7 +664,7 @@ do { \ #define WSTACK_PUSH5(s, A1, A2, A3, A4, A5) \ do { \ if (s.wsp > s.wend - 5) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + erl_grow_wstack(&s, 5); \ } \ *s.wsp++ = (A1); \ *s.wsp++ = (A2); \ @@ -636,7 +676,7 @@ do { \ #define WSTACK_PUSH6(s, A1, A2, A3, A4, A5, A6) \ do { \ if (s.wsp > s.wend - 6) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + erl_grow_wstack(&s, 6); \ } \ *s.wsp++ = (A1); \ *s.wsp++ = (A2); \ @@ -646,9 +686,85 @@ do { \ *s.wsp++ = (A6); \ } while(0) +#define WSTACK_RESERVE(s, push_cnt) \ +do { \ + if (s.wsp > s.wend - (push_cnt)) { \ + erl_grow_wstack(&s, (push_cnt)); \ + } \ +} while(0) + +/* Must be preceded by WSTACK_RESERVE */ +#define WSTACK_FAST_PUSH(s, x) \ +do { \ + ASSERT(s.wsp < s.wend); \ + *s.wsp++ = (x); \ +} while(0) + #define WSTACK_COUNT(s) (s.wsp - s.wstart) #define WSTACK_ISEMPTY(s) (s.wsp == s.wstart) -#define WSTACK_POP(s) (*(--s.wsp)) +#define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp)) + +#define WSTACK_ROLLBACK(s, count) (ASSERT(WSTACK_COUNT(s) >= (count)), \ + s.wsp = s.wstart + (count)) + +/* PSTACK - Stack of any type. + * Usage: + * { + * #define PSTACK_TYPE MyType + * PSTACK_DECLARE(s,16); + * MyType *sp = PSTACK_PUSH(s); + * + * sp->x = .... + * sp->y = .... + * sp = PSTACK_PUSH(s); + * ... + * sp = PSTACK_POP(s); + * if (PSTACK_IS_EMPTY(s)) { + * // sp is invalid when stack is empty after pop + * } + * + * PSTACK_DESTROY(s); + * } + */ + + +typedef struct ErtsPStack_ { + byte* pstart; + byte* psp; + byte* pend; + ErtsAlcType_t alloc_type; +}ErtsPStack; + +void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes); +#define PSTK_CONCAT(a,b) a##b +#define PSTK_DEF_STACK(s) PSTK_CONCAT(s,_default_pstack) + +#define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \ +PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \ +ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ + (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */ \ + (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\ + ERTS_ALC_T_ESTACK /* alloc_type */ \ +} + +#define PSTACK_DESTROY(s) \ +do { \ + if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ + erts_free(s.alloc_type, s.pstart); \ + } \ +} while(0) + +#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) + +#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) + +#define PSTACK_PUSH(s) \ + (s.psp += sizeof(PSTACK_TYPE), \ + ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ + sizeof(PSTACK_TYPE)) : (void)0), \ + ((PSTACK_TYPE*) s.psp)) + +#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) /* binary.c */ @@ -671,9 +787,6 @@ erts_bld_port_info(Eterm **hpp, void erts_bif_info_init(void); /* bif.c */ -Eterm erts_make_ref(Process *); -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]); -void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); ERTS_GLB_INLINE Eterm erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]); @@ -1192,7 +1305,9 @@ erts_alloc_message_heap_state(Uint size, state = erts_smp_atomic32_read_acqb(&receiver->state); if (statep) *statep = state; - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + if (state & (ERTS_PSFLG_OFF_HEAP_MSGS + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_PENDING_EXIT)) goto allocate_in_mbuf; #endif @@ -1212,7 +1327,9 @@ erts_alloc_message_heap_state(Uint size, state = erts_smp_atomic32_read_nob(&receiver->state); if (statep) *statep = state; - if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + if ((state & (ERTS_PSFLG_OFF_HEAP_MSGS + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_PENDING_EXIT)) || (receiver->flags & F_DISABLE_GC) || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) { /* diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 9377237475..1db3a9fba7 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -47,6 +47,7 @@ #include "external.h" #include "dtrace-wrapper.h" #include "erl_map.h" +#include "erl_bif_unique.h" extern ErlDrvEntry fd_driver_entry; #ifndef __OSE__ @@ -4483,7 +4484,7 @@ make_port_info_term(Eterm **hpp_start, int len; int start; static Eterm item[] = ERTS_PORT_INFO_1_ITEMS; - static Eterm value[sizeof(item)/sizeof(item[0])]; + Eterm value[sizeof(item)/sizeof(item[0])]; start = 0; len = sizeof(item)/sizeof(item[0]); @@ -5350,7 +5351,11 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_MAP: { /* int */ ERTS_DDT_CHK_ENOUGH_ARGS(1); if ((int) ptr[0] < 0) ERTS_DDT_FAIL; - need += MAP_HEADER_SIZE + 1 + 2*ptr[0]; + if (ptr[0] > MAP_SMALL_MAP_LIMIT) { + need += hashmap_over_estimated_heap_size(ptr[0]); + } else { + need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0]; + } depth -= 2*ptr[0]; if (depth < 0) ERTS_DDT_FAIL; ptr++; @@ -5594,31 +5599,52 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_MAP: { /* int */ int size = (int)ptr[0]; - Eterm* tp = hp; - Eterm* vp; - map_t *mp; - - *tp = make_arityval(size); - - hp += 1 + size; - mp = (map_t*)hp; - mp->thing_word = MAP_HEADER; - mp->size = size; - mp->keys = make_tuple(tp); - mess = make_map(mp); - - hp += MAP_HEADER_SIZE + size; /* advance "heap" pointer */ - - tp += size; /* point at last key */ - vp = hp - 1; /* point at last value */ - - while(size--) { - *vp-- = ESTACK_POP(stack); - *tp-- = ESTACK_POP(stack); - } - if (!erts_validate_and_sort_map(mp)) - ERTS_DDT_FAIL; - ptr++; + if (size > MAP_SMALL_MAP_LIMIT) { + int ix = 2*size; + ErtsHeapFactory factory; + Eterm* leafs = hp; + + hp += 2*size; + while(ix--) { *--hp = ESTACK_POP(stack); } + + hp += 2*size; + factory.p = NULL; + factory.hp = hp; + /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ + + mess = erts_hashmap_from_array(&factory, leafs, size, 1); + + if (is_non_value(mess)) + ERTS_DDT_FAIL; + + hp = factory.hp; + } else { + Eterm* tp = hp; + Eterm* vp; + flatmap_t *mp; + + *tp = make_arityval(size); + + hp += 1 + size; + mp = (flatmap_t*)hp; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = size; + mp->keys = make_tuple(tp); + mess = make_flatmap(mp); + + hp += MAP_HEADER_FLATMAP_SZ + size; + + tp += size; /* point at last key */ + vp = hp - 1; /* point at last value */ + + while(size--) { + *vp-- = ESTACK_POP(stack); + *tp-- = ESTACK_POP(stack); + } + if (!erts_validate_and_sort_flatmap(mp)) + ERTS_DDT_FAIL; + } + ptr++; break; } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 828f5b427a..251b39508f 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -21,6 +21,25 @@ #define __SYS_H__ +#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) +# undef ERTS_CAN_INLINE +# define ERTS_CAN_INLINE 0 +# undef ERTS_INLINE +# define ERTS_INLINE +#endif + +#if ERTS_CAN_INLINE +#define ERTS_GLB_INLINE static ERTS_INLINE +#else +#define ERTS_GLB_INLINE +#endif + +#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) +# define ERTS_GLB_INLINE_INCL_FUNC_DEF 1 +#else +# define ERTS_GLB_INLINE_INCL_FUNC_DEF 0 +#endif + #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS) # define NO_FPE_SIGNALS #endif @@ -132,24 +151,8 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # endif #endif -#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) -# undef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 0 -# undef ERTS_INLINE -# define ERTS_INLINE -#endif - -#if ERTS_CAN_INLINE -#define ERTS_GLB_INLINE static ERTS_INLINE -#else -#define ERTS_GLB_INLINE -#endif - -#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) -# define ERTS_GLB_INLINE_INCL_FUNC_DEF 1 -#else -# define ERTS_GLB_INLINE_INCL_FUNC_DEF 0 -#endif +#define ERTS_MK_VSN_INT(Major, Minor, Build) \ + ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff)) #ifndef ERTS_EXIT_AFTER_DUMP # define ERTS_EXIT_AFTER_DUMP exit @@ -188,6 +191,16 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f # define ASSERT(e) ((void) 1) #endif +/* ERTS_UNDEF can be used to silence false warnings about + * "variable may be used uninitialized" while keeping the variable + * marked as undefined by valgrind. + */ +#ifdef VALGRIND +# define ERTS_UNDEF(V,I) +#else +# define ERTS_UNDEF(V,I) V = I +#endif + /* * Compile time assert * (the actual compiler error msg can be a bit confusing) @@ -375,17 +388,45 @@ typedef Sint SWord; typedef UWord BeamInstr; #ifndef HAVE_INT64 -#if SIZEOF_LONG == 8 -#define HAVE_INT64 1 +# if SIZEOF_LONG == 8 +# define HAVE_INT64 1 typedef unsigned long Uint64; typedef long Sint64; -#elif SIZEOF_LONG_LONG == 8 -#define HAVE_INT64 1 +# ifdef ULONG_MAX +# define ERTS_UINT64_MAX ULONG_MAX +# endif +# ifdef LONG_MAX +# define ERTS_SINT64_MAX LONG_MAX +# endif +# ifdef LONG_MIN +# define ERTS_SINT64_MIN LONG_MIN +# endif +# elif SIZEOF_LONG_LONG == 8 +# define HAVE_INT64 1 typedef unsigned long long Uint64; typedef long long Sint64; -#else -#define HAVE_INT64 0 +# ifdef ULLONG_MAX +# define ERTS_UINT64_MAX ULLONG_MAX +# endif +# ifdef LLONG_MAX +# define ERTS_SINT64_MAX LLONG_MAX +# endif +# ifdef LLONG_MIN +# define ERTS_SINT64_MIN LLONG_MIN +# endif +# else +# error "No 64-bit integer type found" +# endif +#endif + +#ifndef ERTS_UINT64_MAX +# define ERTS_UINT64_MAX (~((Uint64) 0)) +#endif +#ifndef ERTS_SINT64_MAX +# define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1)) #endif +#ifndef ERTS_SINT64_MIN +# define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63)) #endif #if SIZEOF_LONG == 4 @@ -662,14 +703,43 @@ extern char *erts_default_arg0; extern char os_type[]; -extern int sys_init_time(void); +typedef enum { + ERTS_NO_TIME_WARP_MODE, + ERTS_SINGLE_TIME_WARP_MODE, + ERTS_MULTI_TIME_WARP_MODE +} ErtsTimeWarpMode; + +typedef struct { + int have_os_monotonic_time; + ErtsMonotonicTime os_monotonic_time_unit; + ErtsMonotonicTime sys_clock_resolution; + struct { + Uint64 resolution; + char *func; + char *clock_id; + int locked_use; + int extended; + } os_monotonic_time_info; + struct { + Uint64 resolution; + char *func; + char *clock_id; + int locked_use; + } os_system_time_info; +} ErtsSysInitTimeResult; + +#define ERTS_SYS_INIT_TIME_RESULT_INITER \ + {0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1} + +extern void erts_init_sys_time_sup(void); +extern void sys_init_time(ErtsSysInitTimeResult *); +extern void erts_late_sys_init_time(void); extern void erts_deliver_time(void); extern void erts_time_remaining(SysTimeval *); -extern int erts_init_time_sup(void); +extern int erts_init_time_sup(int, ErtsTimeWarpMode); extern void erts_sys_init_float(void); extern void erts_thread_init_float(void); extern void erts_thread_disable_fpe(void); - ERTS_GLB_INLINE int erts_block_fpe(void); ERTS_GLB_INLINE void erts_unblock_fpe(int); @@ -716,7 +786,7 @@ extern char *erts_sys_ddll_error(int code); void erts_sys_schedule_interrupt(int set); #ifdef ERTS_SMP -void erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec); +void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime); void erts_sys_main_thread(void); #endif @@ -755,6 +825,7 @@ int univ_to_local( int local_to_univ(Sint *year, Sint *month, Sint *day, Sint *hour, Sint *minute, Sint *second, int isdst); void get_now(Uint*, Uint*, Uint*); +ErtsMonotonicTime erts_get_monotonic_time(void); void get_sys_now(Uint*, Uint*, Uint*); void set_break_quit(void (*)(void), void (*)(void)); diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 2fd8e0cf00..2bdda6c8af 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -83,7 +83,8 @@ #define ASSERT_NO_LOCKED_LOCKS #endif -static erts_smp_mtx_t tiw_lock; +#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24) +#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY) /* BEGIN tiw_lock protected variables @@ -91,18 +92,12 @@ static erts_smp_mtx_t tiw_lock; ** The individual timer cells in tiw are also protected by the same mutex. */ +/* timing wheel size NEED to be a power of 2 */ #ifdef SMALL_MEMORY -#define TIW_SIZE 8192 +#define TIW_SIZE (1 << 13) #else -#define TIW_SIZE 65536 /* timing wheel size (should be a power of 2) */ +#define TIW_SIZE (1 << 20) #endif -static ErlTimer** tiw; /* the timing wheel, allocated in init_time() */ -static Uint tiw_pos; /* current position in wheel */ -static Uint tiw_nto; /* number of timeouts in wheel */ -static Uint tiw_min; -static ErlTimer *tiw_min_ptr; - -/* END tiw_lock protected variables */ /* Actual interval time chosen by sys_init_time() */ @@ -114,83 +109,135 @@ static int tiw_itime; /* Constant after init */ # define TIW_ITIME tiw_itime #endif -erts_smp_atomic32_t do_time; /* set at clock interrupt */ -static ERTS_INLINE erts_short_time_t do_time_read(void) +struct ErtsTimerWheel_ { + ErlTimer *w[TIW_SIZE]; + ErtsMonotonicTime pos; + Uint nto; + struct { + ErlTimer *head; + ErlTimer **tail; + Uint nto; + } at_once; + int true_next_timeout_time; + ErtsMonotonicTime next_timeout_time; + erts_atomic64_t next_timeout; + erts_smp_atomic32_t is_bumping; + erts_smp_mtx_t lock; +}; + +ErtsTimerWheel *erts_default_timer_wheel; /* managed by aux thread */ + +static ERTS_INLINE ErtsTimerWheel * +get_timer_wheel(ErlTimer *p) +{ + return (ErtsTimerWheel *) erts_smp_atomic_read_acqb(&p->wheel); +} + +static ERTS_INLINE void +set_timer_wheel(ErlTimer *p, ErtsTimerWheel *tiw) { - return erts_smp_atomic32_read_acqb(&do_time); + erts_smp_atomic_set_relb(&p->wheel, (erts_aint_t) tiw); } -static ERTS_INLINE erts_short_time_t do_time_update(void) +static ERTS_INLINE void +init_next_timeout(ErtsTimerWheel *tiw, + ErtsMonotonicTime time) { - return do_time_read(); + erts_atomic64_init_nob(&tiw->next_timeout, + (erts_aint64_t) time); } -static ERTS_INLINE void do_time_init(void) +static ERTS_INLINE void +set_next_timeout(ErtsTimerWheel *tiw, + ErtsMonotonicTime time, + int true_timeout) { - erts_smp_atomic32_init_nob(&do_time, 0); + tiw->true_next_timeout_time = true_timeout; + tiw->next_timeout_time = time; + erts_atomic64_set_relb(&tiw->next_timeout, + (erts_aint64_t) time); } /* get the time (in units of TIW_ITIME) to the next timeout, or -1 if there are no timeouts */ -static erts_short_time_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ +static ERTS_INLINE ErtsMonotonicTime +find_next_timeout(ErtsTimerWheel *tiw, + ErtsMonotonicTime curr_time, + ErtsMonotonicTime max_search_time) { - int i, tm, nto; - Uint32 min; - ErlTimer* p; - erts_short_time_t dt; - - if (tiw_nto == 0) - return -1; /* no timeouts in wheel */ + int start_ix, tiw_pos_ix; + ErlTimer *p; + int true_min_timeout; + ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos, timeout_limit; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&tiw->lock)); + + if (tiw->true_next_timeout_time) + return tiw->next_timeout_time; - if (tiw_min_ptr) { - min = tiw_min; - dt = do_time_read(); - return ((min >= dt) ? (min - dt) : 0); + /* We never set next timeout beyond timeout_limit */ + timeout_limit = curr_time + ERTS_MONOTONIC_DAY; + + if (tiw->nto == 0) { /* no timeouts in wheel */ + true_min_timeout = tiw->true_next_timeout_time = 0; + min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(timeout_limit); + goto found_next; } - - /* start going through wheel to find next timeout */ - tm = nto = 0; - min = (Uint32) -1; /* max Uint32 */ - i = tiw_pos; + + /* + * Don't want others entering trying to bump + * timers while we are checking... + */ + set_next_timeout(tiw, timeout_limit, 0); + + true_min_timeout = 1; + slot_timeout_pos = tiw->pos; + min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); + + start_ix = tiw_pos_ix = (int) (tiw->pos & (TIW_SIZE-1)); + do { - p = tiw[i]; - while (p != NULL) { - nto++; - if (p->count == 0) { - /* found next timeout */ - dt = do_time_read(); - /* p->count is zero */ - tiw_min_ptr = p; - tiw_min = tm; - return ((tm >= dt) ? (tm - dt) : 0); - } else { - /* keep shortest time in 'min' */ - if (tm + p->count*TIW_SIZE < min) { - min = tm + p->count*TIW_SIZE; - tiw_min_ptr = p; - tiw_min = min; - } + slot_timeout_pos++; + if (slot_timeout_pos >= min_timeout_pos) { + true_min_timeout = 0; + break; + } + + p = tiw->w[tiw_pos_ix]; + + while (p) { + ErtsMonotonicTime timeout_pos; + ASSERT(p != p->next); + timeout_pos = p->timeout_pos; + if (min_timeout_pos > timeout_pos) { + min_timeout_pos = timeout_pos; + if (min_timeout_pos <= slot_timeout_pos) + goto found_next; } p = p->next; } - /* when we have found all timeouts the shortest time will be in min */ - if (nto == tiw_nto) break; - tm++; - i = (i + 1) % TIW_SIZE; - } while (i != tiw_pos); - dt = do_time_read(); - if (min <= (Uint32) dt) - return 0; - if ((min - (Uint32) dt) > (Uint32) ERTS_SHORT_TIME_T_MAX) - return ERTS_SHORT_TIME_T_MAX; - return (erts_short_time_t) (min - (Uint32) dt); + + tiw_pos_ix++; + if (tiw_pos_ix == TIW_SIZE) + tiw_pos_ix = 0; + } while (start_ix != tiw_pos_ix); + +found_next: + + min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); + if (min_timeout != tiw->next_timeout_time) + set_next_timeout(tiw, min_timeout, true_min_timeout); + + return min_timeout; } -static void remove_timer(ErlTimer *p) { +static void +remove_timer(ErtsTimerWheel *tiw, ErlTimer *p) +{ /* first */ if (!p->prev) { - tiw[p->slot] = p->next; + tiw->w[p->slot] = p->next; if(p->next) p->next->prev = NULL; } else { @@ -207,79 +254,164 @@ static void remove_timer(ErlTimer *p) { p->next = NULL; p->prev = NULL; - /* Make sure cancel callback isn't called */ - p->active = 0; - tiw_nto--; + + set_timer_wheel(p, NULL); + tiw->nto--; +} + +ErtsMonotonicTime +erts_check_next_timeout_time(ErtsTimerWheel *tiw, + ErtsMonotonicTime max_search_time) +{ + ErtsMonotonicTime next, curr; + + curr = erts_get_monotonic_time(); + + erts_smp_mtx_lock(&tiw->lock); + + next = find_next_timeout(tiw, curr, max_search_time); + + erts_smp_mtx_unlock(&tiw->lock); + + return next; } -/* Private export to erl_time_sup.c */ -erts_short_time_t erts_next_time(void) +#ifndef DEBUG +#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0) +#else +#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO)) +static void +debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) { - erts_short_time_t ret; + int slots, ix; + ErlTimer *tmr; + ErtsMonotonicTime tmp; + + ix = (int) (tiw->pos & (TIW_SIZE-1)); + tmp = skip_to_pos - tiw->pos; + ASSERT(tmp >= 0); + if (tmp < (ErtsMonotonicTime) TIW_SIZE) + slots = (int) tmp; + else + slots = TIW_SIZE; - erts_smp_mtx_lock(&tiw_lock); - (void)do_time_update(); - ret = next_time_internal(); - erts_smp_mtx_unlock(&tiw_lock); - return ret; + while (slots > 0) { + tmr = tiw->w[ix]; + while (tmr) { + ASSERT(tmr->timeout_pos > skip_to_pos); + tmr = tmr->next; + } + ix++; + if (ix == TIW_SIZE) + ix = 0; + slots--; + } } +#endif -static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lock is write-locked */ +void +erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - Uint keep_pos; - Uint count; - ErlTimer *p, **prev, *timeout_head, **timeout_tail; - Uint dtime = (Uint) dt; - - /* no need to bump the position if there aren't any timeouts */ - if (tiw_nto == 0) { - erts_smp_mtx_unlock(&tiw_lock); - return; + int tiw_pos_ix, slots; + ErlTimer *p, *timeout_head, **timeout_tail; + ErtsMonotonicTime bump_to, tmp_slots; + + if (erts_smp_atomic32_cmpxchg_nob(&tiw->is_bumping, 1, 0) != 0) + return; /* Another thread is currently bumping... */ + + bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + + erts_smp_mtx_lock(&tiw->lock); + + if (tiw->pos >= bump_to) { + timeout_head = NULL; + goto done; } - /* if do_time > TIW_SIZE we want to go around just once */ - count = (Uint)(dtime / TIW_SIZE) + 1; - keep_pos = (tiw_pos + dtime) % TIW_SIZE; - if (dtime > TIW_SIZE) dtime = TIW_SIZE; - - timeout_head = NULL; - timeout_tail = &timeout_head; - while (dtime > 0) { - /* this is to decrease the counters with the right amount */ - /* when dtime >= TIW_SIZE */ - if (tiw_pos == keep_pos) count--; - prev = &tiw[tiw_pos]; - while ((p = *prev) != NULL) { - ASSERT( p != p->next); - if (p->count < count) { /* we have a timeout */ - /* remove min time */ - if (tiw_min_ptr == p) { - tiw_min_ptr = NULL; - tiw_min = 0; - } + /* Don't want others here while we are bumping... */ + set_next_timeout(tiw, curr_time + ERTS_MONOTONIC_DAY, 0); + if (!tiw->at_once.head) { + timeout_head = NULL; + timeout_tail = &timeout_head; + } + else { + ASSERT(tiw->nto >= tiw->at_once.nto); + timeout_head = tiw->at_once.head; + timeout_tail = tiw->at_once.tail; + tiw->nto -= tiw->at_once.nto; + tiw->at_once.head = NULL; + tiw->at_once.tail = &tiw->at_once.head; + tiw->at_once.nto = 0; + } + + if (tiw->nto == 0) { + ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); + tiw->pos = bump_to; + goto done; + } + + if (tiw->true_next_timeout_time) { + ErtsMonotonicTime skip_until_pos; + /* + * No need inspecting slots where we know no timeouts + * to trigger should reside. + */ + + skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + if (skip_until_pos > bump_to) + skip_until_pos = bump_to; + + ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); + ASSERT(skip_until_pos > tiw->pos); + + tiw->pos = skip_until_pos - 1; + } + + tiw_pos_ix = (int) ((tiw->pos+1) & (TIW_SIZE-1)); + tmp_slots = (bump_to - tiw->pos); + if (tmp_slots < (ErtsMonotonicTime) TIW_SIZE) + slots = (int) tmp_slots; + else + slots = TIW_SIZE; + + while (slots > 0) { + p = tiw->w[tiw_pos_ix]; + while (p) { + ErlTimer *next = p->next; + ASSERT(p != next); + if (p->timeout_pos <= bump_to) { /* we have a timeout */ /* Remove from list */ - remove_timer(p); + remove_timer(tiw, p); *timeout_tail = p; /* Insert in timeout queue */ timeout_tail = &p->next; } - else { - /* no timeout, just decrease counter */ - p->count -= count; - prev = &p->next; - } + p = next; } - tiw_pos = (tiw_pos + 1) % TIW_SIZE; - dtime--; + tiw_pos_ix++; + if (tiw_pos_ix == TIW_SIZE) + tiw_pos_ix = 0; + slots--; } - tiw_pos = keep_pos; - if (tiw_min_ptr) - tiw_min -= dt; - - erts_smp_mtx_unlock(&tiw_lock); + + ASSERT(tmp_slots >= (ErtsMonotonicTime) TIW_SIZE + || tiw_pos_ix == (int) ((bump_to+1) & (TIW_SIZE-1))); + + tiw->pos = bump_to; + + /* Search at most two seconds ahead... */ + (void) find_next_timeout(tiw, curr_time, ERTS_SEC_TO_MONOTONIC(2)); + +done: + + erts_smp_mtx_unlock(&tiw->lock); + erts_smp_atomic32_set_nob(&tiw->is_bumping, 0); + /* Call timedout timers callbacks */ while (timeout_head) { + ErlTimeoutProc timeout; + void *arg; p = timeout_head; timeout_head = p->next; /* Here comes hairy use of the timer fields! @@ -288,35 +420,69 @@ static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lo * accesses any field until the ->timeout * callback is called. */ + ASSERT(p->timeout_pos <= bump_to); p->next = NULL; p->prev = NULL; p->slot = 0; - (*p->timeout)(p->arg); + timeout = p->timeout; + arg = p->arg; + (*timeout)(arg); } } -void erts_bump_timer(erts_short_time_t dt) /* dt is value from do_time */ +Uint +erts_timer_wheel_memory_size(void) +{ +#ifdef ERTS_SMP + return sizeof(ErtsTimerWheel)*(1 + erts_no_schedulers); +#else + return sizeof(ErtsTimerWheel); +#endif +} + +ErtsTimerWheel * +erts_create_timer_wheel(int no) { - erts_smp_mtx_lock(&tiw_lock); - bump_timer_internal(dt); + ErtsMonotonicTime mtime; + int i; + ErtsTimerWheel *tiw; + tiw = (ErtsTimerWheel *) erts_alloc(ERTS_ALC_T_TIMER_WHEEL, + sizeof(ErtsTimerWheel)); + for(i = 0; i < TIW_SIZE; i++) + tiw->w[i] = NULL; + + erts_smp_atomic32_init_nob(&tiw->is_bumping, 0); + erts_smp_mtx_init_x(&tiw->lock, "timer_wheel", make_small(no)); + + mtime = erts_get_monotonic_time(); + tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); + tiw->nto = 0; + tiw->at_once.head = NULL; + tiw->at_once.tail = &tiw->at_once.head; + tiw->at_once.nto = 0; + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; + init_next_timeout(tiw, mtime + ERTS_MONOTONIC_DAY); + return tiw; } -Uint -erts_timer_wheel_memory_size(void) +ErtsNextTimeoutRef +erts_get_next_timeout_reference(ErtsTimerWheel *tiw) { - return (Uint) TIW_SIZE * sizeof(ErlTimer*); + return (ErtsNextTimeoutRef) &tiw->next_timeout; } + /* this routine links the time cells into a free list at the start and sets the time queue as empty */ void -erts_init_time(void) +erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) { - int i, itime; + int itime; /* system dependent init; must be done before do_time_init() if timer thread is enabled */ - itime = erts_init_time_sup(); + itime = erts_init_time_sup(time_correction, time_warp_mode); #ifdef TIW_ITIME_IS_CONSTANT if (itime != TIW_ITIME) { erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); @@ -325,117 +491,111 @@ erts_init_time(void) tiw_itime = itime; #endif - erts_smp_mtx_init(&tiw_lock, "timer_wheel"); - - tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL, - TIW_SIZE * sizeof(ErlTimer*)); - for(i = 0; i < TIW_SIZE; i++) - tiw[i] = NULL; - do_time_init(); - tiw_pos = tiw_nto = 0; - tiw_min_ptr = NULL; - tiw_min = 0; + erts_default_timer_wheel = erts_create_timer_wheel(0); } +void +erts_set_timer(ErlTimer *p, ErlTimeoutProc timeout, + ErlCancelProc cancel, void *arg, Uint to) +{ + ErtsMonotonicTime timeout_time, timeout_pos; + ErtsMonotonicTime curr_time; + ErtsTimerWheel *tiw; + ErtsSchedulerData *esdp; + + curr_time = erts_get_monotonic_time(); + esdp = erts_get_scheduler_data(); + if (esdp) + tiw = esdp->timer_wheel; + else + tiw = erts_default_timer_wheel; + erts_smp_mtx_lock(&tiw->lock); + if (get_timer_wheel(p)) + ERTS_INTERNAL_ERROR("Double set timer"); -/* -** Insert a process into the time queue, with a timeout 't' -*/ -static void -insert_timer(ErlTimer* p, Uint t) -{ - Uint tm; - Uint64 ticks; + p->timeout = timeout; + p->cancel = cancel; + p->arg = arg; - /* The current slot (tiw_pos) in timing wheel is the next slot to be - * be processed. Hence no extra time tick is needed. - * - * (x + y - 1)/y is precisely the "number of bins" formula. - */ - ticks = (t + (TIW_ITIME - 1)) / TIW_ITIME; + if (to == 0) { + timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + tiw->nto++; + tiw->at_once.nto++; + *tiw->at_once.tail = p; + tiw->at_once.tail = &p->next; + p->next = NULL; + p->timeout_pos = timeout_pos; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + } + else { + int tm; + ErtsMonotonicTime ticks; - /* - * Ticks must be a Uint64, or the addition may overflow here, - * resulting in an incorrect value for p->count below. - */ - ticks += do_time_update(); /* Add backlog of unprocessed time */ - - /* calculate slot */ - tm = (ticks + tiw_pos) % TIW_SIZE; - p->slot = (Uint) tm; - p->count = (Uint) (ticks / TIW_SIZE); + ticks = ERTS_MSEC_TO_CLKTCKS(to); + timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time - 1) + 1 + ticks; + + /* calculate slot */ + tm = (int) (timeout_pos & (TIW_SIZE-1)); + p->slot = (Uint) tm; - /* insert at head of list at slot */ - p->next = tiw[tm]; - p->prev = NULL; - if (p->next != NULL) - p->next->prev = p; - tiw[tm] = p; + /* insert at head of list at slot */ + p->next = tiw->w[tm]; + p->prev = NULL; + if (p->next != NULL) + p->next->prev = p; + tiw->w[tm] = p; + tiw->nto++; - /* insert min time */ - if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) { - tiw_min = ticks; - tiw_min_ptr = p; - } - if ((tiw_min_ptr == p) && (ticks > tiw_min)) { - /* some other timer might be 'min' now */ - tiw_min = 0; - tiw_min_ptr = NULL; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + p->timeout_pos = timeout_pos; + + ASSERT(ERTS_MSEC_TO_MONOTONIC(to) <= timeout_time - curr_time); + ASSERT(timeout_time - curr_time + < ERTS_MSEC_TO_MONOTONIC(to) + ERTS_CLKTCKS_TO_MONOTONIC(1)); } - tiw_nto++; -} + if (timeout_time < tiw->next_timeout_time) + set_next_timeout(tiw, timeout_time, 1); -void -erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, - void* arg, Uint t) -{ + set_timer_wheel(p, tiw); + + erts_smp_mtx_unlock(&tiw->lock); - erts_deliver_time(); - erts_smp_mtx_lock(&tiw_lock); - if (p->active) { /* XXX assert ? */ - erts_smp_mtx_unlock(&tiw_lock); - return; - } - p->timeout = timeout; - p->cancel = cancel; - p->arg = arg; - p->active = 1; - insert_timer(p, t); - erts_smp_mtx_unlock(&tiw_lock); #if defined(ERTS_SMP) - if (t <= (Uint) ERTS_SHORT_TIME_T_MAX) - erts_sys_schedule_interrupt_timed(1, (erts_short_time_t) t); + if (tiw == erts_default_timer_wheel) + erts_interupt_aux_thread_timed(timeout_time); #endif + } void -erts_cancel_timer(ErlTimer* p) +erts_cancel_timer(ErlTimer *p) { - erts_smp_mtx_lock(&tiw_lock); - if (!p->active) { /* allow repeated cancel (drivers) */ - erts_smp_mtx_unlock(&tiw_lock); + ErtsTimerWheel *tiw; + ErlCancelProc cancel; + void *arg = NULL; /* Shut up faulty warning... */ + + tiw = get_timer_wheel(p); + if (!tiw) return; - } + + erts_smp_mtx_lock(&tiw->lock); + if (tiw != get_timer_wheel(p)) + cancel = NULL; + else { + remove_timer(tiw, p); + p->slot = 0; - /* is it the 'min' timer, remove min */ - if (p == tiw_min_ptr) { - tiw_min_ptr = NULL; - tiw_min = 0; + cancel = p->cancel; + arg = p->arg; } + erts_smp_mtx_unlock(&tiw->lock); - remove_timer(p); - p->slot = p->count = 0; - - if (p->cancel != NULL) { - erts_smp_mtx_unlock(&tiw_lock); - (*p->cancel)(p->arg); - return; - } - erts_smp_mtx_unlock(&tiw_lock); + if (cancel) + (*cancel)(arg); } /* @@ -447,59 +607,58 @@ erts_cancel_timer(ErlTimer* p) Uint erts_time_left(ErlTimer *p) { - Uint left; - erts_short_time_t dt; + ErtsTimerWheel *tiw; + ErtsMonotonicTime current_time, timeout_time; - erts_smp_mtx_lock(&tiw_lock); - - if (!p->active) { - erts_smp_mtx_unlock(&tiw_lock); + tiw = get_timer_wheel(p); + if (!tiw) return 0; - } - if (p->slot < tiw_pos) - left = (p->count + 1) * TIW_SIZE + p->slot - tiw_pos; + erts_smp_mtx_lock(&tiw->lock); + if (tiw != get_timer_wheel(p)) + timeout_time = ERTS_MONOTONIC_TIME_MIN; else - left = p->count * TIW_SIZE + p->slot - tiw_pos; - dt = do_time_read(); - if (left < dt) - left = 0; - else - left -= dt; - - erts_smp_mtx_unlock(&tiw_lock); + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos); + erts_smp_mtx_unlock(&tiw->lock); - return (Uint) left * TIW_ITIME; + current_time = erts_get_monotonic_time(); + if (timeout_time <= current_time) + return 0; + return (Uint) ERTS_MONOTONIC_TO_MSEC(timeout_time - current_time); } #ifdef DEBUG void erts_p_slpq(void) { + ErtsTimerWheel *tiw = erts_default_timer_wheel; + ErtsMonotonicTime current_time = erts_get_monotonic_time(); int i; ErlTimer* p; - erts_smp_mtx_lock(&tiw_lock); + erts_smp_mtx_lock(&tiw->lock); /* print the whole wheel, starting at the current position */ - erts_printf("\ntiw_pos = %d tiw_nto %d\n", tiw_pos, tiw_nto); - i = tiw_pos; - if (tiw[i] != NULL) { + erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n", + current_time, tiw->pos, tiw->nto); + i = tiw->pos; + if (tiw->w[i] != NULL) { erts_printf("%d:\n", i); - for(p = tiw[i]; p != NULL; p = p->next) { - erts_printf(" (count %d, slot %d)\n", - p->count, p->slot); + for(p = tiw->w[i]; p != NULL; p = p->next) { + erts_printf(" (timeout time %bps, slot %d)\n", + ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), + p->slot); } } - for(i = (i+1)%TIW_SIZE; i != tiw_pos; i = (i+1)%TIW_SIZE) { - if (tiw[i] != NULL) { + for(i = ((i+1) & (TIW_SIZE-1)); i != (tiw->pos & (TIW_SIZE-1)); i = ((i+1) & (TIW_SIZE-1))) { + if (tiw->w[i] != NULL) { erts_printf("%d:\n", i); - for(p = tiw[i]; p != NULL; p = p->next) { - erts_printf(" (count %d, slot %d)\n", - p->count, p->slot); + for(p = tiw->w[i]; p != NULL; p = p->next) { + erts_printf(" (timeout time %bps, slot %d)\n", + ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot); } } } - erts_smp_mtx_unlock(&tiw_lock); + erts_smp_mtx_unlock(&tiw->lock); } #endif /* DEBUG */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index b341c4d949..8fc8962e4f 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -49,6 +49,7 @@ #include "beam_bp.h" #include "erl_ptab.h" #include "erl_check_io.h" +#include "erl_bif_unique.h" #ifdef HIPE # include "hipe_mode_switch.h" #endif @@ -190,12 +191,18 @@ erts_set_hole_marker(Eterm* ptr, Uint sz) * Helper function for the ESTACK macros defined in global.h. */ void -erl_grow_estack(ErtsEStack* s, Eterm* default_estack) +erl_grow_estack(ErtsEStack* s, Uint need) { Uint old_size = (s->end - s->start); - Uint new_size = old_size * 2; + Uint new_size; Uint sp_offs = s->sp - s->start; - if (s->start != default_estack) { + + if (need < old_size) + new_size = 2*old_size; + else + new_size = ((need / old_size) + 2) * old_size; + + if (s->start != s->edefault) { s->start = erts_realloc(s->alloc_type, s->start, new_size*sizeof(Eterm)); } else { @@ -210,12 +217,18 @@ erl_grow_estack(ErtsEStack* s, Eterm* default_estack) * Helper function for the WSTACK macros defined in global.h. */ void -erl_grow_wstack(ErtsWStack* s, UWord* default_wstack) +erl_grow_wstack(ErtsWStack* s, Uint need) { Uint old_size = (s->wend - s->wstart); - Uint new_size = old_size * 2; + Uint new_size; Uint sp_offs = s->wsp - s->wstart; - if (s->wstart != default_wstack) { + + if (need < old_size) + new_size = 2 * old_size; + else + new_size = ((need / old_size) + 2) * old_size; + + if (s->wstart != s->wdefault) { s->wstart = erts_realloc(s->alloc_type, s->wstart, new_size*sizeof(UWord)); } else { @@ -227,6 +240,32 @@ erl_grow_wstack(ErtsWStack* s, UWord* default_wstack) s->wsp = s->wstart + sp_offs; } +/* + * Helper function for the PSTACK macros defined in global.h. + */ +void +erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) +{ + Uint old_size = s->pend - s->pstart; + Uint new_size; + Uint sp_offs = s->psp - s->pstart; + + if (need_bytes < old_size) + new_size = 2 * old_size; + else + new_size = ((need_bytes / old_size) + 2) * old_size; + + if (s->pstart != default_pstack) { + s->pstart = erts_realloc(s->alloc_type, s->pstart, new_size); + } else { + byte* new_ptr = erts_alloc(s->alloc_type, new_size); + sys_memcpy(new_ptr, s->pstart, old_size); + s->pstart = new_ptr; + } + s->pend = s->pstart + new_size; + s->psp = s->pstart + sp_offs; +} + /* CTYPE macros */ #define LATIN1 @@ -314,6 +353,17 @@ int erts_fit_in_bits_int32(Sint32 value) return fit_in_bits((Sint64) (Uint32) value, 4); } +int erts_fit_in_bits_uint(Uint value) +{ +#if ERTS_SIZEOF_ETERM == 4 + return fit_in_bits((Sint64) (Uint32) value, 4); +#elif ERTS_SIZEOF_ETERM == 8 + return fit_in_bits(value, 5); +#else +# error "No way, Jose" +#endif +} + int erts_print(int to, void *arg, char *format, ...) { @@ -710,7 +760,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, ** If N < 0, Y = FUNNY_NUMBER4 else Y = FUNNY_NUMBER3. ** The hash value is Y*h(J) mod 2^32 where h(J) is calculated like ** h(0) = <initial hash> -** h(i) = h(i-i)*X + B(i-1) +** h(i) = h(i-1)*X + B(i-1) ** The above should hold regardless of internal representation. ** Pids are hashed like small numbers but with differrent constants, as are ** ports. @@ -792,10 +842,10 @@ Uint32 make_hash(Eterm term_arg) unsigned op; /* Must not collide with the real tag_val_def's: */ -#define MAKE_HASH_TUPLE_OP 0x11 -#define MAKE_HASH_TERM_ARRAY_OP 0x12 -#define MAKE_HASH_CDR_PRE_OP 0x13 -#define MAKE_HASH_CDR_POST_OP 0x14 +#define MAKE_HASH_TUPLE_OP (FIRST_VACANT_TAG_DEF) +#define MAKE_HASH_TERM_ARRAY_OP (FIRST_VACANT_TAG_DEF+1) +#define MAKE_HASH_CDR_PRE_OP (FIRST_VACANT_TAG_DEF+2) +#define MAKE_HASH_CDR_POST_OP (FIRST_VACANT_TAG_DEF+3) /* ** Convenience macro for calculating a bytewise hash on an unsigned 32 bit @@ -975,23 +1025,8 @@ tail_recur: break; } case MAP_DEF: - { - map_t *mp = (map_t *)map_val(term); - int size = map_get_size(mp); - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - - /* Use a prime with size to remedy some of - * the {} and <<>> hash problems */ - hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + size; - if (size == 0) - break; - - /* push values first */ - WSTACK_PUSH3(stack, (UWord)vs, (UWord) size, MAKE_HASH_TERM_ARRAY_OP); - WSTACK_PUSH3(stack, (UWord)ks, (UWord) size, MAKE_HASH_TERM_ARRAY_OP); - break; - } + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); + break; case TUPLE_DEF: { Eterm* ptr = tuple_val(term); @@ -1095,10 +1130,11 @@ Uint32 make_hash2(Eterm term) { Uint32 hash; - Uint32 hash_xor_keys = 0; - Uint32 hash_xor_values = 0; + Uint32 hash_xor_pairs; DeclareTmpHeapNoproc(tmp_big,2); + ERTS_UNDEF(hash_xor_pairs, 0); + /* (HCONST * {2, ..., 16}) mod 2^32 */ #define HCONST_2 0x3c6ef372UL #define HCONST_3 0xdaa66d2bUL @@ -1115,10 +1151,15 @@ make_hash2(Eterm term) #define HCONST_14 0xa708a81eUL #define HCONST_15 0x454021d7UL #define HCONST_16 0xe3779b90UL +#define HCONST_17 0x81af1549UL +#define HCONST_18 0x1fe68f02UL +#define HCONST_19 0xbe1e08bbUL +#define HCONST_20 0x5c558274UL +#define HCONST_21 0xfa8cfc2dUL #define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF)) -#define HASH_MAP_KEY (_make_header(2,_TAG_HEADER_REF)) -#define HASH_MAP_VAL (_make_header(3,_TAG_HEADER_REF)) +#define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF)) +#define HASH_CDR (_make_header(3,_TAG_HEADER_REF)) #define UINT32_HASH_2(Expr1, Expr2, AConst) \ do { \ @@ -1141,6 +1182,13 @@ make_hash2(Eterm term) } while(0) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) + +#ifdef ARCH_64 +# define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst) +#else +# define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst) +#endif + /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { switch (term & _TAG_IMMED1_MASK) { @@ -1195,9 +1243,9 @@ make_hash2(Eterm term) if (c > 0) UINT32_HASH(sh, HCONST_4); if (is_list(term)) { - term = *ptr; - tmp = *++ptr; - ESTACK_PUSH(s, tmp); + tmp = CDR(ptr); + ESTACK_PUSH(s, tmp); + term = CAR(ptr); } } break; @@ -1214,46 +1262,90 @@ make_hash2(Eterm term) UINT32_HASH(arity, HCONST_9); if (arity == 0) /* Empty tuple */ goto hash2_common; - for (i = arity; i >= 1; i--) { - tmp = elem[i]; - ESTACK_PUSH(s, tmp); - } - goto hash2_common; - } - break; - case MAP_SUBTAG: - { - map_t *mp = (map_t *)map_val(term); - int i; - int size = map_get_size(mp); - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - UINT32_HASH(size, HCONST_16); - if (size == 0) { - goto hash2_common; - } - ESTACK_PUSH4(s, hash_xor_values, hash_xor_keys, hash, HASH_MAP_TAIL); - hash = 0; - hash_xor_keys = 0; - hash_xor_values = 0; - for (i = size - 1; i >= 0; i--) { - tmp = vs[i]; - ESTACK_PUSH2(s, HASH_MAP_VAL, tmp); + for (i = arity; ; i--) { + term = elem[i]; + if (i == 1) + break; + ESTACK_PUSH(s, term); } - /* We do not want to expose the tuple representation. - * Do not push the keys as a tuple. - */ - for (i = size - 1; i >= 0; i--) { - tmp = ks[i]; - ESTACK_PUSH2(s, HASH_MAP_KEY, tmp); - } - goto hash2_common; } break; + case MAP_SUBTAG: + { + Eterm* ptr = boxed_val(term) + 1; + Uint size; + int i; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + { + flatmap_t *mp = (flatmap_t *)flatmap_val(term); + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + size = flatmap_get_size(mp); + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto hash2_common; + + /* We want a portable hash function that is *independent* of + * the order in which keys and values are encountered. + * We therefore calculate context independent hashes for all . + * key-value pairs and then xor them together. + */ + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, vs[i]); + ESTACK_PUSH(s, ks[i]); + } + goto hash2_common; + } + + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + size = *ptr++; + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto hash2_common; + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + } + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + i = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erl_exit(1, "bad header"); + } + while (i) { + if (is_list(*ptr)) { + Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, CDR(cons)); + ESTACK_PUSH(s, CAR(cons)); + } + else { + ASSERT(is_boxed(*ptr)); + ESTACK_PUSH(s, *ptr); + } + i--; ptr++; + } + goto hash2_common; + } + break; case EXPORT_SUBTAG: { Export* ep = *((Export **) (export_val(term) + 1)); - UINT32_HASH_2 (ep->code[2], atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, @@ -1268,7 +1360,6 @@ make_hash2(Eterm term) { ErlFunThing* funp = (ErlFunThing *) fun_val(term); Uint num_free = funp->num_free; - UINT32_HASH_2 (num_free, atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue, @@ -1349,7 +1440,8 @@ make_hash2(Eterm term) do { Uint t; Uint32 x, y; - t = i < n ? BIG_DIGIT(ptr, i++) : 0; + ASSERT(i < n); + t = BIG_DIGIT(ptr, i++); x = t & 0xffffffff; y = t >> 32; UINT32_HASH_2(x, y, con); @@ -1457,20 +1549,395 @@ make_hash2(Eterm term) switch (term) { case HASH_MAP_TAIL: { hash = (Uint32) ESTACK_POP(s); - UINT32_HASH(hash_xor_keys, HCONST_16); - UINT32_HASH(hash_xor_values, HCONST_16); - hash_xor_keys = (Uint32) ESTACK_POP(s); - hash_xor_values = (Uint32) ESTACK_POP(s); + UINT32_HASH(hash_xor_pairs, HCONST_19); + hash_xor_pairs = (Uint32) ESTACK_POP(s); goto hash2_common; } - case HASH_MAP_KEY: - hash_xor_keys ^= hash; + case HASH_MAP_PAIR: + hash_xor_pairs ^= hash; hash = 0; goto hash2_common; - case HASH_MAP_VAL: - hash_xor_values ^= hash; + default: + break; + } + } + } + } +} + +/* Term hash function for internal use. + * + * Limitation #1: Is not "portable" in any way between different VM instances. + * + * Limitation #2: The hash value is only valid as long as the term exists + * somewhere in the VM. Why? Because external pids, ports and refs are hashed + * by mixing the node *pointer* value. If a node disappears and later reappears + * with a new ErlNode struct, externals from that node will hash different than + * before. + * + * One IMPORTANT property must hold (for hamt). + * EVERY BIT of the term that is significant for equality (see EQ) + * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a + * chance of hashing different when salted: hash([Salt|A]) vs hash([Salt|B]). + * + * This is why we can not use cached hash values for atoms for example. + * + */ + +#define CONST_HASH(AConst) \ +do { /* Lightweight mixing of constant (type info) */ \ + hash ^= AConst; \ + hash = (hash << 17) ^ (hash >> (32-17)); \ +} while (0) + +Uint32 +make_internal_hash(Eterm term) +{ + Uint32 hash; + Uint32 hash_xor_pairs; + + ERTS_UNDEF(hash_xor_pairs, 0); + + /* Optimization. Simple cases before declaration of estack. */ + if (primary_tag(term) == TAG_PRIMARY_IMMED1) { + hash = 0; + #if ERTS_SIZEOF_ETERM == 8 + UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); + #elif ERTS_SIZEOF_ETERM == 4 + UINT32_HASH(term, HCONST); + #else + # error "No you don't" + #endif + return hash; + } + { + Eterm tmp; + DECLARE_ESTACK(s); + + UseTmpHeapNoproc(2); + hash = 0; + for (;;) { + switch (primary_tag(term)) { + case TAG_PRIMARY_LIST: + { + int c = 0; + Uint32 sh = 0; + Eterm* ptr = list_val(term); + while (is_byte(*ptr)) { + /* Optimization for strings. */ + sh = (sh << 8) + unsigned_val(*ptr); + if (c == 3) { + UINT32_HASH(sh, HCONST_4); + c = sh = 0; + } else { + c++; + } + term = CDR(ptr); + if (is_not_list(term)) + break; + ptr = list_val(term); + } + if (c > 0) + UINT32_HASH(sh, HCONST_4); + if (is_list(term)) { + tmp = CDR(ptr); + CONST_HASH(HCONST_17); /* Hash CAR in cons cell */ + ESTACK_PUSH(s, tmp); + if (is_not_list(tmp)) { + ESTACK_PUSH(s, HASH_CDR); + } + term = CAR(ptr); + } + } + break; + case TAG_PRIMARY_BOXED: + { + Eterm hdr = *boxed_val(term); + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + { + int i; + int arity = header_arity(hdr); + Eterm* elem = tuple_val(term); + UINT32_HASH(arity, HCONST_9); + if (arity == 0) /* Empty tuple */ + goto pop_next; + for (i = arity; ; i--) { + term = elem[i]; + if (i == 1) + break; + ESTACK_PUSH(s, term); + } + } + break; + + case MAP_SUBTAG: + { + Eterm* ptr = boxed_val(term) + 1; + Uint size; + int i; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + { + flatmap_t *mp = (flatmap_t *)flatmap_val(term); + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + size = flatmap_get_size(mp); + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto pop_next; + + /* We want a hash function that is *independent* of + * the order in which keys and values are encountered. + * We therefore calculate context independent hashes for all . + * key-value pairs and then xor them together. + */ + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, vs[i]); + ESTACK_PUSH(s, ks[i]); + } + goto pop_next; + } + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + size = *ptr++; + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto pop_next; + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + } + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + i = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erl_exit(1, "bad header"); + } + while (i) { + if (is_list(*ptr)) { + Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, CDR(cons)); + ESTACK_PUSH(s, CAR(cons)); + } + else { + ASSERT(is_boxed(*ptr)); + ESTACK_PUSH(s, *ptr); + } + i--; ptr++; + } + goto pop_next; + } + break; + case EXPORT_SUBTAG: + { + Export* ep = *((Export **) (export_val(term) + 1)); + /* Assumes Export entries never moves */ + POINTER_HASH(ep, HCONST_14); + goto pop_next; + } + + case FUN_SUBTAG: + { + ErlFunThing* funp = (ErlFunThing *) fun_val(term); + Uint num_free = funp->num_free; + UINT32_HASH_2(num_free, funp->fe->module, HCONST_20); + UINT32_HASH_2(funp->fe->old_index, funp->fe->old_uniq, HCONST_21); + if (num_free == 0) { + goto pop_next; + } else { + Eterm* bptr = funp->env + num_free - 1; + while (num_free-- > 1) { + term = *bptr--; + ESTACK_PUSH(s, term); + } + term = *bptr; + } + } + break; + case REFC_BINARY_SUBTAG: + case HEAP_BINARY_SUBTAG: + case SUB_BINARY_SUBTAG: + { + byte* bptr; + unsigned sz = binary_size(term); + Uint32 con = HCONST_13 + hash; + Uint bitoffs; + Uint bitsize; + + ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize); + if (sz == 0 && bitsize == 0) { + hash = con; + } else { + if (bitoffs == 0) { + hash = block_hash(bptr, sz, con); + if (bitsize > 0) { + UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)), + HCONST_15); + } + } else { + byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, + sz + (bitsize != 0)); + erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize); + hash = block_hash(buf, sz, con); + if (bitsize > 0) { + UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_TMP, (void *) buf); + } + } + goto pop_next; + } + break; + case POS_BIG_SUBTAG: + case NEG_BIG_SUBTAG: + { + Eterm* ptr = big_val(term); + Uint i = 0; + Uint n = BIG_SIZE(ptr); + Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11; +#if D_EXP == 16 + do { + Uint32 x, y; + x = i < n ? BIG_DIGIT(ptr, i++) : 0; + x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; + y = i < n ? BIG_DIGIT(ptr, i++) : 0; + y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; + UINT32_HASH_2(x, y, con); + } while (i < n); +#elif D_EXP == 32 + do { + Uint32 x, y; + x = i < n ? BIG_DIGIT(ptr, i++) : 0; + y = i < n ? BIG_DIGIT(ptr, i++) : 0; + UINT32_HASH_2(x, y, con); + } while (i < n); +#elif D_EXP == 64 + do { + Uint t; + Uint32 x, y; + ASSERT(i < n); + t = BIG_DIGIT(ptr, i++); + x = t & 0xffffffff; + y = t >> 32; + UINT32_HASH_2(x, y, con); + } while (i < n); +#else +#error "unsupported D_EXP size" +#endif + goto pop_next; + } + break; + case REF_SUBTAG: + UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7); + ASSERT(internal_ref_no_of_numbers(term) == 3); + UINT32_HASH_2(internal_ref_numbers(term)[1], + internal_ref_numbers(term)[2], HCONST_8); + goto pop_next; + + case EXTERNAL_REF_SUBTAG: + { + ExternalThing* thing = external_thing_ptr(term); + + ASSERT(external_thing_ref_no_of_numbers(thing) == 3); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_7); + UINT32_HASH(external_thing_ref_numbers(thing)[0], HCONST_7); + #else + UINT32_HASH_2(thing->node, + external_thing_ref_numbers(thing)[0], HCONST_7); + #endif + UINT32_HASH_2(external_thing_ref_numbers(thing)[1], + external_thing_ref_numbers(thing)[2], HCONST_8); + goto pop_next; + } + case EXTERNAL_PID_SUBTAG: { + ExternalThing* thing = external_thing_ptr(term); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_5); + UINT32_HASH(thing->data.ui[0], HCONST_5); + #else + UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_5); + #endif + goto pop_next; + } + case EXTERNAL_PORT_SUBTAG: { + ExternalThing* thing = external_thing_ptr(term); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_6); + UINT32_HASH(thing->data.ui[0], HCONST_6); + #else + UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_6); + #endif + goto pop_next; + } + case FLOAT_SUBTAG: + { + FloatDef ff; + GET_DOUBLE(term, ff); + UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); + goto pop_next; + } + + default: + erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + } + } + break; + case TAG_PRIMARY_IMMED1: + #if ERTS_SIZEOF_ETERM == 8 + UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); + #else + UINT32_HASH(term, HCONST); + #endif + goto pop_next; + + default: + erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + + pop_next: + if (ESTACK_ISEMPTY(s)) { + DESTROY_ESTACK(s); + UnUseTmpHeapNoproc(2); + return hash; + } + + term = ESTACK_POP(s); + + switch (term) { + case HASH_MAP_TAIL: { + hash = (Uint32) ESTACK_POP(s); + UINT32_HASH(hash_xor_pairs, HCONST_19); + hash_xor_pairs = (Uint32) ESTACK_POP(s); + goto pop_next; + } + case HASH_MAP_PAIR: + hash_xor_pairs ^= hash; hash = 0; - goto hash2_common; + goto pop_next; + + case HASH_CDR: + CONST_HASH(HCONST_18); /* Hash CDR i cons cell */ + goto pop_next; default: break; } @@ -1478,9 +1945,10 @@ make_hash2(Eterm term) } } +#undef CONST_HASH #undef HASH_MAP_TAIL -#undef HASH_MAP_KEY -#undef HASH_MAP_VAL +#undef HASH_MAP_PAIR +#undef HASH_CDR #undef UINT32_HASH_2 #undef UINT32_HASH @@ -1697,23 +2165,8 @@ tail_recur: break; case MAP_DEF: - { - map_t *mp = (map_t *)map_val(term); - int size = map_get_size(mp); - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - - /* Use a prime with size to remedy some of - * the {} and <<>> hash problems */ - hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + size; - if (size == 0) - break; - - /* push values first */ - WSTACK_PUSH3(stack, (UWord)vs, (UWord) size, MAKE_HASH_TERM_ARRAY_OP); - WSTACK_PUSH3(stack, (UWord)ks, (UWord) size, MAKE_HASH_TERM_ARRAY_OP); - break; - } + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); + break; case TUPLE_DEF: { Eterm* ptr = tuple_val(term); @@ -2118,22 +2571,6 @@ tailrecur_ne: ++bb; goto term_array; } - case MAP_SUBTAG: - { - aa = map_val_rel(a, a_base); - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) - goto not_equal; - bb = map_val_rel(b,b_base); - sz = map_get_size((map_t*)aa); - - if (sz != map_get_size((map_t*)bb)) goto not_equal; - if (sz == 0) goto pop_next; - - aa += 2; - bb += 2; - sz += 1; /* increment for tuple-keys */ - goto term_array; - } case REFC_BINARY_SUBTAG: case HEAP_BINARY_SUBTAG: case SUB_BINARY_SUBTAG: @@ -2325,6 +2762,46 @@ tailrecur_ne: } break; /* not equal */ } + case MAP_SUBTAG: + if (is_flatmap_rel(a, a_base)) { + aa = flatmap_val_rel(a, a_base); + if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) + goto not_equal; + bb = flatmap_val_rel(b,b_base); + sz = flatmap_get_size((flatmap_t*)aa); + + if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal; + if (sz == 0) goto pop_next; + + aa += 2; + bb += 2; + sz += 1; /* increment for tuple-keys */ + goto term_array; + + } else { + if (!is_boxed(b) || *boxed_val_rel(b,b_base) != hdr) + goto not_equal; + + aa = hashmap_val_rel(a, a_base) + 1; + bb = hashmap_val_rel(b, b_base) + 1; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + aa++; bb++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + aa++; bb++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz > 0 && sz < 17); + break; + default: + erl_exit(1, "Unknown hashmap subsubtag\n"); + } + goto term_array; + } + default: + ASSERT(!"Unknown boxed subtab in EQ"); } break; } @@ -2448,7 +2925,18 @@ Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base, int exact) Sint erts_cmp(Eterm a, Eterm b, int exact) #endif { - DECLARE_WSTACK(stack); +#define PSTACK_TYPE struct erts_cmp_hashmap_state + struct erts_cmp_hashmap_state { + Sint wstack_rollback; + int was_exact; + Eterm *ap; + Eterm *bp; + Eterm min_key; + Sint cmp_res; /* result so far -1,0,+1 */ + }; + PSTACK_DECLARE(hmap_stack, 1); + WSTACK_DECLARE(stack); + WSTACK_DECLARE(b_stack); /* only used by hashmaps */ Eterm* aa; Eterm* bb; int i; @@ -2464,6 +2952,26 @@ Sint erts_cmp(Eterm a, Eterm b, int exact) Uint32 *anum; Uint32 *bnum; +/* The WSTACK contains naked Eterms and Operations marked with header-tags */ +#define OP_BITS 4 +#define OP_MASK 0xF +#define TERM_ARRAY_OP 0 +#define SWITCH_EXACT_OFF_OP 1 +#define HASHMAP_PHASE1_ARE_KEYS_EQUAL 2 +#define HASHMAP_PHASE1_IS_MIN_KEY 3 +#define HASHMAP_PHASE1_CMP_VALUES 4 +#define HASHMAP_PHASE2_ARE_KEYS_EQUAL 5 +#define HASHMAP_PHASE2_IS_MIN_KEY_A 6 +#define HASHMAP_PHASE2_IS_MIN_KEY_B 7 + + +#define OP_WORD(OP) (((OP) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP_WORD(SZ) OP_WORD(((SZ) << OP_BITS) | TERM_ARRAY_OP) + +#define GET_OP(WORD) (ASSERT(is_header(WORD)), ((WORD) >> _TAG_PRIMARY_SIZE) & OP_MASK) +#define GET_OP_ARG(WORD) (ASSERT(is_header(WORD)), ((WORD) >> (OP_BITS + _TAG_PRIMARY_SIZE))) + + #define RETURN_NEQ(cmp) { j=(cmp); ASSERT(j != 0); goto not_equal; } #define ON_CMP_GOTO(cmp) if ((j=(cmp)) == 0) goto pop_next; else goto not_equal @@ -2479,6 +2987,8 @@ Sint erts_cmp(Eterm a, Eterm b, int exact) } while (0) +bodyrecur: + j = 0; tailrecur: if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */ goto pop_next; @@ -2605,25 +3115,96 @@ tailrecur_ne: ++aa; ++bb; goto term_array; - case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) : - if (!is_map_rel(b,b_base)) { - a_tag = MAP_DEF; - goto mixed_types; - } - aa = (Eterm *)map_val_rel(a,a_base); - bb = (Eterm *)map_val_rel(b,b_base); - - i = map_get_size((map_t*)aa); - if (i != map_get_size((map_t*)bb)) { - RETURN_NEQ((int)(i - map_get_size((map_t*)bb))); - } - if (i == 0) { - goto pop_next; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) : + { + struct erts_cmp_hashmap_state* sp; + if (is_flatmap_header(ahdr)) { + if (!is_flatmap_rel(b,b_base)) { + if (is_hashmap_rel(b,b_base)) { + aa = (Eterm *)flatmap_val_rel(a,a_base); + i = flatmap_get_size((flatmap_t*)aa) - hashmap_size_rel(b,b_base); + ASSERT(i != 0); + RETURN_NEQ(i); + } + a_tag = MAP_DEF; + goto mixed_types; + } + aa = (Eterm *)flatmap_val_rel(a,a_base); + bb = (Eterm *)flatmap_val_rel(b,b_base); + + i = flatmap_get_size((flatmap_t*)aa); + if (i != flatmap_get_size((flatmap_t*)bb)) { + RETURN_NEQ((int)(i - flatmap_get_size((flatmap_t*)bb))); + } + if (i == 0) { + goto pop_next; + } + aa += 2; + bb += 2; + if (exact) { + i += 1; /* increment for tuple-keys */ + goto term_array; + } + else { + /* Value array */ + WSTACK_PUSH3(stack,(UWord)(bb+1),(UWord)(aa+1),TERM_ARRAY_OP_WORD(i)); + /* Switch back from 'exact' key compare */ + WSTACK_PUSH(stack,OP_WORD(SWITCH_EXACT_OFF_OP)); + /* Now do 'exact' compare of key tuples */ + a = *aa; + b = *bb; + exact = 1; + goto bodyrecur; + } + } + if (!is_hashmap_rel(b,b_base)) { + if (is_flatmap_rel(b,b_base)) { + bb = (Eterm *)flatmap_val_rel(b,b_base); + i = hashmap_size_rel(a,a_base) - flatmap_get_size((flatmap_t*)bb); + ASSERT(i != 0); + RETURN_NEQ(i); + } + a_tag = MAP_DEF; + goto mixed_types; + } + i = hashmap_size_rel(a,a_base) - hashmap_size_rel(b,b_base); + if (i) { + RETURN_NEQ(i); + } + if (hashmap_size_rel(a,a_base) == 0) { + goto pop_next; + } + + /* Hashmap compare strategy: + Phase 1. While keys are identical + Do synchronous stepping through leafs of both trees in hash + order. Maintain value compare result of minimal key. + + Phase 2. If key diff was found in phase 1 + Ignore values from now on. + Continue iterate trees by always advancing the one + lagging behind hash-wise. Identical keys are skipped. + A minimal key can only be candidate as tie-breaker if we + have passed that hash value in the other tree (which means + the key did not exist in the other tree). + */ + + sp = PSTACK_PUSH(hmap_stack); + hashmap_iterator_init(&stack, a, 0); + hashmap_iterator_init(&b_stack, b, 0); + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + sp->cmp_res = 0; + ASSERT(sp->ap && sp->bp); + + a = CAR(sp->ap); + b = CAR(sp->bp); + sp->was_exact = exact; + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; } - aa += 2; - bb += 2; - i += 1; /* increment for tuple-keys */ - goto term_array; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): if (!is_float_rel(b,b_base)) { a_tag = FLOAT_DEF; @@ -2983,8 +3564,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ goto not_equal; } } else { - /* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */ - WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER); + WSTACK_PUSH3(stack, (UWord)bb, (UWord)aa, TERM_ARRAY_OP_WORD(i)); goto tailrecur_ne; } } @@ -2996,22 +3576,179 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ pop_next: if (!WSTACK_ISEMPTY(stack)) { UWord something = WSTACK_POP(stack); - if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */ - aa = (Eterm*) something; - bb = (Eterm*) WSTACK_POP(stack); - i = WSTACK_POP(stack); - goto term_array; + struct erts_cmp_hashmap_state* sp; + if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* an operation */ + switch (GET_OP(something)) { + case TERM_ARRAY_OP: + i = GET_OP_ARG(something); + aa = (Eterm*)WSTACK_POP(stack); + bb = (Eterm*) WSTACK_POP(stack); + goto term_array; + + case SWITCH_EXACT_OFF_OP: + /* Done with exact compare of map keys, switch back */ + ASSERT(exact); + exact = 0; + goto pop_next; + + case HASHMAP_PHASE1_ARE_KEYS_EQUAL: { + sp = PSTACK_TOP(hmap_stack); + if (j) { + /* Key diff found, enter phase 2 */ + if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) { + sp->min_key = CAR(sp->ap); + sp->cmp_res = -1; + sp->ap = hashmap_iterator_next(&stack); + } + else { + sp->min_key = CAR(sp->bp); + sp->cmp_res = 1; + sp->bp = hashmap_iterator_next(&b_stack); + } + exact = 1; /* only exact key compares in phase 2 */ + goto case_HASHMAP_PHASE2_LOOP; + } + + /* No key diff found so far, compare values if min key */ + + if (sp->cmp_res) { + a = CAR(sp->ap); + b = sp->min_key; + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_IS_MIN_KEY)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + /* no min key-value found yet */ + a = CDR(sp->ap); + b = CDR(sp->bp); + exact = sp->was_exact; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + case HASHMAP_PHASE1_IS_MIN_KEY: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + a = CDR(sp->ap); + b = CDR(sp->bp); + exact = sp->was_exact; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + goto case_HASHMAP_PHASE1_LOOP; + + case HASHMAP_PHASE1_CMP_VALUES: + sp = PSTACK_TOP(hmap_stack); + if (j) { + sp->cmp_res = j; + sp->min_key = CAR(sp->ap); + } + case_HASHMAP_PHASE1_LOOP: + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + if (!sp->ap) { + /* end of maps with identical keys */ + ASSERT(!sp->bp); + j = sp->cmp_res; + exact = sp->was_exact; + (void) PSTACK_POP(hmap_stack); + ON_CMP_GOTO(j); + } + a = CAR(sp->ap); + b = CAR(sp->bp); + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + + case_HASHMAP_PHASE2_LOOP: + if (sp->ap && sp->bp) { + a = CAR(sp->ap); + b = CAR(sp->bp); + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + goto case_HASHMAP_PHASE2_NEXT_STEP; + + case HASHMAP_PHASE2_ARE_KEYS_EQUAL: + sp = PSTACK_TOP(hmap_stack); + if (j == 0) { + /* keys are equal, skip them */ + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + goto case_HASHMAP_PHASE2_LOOP; + } + /* fall through */ + case_HASHMAP_PHASE2_NEXT_STEP: + if (sp->ap || sp->bp) { + if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) { + ASSERT(sp->ap); + a = CAR(sp->ap); + b = sp->min_key; + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_A)); + } + else { /* hash_cmp > 0 */ + ASSERT(sp->bp); + a = CAR(sp->bp); + b = sp->min_key; + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_B)); + } + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + /* End of both maps */ + j = sp->cmp_res; + exact = sp->was_exact; + (void) PSTACK_POP(hmap_stack); + ON_CMP_GOTO(j); + + case HASHMAP_PHASE2_IS_MIN_KEY_A: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + sp->min_key = CAR(sp->ap); + sp->cmp_res = -1; + } + sp->ap = hashmap_iterator_next(&stack); + goto case_HASHMAP_PHASE2_LOOP; + + case HASHMAP_PHASE2_IS_MIN_KEY_B: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + sp->min_key = CAR(sp->bp); + sp->cmp_res = 1; + } + sp->bp = hashmap_iterator_next(&b_stack); + goto case_HASHMAP_PHASE2_LOOP; + + default: + ASSERT(!"Invalid cmp op"); + } /* switch */ } a = (Eterm) something; b = (Eterm) WSTACK_POP(stack); goto tailrecur; } - DESTROY_WSTACK(stack); + ASSERT(PSTACK_IS_EMPTY(hmap_stack)); + PSTACK_DESTROY(hmap_stack); + WSTACK_DESTROY(stack); + WSTACK_DESTROY(b_stack); return 0; not_equal: - DESTROY_WSTACK(stack); + if (!PSTACK_IS_EMPTY(hmap_stack)) { + WSTACK_ROLLBACK(stack, PSTACK_TOP(hmap_stack)->wstack_rollback); + goto pop_next; + } + PSTACK_DESTROY(hmap_stack); + WSTACK_DESTROY(stack); + WSTACK_DESTROY(b_stack); return j; #undef CMP_NODES @@ -3774,7 +4511,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, res->timer.timeout_func = timeout_func; res->timer.timer_ref = timer_ref; res->timer.id = id; - res->timer.tm.active = 0; /* MUST be initalized */ + erts_init_timer(&res->timer.tm); ASSERT(!*timer_ref); @@ -4351,8 +5088,8 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) */ Uint64 erts_timestamp_millis(void) { -#ifdef HAVE_GETHRTIME - return (Uint64) (sys_gethrtime() / 1000000); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return ERTS_MONOTONIC_TO_MSEC(erts_os_monotonic_time()); #else Uint64 res; SysTimeval tv; |