1 files changed, 313 insertions, 141 deletions

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index cbdaa459de..3173d3510e 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -78,11 +78,19 @@ enum DbIterSafety {
 ** The main meta table, containing all ets tables.
 */
 #ifdef ERTS_SMP
-#  define META_MAIN_TAB_LOCK_CNT 16
-static union {
-    erts_smp_spinlock_t lck;
-    byte _cache_line_alignment[64];
-}meta_main_tab_locks[META_MAIN_TAB_LOCK_CNT];
+
+#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8
+#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS)
+#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1)
+
+typedef union {
+    erts_smp_rwmtx_t rwmtx;
+    byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
+				sizeof(erts_smp_rwmtx_t))];
+} erts_meta_main_tab_lock_t;
+
+static erts_meta_main_tab_lock_t *meta_main_tab_locks;
+
 #endif
 static struct {
     union {
@@ -104,17 +112,13 @@ static struct {
 #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2)
 #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */
 
-static ERTS_INLINE void meta_main_tab_lock(unsigned slot)
-{
-#ifdef ERTS_SMP
-    erts_smp_spin_lock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck);
-#endif
-}
-
-static ERTS_INLINE void meta_main_tab_unlock(unsigned slot)
+static ERTS_INLINE erts_smp_rwmtx_t *
+get_meta_main_tab_lock(unsigned slot)
 {
 #ifdef ERTS_SMP
-    erts_smp_spin_unlock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck);
+    return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx;
+#else
+    return NULL;
 #endif
 }
 
@@ -166,7 +170,8 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name,
 typedef enum {
     LCK_READ=1,     /* read only access */
     LCK_WRITE=2,    /* exclusive table write access */
-    LCK_WRITE_REC=3 /* record write access */
+    LCK_WRITE_REC=3, /* record write access */
+    LCK_NONE=4
 } db_lock_kind_t;
 
 extern DbTableMethod db_hash;
@@ -174,6 +179,7 @@ extern DbTableMethod db_tree;
 
 int user_requested_db_max_tabs;
 int erts_ets_realloc_always_moves;
+int erts_ets_always_compress;
 static int db_max_tabs;
 static DbTable *meta_pid_to_tab; /* Pid mapped to owned tables */
 static DbTable *meta_pid_to_fixed_tab; /* Pid mapped to fixed tables */
@@ -213,61 +219,87 @@ Export ets_select_continue_exp;
  * Static traps
  */
 static Export ets_delete_continue_exp;
-
-static ERTS_INLINE DbTable* db_ref(DbTable* tb)
-{
-    if (tb != NULL) {
-	erts_refc_inc(&tb->common.ref, 2);
-    }
-    return tb;
-}
-
-static ERTS_INLINE DbTable* db_unref(DbTable* tb)
+	
+static void
+free_dbtable(DbTable* tb)
 {
-    if (!erts_refc_dectest(&tb->common.ref, 0)) {
 #ifdef HARDDEBUG
 	if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) {
-	    erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n",
-			 erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), 
+	    erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n",
+			 erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable),
 			 tb->common.fixations);
 	}
-	erts_fprintf(stderr, "ets: db_unref(%T) deleted!!!\r\n", 
+	erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n",
 		     tb->common.id);
 
-	erts_fprintf(stderr, "ets: db_unref: meta_pid_to_tab common.memory_size = %ld\n",
+	erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n",
 		     erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size));
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab);
 
 
-	erts_fprintf(stderr, "ets: db_unref: meta_pid_to_fixed_tab common.memory_size = %ld\n",
+	erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n",
 		     erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab);
-	
 #endif
 #ifdef ERTS_SMP
 	erts_smp_rwmtx_destroy(&tb->common.rwlock);
 	erts_smp_mtx_destroy(&tb->common.fixlock);
 #endif
 	ASSERT(is_immed(tb->common.heir_data));
-	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));		     
+	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
-	return NULL;
-    }
-    return tb;
 }
 
-static ERTS_INLINE void db_init_lock(DbTable* tb, char *rwname, char* fixname)
+#ifdef ERTS_SMP
+static void
+chk_free_dbtable(void *vtb)
+{
+    DbTable * tb = (DbTable *) vtb;
+    ERTS_THR_MEMORY_BARRIER;
+    if (erts_refc_dectest(&tb->common.ref, 0) == 0)
+	free_dbtable(tb);
+}
+#endif
+
+static void schedule_free_dbtable(DbTable* tb)
+{
+    /*
+     * NON-SMP case: Caller is *not* allowed to access the *tb
+     *               structure after this function has returned!          
+     * SMP case:     Caller is allowed to access the *tb structure
+     *               until the bif has returned (we typically
+     *               need to unlock the table lock after this
+     *               function has returned).
+     */
+#ifdef ERTS_SMP
+    int scheds = erts_get_max_no_executing_schedulers();
+    ASSERT(scheds >= 1);
+    ASSERT(erts_refc_read(&tb->common.ref, 0) == 0);
+    erts_refc_init(&tb->common.ref, scheds);
+    ERTS_THR_MEMORY_BARRIER;
+    erts_smp_schedule_misc_aux_work(0, scheds, chk_free_dbtable, tb);
+#else
+    free_dbtable(tb);
+#endif
+}
+
+static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
+				     char *rwname, char* fixname)
 {
-    erts_refc_init(&tb->common.ref, 1);
-    erts_refc_init(&tb->common.fixref, 0);
 #ifdef ERTS_SMP
-    erts_smp_rwmtx_init_x(&tb->common.rwlock, rwname, tb->common.the_name);
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    if (use_frequent_read_lock)
+	rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+#endif
+#ifdef ERTS_SMP
+    erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt,
+			      rwname, tb->common.the_name);
     erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name);
     tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED);
 #endif
 }
 
-static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind)
+static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
 {
 #ifdef ERTS_SMP
     ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
@@ -295,16 +327,13 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind)
 #endif
 }
 
-static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
-{
-    (void) db_ref(tb);
-#ifdef ERTS_SMP
-    db_lock_take_over_ref(tb, kind);
-#endif
-}
-
 static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 {
+    /*
+     * In NON-SMP case tb may refer to an already deallocated
+     * DbTable structure. That is, ONLY the SMP case is allowed
+     * to follow the tb pointer!
+     */
 #ifdef ERTS_SMP
     ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
 
@@ -331,7 +360,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 	}
     }
 #endif
-    (void) db_unref(tb); /* May delete table... */
 }
 
 
@@ -349,56 +377,85 @@ static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind)
 }
 
 static ERTS_INLINE
-DbTable* db_get_table(Process *p,
-		      Eterm id,
-		      int what,
-		      db_lock_kind_t kind)
+DbTable* db_get_table_aux(Process *p,
+			  Eterm id,
+			  int what,
+			  db_lock_kind_t kind,
+			  int meta_already_locked)
 {
     DbTable *tb = NULL;
+    erts_smp_rwmtx_t *mtl = NULL;
+
+    /*
+     * IMPORTANT: Only scheduler threads are allowed
+     *            to access tables. Memory management
+     *            depend on it.
+     */
+    ASSERT(erts_get_scheduler_data());
 
     if (is_small(id)) {
 	Uint slot = unsigned_val(id) & meta_main_tab_slot_mask;
-	meta_main_tab_lock(slot);
-	if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) {
-	    /* SMP: inc to prevent race, between unlock of meta_main_tab_lock
-	     * and the table locking outside the meta_main_tab_lock
-	     */
-	    tb = db_ref(meta_main_tab[slot].u.tb);
+	if (!meta_already_locked) {
+	    mtl = get_meta_main_tab_lock(slot);
+	    erts_smp_rwmtx_rlock(mtl);
+	}
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+	else {
+	    erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot);
+	    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl)
+			       || erts_lc_rwmtx_is_rwlocked(test_mtl));
 	}
-	meta_main_tab_unlock(slot);
+#endif
+	if (slot < db_max_tabs && IS_SLOT_ALIVE(slot))
+	    tb = meta_main_tab[slot].u.tb;
     }
     else if (is_atom(id)) {
-	erts_smp_rwmtx_t* rwlock;
-	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&rwlock);
-	erts_smp_rwmtx_rlock(rwlock);
+	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl);
+	if (!meta_already_locked)
+	    erts_smp_rwmtx_rlock(mtl);
+	else{
+	    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl)
+			       || erts_lc_rwmtx_is_rwlocked(mtl));
+	    mtl = NULL;
+	}
+
 	if (bucket->pu.tb != NULL) {
 	    if (is_atom(bucket->u.name_atom)) { /* single */
-		if (bucket->u.name_atom == id) {
-		    tb = db_ref(bucket->pu.tb);
-		}
+		if (bucket->u.name_atom == id)
+		    tb = bucket->pu.tb;
 	    }
 	    else { /* multi */
 		Uint cnt = unsigned_val(bucket->u.mcnt);
 		Uint i;
 		for (i=0; i<cnt; i++) {
 		    if (bucket->pu.mvec[i].u.name_atom == id) {
-			tb = db_ref(bucket->pu.mvec[i].pu.tb);
+			tb = bucket->pu.mvec[i].pu.tb;
 			break;
 		    }
 		}
 	    }
 	}
-	erts_smp_rwmtx_runlock(rwlock);
     }
     if (tb) {
-	db_lock_take_over_ref(tb, kind);
-	if (tb->common.id == id && ((tb->common.status & what) != 0 || 
-				    p->id == tb->common.owner)) {
-	    return tb;
+	db_lock(tb, kind);
+	if (tb->common.id != id
+	    || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) {
+	    db_unlock(tb, kind);
+	    tb = NULL;
 	}
-	db_unlock(tb, kind);
     }
-    return NULL;
+    if (mtl)
+	erts_smp_rwmtx_runlock(mtl);
+    return tb;
+}
+
+static ERTS_INLINE
+DbTable* db_get_table(Process *p,
+		      Eterm id,
+		      int what,
+		      db_lock_kind_t kind)
+{
+    return db_get_table_aux(p, id, what, kind, 0);
 }
 
 /* Requires meta_main_tab_locks[slot] locked.
@@ -413,15 +470,15 @@ static ERTS_INLINE void free_slot(int slot)
     erts_smp_spin_unlock(&meta_main_tab_main_lock);
 }
 
-static int insert_named_tab(Eterm name_atom, DbTable* tb)
+static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock)
 {
     int ret = 0;
     erts_smp_rwmtx_t* rwlock;
     struct meta_name_tab_entry* new_entry;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
-
-    erts_smp_rwmtx_rwlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwlock(rwlock);
 
     if (bucket->pu.tb == NULL) { /* empty */
 	new_entry = bucket;
@@ -468,17 +525,32 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb)
     ret = 1; /* Ok */
 
 done:
-    erts_smp_rwmtx_rwunlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwunlock(rwlock);
     return ret;
 }
 
-static int remove_named_tab(Eterm name_atom)
+static int remove_named_tab(DbTable *tb, int have_lock)
 {
     int ret = 0;
     erts_smp_rwmtx_t* rwlock;
+    Eterm name_atom = tb->common.id;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
-    erts_smp_rwmtx_rwlock(rwlock);
+#ifdef ERTS_SMP
+    if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) {
+	/*
+	 * We keep our increased refc over this op in order to
+	 * prevent the table from disapearing.
+	 */
+	erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	erts_smp_rwmtx_rwlock(rwlock);
+	erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+    }
+#endif
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock));
+
     if (bucket->pu.tb == NULL) {
 	goto done;
     }
@@ -529,7 +601,8 @@ static int remove_named_tab(Eterm name_atom)
     ret = 1; /* Ok */
 
 done:
-    erts_smp_rwmtx_rwunlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwunlock(rwlock);
     return ret;
 }
 
@@ -538,11 +611,11 @@ done:
 */
 static ERTS_INLINE void local_fix_table(DbTable* tb)
 {
-    erts_refc_inc(&tb->common.fixref, 1);
+    erts_refc_inc(&tb->common.ref, 1);
 }	    
 static ERTS_INLINE void local_unfix_table(DbTable* tb)
 {	
-    if (erts_refc_dectest(&tb->common.fixref, 0) == 0) {
+    if (erts_refc_dectest(&tb->common.ref, 0) == 0) {
 	ASSERT(IS_HASH_TABLE(tb->common.status));
 	db_unfix_table_hash(&(tb->hash));
     }
@@ -876,7 +949,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 		position > arityval(handle.dbterm->tpl[0])) {
 		goto finalize;
 	    }
-	    oldcnt = handle.dbterm->tpl[position];
+	    oldcnt = db_do_read_element(&handle, position);
 	    if (is_big(oldcnt)) {
 		halloc_size += BIG_NEED_SIZE(big_arity(oldcnt));
 	    }
@@ -1133,6 +1206,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 {
     DbTable* tb;
     Eterm ret;
+    erts_smp_rwmtx_t *lck1, *lck2;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -1141,34 +1215,65 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 		BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
 #endif
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
+
+    if (is_not_atom(BIF_ARG_2)) {
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    if (is_not_atom(BIF_ARG_2)) {
-	goto badarg;
+    (void) meta_name_tab_bucket(BIF_ARG_2, &lck1);
+
+    if (is_small(BIF_ARG_1)) {
+	Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask;
+	lck2 = get_meta_main_tab_lock(slot);
     }
+    else if (is_atom(BIF_ARG_1)) {
+	(void) meta_name_tab_bucket(BIF_ARG_1, &lck2);
+	if (lck1 == lck2)
+	    lck2 = NULL;
+	else if (lck1 > lck2) {
+	    erts_smp_rwmtx_t *tmp = lck1;
+	    lck1 = lck2;
+	    lck2 = tmp;
+	}
+    }
+    else {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
+    erts_smp_rwmtx_rwlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwlock(lck2);
+
+    tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1);
+    if (!tb)
+	goto badarg;
 
     if (is_not_atom(tb->common.id)) { /* Not a named table */
 	tb->common.the_name = BIF_ARG_2;
 	goto done;
     }
 
-    if (!insert_named_tab(BIF_ARG_2,tb)) {
+    if (!insert_named_tab(BIF_ARG_2, tb, 1))
 	goto badarg;
-    }
-    if (!remove_named_tab(tb->common.id)) {
+
+    if (!remove_named_tab(tb, 1))
 	erl_exit(1,"Could not find named tab %s", tb->common.id);
-    }
 
     tb->common.id = tb->common.the_name = BIF_ARG_2;
 
  done:
     ret = tb->common.id;
     db_unlock(tb, LCK_WRITE);
+    erts_smp_rwmtx_rwunlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwunlock(lck2);
     BIF_RET(ret);
  badarg:
-    db_unlock(tb, LCK_WRITE);
+    if (tb)
+	db_unlock(tb, LCK_WRITE);
+    erts_smp_rwmtx_rwunlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwunlock(lck2);
     BIF_ERROR(BIF_P, BADARG);    
 }
 
@@ -1189,10 +1294,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     UWord heir_data;
     Uint32 status;
     Sint keypos;
-    int is_named, is_fine_locked;
+    int is_named, is_fine_locked, frequent_read, is_compressed;
     int cret;
     DeclareTmpHeap(meta_tuple,3,BIF_P);
     DbTableMethod* meth;
+    erts_smp_rwmtx_t *mmtl;
 
     if (is_not_atom(BIF_ARG_1)) {
 	BIF_ERROR(BIF_P, BADARG);
@@ -1205,8 +1311,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     keypos = 1;
     is_named = 0;
     is_fine_locked = 0;
+    frequent_read = 0;
     heir = am_none;
     heir_data = (UWord) am_undefined;
+    is_compressed = erts_ets_always_compress;
 
     list = BIF_ARG_2;
     while(is_list(list)) {
@@ -1238,6 +1346,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 			is_fine_locked = 0;
 		    } else break;
 		}
+		else if (tp[1] == am_read_concurrency) {
+		    if (tp[2] == am_true) {
+			frequent_read = 1;
+		    } else if (tp[2] == am_false) {
+			frequent_read = 0;
+		    } else break;
+		}
 		else if (tp[1] == am_heir && tp[2] == am_none) {
 		    heir = am_none;
 		    heir_data = am_undefined;
@@ -1262,6 +1377,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	else if (val == am_named_table) {
 	    is_named = 1;
 	}
+	else if (val == am_compressed) {
+	    is_compressed = 1;
+	}
 	else if (val == am_set || val == am_protected)
 	    ;
 	else break;
@@ -1286,6 +1404,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
+#ifdef ERTS_SMP
+    if (frequent_read && !(status & DB_PRIVATE))
+	status |= DB_FREQ_READ;
+#endif
+
     /* we create table outside any table lock
      * and take the unusal cost of destroy table if it
      * fails to find a slot 
@@ -1308,7 +1431,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.type = status & ERTS_ETS_TABLE_TYPES;
     /* Note, 'type' is *read only* from now on... */
 #endif
-    db_init_lock(tb, "db_tab", "db_tab_fix");
+    erts_refc_init(&tb->common.ref, 0);
+    db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ),
+		 "db_tab", "db_tab_fix");
     tb->common.keypos = keypos;
     tb->common.owner = BIF_P->id;
     set_heir(BIF_P, tb, heir, heir_data);
@@ -1316,6 +1441,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     erts_smp_atomic_init(&tb->common.nitems, 0);
 
     tb->common.fixations = NULL;
+    tb->common.compress = is_compressed;
 
     cret = meth->db_create(BIF_P, tb);
     ASSERT(cret == DB_ERROR_NONE);
@@ -1328,8 +1454,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 				      "** Too many db tables **\n");
 	free_heir_data(tb);
 	tb->common.meth->db_free_table(tb);
-	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
-	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
+	free_dbtable(tb);
 	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
     }
 
@@ -1351,19 +1476,22 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.id = ret;
     tb->common.slot = slot;           /* store slot for erase */
 
-    meta_main_tab_lock(slot);
+    mmtl = get_meta_main_tab_lock(slot);
+    erts_smp_rwmtx_rwlock(mmtl);
     meta_main_tab[slot].u.tb = tb;
     ASSERT(IS_SLOT_ALIVE(slot));
-    meta_main_tab_unlock(slot);
+    erts_smp_rwmtx_rwunlock(mmtl);
 
-    if (is_named && !insert_named_tab(BIF_ARG_1, tb)) {
-	meta_main_tab_lock(slot);
+    if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) {
+	mmtl = get_meta_main_tab_lock(slot);
+	erts_smp_rwmtx_rwlock(mmtl);
 	free_slot(slot);
-	meta_main_tab_unlock(slot);
+	erts_smp_rwmtx_rwunlock(mmtl);
 
-	db_lock_take_over_ref(tb,LCK_WRITE);
+	db_lock(tb,LCK_WRITE);
 	free_heir_data(tb);
 	tb->common.meth->db_free_table(tb);
+	schedule_free_dbtable(tb);
 	db_unlock(tb,LCK_WRITE);
 	BIF_ERROR(BIF_P, BADARG);
     }
@@ -1499,6 +1627,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 {
     int trap;
     DbTable* tb;
+    erts_smp_rwmtx_t *mmtl;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -1520,13 +1649,23 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE);
     tb->common.status |= DB_DELETE;
 
-    meta_main_tab_lock(tb->common.slot);
+    mmtl = get_meta_main_tab_lock(tb->common.slot);
+#ifdef ERTS_SMP
+    if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
+	/*
+	 * We keep our increased refc over this op in order to
+	 * prevent the table from disapearing.
+	 */
+	erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	erts_smp_rwmtx_rwlock(mmtl);
+	erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+    }
+#endif
     /* We must keep the slot, to be found by db_proc_dead() if process dies */
     MARK_SLOT_DEAD(tb->common.slot);
-    meta_main_tab_unlock(tb->common.slot);
-    if (is_atom(tb->common.id)) {
-	remove_named_tab(tb->common.id);
-    }
+    erts_smp_rwmtx_rwunlock(mmtl);
+    if (is_atom(tb->common.id))
+	remove_named_tab(tb, 0);
     
     if (tb->common.owner != BIF_P->id) {
 	DeclareTmpHeap(meta_tuple,3,BIF_P);
@@ -1919,14 +2058,15 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0)
     previous = NIL;
     j = 0;
     for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) {
-	meta_main_tab_lock(i);
+	erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i);
+	erts_smp_rwmtx_rlock(mmtl);
 	if (IS_SLOT_ALIVE(i)) {
 	    j++;
 	    tb = meta_main_tab[i].u.tb;
 	    previous = CONS(hp, tb->common.id, previous);
 	    hp += 2;
 	}
-	meta_main_tab_unlock(i);
+	erts_smp_rwmtx_runlock(mmtl);
     }
     HRelease(BIF_P, hendp, hp);
     BIF_RET(previous);
@@ -2456,7 +2596,7 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
 BIF_RETTYPE ets_info_1(BIF_ALIST_1)
 {
     static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table,
-	am_node, am_size, am_name, am_heir, am_owner, am_memory};
+	am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed};
     Eterm results[sizeof(fields)/sizeof(Eterm)];
     DbTable* tb;
     Eterm res;
@@ -2630,12 +2770,30 @@ void init_db(void)
     size_t size;
 
 #ifdef ERTS_SMP
-    for (i=0; i<META_MAIN_TAB_LOCK_CNT; i++) {
-	erts_smp_spinlock_init_x(&meta_main_tab_locks[i].lck, "meta_main_tab_slot", make_small(i));
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+    meta_main_tab_locks = erts_alloc(ERTS_ALC_T_DB_TABLES,
+				     (sizeof(erts_meta_main_tab_lock_t)
+				      * (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE+1)));
+
+    if ((((UWord) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) != 0)
+	meta_main_tab_locks = ((erts_meta_main_tab_lock_t *)
+			       ((((UWord) meta_main_tab_locks)
+				 & ~ERTS_CACHE_LINE_MASK)
+				+ ERTS_CACHE_LINE_SIZE));
+
+    ASSERT((((UWord) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) == 0);
+
+    for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) {
+	erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt,
+				  "meta_main_tab_slot", make_small(i));
     }
     erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main");
     for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) {
-	erts_smp_rwmtx_init_x(&meta_name_tab_rwlocks[i].lck, "meta_name_tab", make_small(i));
+	erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt,
+				  "meta_name_tab", make_small(i));
     }
 #endif
 
@@ -2703,9 +2861,9 @@ void init_db(void)
     erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0);
     meta_pid_to_tab->common.slot   = -1;
     meta_pid_to_tab->common.meth   = &db_hash;
+    meta_pid_to_tab->common.compress = 0;
 
-    erts_refc_init(&meta_pid_to_tab->common.ref, 1);
-    erts_refc_init(&meta_pid_to_tab->common.fixref, 0);
+    erts_refc_init(&meta_pid_to_tab->common.ref, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
 
@@ -2735,9 +2893,9 @@ void init_db(void)
     erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0);
     meta_pid_to_fixed_tab->common.slot   = -1;
     meta_pid_to_fixed_tab->common.meth   = &db_hash;
+    meta_pid_to_fixed_tab->common.compress = 0;
 
-    erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 1);
-    erts_refc_init(&meta_pid_to_fixed_tab->common.fixref, 0);
+    erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
 
@@ -2895,12 +3053,10 @@ retry:
 				to_pid, to_locks,
 				ERTS_P2P_FLG_TRY_LOCK);
     if (to_proc == ERTS_PROC_LOCK_BUSY) {
-	db_ref(tb); /* while unlocked */
 	db_unlock(tb,LCK_WRITE);    
 	to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
 				to_pid, to_locks);    
 	db_lock(tb,LCK_WRITE);
-	tb = db_unref(tb);
 	ASSERT(tb != NULL);
     
 	if (tb->common.owner != p->id) {
@@ -2943,7 +3099,7 @@ retry:
     db_unlock(tb,LCK_WRITE);
     heir_data = tb->common.heir_data;
     if (!is_immed(heir_data)) {
-	Eterm* tpv = DBTERM_BUF((DbTerm*)heir_data); /* tuple_val */
+	Eterm* tpv = ((DbTerm*)heir_data)->tpl; /* tuple_val */
 	ASSERT(arityval(*tpv) == 1);
 	heir_data = tpv[1];
     }
@@ -3008,15 +3164,16 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 	    while (state->slots.ix < state->slots.size) {
 		DbTable *tb = NULL;
 		Sint ix = unsigned_val(state->slots.arr[state->slots.ix]);
-		meta_main_tab_lock(ix);
+		erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
+		erts_smp_rwmtx_rlock(mmtl);
 		if (!IS_SLOT_FREE(ix)) {
-		    tb = db_ref(GET_ANY_SLOT_TAB(ix));
+		    tb = GET_ANY_SLOT_TAB(ix);
 		    ASSERT(tb);
 		}
-		meta_main_tab_unlock(ix);
+		erts_smp_rwmtx_runlock(mmtl);
 		if (tb) {
 		    int do_yield;
-		    db_lock_take_over_ref(tb, LCK_WRITE);
+		    db_lock(tb, LCK_WRITE);
 		    /* Ownership may have changed since
 		       we looked up the table. */
 		    if (tb->common.owner != pid) {
@@ -3045,7 +3202,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 			    tb->common.status |= DB_DELETE;
 
 			    if (is_atom(tb->common.id))
-				remove_named_tab(tb->common.id);
+				remove_named_tab(tb, 0);
 
 			    free_heir_data(tb);
 			    free_fixations_locked(tb);
@@ -3095,17 +3252,18 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 	    while (state->slots.ix < state->slots.size) {
 		DbTable *tb = NULL;
 		Sint ix = unsigned_val(state->slots.arr[state->slots.ix]);
-		meta_main_tab_lock(ix);
+		erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
+		erts_smp_rwmtx_rlock(mmtl);
 		if (IS_SLOT_ALIVE(ix)) {
-		    tb = db_ref(meta_main_tab[ix].u.tb);
+		    tb = meta_main_tab[ix].u.tb;
 		    ASSERT(tb);
 		}
-		meta_main_tab_unlock(ix);
+		erts_smp_rwmtx_runlock(mmtl);
 		if (tb) {
 		    int reds;
 		    DbFixation** pp;
 
-		    db_lock_take_over_ref(tb, LCK_WRITE_REC);
+		    db_lock(tb, LCK_WRITE_REC);
 		    #ifdef ERTS_SMP
 		    erts_smp_mtx_lock(&tb->common.fixlock);
 		    #endif
@@ -3115,7 +3273,8 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 			  pp = &(*pp)->next) {
 			if ((*pp)->pid == pid) {
 			    DbFixation* fix = *pp;
-			    erts_refc_add(&tb->common.fixref,-fix->counter,0);
+			    erts_aint_t diff = -((erts_aint_t) fix->counter);
+			    erts_refc_add(&tb->common.ref,diff,0);
 			    *pp = fix->next;
 			    erts_db_free(ERTS_ALC_T_DB_FIXATION,
 					 tb, fix, sizeof(DbFixation));
@@ -3190,7 +3349,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
 #ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
 #endif
-    erts_refc_inc(&tb->common.fixref,1);
+    erts_refc_inc(&tb->common.ref,1);
     fix = tb->common.fixations;
     if (fix == NULL) { 
 	get_now(&(tb->common.megasec),
@@ -3244,7 +3403,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
     for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) {
 	if ((*pp)->pid == p->id) {
 	    DbFixation* fix = *pp;
-	    erts_refc_dec(&tb->common.fixref,0);
+	    erts_refc_dec(&tb->common.ref,0);
 	    --(fix->counter);
 	    ASSERT(fix->counter >= 0);
 	    if (fix->counter > 0) {
@@ -3270,7 +3429,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 unlocked:
 
     if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)
-	&& erts_smp_atomic_read(&tb->hash.fixdel) != (long)NULL) {
+	&& erts_smp_atomic_read(&tb->hash.fixdel) != (erts_aint_t)NULL) {
 #ifdef ERTS_SMP
 	if (*kind_p == LCK_READ && tb->common.is_thread_safe) {
 	    /* Must have write lock while purging pseudo-deleted (OTP-8166) */
@@ -3292,6 +3451,8 @@ static void free_fixations_locked(DbTable *tb)
 
     fix = tb->common.fixations;
     while (fix != NULL) {
+	erts_aint_t diff = -((erts_aint_t) fix->counter);
+	erts_refc_add(&tb->common.ref,diff,0);
 	next_fix = fix->next;
 	db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
 	db_erase_bag_exact2(meta_pid_to_fixed_tab,
@@ -3332,8 +3493,8 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data)
 
 	UseTmpHeap(2,me);
 	/* Make a dummy 1-tuple around data to use db_get_term() */
-	heir_data = (UWord) db_get_term(&tb->common, NULL, 0,
-					TUPLE1(tmp,heir_data));
+	heir_data = (UWord) db_store_term(&tb->common, NULL, 0,
+					  TUPLE1(tmp,heir_data));
 	UnUseTmpHeap(2,me);
 	ASSERT(!is_immed(heir_data));
     }
@@ -3344,7 +3505,7 @@ static void free_heir_data(DbTable* tb)
 {
     if (tb->common.heir != am_none && !is_immed(tb->common.heir_data)) {
 	DbTerm* p = (DbTerm*) tb->common.heir_data;
-	db_free_term_data(p);
+	db_cleanup_offheap_comp(p);
 	erts_db_free(ERTS_ALC_T_DB_TERM, tb, (void *)p,
 		     sizeof(DbTerm) + (p->size-1)*sizeof(Eterm));
     }
@@ -3386,6 +3547,7 @@ static int free_table_cont(Process *p,
 			   int clean_meta_tab)
 {
     Eterm result;
+    erts_smp_rwmtx_t *mmtl;
 
 #ifdef HARDDEBUG
     if (!first) {
@@ -3411,9 +3573,16 @@ static int free_table_cont(Process *p,
 		     tb->common.id);
 #endif
 	/* Completely done - we will not get called again. */
-	meta_main_tab_lock(tb->common.slot);
+	mmtl = get_meta_main_tab_lock(tb->common.slot);
+#ifdef ERTS_SMP
+	if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
+	    erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	    erts_smp_rwmtx_rwlock(mmtl);
+	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+	}
+#endif
 	free_slot(tb->common.slot);
-	meta_main_tab_unlock(tb->common.slot);
+	erts_smp_rwmtx_rwunlock(mmtl);
 
 	if (clean_meta_tab) {
 	    db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
@@ -3421,7 +3590,7 @@ static int free_table_cont(Process *p,
 				make_small(tb->common.slot));
 	    db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 	}
-	db_unref(tb);
+	schedule_free_dbtable(tb);
 	BUMP_REDS(p, 100);
 	return 0;
     }
@@ -3469,10 +3638,13 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	ret = erts_this_dist_entry->sysname;
     } else if (What == am_named_table) {
 	ret = is_atom(tb->common.id) ? am_true : am_false;
+    } else if (What == am_compressed) {
+	ret = tb->common.compress ? am_true : am_false;
+    }
     /*
      * For debugging purposes
      */
-    } else if (What == am_data) { 
+    else if (What == am_data) {
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, tb);
 	ret = am_true;
     } else if (What == am_atom_put("fixed",5)) {