Add annotations for likely/unlikely

In a correct Erlang programs, we can expect that:

* A GC test instruction (such as test_heap) is more likely
  not to do the GC.
* A BIF is more likely to succeed than to fail.
* A BIF is more likely to fail in a guard than in a body.
* An apply or fun call is likely to succeed.

Annotate conditions accordingly.

1 files changed, 11 insertions, 5 deletions

diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 41dc761e90..bac96be7d3 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -42,7 +42,7 @@ JUMP(Fail) {
 
 GC_TEST(Ns, Nh, Live) {
     Uint need = $Nh + $Ns;
-    if (E - HTOP < need) {
+    if (ERTS_UNLIKELY(E - HTOP < need)) {
        SWAPOUT;
        PROCESS_MAIN_CHK_LOCKS(c_p);
        FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
@@ -55,7 +55,7 @@ GC_TEST(Ns, Nh, Live) {
 
 GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) {
     Uint need = $NeedHeap;
-    if (E - HTOP < need) {
+    if (ERTS_UNLIKELY(E - HTOP < need)) {
        SWAPOUT;
        reg[$Live] = $PreserveTerm;
        PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -99,7 +99,13 @@ FAIL_BODY() {
 
 FAIL_HEAD_OR_BODY(Fail) {
     //| -no_prefetch
-    if ($Fail) {
+
+    /*
+     * In a correctly working program, we expect failures in
+     * guards to be more likely than failures in bodies.
+     */
+
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     goto find_func_info;
@@ -122,7 +128,7 @@ SYSTEM_LIMIT(Fail) {
 
 BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
     //| -no_prefetch
-    if ($Fail) {
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     reg[0] = $Op1;
@@ -133,7 +139,7 @@ BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
 
 BIF_ERROR_ARITY_2(Fail, BIF, Op1, Op2) {
     //| -no_prefetch
-    if ($Fail) {
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     reg[0] = $Op1;