Implement packing of 'f' and 'j'

4 files changed, 160 insertions, 19 deletions

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 0ff4ed39b4..39e96126bf 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -425,7 +425,9 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	while (start_prog < prog) {
 	    prog--;
 	    switch (*prog) {
+	    case 'f':
 	    case 'g':
+	    case 'q':
 		*ap++ = *--sp;
 		break;
 	    case 'i':		/* Initialize packing accumulator. */
@@ -805,7 +807,7 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif)
 
 static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap)
 {
-    return base - 1 + opc[op].adjust + *ap;
+    return base - 1 + opc[op].adjust + (Sint32) *ap;
 }
 
 
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 55990362ff..1baf3ff7ab 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -235,6 +235,8 @@ void** beam_ops;
      ERTS_UNREQ_PROC_MAIN_LOCK((P))
 
 #define db(N) (N)
+#define fb(N) ((Sint)(Sint32)(N))
+#define jb(N) ((Sint)(Sint32)(N))
 #define tb(N) (N)
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 2888158c7b..2acf3a2c1c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -87,6 +87,8 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 
 typedef struct {
     Uint pos;                   /* Position of label reference to patch. */
+    Uint offset;                /* Offset from patch location.  */
+    int packed;                 /* 0 (not packed), 1 (lsw), 2 (msw) */
 } LabelPatch;
 
 /*
@@ -237,7 +239,7 @@ typedef struct {
 
 typedef struct literal_patch LiteralPatch;
 struct literal_patch {
-    int pos;			/* Position in code */
+    Uint pos;			/* Position in code */
     LiteralPatch* next;
 };
 
@@ -1948,8 +1950,11 @@ register_label_patch(LoaderState* stp, Uint label, Uint ci, Uint offset)
                                    (void *) lp->patches,
                                    lp->num_allocated * sizeof(LabelPatch));
     }
-    lp->patches[lp->num_patches++].pos = ci;
-    stp->codev[ci] = offset;
+    lp->patches[lp->num_patches].pos = ci;
+    lp->patches[lp->num_patches].offset = offset;
+    lp->patches[lp->num_patches].packed = 0;
+    lp->num_patches++;
+    stp->codev[ci] = label;
 }
 
 static int
@@ -2519,23 +2524,58 @@ load_code(LoaderState* stp)
 	    char* prog;            /* Program for packing engine. */
 	    struct pack_stack {
                 BeamInstr instr;
-                LiteralPatch* patch;
+                Uint* patch_pos;
             } stack[8];            /* Stack. */
 	    struct pack_stack* sp = stack; /* Points to next free position. */
 	    BeamInstr packed = 0; /* Accumulator for packed operations. */
+            LabelPatch* packed_label = 0;
 
 	    for (prog = opc[stp->specific_op].pack; *prog; prog++) {
 		switch (*prog) {
-		case 'g':	/* Get instruction; push on stack. */
+		case 'g':	/* Get operand and push on stack. */
+                    ci--;
+                    sp->instr = code[ci];
+                    sp->patch_pos = 0;
+                    sp++;
+                    break;
+		case 'f':	/* Get possible 'f' operand and push on stack. */
+                    {
+                        Uint w = code[--ci];
+                        sp->instr = w;
+                        sp->patch_pos = 0;
+
+                        if (w != 0) {
+                            LabelPatch* lbl_p;
+                            int num_patches;
+                            int patch;
+
+                            ASSERT(w < stp->num_labels);
+                            lbl_p = stp->labels[w].patches;
+                            num_patches = stp->labels[w].num_patches;
+                            for (patch = num_patches - 1; patch >= 0; patch--) {
+                                if (lbl_p[patch].pos == ci) {
+                                    sp->patch_pos = &lbl_p[patch].pos;
+                                    break;
+                                }
+                            }
+                            ASSERT(sp->patch_pos);
+                        }
+                        sp++;
+                    }
+                    break;
+		case 'q':	/* Get possible 'q' operand and push on stack. */
                     {
                         LiteralPatch* lp;
 
                         ci--;
                         sp->instr = code[ci];
-                        sp->patch = 0;
-                        for (lp = stp->literal_patches; lp && lp->pos > ci-MAX_OPARGS; lp = lp->next) {
+                        sp->patch_pos = 0;
+
+                        for (lp = stp->literal_patches;
+                             lp && lp->pos > ci-MAX_OPARGS;
+                             lp = lp->next) {
                             if (lp->pos == ci) {
-                                sp->patch = lp;
+                                sp->patch_pos = &lp->pos;
                                 break;
                             }
                         }
@@ -2547,28 +2587,68 @@ load_code(LoaderState* stp)
 		    break;
 		case '0':	/* Tight shift */
 		    packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci];
+                    if (packed_label) {
+                        packed_label->packed++;
+                    }
 		    break;
 		case '6':	/* Shift 16 steps */
 		    packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci];
+                    if (packed_label) {
+                        packed_label->packed++;
+                    }
 		    break;
 #ifdef ARCH_64
 		case 'w':	/* Shift 32 steps */
-		    packed = (packed << BEAM_WIDE_SHIFT) | code[--ci];
-		    break;
+                    {
+                        Uint w = code[--ci];
+
+                        if (packed_label) {
+                            packed_label->packed++;
+                        }
+
+                        /*
+                         * 'w' can handle both labels ('f' and 'j'), as well
+                         * as 'I'. Test whether this is a label.
+                         */
+
+                        if (w < stp->num_labels) {
+                            /*
+                             * Probably a label. Look for patch pointing to this
+                             * position.
+                             */
+                            LabelPatch* lp = stp->labels[w].patches;
+                            int num_patches = stp->labels[w].num_patches;
+                            int patch;
+                            for (patch = num_patches - 1; patch >= 0; patch--) {
+                                if (lp[patch].pos == ci) {
+                                    lp[patch].packed = 1;
+                                    packed_label = &lp[patch];
+                                    break;
+                                }
+                            }
+                        }
+                        packed = (packed << BEAM_WIDE_SHIFT) |
+                            (code[ci] & BEAM_WIDE_MASK);
+                    }
+                    break;
 #endif
 		case 'p':	/* Put instruction (from stack). */
                     --sp;
                     code[ci] = sp->instr;
-                    if (sp->patch) {
-                        sp->patch->pos = ci;
+                    if (sp->patch_pos) {
+                        *sp->patch_pos = ci;
                     }
                     ci++;
 		    break;
 		case 'P':	/* Put packed operands. */
                     sp->instr = packed;
-                    sp->patch = 0;
+                    sp->patch_pos = 0;
                     sp++;
 		    packed = 0;
+                    if (packed_label) {
+                        packed_label->pos = ci;
+                        packed_label = 0;
+                    }
 		    break;
 		default:
 		    ASSERT(0);
@@ -4873,7 +4953,8 @@ freeze_code(LoaderState* stp)
 	}
 	ASSERT(value < stp->ci);
         for (patch = 0; patch < stp->labels[i].num_patches; patch++) {
-            Uint pos = stp->labels[i].patches[patch].pos;
+            LabelPatch* lp = &stp->labels[i].patches[patch];
+            Uint pos = lp->pos;
 	    ASSERT(pos < stp->ci);
             if (pos < stp->num_functions) {
                 /*
@@ -4882,7 +4963,38 @@ freeze_code(LoaderState* stp)
                  */
                 codev[pos] = (BeamInstr) (codev + value);
             } else {
-                codev[pos] += value;
+#ifdef DEBUG
+                Uint w;
+#endif
+                Sint32 rel = lp->offset + value;
+                switch (lp->packed) {
+                case 0:         /* Not packed */
+                    ASSERT(codev[pos] == i);
+                    codev[pos] = rel;
+                    break;
+#ifdef BEAM_WIDE_MASK
+                case 1:         /* Least significant word. */
+#ifdef DEBUG
+                    w = codev[pos] & BEAM_WIDE_MASK;
+                    /* Correct label in least significant word? */
+                    ASSERT(w == i);
+#endif
+                    codev[pos] = (codev[pos] & ~BEAM_WIDE_MASK) |
+                        (rel & BEAM_WIDE_MASK);
+                    break;
+                case 2:         /* Most significant word */
+#ifdef DEBUG
+                    w = (codev[pos] >> BEAM_WIDE_SHIFT) & BEAM_WIDE_MASK;
+                    /* Correct label in most significant word? */
+                    ASSERT(w == i);
+#endif
+                    codev[pos] = ((Uint)rel << BEAM_WIDE_SHIFT) |
+                        (codev[pos] & BEAM_WIDE_MASK);
+                    break;
+#endif
+                default:
+                    ASSERT(0);
+                }
             }
 	}
     }
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 6248583b1c..d156bd83c0 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1232,7 +1232,7 @@ sub basic_generator {
     #
 
     my $c_code_ref = $c_code{$name};
-    if ($hot and defined $c_code_ref) {
+    if ($hot and defined $c_code_ref and $name ne 'catch') {
         ($var_decls, $pack_spec, @args) = do_pack(@args);
     }
 
@@ -1517,6 +1517,23 @@ sub do_pack {
     }
 
     #
+    # Try to pack 'f' and 'j', but not at expense at worse packing
+    # for other operands. For example, given the arguments "f x x", we
+    # want the 'x' operands to be packed, not 'f' and 'x' packed and
+    # the final 'x' not packed.
+    #
+
+    if ($wordsize == 64 and $packable_args == 1) {
+        for (my $i = 0; $i < @args; $i++) {
+            if ($args[$i] =~ /^[fj]$/) {
+                $bits_needed[$i] = 32;
+                $packable_args++;
+                last;
+            }
+        }
+    }
+
+    #
     # Nothing to pack unless there are at least 2 packable arguments.
     #
     return ('', '', @args) if $packable_args < 2;
@@ -1602,7 +1619,15 @@ sub do_pack {
 
         if ($arg_size{$arg} and $did_some_packing) {
             # Save the argument on the pack engine's stack.
-            $down = "g${down}";
+            my $push = 'g';
+            if ($type_bit{$arg} & $type_bit{'q'}) {
+                # The operand may be a literal.
+                $push = 'q';
+            } elsif ($type_bit{$arg} & $type_bit{'f'}) {
+                # The operand may be a failure label.
+                $push = 'f';
+            }
+            $down = "$push${down}";
             $up = "${up}p";
         } else {
             # The argument has either zero size (e.g. r(0)),
@@ -1630,7 +1655,7 @@ sub do_pack {
         if ($need_wide_mask[$word]) {
             @shift = ('0', 'BEAM_WIDE_SHIFT');
             @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
-            @instr = ('w', 'i');
+            @instr = ('w', 'w');
         } else {
             @shift = @{$pack_shift[$args_per_word]};
             @mask = @{$pack_mask[$args_per_word]};