Much faster VP8 mv and mode prediction
[ffmpeg.git] / libavcodec / vp8.c
index 0b4434b..bc95fb0 100644 (file)
@@ -520,39 +520,45 @@ static inline void clamp_mv(VP8Context *s, VP56mv *dst, const VP56mv *src,
 }
 
 static void find_near_mvs(VP8Context *s, VP8Macroblock *mb, int mb_x, int mb_y,
-                          VP56mv near[2], VP56mv *best, int cnt[4])
+                          VP56mv near[2], VP56mv *best, uint8_t cnt[4])
 {
-    VP8Macroblock *mb_edge[3] = { mb - s->mb_stride     /* top */,
-                                  mb - 1                /* left */,
-                                  mb - s->mb_stride - 1 /* top-left */ };
+    int mb_stride = s->mb_stride;
+    VP8Macroblock *mb_edge[3] = { mb - mb_stride     /* top */,
+                                  mb - 1             /* left */,
+                                  mb - mb_stride - 1 /* top-left */ };
     enum { EDGE_TOP, EDGE_LEFT, EDGE_TOPLEFT };
     VP56mv near_mv[4]  = {{ 0 }};
     enum { CNT_ZERO, CNT_NEAREST, CNT_NEAR, CNT_SPLITMV };
-    int idx = CNT_ZERO, n;
+    int idx = CNT_ZERO;
     int best_idx = CNT_ZERO;
+    int cur_sign_bias = s->sign_bias[mb->ref_frame];
+    int *sign_bias = s->sign_bias;
 
     /* Process MB on top, left and top-left */
-    for (n = 0; n < 3; n++) {
-        VP8Macroblock *edge = mb_edge[n];
-        if (edge->ref_frame != VP56_FRAME_CURRENT) {
-            if (edge->mv.x | edge->mv.y) {
-                VP56mv tmp = edge->mv;
-                if (s->sign_bias[mb->ref_frame] != s->sign_bias[edge->ref_frame]) {
-                    tmp.x *= -1;
-                    tmp.y *= -1;
-                }
-                if ((tmp.x ^ near_mv[idx].x) | (tmp.y ^ near_mv[idx].y))
-                    near_mv[++idx] = tmp;
-                cnt[idx]       += 1 + (n != 2);
-            } else
-                cnt[CNT_ZERO] += 1 + (n != 2);
-        }
+    #define MV_EDGE_CHECK(n)\
+    {\
+        VP8Macroblock *edge = mb_edge[n];\
+        int edge_ref = edge->ref_frame;\
+        if (edge_ref != VP56_FRAME_CURRENT) {\
+            uint32_t mv = AV_RN32A(&edge->mv);\
+            if (mv) {\
+                if (cur_sign_bias != sign_bias[edge_ref]) {\
+                    /* SWAR negate of the values in mv. */\
+                    mv = ((mv&0x80008000) + 0x00010001) ^ (mv&0x7fff7fff);\
+                }\
+                if (!n || mv != AV_RN32A(&near_mv[idx]))\
+                    AV_WN32A(&near_mv[++idx], mv);\
+                cnt[idx]      += 1 + (n != 2);\
+            } else\
+                cnt[CNT_ZERO] += 1 + (n != 2);\
+        }\
     }
+    MV_EDGE_CHECK(0)
+    MV_EDGE_CHECK(1)
+    MV_EDGE_CHECK(2)
 
-    /* If we have three distinct MV's, merge first and last if they're the same */
-    if (cnt[CNT_SPLITMV] &&
-        !((near_mv[1+EDGE_TOP].x ^ near_mv[1+EDGE_TOPLEFT].x) |
-          (near_mv[1+EDGE_TOP].y ^ near_mv[1+EDGE_TOPLEFT].y)))
+    /* If we have three distinct MVs, merge first and last if they're the same */
+    if (cnt[CNT_SPLITMV] && AV_RN32A(&near_mv[1+EDGE_TOP]) == AV_RN32A(&near_mv[1+EDGE_TOPLEFT]))
         cnt[CNT_NEAREST] += 1;
 
     cnt[CNT_SPLITMV] = ((mb_edge[EDGE_LEFT]->mode   == VP8_MVMODE_SPLIT) +
@@ -561,8 +567,8 @@ static void find_near_mvs(VP8Context *s, VP8Macroblock *mb, int mb_x, int mb_y,
 
     /* Swap near and nearest if necessary */
     if (cnt[CNT_NEAR] > cnt[CNT_NEAREST]) {
-        FFSWAP(int,    cnt[CNT_NEAREST],     cnt[CNT_NEAR]);
-        FFSWAP(VP56mv, near_mv[CNT_NEAREST], near_mv[CNT_NEAR]);
+        FFSWAP(uint8_t,     cnt[CNT_NEAREST],     cnt[CNT_NEAR]);
+        FFSWAP( VP56mv, near_mv[CNT_NEAREST], near_mv[CNT_NEAR]);
     }
 
     /* Choose the best mv out of 0,0 and the nearest mv */
@@ -684,7 +690,6 @@ static void decode_mb_mode(VP8Context *s, VP8Macroblock *mb, int mb_x, int mb_y,
                            uint8_t *intra4x4)
 {
     VP56RangeCoder *c = &s->c;
-    int n;
 
     if (s->segmentation.update_map)
         mb->segment = vp8_rac_get_tree(c, vp8_segmentid_tree, s->prob->segmentid);
@@ -703,7 +708,7 @@ static void decode_mb_mode(VP8Context *s, VP8Macroblock *mb, int mb_x, int mb_y,
         mb->ref_frame = VP56_FRAME_CURRENT;
     } else if (vp56_rac_get_prob(c, s->prob->intra)) {
         VP56mv near[2], best;
-        int cnt[4] = { 0 };
+        uint8_t cnt[4] = { 0 };
         uint8_t p[4];
 
         // inter MB, 16.2
@@ -715,8 +720,10 @@ static void decode_mb_mode(VP8Context *s, VP8Macroblock *mb, int mb_x, int mb_y,
 
         // motion vectors, 16.3
         find_near_mvs(s, mb, mb_x, mb_y, near, &best, cnt);
-        for (n = 0; n < 4; n++)
-            p[n] = vp8_mode_contexts[cnt[n]][n];
+        p[0] = vp8_mode_contexts[cnt[0]][0];
+        p[1] = vp8_mode_contexts[cnt[1]][1];
+        p[2] = vp8_mode_contexts[cnt[2]][2];
+        p[3] = vp8_mode_contexts[cnt[3]][3];
         mb->mode = vp8_rac_get_tree(c, vp8_pred16x16_tree_mvinter, p);
         switch (mb->mode) {
         case VP8_MVMODE_SPLIT:
@@ -882,9 +889,10 @@ void xchg_mb_border(uint8_t *top_border, uint8_t *src_y, uint8_t *src_cb, uint8_
     src_cb -= uvlinesize;
     src_cr -= uvlinesize;
 
-#define XCHG(a,b,xchg)\
-if (xchg) AV_SWAP64(b,a);\
-else      AV_COPY64(b,a);
+#define XCHG(a,b,xchg) do {                     \
+        if (xchg) AV_SWAP64(b,a);               \
+        else      AV_COPY64(b,a);               \
+    } while (0)
 
     XCHG(top_border_m1+8, src_y-8, xchg);
     XCHG(top_border,      src_y,   xchg);
@@ -1053,6 +1061,21 @@ static inline void vp8_mc_part(VP8Context *s, uint8_t *dst[3],
            s->put_pixels_tab[1 + (block_w == 4)]);
 }
 
+/* Fetch pixels for estimated mv 4 macroblocks ahead.
+ * Optimized for 64-byte cache lines.  Inspired by ffh264 prefetch_motion. */
+static inline void prefetch_motion(VP8Context *s, VP8Macroblock *mb, int mb_x, int mb_y, int x_off, int y_off, int ref)
+{
+    if (mb->ref_frame != VP56_FRAME_CURRENT) {
+        int mx = mb->mv.x + x_off + 8;
+        int my = mb->mv.y + y_off;
+        uint8_t **src= s->framep[mb->ref_frame]->data;
+        int off= mx + (my + (mb_x&3)*4)*s->linesize + 64;
+        s->dsp.prefetch(src[0]+off, s->linesize, 4);
+        off= (mx>>1) + ((my>>1) + (mb_x&7))*s->uvlinesize + 64;
+        s->dsp.prefetch(src[1]+off, src[2]-src[1], 2);
+    }
+}
+
 /**
  * Apply motion vectors to prediction buffer, chapter 18.
  */
@@ -1062,6 +1085,8 @@ static void inter_predict(VP8Context *s, uint8_t *dst[3], VP8Macroblock *mb,
     int x_off = mb_x << 4, y_off = mb_y << 4;
     int width = 16*s->mb_width, height = 16*s->mb_height;
 
+    prefetch_motion(s, mb, mb_x, mb_y, x_off, y_off, VP56_FRAME_PREVIOUS);
+
     if (mb->mode < VP8_MVMODE_SPLIT) {
         vp8_mc_part(s, dst, s->framep[mb->ref_frame], x_off, y_off,
                     0, 0, 16, 16, width, height, &mb->mv);
@@ -1136,6 +1161,8 @@ static void inter_predict(VP8Context *s, uint8_t *dst[3], VP8Macroblock *mb,
                     8, 8, 8, 8, width, height, &mb->bmv[3]);
         break;
     }
+
+    prefetch_motion(s, mb, mb_x, mb_y, x_off, y_off, VP56_FRAME_GOLDEN);
 }
 
 static void idct_mb(VP8Context *s, uint8_t *y_dst, uint8_t *u_dst, uint8_t *v_dst,
@@ -1245,31 +1272,45 @@ static void filter_mb(VP8Context *s, uint8_t *dst[3], VP8Macroblock *mb, int mb_
      bedge_lim = 2* filter_level    + inner_limit;
 
     if (mb_x) {
-        s->vp8dsp.vp8_h_loop_filter16(dst[0], s->linesize,   mbedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_h_loop_filter8 (dst[1], s->uvlinesize, mbedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_h_loop_filter8 (dst[2], s->uvlinesize, mbedge_lim, inner_limit, hev_thresh);
+        s->vp8dsp.vp8_h_loop_filter16y(dst[0],     s->linesize,
+                                       mbedge_lim, inner_limit, hev_thresh);
+        s->vp8dsp.vp8_h_loop_filter8uv(dst[1],     dst[2],      s->uvlinesize,
+                                       mbedge_lim, inner_limit, hev_thresh);
     }
 
     if (!mb->skip || mb->mode == MODE_I4x4 || mb->mode == VP8_MVMODE_SPLIT) {
-        s->vp8dsp.vp8_h_loop_filter16_inner(dst[0]+ 4, s->linesize,   bedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_h_loop_filter16_inner(dst[0]+ 8, s->linesize,   bedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_h_loop_filter16_inner(dst[0]+12, s->linesize,   bedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_h_loop_filter8_inner (dst[1]+ 4, s->uvlinesize, bedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_h_loop_filter8_inner (dst[2]+ 4, s->uvlinesize, bedge_lim, inner_limit, hev_thresh);
+        s->vp8dsp.vp8_h_loop_filter16y_inner(dst[0]+ 4, s->linesize, bedge_lim,
+                                             inner_limit,   hev_thresh);
+        s->vp8dsp.vp8_h_loop_filter16y_inner(dst[0]+ 8, s->linesize, bedge_lim,
+                                             inner_limit,   hev_thresh);
+        s->vp8dsp.vp8_h_loop_filter16y_inner(dst[0]+12, s->linesize, bedge_lim,
+                                             inner_limit,   hev_thresh);
+        s->vp8dsp.vp8_h_loop_filter8uv_inner(dst[1] + 4,    dst[2] + 4,
+                                             s->uvlinesize, bedge_lim,
+                                             inner_limit,   hev_thresh);
     }
 
     if (mb_y) {
-        s->vp8dsp.vp8_v_loop_filter16(dst[0], s->linesize,   mbedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_v_loop_filter8 (dst[1], s->uvlinesize, mbedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_v_loop_filter8 (dst[2], s->uvlinesize, mbedge_lim, inner_limit, hev_thresh);
+        s->vp8dsp.vp8_v_loop_filter16y(dst[0],     s->linesize,
+                                       mbedge_lim, inner_limit, hev_thresh);
+        s->vp8dsp.vp8_v_loop_filter8uv(dst[1],     dst[2],      s->uvlinesize,
+                                       mbedge_lim, inner_limit, hev_thresh);
     }
 
     if (!mb->skip || mb->mode == MODE_I4x4 || mb->mode == VP8_MVMODE_SPLIT) {
-        s->vp8dsp.vp8_v_loop_filter16_inner(dst[0]+ 4*s->linesize,   s->linesize,   bedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_v_loop_filter16_inner(dst[0]+ 8*s->linesize,   s->linesize,   bedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_v_loop_filter16_inner(dst[0]+12*s->linesize,   s->linesize,   bedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_v_loop_filter8_inner (dst[1]+ 4*s->uvlinesize, s->uvlinesize, bedge_lim, inner_limit, hev_thresh);
-        s->vp8dsp.vp8_v_loop_filter8_inner (dst[2]+ 4*s->uvlinesize, s->uvlinesize, bedge_lim, inner_limit, hev_thresh);
+        s->vp8dsp.vp8_v_loop_filter16y_inner(dst[0]+ 4*s->linesize,
+                                             s->linesize,   bedge_lim,
+                                             inner_limit,   hev_thresh);
+        s->vp8dsp.vp8_v_loop_filter16y_inner(dst[0]+ 8*s->linesize,
+                                             s->linesize,   bedge_lim,
+                                             inner_limit,   hev_thresh);
+        s->vp8dsp.vp8_v_loop_filter16y_inner(dst[0]+12*s->linesize,
+                                             s->linesize,   bedge_lim,
+                                             inner_limit, hev_thresh);
+        s->vp8dsp.vp8_v_loop_filter8uv_inner(dst[1] + 4 * s->uvlinesize,
+                                             dst[2] + 4 * s->uvlinesize,
+                                             s->uvlinesize, bedge_lim,
+                                             inner_limit,   hev_thresh);
     }
 }
 
@@ -1416,6 +1457,10 @@ static int vp8_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
             memset(s->top_border, 129, sizeof(*s->top_border));
 
         for (mb_x = 0; mb_x < s->mb_width; mb_x++) {
+            /* Prefetch the current frame, 4 MBs ahead */
+            s->dsp.prefetch(dst[0] + (mb_x&3)*4*s->linesize + 64, s->linesize, 4);
+            s->dsp.prefetch(dst[1] + (mb_x&7)*s->uvlinesize + 64, dst[2] - dst[1], 2);
+
             decode_mb_mode(s, mb, mb_x, mb_y, intra4x4 + 4*mb_x);
 
             if (!mb->skip)