avcodec/vc1: fix overlap smoothing filter for P frames
authorJerome Borsboom <jerome.borsboom@carpalis.nl>
Wed, 30 May 2018 11:53:48 +0000 (13:53 +0200)
committerCarl Eugen Hoyos <ceffmpeg@gmail.com>
Sat, 16 Jun 2018 17:15:07 +0000 (19:15 +0200)
The v_overlap_filter needs to run on the colocated block of the previous
macroblock. For the luma plane, the colocated block is located two blocks
on the left instead of one. In addition, the overlap filter needs to run
on the non-edge blocks of the first macroblock row and column.

Fixes ticket #7171.

Signed-off-by: Jerome Borsboom <jerome.borsboom@carpalis.nl>
libavcodec/vc1_loopfilter.c

index 4c0de7c..aceb1f7 100644 (file)
@@ -64,27 +64,23 @@ void ff_vc1_loop_filter_iblk(VC1Context *v, int pq)
 static av_always_inline void vc1_h_overlap_filter(VC1Context *v, int16_t (*left_block)[64],
                                                   int16_t (*right_block)[64], int block_num)
 {
-    if (left_block != right_block || (block_num & 5) == 1) {
-        if (block_num > 3)
-            v->vc1dsp.vc1_h_s_overlap(left_block[block_num], right_block[block_num]);
-        else if (block_num & 1)
-            v->vc1dsp.vc1_h_s_overlap(right_block[block_num - 1], right_block[block_num]);
-        else
-            v->vc1dsp.vc1_h_s_overlap(left_block[block_num + 1], right_block[block_num]);
-    }
+    if (block_num > 3)
+        v->vc1dsp.vc1_h_s_overlap(left_block[block_num], right_block[block_num]);
+    else if (block_num & 1)
+        v->vc1dsp.vc1_h_s_overlap(right_block[block_num - 1], right_block[block_num]);
+    else
+        v->vc1dsp.vc1_h_s_overlap(left_block[block_num + 1], right_block[block_num]);
 }
 
 static av_always_inline void vc1_v_overlap_filter(VC1Context *v, int16_t (*top_block)[64],
                                                   int16_t (*bottom_block)[64], int block_num)
 {
-    if (top_block != bottom_block || block_num & 2) {
-        if (block_num > 3)
-            v->vc1dsp.vc1_v_s_overlap(top_block[block_num], bottom_block[block_num]);
-        else if (block_num & 2)
-            v->vc1dsp.vc1_v_s_overlap(bottom_block[block_num - 2], bottom_block[block_num]);
-        else
-            v->vc1dsp.vc1_v_s_overlap(top_block[block_num + 2], bottom_block[block_num]);
-    }
+    if (block_num > 3)
+        v->vc1dsp.vc1_v_s_overlap(top_block[block_num], bottom_block[block_num]);
+    else if (block_num & 2)
+        v->vc1dsp.vc1_v_s_overlap(bottom_block[block_num - 2], bottom_block[block_num]);
+    else
+        v->vc1dsp.vc1_v_s_overlap(top_block[block_num + 2], bottom_block[block_num]);
 }
 
 void ff_vc1_i_overlap_filter(VC1Context *v)
@@ -108,21 +104,28 @@ void ff_vc1_i_overlap_filter(VC1Context *v)
      * borders. Therefore, the H overlap trails by one MB col and the
      * V overlap trails by one MB row. This is reflected in the time at which
      * we run the put_pixels loop, i.e. delayed by one row and one column. */
-    for (i = 0; i < block_count; i++)
+    for (i = 0; i < block_count; i++) {
+        if (s->mb_x == 0 && (i & 5) != 1)
+            continue;
+
         if (v->pq >= 9 || v->condover == CONDOVER_ALL ||
-            (v->over_flags_plane[mb_pos] && ((i & 5) == 1 || (s->mb_x && v->over_flags_plane[mb_pos - 1]))))
+            (v->over_flags_plane[mb_pos] && ((i & 5) == 1 || v->over_flags_plane[mb_pos - 1])))
             vc1_h_overlap_filter(v, s->mb_x ? left_blk : cur_blk, cur_blk, i);
+    }
 
     if (v->fcm != ILACE_FRAME)
         for (i = 0; i < block_count; i++) {
+            if (s->first_slice_line && !(i & 2))
+                continue;
+
             if (s->mb_x && (v->pq >= 9 || v->condover == CONDOVER_ALL ||
                 (v->over_flags_plane[mb_pos - 1] &&
-                 ((i & 2) || (!s->first_slice_line && v->over_flags_plane[mb_pos - 1 - s->mb_stride])))))
+                 ((i & 2) || v->over_flags_plane[mb_pos - 1 - s->mb_stride]))))
                 vc1_v_overlap_filter(v, s->first_slice_line ? left_blk : topleft_blk, left_blk, i);
             if (s->mb_x == s->mb_width - 1)
                 if (v->pq >= 9 || v->condover == CONDOVER_ALL ||
                     (v->over_flags_plane[mb_pos] &&
-                     ((i & 2) || (!s->first_slice_line && v->over_flags_plane[mb_pos - s->mb_stride]))))
+                     ((i & 2) || v->over_flags_plane[mb_pos - s->mb_stride])))
                     vc1_v_overlap_filter(v, s->first_slice_line ? cur_blk : top_blk, cur_blk, i);
         }
 }
@@ -139,18 +142,25 @@ void ff_vc1_p_overlap_filter(VC1Context *v)
     left_blk = v->block[v->left_blk_idx];
     cur_blk = v->block[v->cur_blk_idx];
 
-    for (i = 0; i < block_count; i++)
-        if (v->mb_type[0][s->block_index[i]] && (s->mb_x == 0 || v->mb_type[0][s->block_index[i] - 1]))
+    for (i = 0; i < block_count; i++) {
+        if (s->mb_x == 0 && (i & 5) != 1)
+            continue;
+
+        if (v->mb_type[0][s->block_index[i]] && v->mb_type[0][s->block_index[i] - 1])
             vc1_h_overlap_filter(v, s->mb_x ? left_blk : cur_blk, cur_blk, i);
+    }
 
     if (v->fcm != ILACE_FRAME)
         for (i = 0; i < block_count; i++) {
-            if (s->mb_x && v->mb_type[0][s->block_index[i] - 1] &&
-                (s->first_slice_line || v->mb_type[0][s->block_index[i] - s->block_wrap[i] - 1]))
+            if (s->first_slice_line && !(i & 2))
+                continue;
+
+            if (s->mb_x && v->mb_type[0][s->block_index[i] - 2 + (i > 3)] &&
+                v->mb_type[0][s->block_index[i] - s->block_wrap[i] - 2 + (i > 3)])
                 vc1_v_overlap_filter(v, s->first_slice_line ? left_blk : topleft_blk, left_blk, i);
             if (s->mb_x == s->mb_width - 1)
                 if (v->mb_type[0][s->block_index[i]] &&
-                    (s->first_slice_line || v->mb_type[0][s->block_index[i] - s->block_wrap[i]]))
+                    v->mb_type[0][s->block_index[i] - s->block_wrap[i]])
                     vc1_v_overlap_filter(v, s->first_slice_line ? cur_blk : top_blk, cur_blk, i);
         }
 }