avcodec/nvdec_hevc: fix scaling lists
authorPhilip Langdale <philipl@overt.org>
Thu, 10 May 2018 01:48:59 +0000 (18:48 -0700)
committerTimo Rothenpieler <timo@rothenpieler.org>
Thu, 10 May 2018 16:41:56 +0000 (18:41 +0200)
The main issue here was the use of [i] instead of [i * 3] for the 32x32
matrix. As part of fixing this, I changed the code to match that used
in vdpau_hevc, which I spent a lot of time verifying.

I also changed to calculating NumPocTotalCurr using the existing helper,
which is what vdpau does.

Signed-off-by: Timo Rothenpieler <timo@rothenpieler.org>
libavcodec/nvdec_hevc.c

index 0089631..e04a701 100644 (file)
@@ -58,12 +58,13 @@ static void fill_scaling_lists(CUVIDHEVCPICPARAMS *ppc, const HEVCContext *s)
             ppc->ScalingList16x16[i][j] = sl->sl[2][i][pos];
 
             if (i < 2)
-                ppc->ScalingList32x32[i][j] = sl->sl[3][i][pos];
+                ppc->ScalingList32x32[i][j] = sl->sl[3][i * 3][pos];
         }
-    }
 
-    memcpy(ppc->ScalingListDCCoeff16x16, sl->sl_dc[0], sizeof(ppc->ScalingListDCCoeff16x16));
-    memcpy(ppc->ScalingListDCCoeff32x32, sl->sl_dc[1], sizeof(ppc->ScalingListDCCoeff32x32));
+        ppc->ScalingListDCCoeff16x16[i] = sl->sl_dc[0][i];
+        if (i < 2)
+            ppc->ScalingListDCCoeff32x32[i] = sl->sl_dc[1][i * 3];
+    }
 }
 
 static int nvdec_hevc_start_frame(AVCodecContext *avctx,
@@ -166,8 +167,7 @@ static int nvdec_hevc_start_frame(AVCodecContext *avctx,
 
             .NumBitsForShortTermRPSInSlice                = s->sh.short_term_rps ? s->sh.short_term_ref_pic_set_size : 0,
             .NumDeltaPocsOfRefRpsIdx                      = s->sh.short_term_rps ? s->sh.short_term_rps->rps_idx_num_delta_pocs : 0,
-            .NumPocTotalCurr                              = s->rps[ST_CURR_BEF].nb_refs + s->rps[ST_CURR_AFT].nb_refs +
-                                                            s->rps[LT_CURR].nb_refs,
+            .NumPocTotalCurr                              = ff_hevc_frame_nb_refs(s),
             .NumPocStCurrBefore                           = s->rps[ST_CURR_BEF].nb_refs,
             .NumPocStCurrAfter                            = s->rps[ST_CURR_AFT].nb_refs,
             .NumPocLtCurr                                 = s->rps[LT_CURR].nb_refs,