shithub: h264bsd

Download patch

ref: 73e0ce9f876fc6ea042072addeb21cd9cbac443b
parent: 7d0cb59987f5c5c70cf55e35e65aa6b7a5bc460e
author: Sigrid Haflínudóttir <ftrvxmtrx@gmail.com>
date: Wed Sep 16 05:50:48 EDT 2020

report more errors; give up on non-baseline profile immediately

--- a/src/h264bsd_byte_stream.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_byte_stream.c	Wed Sep 16 05:50:48 2020
@@ -117,6 +117,7 @@
             {
                 /* no start code prefix found -> error */
                 *readBytes = len;
+                werrstr("no start code prefix");
                 return(HANTRO_NOK);
             }
 
@@ -187,6 +188,7 @@
 
     if (invalidStream)
     {
+        werrstr("invalid stream");
         return(HANTRO_NOK);
     }
 
@@ -204,8 +206,10 @@
                  * following bytes: 0x00, 0x01, 0x02, 0x03. This implies that
                  * emulation prevention 0x03 byte shall not be the last byte
                  * of the stream. */
-                if ( (i == 0) || (*(readPtr+1) > 0x03) )
+                if ( (i == 0) || (*(readPtr+1) > 0x03) ) {
+                    werrstr("emulation byte last in the stream");
                     return(HANTRO_NOK);
+                }
 
                 /* do not write emulation prevention byte */
                 readPtr++;
@@ -215,8 +219,10 @@
             {
                 /* NAL unit shall not contain byte sequences 0x000000,
                  * 0x000001 or 0x000002 */
-                if ( (zeroCount == 2) && (*readPtr <= 0x02) )
+                if ( (zeroCount == 2) && (*readPtr <= 0x02) ) {
+                    werrstr("start sequence in NAL unit");
                     return(HANTRO_NOK);
+                }
 
                 if (*readPtr == 0)
                     zeroCount++;
--- a/src/h264bsd_cavlc.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_cavlc.c	Wed Sep 16 05:50:48 2020
@@ -784,8 +784,10 @@
     BUFFER_FLUSH(bufferValue, bufferBits, LENGTH_TC(tmp));
 
     totalCoeff = TOTAL_COEFF(tmp);
-    if (totalCoeff > maxNumCoeff)
+    if (totalCoeff > maxNumCoeff) {
+        werrstr("invalid total coeff: %d", totalCoeff);
         return(HANTRO_NOK);
+    }
     trailingOnes = TRAILING_ONES(tmp);
 
     if (totalCoeff != 0)
@@ -813,8 +815,10 @@
         {
             BUFFER_SHOW(bufferValue, bufferBits, bit, 16);
             levelPrefix = DecodeLevelPrefix(bit);
-            if (levelPrefix == VLC_NOT_FOUND)
+            if (levelPrefix == VLC_NOT_FOUND) {
+                werrstr("level prefix unknown");
                 return(HANTRO_NOK);
+            }
             BUFFER_FLUSH(bufferValue, bufferBits, levelPrefix+1);
 
             if (levelPrefix < 14)
@@ -865,8 +869,10 @@
             BUFFER_SHOW(bufferValue, bufferBits, bit,9);
             zerosLeft = DecodeTotalZeros(bit, totalCoeff,
                                         (u32)(maxNumCoeff == 4));
-            if (!zerosLeft)
+            if (!zerosLeft) {
+                werrstr("no zeros left");
                 return(HANTRO_NOK);
+            }
             BUFFER_FLUSH(bufferValue, bufferBits, LENGTH(zerosLeft));
             zerosLeft = INFO(zerosLeft);
         }
--- a/src/h264bsd_decoder.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_decoder.c	Wed Sep 16 05:50:48 2020
@@ -103,8 +103,10 @@
     size = (sizeof(macroblockLayer_t) + 63) & ~0x3F;
 
     pStorage->mbLayer = (macroblockLayer_t*)malloc(size);
-    if (!pStorage->mbLayer)
+    if (!pStorage->mbLayer) {
+        werrstr("no memory");
         return HANTRO_NOK;
+    }
 
     if (noOutputReordering)
         pStorage->noReordering = HANTRO_TRUE;
--- a/src/h264bsd_dpb.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_dpb.c	Wed Sep 16 05:50:48 2020
@@ -285,8 +285,10 @@
         }
         /* find corresponding picture from dpb */
         index = FindDpbPic(dpb, picNum, isShortTerm);
-        if (index < 0 || !IS_EXISTING(dpb->buffer[index]))
+        if (index < 0 || !IS_EXISTING(dpb->buffer[index])) {
+            werrstr("dpb: picture not found");
             return(HANTRO_NOK);
+        }
 
         /* shift pictures */
         for (j = numRefIdxActive; j > refIdx; j--)
@@ -652,7 +654,7 @@
 
     if (image->data != dpb->currentOut->data)
     {
-        EPRINT("TRYING TO MARK NON-ALLOCATED IMAGE");
+        werrstr("trying to mark non-allocated image");
         return(HANTRO_NOK);
     }
 
@@ -760,6 +762,7 @@
                         break;
 
                     default: /* invalid memory management control operation */
+                        werrstr("invalid mem management control op %d", mark->operation[i].memoryManagementControlOperation);
                         status = HANTRO_NOK;
                         break;
                 }
@@ -792,6 +795,7 @@
             /* no room */
             else
             {
+                werrstr("no room for pic in dpb");
                 status = HANTRO_NOK;
             }
         }
--- a/src/h264bsd_inter_prediction.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_inter_prediction.c	Wed Sep 16 05:50:48 2020
@@ -535,13 +535,17 @@
         mv.ver += mvPred.ver;
 
         /* horizontal motion vector range [-2048, 2047.75] */
-        if ((u32)(i32)(mv.hor+8192) >= (16384))
+        if ((u32)(i32)(mv.hor+8192) >= (16384)) {
+            werrstr("horizontal motion vector out of range");
             return(HANTRO_NOK);
+        }
 
         /* vertical motion vector range [-512, 511.75]
          * (smaller for low levels) */
-        if ((u32)(i32)(mv.ver+2048) >= (4096))
+        if ((u32)(i32)(mv.ver+2048) >= (4096)) {
+            werrstr("vertical motion vector out of range");
             return(HANTRO_NOK);
+        }
     }
 
     tmp = h264bsdGetRefPicData(dpb, refIndex);
@@ -623,12 +627,16 @@
     mv.ver += mvPred.ver;
 
     /* horizontal motion vector range [-2048, 2047.75] */
-    if ((u32)(i32)(mv.hor+8192) >= (16384))
+    if ((u32)(i32)(mv.hor+8192) >= (16384)) {
+        werrstr("horizontal motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
-    if ((u32)(i32)(mv.ver+2048) >= (4096))
+    if ((u32)(i32)(mv.ver+2048) >= (4096)) {
+        werrstr("vertical motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     tmp = h264bsdGetRefPicData(dpb, refIndex);
     if (tmp == NULL)
@@ -669,12 +677,16 @@
     mv.ver += mvPred.ver;
 
     /* horizontal motion vector range [-2048, 2047.75] */
-    if ((u32)(i32)(mv.hor+8192) >= (16384))
+    if ((u32)(i32)(mv.hor+8192) >= (16384)) {
+        werrstr("horizontal motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
-    if ((u32)(i32)(mv.ver+2048) >= (4096))
+    if ((u32)(i32)(mv.ver+2048) >= (4096)) {
+        werrstr("vertical motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     tmp = h264bsdGetRefPicData(dpb, refIndex);
     if (tmp == NULL)
@@ -742,12 +754,16 @@
     mv.ver += mvPred.ver;
 
     /* horizontal motion vector range [-2048, 2047.75] */
-    if ((u32)(i32)(mv.hor+8192) >= (16384))
+    if ((u32)(i32)(mv.hor+8192) >= (16384)) {
+        werrstr("horizontal motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
-    if ((u32)(i32)(mv.ver+2048) >= (4096))
+    if ((u32)(i32)(mv.ver+2048) >= (4096)) {
+        werrstr("vertical motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     tmp = h264bsdGetRefPicData(dpb, refIndex);
     if (tmp == NULL)
@@ -791,12 +807,16 @@
     mv.ver += mvPred.ver;
 
     /* horizontal motion vector range [-2048, 2047.75] */
-    if ((u32)(i32)(mv.hor+8192) >= (16384))
+    if ((u32)(i32)(mv.hor+8192) >= (16384)) {
+        werrstr("horizontal motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
-    if ((u32)(i32)(mv.ver+2048) >= (4096))
+    if ((u32)(i32)(mv.ver+2048) >= (4096)) {
+        werrstr("vertical motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     tmp = h264bsdGetRefPicData(dpb, refIndex);
     if (tmp == NULL)
@@ -909,12 +929,16 @@
     mv.ver += mvPred.ver;
 
     /* horizontal motion vector range [-2048, 2047.75] */
-    if (((u32)(i32)(mv.hor+8192) >= (16384)))
+    if (((u32)(i32)(mv.hor+8192) >= (16384))) {
+        werrstr("horizontal motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
-    if (((u32)(i32)(mv.ver+2048) >= (4096)))
+    if (((u32)(i32)(mv.ver+2048) >= (4096))) {
+        werrstr("vertical motion vector out of range");
         return(HANTRO_NOK);
+    }
 
     switch (subMbPartMode)
     {
--- a/src/h264bsd_nal_unit.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_nal_unit.c	Wed Sep 16 05:50:48 2020
@@ -83,8 +83,10 @@
     tmp = h264bsdGetBits(pStrmData, 1);
     /* Assuming that NAL unit starts from byte boundary �> don't have to check
      * following 7 bits for END_OF_STREAM */
-    if (tmp == END_OF_STREAM)
+    if (tmp == END_OF_STREAM) {
+        werrstr("end of stream");
         return(HANTRO_NOK);
+    }
 
     tmp = h264bsdGetBits(pStrmData, 2);
     pNalUnit->nalRefIdc = tmp;
--- a/src/h264bsd_pic_param_set.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_pic_param_set.c	Wed Sep 16 05:50:48 2020
@@ -109,7 +109,7 @@
         return(tmp);
     if (pPicParamSet->picParameterSetId >= MAX_NUM_PIC_PARAM_SETS)
     {
-        EPRINT("pic_parameter_set_id");
+        werrstr("pic_parameter_set_id out of range: %d", pPicParamSet->picParameterSetId);
         return(HANTRO_NOK);
     }
 
@@ -119,7 +119,7 @@
         return(tmp);
     if (pPicParamSet->seqParameterSetId >= MAX_NUM_SEQ_PARAM_SETS)
     {
-        EPRINT("seq_param_set_id");
+        werrstr("seq_param_set_id out of range: %d", pPicParamSet->seqParameterSetId);
         return(HANTRO_NOK);
     }
 
@@ -127,7 +127,7 @@
     tmp = h264bsdGetBits(pStrmData, 1);
     if (tmp)
     {
-        EPRINT("entropy_coding_mode_flag");
+        werrstr("unexpected entropy_coding_mode_flag");
         return(HANTRO_NOK);
     }
 
@@ -143,7 +143,7 @@
     pPicParamSet->numSliceGroups = value + 1;
     if (pPicParamSet->numSliceGroups > MAX_NUM_SLICE_GROUPS)
     {
-        EPRINT("num_slice_groups_minus1");
+        werrstr("num_slice_groups_minus1 out of range: %d", pPicParamSet->numSliceGroups);
         return(HANTRO_NOK);
     }
 
@@ -156,7 +156,7 @@
             return(tmp);
         if (pPicParamSet->sliceGroupMapType > 6)
         {
-            EPRINT("slice_group_map_type");
+            werrstr("slice_group_map_type out of range: %d", pPicParamSet->sliceGroupMapType);
             return(HANTRO_NOK);
         }
 
@@ -234,7 +234,7 @@
                 if ( pPicParamSet->sliceGroupId[i] >=
                      pPicParamSet->numSliceGroups )
                 {
-                    EPRINT("slice_group_id");
+                    werrstr("slice_group_id out of range");
                     return(HANTRO_NOK);
                 }
             }
@@ -258,7 +258,7 @@
         return(tmp);
     if (value > 31)
     {
-        EPRINT("num_ref_idx_l1_active_minus1");
+        werrstr("num_ref_idx_l1_active_minus1 out of range: %d", value);
         return(HANTRO_NOK);
     }
 
@@ -266,7 +266,7 @@
     tmp = h264bsdGetBits(pStrmData, 1);
     if (tmp)
     {
-        EPRINT("weighted_pred_flag");
+        werrstr("unexpected weighted_pred_flag");
         return(HANTRO_NOK);
     }
 
@@ -274,7 +274,7 @@
     tmp = h264bsdGetBits(pStrmData, 2);
     if (tmp > 2)
     {
-        EPRINT("weighted_bipred_idc");
+        werrstr("weighted_bipred_idc out of range: %d", tmp);
         return(HANTRO_NOK);
     }
 
@@ -284,7 +284,7 @@
         return(tmp);
     if ((itmp < -26) || (itmp > 25))
     {
-        EPRINT("pic_init_qp_minus26");
+        werrstr("pic_init_qp_minus26 out of range: %d", itmp);
         return(HANTRO_NOK);
     }
     pPicParamSet->picInitQp = (u32)(itmp + 26);
@@ -295,7 +295,7 @@
         return(tmp);
     if ((itmp < -26) || (itmp > 25))
     {
-        EPRINT("pic_init_qs_minus26");
+        werrstr("pic_init_qs_minus26 out of range: %d", itmp);
         return(HANTRO_NOK);
     }
 
@@ -304,7 +304,7 @@
         return(tmp);
     if ((itmp < -12) || (itmp > 12))
     {
-        EPRINT("chroma_qp_index_offset");
+        werrstr("chroma_qp_index_offset out of range: %d", itmp);
         return(HANTRO_NOK);
     }
     pPicParamSet->chromaQpIndexOffset = itmp;
--- a/src/h264bsd_sei.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_sei.c	Wed Sep 16 05:50:48 2020
@@ -418,8 +418,10 @@
       &pBufferingPeriod->seqParameterSetId);
     if (tmp != HANTRO_OK)
         return(tmp);
-    if (pBufferingPeriod->seqParameterSetId > 31)
+    if (pBufferingPeriod->seqParameterSetId > 31) {
+        werrstr("seqParameterSetId out of range: %d", pBufferingPeriod->seqParameterSetId);
         return(HANTRO_NOK);
+    }
 
     if (nalHrdBpPresentFlag)
     {
--- a/src/h264bsd_seq_param_set.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_seq_param_set.c	Wed Sep 16 05:50:48 2020
@@ -99,9 +99,9 @@
     tmp = h264bsdGetBits(pStrmData, 8);
     if (tmp == END_OF_STREAM)
         return(HANTRO_NOK);
-    if (tmp != 66)
-    {
-        DEBUG(("NOT BASELINE PROFILE %d\n", tmp));
+    if (tmp != 66) {
+        werrstr("not baseline profile: %d", tmp);
+        return(HANTRO_NOK);
     }
     pSeqParamSet->profileIdc = tmp;
 
@@ -131,7 +131,7 @@
         return(tmp);
     if (pSeqParamSet->seqParameterSetId >= MAX_NUM_SEQ_PARAM_SETS)
     {
-        EPRINT("seq_param_set_id");
+        werrstr("seq_param_set_id out of range: %d", pSeqParamSet->seqParameterSetId);
         return(HANTRO_NOK);
     }
 
@@ -141,7 +141,7 @@
         return(tmp);
     if (value > 12)
     {
-        EPRINT("log2_max_frame_num_minus4");
+        werrstr("log2_max_frame_num_minus4 out of range: %d", value);
         return(HANTRO_NOK);
     }
     /* maxFrameNum = 2^(log2_max_frame_num_minus4 + 4) */
@@ -153,7 +153,7 @@
         return(tmp);
     if (value > 2)
     {
-        EPRINT("pic_order_cnt_type");
+        werrstr("pic_order_cnt_type out of range: %d", value);
         return(HANTRO_NOK);
     }
     pSeqParamSet->picOrderCntType = value;
@@ -166,7 +166,7 @@
             return(tmp);
         if (value > 12)
         {
-            EPRINT("log2_max_pic_order_cnt_lsb_minus4");
+            werrstr("log2_max_pic_order_cnt_lsb_minus4 out of range: %d", value);
             return(HANTRO_NOK);
         }
         /* maxPicOrderCntLsb = 2^(log2_max_pic_order_cnt_lsb_minus4 + 4) */
@@ -196,7 +196,7 @@
             return(tmp);
         if (pSeqParamSet->numRefFramesInPicOrderCntCycle > 255)
         {
-            EPRINT("num_ref_frames_in_pic_order_cnt_cycle");
+            werrstr("num_ref_frames_in_pic_order_cnt_cycle out of range: %d", pSeqParamSet->numRefFramesInPicOrderCntCycle);
             return(HANTRO_NOK);
         }
 
@@ -228,7 +228,7 @@
         return(tmp);
     if (pSeqParamSet->numRefFrames > MAX_NUM_REF_PICS)
     {
-        EPRINT("num_ref_frames");
+        werrstr("num_ref_frames out of range: %d", pSeqParamSet->numRefFrames);
         return(HANTRO_NOK);
     }
 
@@ -254,7 +254,7 @@
         return(HANTRO_NOK);
     if (!tmp)
     {
-        EPRINT("frame_mbs_only_flag");
+        werrstr("frame_mbs_only_flag not set");
         return(HANTRO_NOK);
     }
 
@@ -296,7 +296,7 @@
                ( 8 * (i32)pSeqParamSet->picHeightInMbs -
                  ((i32)pSeqParamSet->frameCropBottomOffset + 1) ) ) )
         {
-            EPRINT("frame_cropping");
+            werrstr("frame_cropping invalid");
             return(HANTRO_NOK);
         }
     }
@@ -323,8 +323,10 @@
     if (pSeqParamSet->vuiParametersPresentFlag)
     {
         ALLOCATE(pSeqParamSet->vuiParameters, 1, vuiParameters_t);
-        if (pSeqParamSet->vuiParameters == NULL)
+        if (pSeqParamSet->vuiParameters == NULL) {
+            werrstr("no memory");
             return(MEMORY_ALLOCATION_ERROR);
+        }
         tmp = h264bsdDecodeVuiParameters(pStrmData,
             pSeqParamSet->vuiParameters);
         if (tmp != HANTRO_OK)
--- a/src/h264bsd_vlc.c	Tue Sep 15 14:54:25 2020
+++ b/src/h264bsd_vlc.c	Wed Sep 16 05:50:48 2020
@@ -175,10 +175,12 @@
                 else if (bits == 1)
                 {
                     *codeNum = BIG_CODE_NUM;
+                    werrstr("big_code_num");
                     return(HANTRO_NOK);
                 }
             }
             /* if more zeros than 32, it is an error */
+            werrstr("zeros > 32");
             return(HANTRO_NOK);
         }
         else
@@ -267,6 +269,8 @@
         return(HANTRO_OK);
     }
 
+    werrstr("h264bsdDecodeExpGolombSigned failed");
+
     return(HANTRO_NOK);
 
 }
@@ -315,8 +319,10 @@
     else
     {
         /* range of valid codeNums [0,47] */
-        if (codeNum > 47)
+        if (codeNum > 47) {
+            werrstr("invalid codeNum %d", codeNum);
             return (HANTRO_NOK);
+        }
         if (isIntra)
             *value = codedBlockPatternIntra4x4[codeNum];
         else