[FFmpeg-trac] #5886(swscale:new): Crash in swscale

[FFmpeg]

#5886: Crash in swscale
---------------------------------+-------------------------------------
             Reporter:  w3sip    |                     Type:  defect
               Status:  new      |                 Priority:  important
            Component:  swscale  |                  Version:  3.0.3
             Keywords:           |               Blocked By:
             Blocking:           |  Reproduced by developer:  0
Analyzed by developer:  0        |
---------------------------------+-------------------------------------
 Please consider the following code:
 {{{
 void pixfmt_conv_test()
 {
     struct SwsContext*  ctx;
     int   srcH = 800;
     int   srcW = 1280;
     enum AVPixelFormat   srcFMT = AV_PIX_FMT_RGB24;
     enum AVPixelFormat   dstFMT = AV_PIX_FMT_YUV420P;
     int padding = 0; // the actual test case ... crashes
     //int padding = 1; // doesn't crash

     uint8_t* rgbSrc = (uint8_t*)malloc(srcH*srcW*3 + padding);
     memset(rgbSrc, 255, srcW*srcH*3);

     printf("Addr=0x%x size=%d\n", rgbSrc, srcW*srcH*3);
     ctx = sws_getContext(srcW,
                         srcH,
                         srcFMT,
                         srcW,
                         srcH,
                         dstFMT,
                         SWS_FAST_BILINEAR,
                         NULL,
                         NULL,
                         NULL);


     AVFrame* srcFrame = av_frame_alloc();
     AVFrame* dstFrame = av_frame_alloc();

     int outputBufferSize = av_image_get_buffer_size(dstFMT,
                                srcW,
                                srcH,
                                1);
     uint8_t* buffer = (uint8_t*)av_malloc(outputBufferSize);

     av_image_fill_arrays(dstFrame->data,
                    dstFrame->linesize,
                    buffer,
                    dstFMT,
                    srcW,
                    srcH,
                    1);

     av_image_fill_arrays(srcFrame->data,
                    srcFrame->linesize,
                    rgbSrc,
                    srcFMT,
                    srcW,
                    srcH,
                    1 );

     srcFrame->width = srcW;
     srcFrame->height = srcH;
     srcFrame->format = srcFMT;


     sws_scale(ctx,
               (const uint8_t* const*)srcFrame,
               srcFrame->linesize,
               0,
               srcH,
               dstFrame->data,
               dstFrame->linesize);
 }
 }}}

 It crashes with OSX 10.10.5, 32-bit app and ffmpeg 3.1.4 built from
 source. The lldb output looks like so:

 {{{
 (lldb) r
 Process 94178 launched: './videoLib2/INSTALL/i386-apple-
 darwin/bin/videolib_test' (i386)
 Addr=0x1500000 size=3072000
 Process 94178 stopped
 * thread #1: tid = 0x1276311, 0x003cbf55
 libswscale.4.dylib`ff_rgb24ToY_avx + 53, queue = 'com.apple.main-thread',
 stop reason = EXC_BAD_ACCESS (code=1, address=0x17ee000)
     frame #0: 0x003cbf55 libswscale.4.dylib`ff_rgb24ToY_avx + 53
 libswscale.4.dylib`ff_rgb24ToY_avx:
 ->  0x3cbf55 <+53>: vmovdqu 0xc(%ecx), %xmm2
     0x3cbf5a <+58>: vpshufb 0x3faf50, %xmm0, %xmm1
     0x3cbf63 <+67>: vpshufb %xmm7, %xmm0, %xmm0
     0x3cbf68 <+72>: vpshufb 0x3faf50, %xmm2, %xmm3
 (lldb) bt
 * thread #1: tid = 0x1276311, 0x003cbf55
 libswscale.4.dylib`ff_rgb24ToY_avx + 53, queue = 'com.apple.main-thread',
 stop reason = EXC_BAD_ACCESS (code=1, address=0x17ee000)
   * frame #0: 0x003cbf55 libswscale.4.dylib`ff_rgb24ToY_avx + 53
     frame #1: 0x0036eec9 libswscale.4.dylib`lum_convert(c=<unavailable>,
 desc=<unavailable>, sliceY=<unavailable>, sliceH=<unavailable>) + 297 at
 hscale.c:108
     frame #2: 0x003abbb9 libswscale.4.dylib`swscale(c=0x01802400,
 src=<unavailable>, srcStride=<unavailable>, srcSliceY=<unavailable>,
 srcSliceH=<unavailable>, dst=<unavailable>, dstStride=<unavailable>) +
 2809 at swscale.c:460
     frame #3: 0x003ad1fb libswscale.4.dylib`sws_scale(c=<unavailable>,
 srcSlice=<unavailable>, srcStride=<unavailable>, srcSliceY=<unavailable>,
 srcSliceH=<unavailable>, dst=<unavailable>, dstStride=<unavailable>) +
 3339 at swscale.c:962
     frame #4: 0x0007e32c libvideolib.dylib`pixfmt_conv_test() + 620 at
 stream_ffmpeg_resize_filter.cpp:559
     frame #5: 0x0007e35c libvideolib.dylib`::get_resize_filter_api() + 28
 at stream_ffmpeg_resize_filter.cpp:572
     frame #6: 0x00014f39 videolib_test`main(argc=2, argv=0xbffff75c) + 841
 at videolib_tests.cpp:1688
     frame #7: 0x00004df5 videolib_test`start + 53
 (lldb) disassemble --start-address $pc-32 --end-address $pc+32
 libswscale.4.dylib`ff_rgb24ToY_avx:
     0x3cbf35 <+21>: andb   $0x20, %al
     0x3cbf37 <+23>: movl   0x24(%esp), %edi
     0x3cbf3b <+27>: vmovdqa 0x3faf40, %xmm7
     0x3cbf43 <+35>: addl   %esi, %esi
     0x3cbf45 <+37>: addl   %esi, %eax
     0x3cbf47 <+39>: negl   %esi
     0x3cbf49 <+41>: vmovdqa 0x3faf20, %xmm4
     0x3cbf51 <+49>: vmovdqu (%ecx), %xmm0
 ->  0x3cbf55 <+53>: vmovdqu 0xc(%ecx), %xmm2
     0x3cbf5a <+58>: vpshufb 0x3faf50, %xmm0, %xmm1
     0x3cbf63 <+67>: vpshufb %xmm7, %xmm0, %xmm0
     0x3cbf68 <+72>: vpshufb 0x3faf50, %xmm2, %xmm3
 (lldb) register read
 General Purpose Registers:
        eax = 0x02810800
        ebx = 0x017ed100
        ecx = 0x017edfe8
        edx = 0x017ed100
        edi = 0x01807d60
        esi = 0xfffffff0
        ebp = 0x017ed100
        esp = 0xbffff1b0
         ss = 0x00000023
     eflags = 0x00010286  videolib_test`testClipRecode(int, char const**,
 unsigned long long*, unsigned long long, unsigned long long, char const*)
 + 118 at videolib_tests.cpp:1234
        eip = 0x003cbf55  libswscale.4.dylib`ff_rgb24ToY_avx + 53
         cs = 0x0000001b
         ds = 0x00000023
         es = 0x00000023
         fs = 0x00000000
         gs = 0x0000000f

 (lldb)
 }}}

 Note that the crash occurs accessing address 0x17ee000, which is 1 byte
 after the allocated buffer. It occurs intermittently, though we've seen it
 happen at least 50% of the time.
 Overallocating the source buffer by one byte eliminates the crash from the
 above test case.

--
Ticket URL: <https://trac.ffmpeg.org/ticket/5886>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker