[FFmpeg-trac] #72(swscale:new): Invalid pal8 sample crashes libswscale

[FFmpeg]

#72: Invalid pal8 sample crashes libswscale
------------------------+---------------------
  Reporter:  cehoyos    |      Owner:  michael
      Type:  defect     |     Status:  new
  Priority:  important  |  Component:  swscale
   Version:  git        |   Keywords:
Blocked By:             |   Blocking:
Reproduced:  1          |   Analyzed:  0
------------------------+---------------------
 The sample from issue 2497 (that originally crashed the bfi decoder) now
 crashes libswscale.
 {{{
 (gdb) r -i bfi_buffer_overread.bfi out.avi

 FFmpeg version git-N-29196-ge61b83d, Copyright (c) 2000-2011 the FFmpeg
 developers
   built on Apr 19 2011 19:44:16 with gcc 4.4.5
   configuration: --cc='/usr/local/gcc-4.4.5/bin/gcc -m32' --disable-asm
   libavutil    50. 40. 1 / 50. 40. 1
   libavcodec   52.120. 0 / 52.120. 0
   libavformat  52.108. 0 / 52.108. 0
   libavdevice  52.  4. 0 / 52.  4. 0
   libavfilter   1. 79. 1 /  1. 79. 1
   libswscale    0. 13. 0 /  0. 13. 0
 [bfi @ 0x8c66de0] Estimating duration from bitrate, this may be inaccurate
 Input #0, bfi, from 'bfi_buffer_overread.bfi':
   Duration: 00:00:01.88, start: 0.000000, bitrate: 86 kb/s
     Stream #0.0: Video: bfi, pal8, 320x131212, 9 tbr, 9 tbn, 9 tbc
     Stream #0.1: Audio: pcm_u8, 11025 Hz, 1 channels, u8, 88 kb/s
 Incompatible pixel format 'pal8' for codec 'mpeg4', auto-selecting format
 'yuv420p'
 Incompatible sample format 'u8' for codec 'mp2', auto-selecting format
 's16'
 [NULL @ 0x8c6ef30] Requested sampling rate unsupported using closest
 supported (16000)
 [buffer @ 0x8c6f630] w:320 h:131212 pixfmt:pal8
 [ffsink @ 0x8c6f880] auto-inserting filter 'auto-inserted scaler 0'
 between the filter 'src' and the filter 'out'
 [scale @ 0x8c6fc40] w:320 h:131212 fmt:pal8 -> w:320 h:131212 fmt:yuv420p
 flags:0x4

 Program received signal SIGSEGV, Segmentation fault.
 0x085a7753 in sws_init_context (c=0x8c87d40, srcFilter=0xffffbe90,
 dstFilter=0xffffbe90) at libswscale/utils.c:1000
 1000            int nextSlice= FFMAX(c->vLumFilterPos[i   ] +
 c->vLumFilterSize - 1,
 (gdb) bt
 #0  0x085a7753 in sws_init_context (c=0x8c87d40, srcFilter=0xffffbe90,
 dstFilter=0xffffbe90) at libswscale/utils.c:1000
 #1  0x085a8b62 in sws_getContext (srcW=320, srcH=131212,
 srcFormat=PIX_FMT_PAL8, dstW=320, dstH=131212, dstFormat=PIX_FMT_YUV420P,
 flags=4, srcFilter=0x0, dstFilter=0x0, param=0x0) at
 libswscale/utils.c:1166
 #2  0x0806ac4f in config_props (outlink=0x8c6fca0) at
 libavfilter/vf_scale.c:219
 #3  0x08060cac in avfilter_config_links (filter=0x8c6f880) at
 libavfilter/avfilter.c:190
 #4  0x08062b4a in ff_avfilter_graph_config_links (log_ctx=0x0,
 graph=<value optimized out>) at libavfilter/avfiltergraph.c:119
 #5  avfilter_graph_config (log_ctx=0x0, graph=<value optimized out>) at
 libavfilter/avfiltergraph.c:238
 #6  0x08055811 in configure_video_filters (ost=<value optimized out>,
 ist=<value optimized out>) at ffmpeg.c:426
 #7  transcode (ost=<value optimized out>, ist=<value optimized out>) at
 ffmpeg.c:2321
 #8  0x08055cab in main (argc=4, argv=0xffffd004) at ffmpeg.c:4463
 (gdb) disass $pc-32 $pc+32
 Dump of assembler code from 0x85a7733 to 0x85a7773:
 0x085a7733 <sws_init_context+4707>:     je     0x85a7759
 <sws_init_context+4745>
 0x085a7735 <sws_init_context+4709>:     pop    %eax
 0x085a7736 <sws_init_context+4710>:     mov    0x8c(%esp),%edx
 0x085a773d <sws_init_context+4717>:     imul   0x60(%esp),%edx
 0x085a7742 <sws_init_context+4722>:     mov    0x70(%esp),%esi
 0x085a7746 <sws_init_context+4726>:     mov    0x60(%esp),%edi
 0x085a774a <sws_init_context+4730>:     mov    %edx,%eax
 0x085a774c <sws_init_context+4732>:     sar    $0x1f,%edx
 0x085a774f <sws_init_context+4735>:     idivl  0x4c(%esp)
 0x085a7753 <sws_init_context+4739>:     movswl (%esi,%eax,2),%edx
 0x085a7757 <sws_init_context+4743>:     mov    %eax,0x68(%esp)
 0x085a775b <sws_init_context+4747>:     mov    0x78(%esp),%eax
 0x085a775f <sws_init_context+4751>:     mov    0x88(%esp),%esi
 0x085a7766 <sws_init_context+4758>:     movswl (%eax,%edi,2),%edi
 0x085a776a <sws_init_context+4762>:     lea    (%edx,%esi,1),%eax
 0x085a776d <sws_init_context+4765>:     shl    %cl,%eax
 0x085a776f <sws_init_context+4767>:     mov    %edi,0x64(%esp)
 End of assembler dump.
 (gdb) info register
 eax            0xffffcff0       -12304
 ecx            0x0      0
 edx            0xfffeec96       -70506
 ebx            0x8c87d40        147356992
 esp            0xffffbe00       0xffffbe00
 ebp            0x4      0x4
 esi            0xf7fbc020       -134496224
 edi            0x9f99   40857
 eip            0x85a7753        0x85a7753 <sws_init_context+4739>
 eflags         0x10a86  [ PF SF IF OF RF ]
 cs             0x23     35
 ss             0x2b     43
 ds             0x2b     43
 es             0x2b     43
 fs             0x0      0
 gs             0x63     99
 }}}

-- 
Ticket URL: <https://avcodec.org/trac/ffmpeg/ticket/72>
FFmpeg <http://ffmpeg.org>
FFmpeg issue tracker