videoplayer/src/videodec.rs

   1 use std::thread::JoinHandle;
   2 use std::sync::mpsc::{Receiver, SyncSender, TrySendError};
   3 use std::thread;
   4
   5 use sdl2::render::Texture;
   6
   7 use nihav_core::frame::{NABufferType, NAVideoBuffer};
   8 use nihav_core::formats::*;
   9 use nihav_core::codecs::*;
  10 use nihav_core::scale::*;
  11
  12 use super::{DecoderStuff, DecoderType, DecoderState, DecodingState, DispQueue, FrameRecord, PktSendEvent, FRAME_QUEUE_LEN};
  13
  14 static VDEC_STATE: DecoderState = DecoderState::new();
  15
  16 pub const FRAME_QUEUE_SIZE: usize = 25;
  17
  18 pub const SDL_RGB_FMT: NAPixelFormaton = NAPixelFormaton { model: ColorModel::RGB(RGBSubmodel::RGB), components: 3,
  19     comp_info: [
  20         Some(NAPixelChromaton { h_ss: 0, v_ss: 0, packed: true, depth: 8, shift: 0, comp_offs: 0, next_elem: 3 }),
  21         Some(NAPixelChromaton { h_ss: 0, v_ss: 0, packed: true, depth: 8, shift: 0, comp_offs: 1, next_elem: 3 }),
  22         Some(NAPixelChromaton { h_ss: 0, v_ss: 0, packed: true, depth: 8, shift: 0, comp_offs: 2, next_elem: 3 }),
  23         None, None
  24     ], elem_size: 3, be: false, alpha: false, palette: false };
  25
  26 pub struct VideoDecoder {
  27     yuv_pool:   NAVideoBufferPool<u8>,
  28     rgb_pool:   NAVideoBufferPool<u8>,
  29     tb_num:     u32,
  30     tb_den:     u32,
  31     dec:        DecoderStuff,
  32     ifmt:       NAVideoInfo,
  33     scaler:     NAScale,
  34     ofmt_rgb:   ScaleInfo,
  35     ofmt_yuv:   ScaleInfo,
  36     oinfo_yuv:  NAVideoInfo,
  37     oinfo_rgb:  NAVideoInfo,
  38 }
  39
  40 impl VideoDecoder {
  41     pub fn new(width: usize, height: usize, tb_num: u32, tb_den: u32, dec: DecoderStuff) -> Self {
  42         let ofmt_rgb = ScaleInfo { width, height, fmt: SDL_RGB_FMT };
  43         let ofmt_yuv = ScaleInfo { width, height, fmt: YUV420_FORMAT };
  44         let oinfo_rgb = NAVideoInfo { width, height, flipped: false, format: SDL_RGB_FMT, bits: 24 };
  45         let oinfo_yuv = NAVideoInfo { width, height, flipped: false, format: YUV420_FORMAT, bits: 12 };
  46         Self {
  47             yuv_pool:   NAVideoBufferPool::new(FRAME_QUEUE_SIZE),
  48             rgb_pool:   NAVideoBufferPool::new(FRAME_QUEUE_SIZE),
  49             tb_num, tb_den,
  50             dec, ofmt_yuv, ofmt_rgb, oinfo_yuv, oinfo_rgb,
  51             scaler:     NAScale::new(ofmt_rgb, ofmt_rgb).expect("creating scaler failed"),
  52             ifmt:       NAVideoInfo { width: 0, height: 0, flipped: false, format: SDL_RGB_FMT, bits: 24 },
  53         }
  54     }
  55     fn convert_buf(&mut self, bt: NABufferType, ts: u64) -> Option<FrameRecord> {
  56         let vinfo = bt.get_video_info().expect("this should be a video buffer");
  57         if self.ifmt.get_width() != vinfo.get_width() ||
  58             self.ifmt.get_height() != vinfo.get_height() ||
  59             self.ifmt.get_format() != vinfo.get_format() {
  60             self.ifmt = vinfo;
  61             let sc_ifmt = ScaleInfo { width: self.ifmt.get_width(), height: self.ifmt.get_height(), fmt: self.ifmt.get_format() };
  62             let do_yuv = if let ColorModel::YUV(_) = self.ifmt.get_format().get_model() { true } else { false };
  63             let ofmt = if do_yuv { self.ofmt_yuv } else { self.ofmt_rgb };
  64             self.scaler = NAScale::new(sc_ifmt, ofmt).expect("scaling should not fail");
  65         }
  66         let mut opic = if let ColorModel::YUV(_) = self.ifmt.get_format().get_model() {
  67                 self.yuv_pool.prealloc_video(self.oinfo_yuv, 2).expect("video frame pool allocation failure");
  68                 while self.yuv_pool.get_free().is_none() {
  69                     if VDEC_STATE.is_flushing() {
  70                         return None;
  71                     }
  72                     std::thread::yield_now();
  73                 }
  74                 NABufferType::Video(self.yuv_pool.get_free().expect("video frame pool should have a free frame"))
  75             } else {
  76                 self.rgb_pool.prealloc_video(self.oinfo_rgb, 0).expect("video frame pool allocation failure");
  77                 while self.rgb_pool.get_free().is_none() {
  78                     if VDEC_STATE.is_flushing() {
  79                         return None;
  80                     }
  81                     std::thread::yield_now();
  82                 }
  83                 NABufferType::VideoPacked(self.rgb_pool.get_free().expect("video frame pool should have a free frame"))
  84             };
  85         let ret = self.scaler.convert(&bt, &mut opic);
  86         if ret.is_err() { println!(" scaler error {:?}", ret.err()); return None; }
  87         ret.unwrap();
  88         let time = NATimeInfo::ts_to_time(ts, 1000, self.tb_num, self.tb_den);
  89         Some((opic, time))
  90     }
  91     pub fn next_frame(&mut self, pkt: &NAPacket) -> Option<FrameRecord> {
  92         match self.dec.dec {
  93             DecoderType::Video(ref mut vdec, ref mut reord) => {
  94                 if let Ok(frm) = vdec.decode(&mut self.dec.dsupp, pkt) {
  95                     reord.add_frame(frm);
  96                     while let Some(frm) = reord.get_frame() {
  97                         let bt = frm.get_buffer();
  98                         if let NABufferType::None = bt { continue; }
  99                         let ts = frm.get_dts().unwrap_or_else(|| frm.get_pts().unwrap_or(0));
 100                         return self.convert_buf(bt, ts);
 101                     }
 102                 }
 103             },
 104             DecoderType::VideoMT(ref mut vdec, ref mut reord) => {
 105                 let queue_id = reord.register_frame();
 106                 match vdec.queue_pkt(&mut self.dec.dsupp, &pkt, queue_id) {
 107                     Ok(true) => {},
 108                     Ok(false) => {
 109                         while !vdec.can_take_input() || vdec.has_output() {
 110                             match vdec.get_frame() {
 111                                 (Ok(frm), id) => {
 112                                     reord.add_frame(frm, id);
 113                                 },
 114                                 (Err(err), id) => {
 115                                     reord.drop_frame(id);
 116                                     println!("frame {} decoding error {:?}", id, err);
 117                                 },
 118                             };
 119                         }
 120                         match vdec.queue_pkt(&mut self.dec.dsupp, &pkt, queue_id) {
 121                             Ok(true) => {},
 122                             Ok(false) => {
 123                                 println!("still can't queue frame!");
 124                                 VDEC_STATE.set_state(DecodingState::Error);
 125                             },
 126                             Err(err) => println!("queueing error {:?}", err),
 127                         };
 128                     },
 129                     Err(err) => println!("queueing error {:?}", err),
 130                 };
 131                 while let Some(frm) = reord.get_frame() {
 132                     let bt = frm.get_buffer();
 133                     if let NABufferType::None = bt { continue; }
 134                     let ts = frm.get_dts().unwrap_or_else(|| frm.get_pts().unwrap_or(0));
 135                     return self.convert_buf(bt, ts);
 136                 }
 137             },
 138             _ => panic!("not a video decoder!"),
 139         };
 140         None
 141     }
 142     pub fn more_frames(&mut self, do_not_wait: bool) -> Option<FrameRecord> {
 143         match self.dec.dec {
 144             DecoderType::Video(ref mut _dec, ref mut reord) => {
 145                 while let Some(frm) = reord.get_frame() {
 146                     let bt = frm.get_buffer();
 147                     if let NABufferType::None = bt { continue; }
 148                     let ts = frm.get_dts().unwrap_or_else(|| frm.get_pts().unwrap_or(0));
 149                     return self.convert_buf(bt, ts);
 150                 }
 151             },
 152             DecoderType::VideoMT(ref mut vdec, ref mut reord) => {
 153                 let mut got_some = false;
 154                 while vdec.has_output() {
 155                     match vdec.get_frame() {
 156                         (Ok(frm), id) => {
 157                             reord.add_frame(frm, id);
 158                             got_some = true;
 159                         },
 160                         (Err(err), id) => {
 161                             reord.drop_frame(id);
 162                             println!("frame {} decoding error {:?}", id, err);
 163                         },
 164                     };
 165                 }
 166                 if !got_some && !do_not_wait {
 167                     match vdec.get_frame() {
 168                         (Ok(frm), id) => {
 169                             reord.add_frame(frm, id);
 170                         },
 171                         (Err(DecoderError::NoFrame), _) => {},
 172                         (Err(err), id) => {
 173                             reord.drop_frame(id);
 174                             println!("frame {} decoding error {:?}", id, err);
 175                         },
 176                     };
 177                 }
 178                 while let Some(frm) = reord.get_frame() {
 179                     let bt = frm.get_buffer();
 180                     if let NABufferType::None = bt { continue; }
 181                     let ts = frm.get_dts().unwrap_or_else(|| frm.get_pts().unwrap_or(0));
 182                     return self.convert_buf(bt, ts);
 183                 }
 184             },
 185             _ => {},
 186         };
 187         None
 188     }
 189     pub fn last_frame(&mut self) -> Option<FrameRecord> {
 190         match self.dec.dec {
 191             DecoderType::Video(ref mut _dec, ref mut reord) => {
 192                 while let Some(frm) = reord.get_last_frames() {
 193                     let bt = frm.get_buffer();
 194                     if let NABufferType::None = bt { continue; }
 195                     let ts = frm.get_dts().unwrap_or_else(|| frm.get_pts().unwrap_or(0));
 196                     return self.convert_buf(bt, ts);
 197                 }
 198             },
 199             DecoderType::VideoMT(ref mut _dec, ref mut reord) => {
 200                 while let Some(frm) = reord.get_last_frames() {
 201                     let bt = frm.get_buffer();
 202                     if let NABufferType::None = bt { continue; }
 203                     let ts = frm.get_dts().unwrap_or_else(|| frm.get_pts().unwrap_or(0));
 204                     return self.convert_buf(bt, ts);
 205                 }
 206             },
 207             _ => {},
 208         };
 209         None
 210     }
 211     pub fn flush(&mut self) {
 212         match self.dec.dec {
 213             DecoderType::Video(ref mut dec, ref mut reord) => {
 214                 dec.flush();
 215                 reord.flush();
 216             },
 217             DecoderType::VideoMT(ref mut dec, ref mut reord) => {
 218                 dec.flush();
 219                 reord.flush();
 220             },
 221             _ => {},
 222         };
 223     }
 224 }
 225
 226 fn start_video_decoding(width: usize, height: usize, tb_num: u32, tb_den: u32, video_dec: DecoderStuff, vprecv: Receiver<PktSendEvent>, vfsend: SyncSender<(NABufferType, u64)>) -> JoinHandle<()> {
 227     std::thread::Builder::new().name("vdecoder".to_string()).spawn(move ||{
 228             VDEC_STATE.set_state(DecodingState::Waiting);
 229             let mut vdec = VideoDecoder::new(width, height, tb_num, tb_den, video_dec);
 230             let mut skip_mode = FrameSkipMode::None;
 231             loop {
 232                 match vprecv.recv() {
 233                     Ok(PktSendEvent::Packet(pkt)) => {
 234                         if !VDEC_STATE.is_flushing() {
 235                             if let Some((buf, time)) = vdec.next_frame(&pkt) {
 236                                 vfsend.send((buf, time)).expect("video frame should be sent");
 237                             }
 238                             while let Some((buf, time)) = vdec.more_frames(true) {
 239                                 vfsend.send((buf, time)).expect("video frame should be sent");
 240                             }
 241                         }
 242                     },
 243                     Ok(PktSendEvent::GetFrames) => {
 244                         while let Some((buf, time)) = vdec.more_frames(false) {
 245                             vfsend.send((buf, time)).expect("video frame should be sent");
 246                         }
 247                         VDEC_STATE.set_state(DecodingState::Waiting);
 248                     },
 249                     Ok(PktSendEvent::Flush) => {
 250                         vdec.flush();
 251                         VDEC_STATE.set_state(DecodingState::Waiting);
 252                     },
 253                     Ok(PktSendEvent::End) => {
 254                         while vdec.yuv_pool.get_free().is_some() && vdec.rgb_pool.get_free().is_some() {
 255                             if let Some(frm) = vdec.last_frame() {
 256                                 vfsend.send(frm).expect("video frame should be sent");
 257                             } else {
 258                                 break;
 259                             }
 260                         }
 261                         VDEC_STATE.set_state(DecodingState::End);
 262                         break;
 263                     },
 264                     Ok(PktSendEvent::ImmediateEnd) => {
 265                         VDEC_STATE.set_state(DecodingState::End);
 266                         break;
 267                     },
 268                     Ok(PktSendEvent::HurryUp) => {
 269                         skip_mode = skip_mode.advance();
 270                         if let DecoderType::Video(ref mut dec, ref mut _reord) = vdec.dec.dec {
 271                             dec.set_options(&[NAOption{
 272                                 name: FRAME_SKIP_OPTION,
 273                                 value: NAValue::String(skip_mode.to_string()),
 274                             }]);
 275                         }
 276                     },
 277                     Err(_) => {
 278                         break;
 279                     },
 280                 };
 281             }
 282         }).unwrap()
 283 }
 284
 285 trait Advance {
 286     fn advance(&self) -> Self;
 287 }
 288
 289 impl Advance for FrameSkipMode {
 290     fn advance(&self) -> Self {
 291         match *self {
 292             FrameSkipMode::None             => FrameSkipMode::KeyframesOnly,
 293             FrameSkipMode::KeyframesOnly    => FrameSkipMode::IntraOnly,
 294             FrameSkipMode::IntraOnly        => FrameSkipMode::None,
 295         }
 296     }
 297 }
 298
 299 fn output_yuv(yuv_texture: &mut Texture, buf: &NAVideoBuffer<u8>, width: usize, height: usize) {
 300     let src = buf.get_data();
 301     let ysstride = buf.get_stride(0);
 302     let ysrc = &src[buf.get_offset(0)..];
 303     let usstride = buf.get_stride(2);
 304     let usrc = &src[buf.get_offset(2)..];
 305     let vsstride = buf.get_stride(1);
 306     let vsrc = &src[buf.get_offset(1)..];
 307     yuv_texture.with_lock(None, |buffer: &mut [u8], pitch: usize| {
 308             let csize = pitch.min(width);
 309             for (dline, sline) in buffer.chunks_exact_mut(pitch).take(height).zip(ysrc.chunks_exact(ysstride)) {
 310                 dline[..csize].copy_from_slice(&sline[..csize]);
 311             }
 312             let coff = pitch * height;
 313             let csize = (pitch / 2).min(width / 2);
 314             for (dline, sline) in buffer[coff..].chunks_exact_mut(pitch / 2).take(height/2).zip(vsrc.chunks(vsstride)) {
 315                 dline[..csize].copy_from_slice(&sline[..csize]);
 316             }
 317             let coff = pitch * height + (pitch / 2) * (height / 2);
 318             for (dline, sline) in buffer[coff..].chunks_exact_mut(pitch / 2).take(height/2).zip(usrc.chunks(usstride)) {
 319                 dline[..csize].copy_from_slice(&sline[..csize]);
 320             }
 321         }).expect("surface should be locked");
 322 }
 323
 324
 325 pub struct VideoControl {
 326     vqueue:         Vec<PktSendEvent>,
 327     vpsend:         SyncSender<PktSendEvent>,
 328     vfrecv:         Receiver<FrameRecord>,
 329     do_yuv:         bool,
 330     vthread:        JoinHandle<()>,
 331 }
 332
 333 impl VideoControl {
 334     pub fn new(video_dec: Option<DecoderStuff>, width: usize, height: usize, tb_num: u32, tb_den: u32) -> Self {
 335         let (vpsend, vprecv) = std::sync::mpsc::sync_channel::<PktSendEvent>(0);
 336         let (vfsend, vfrecv) = std::sync::mpsc::sync_channel::<FrameRecord>(FRAME_QUEUE_SIZE - 1);
 337
 338         VDEC_STATE.set_state(DecodingState::Normal);
 339
 340         let vthread = if let Some(video_dec) = video_dec {
 341                 start_video_decoding(width, height, tb_num, tb_den, video_dec, vprecv, vfsend)
 342             } else {
 343                 thread::Builder::new().name("vdecoder-dummy".to_string()).spawn(move ||{
 344                         loop {
 345                             match vprecv.recv() {
 346                                 Ok(PktSendEvent::End) => break,
 347                                 Ok(PktSendEvent::ImmediateEnd) => break,
 348                                 Err(_) => {
 349                                     break;
 350                                 },
 351                                 _ => {},
 352                             };
 353                         }
 354                         VDEC_STATE.set_state(DecodingState::End);
 355                     }).unwrap()
 356             };
 357
 358
 359         Self {
 360             vqueue:         Vec::with_capacity(FRAME_QUEUE_LEN),
 361             vpsend, vfrecv,
 362             do_yuv:         false,
 363             vthread,
 364         }
 365     }
 366     pub fn flush(&mut self) {
 367         self.vqueue.clear();
 368         VDEC_STATE.set_state(DecodingState::Flush);
 369         for _ in 0..8 {
 370             let _ = self.vfrecv.try_recv();
 371         }
 372         let _ = self.vpsend.send(PktSendEvent::Flush);
 373         while self.vfrecv.try_recv().is_ok() { }
 374     }
 375     pub fn get_queue_size(&self) -> usize { self.vqueue.len() }
 376     pub fn is_filled(&self, size: usize) -> bool {
 377         self.vqueue.len() >= size
 378     }
 379     pub fn try_send_video(&mut self, evt: PktSendEvent) -> bool {
 380         if self.vqueue.len() > 0 {
 381             self.vqueue.push(evt);
 382             false
 383         } else {
 384             self.try_send_event(evt)
 385         }
 386     }
 387     fn try_send_event(&mut self, evt: PktSendEvent) -> bool {
 388         if let Err(TrySendError::Full(evt)) = self.vpsend.try_send(evt) {
 389             self.vqueue.insert(0, evt);
 390             false
 391         } else {
 392             true
 393         }
 394     }
 395     pub fn try_send_queued(&mut self) -> bool {
 396         while !self.vqueue.is_empty() {
 397             let pkt = self.vqueue.remove(0);
 398             if !self.try_send_event(pkt) {
 399                  return false;
 400             }
 401         }
 402         true
 403     }
 404     pub fn is_video_end(&self) -> bool {
 405         matches!(VDEC_STATE.get_state(), DecodingState::End | DecodingState::Error)
 406     }
 407     pub fn wait_for_frames(&mut self) -> Result<(), ()> {
 408         VDEC_STATE.set_state(DecodingState::Prefetch);
 409         self.try_send_event(PktSendEvent::GetFrames);
 410         while !self.try_send_queued() {
 411         }
 412         loop {
 413             match VDEC_STATE.get_state() {
 414                 DecodingState::Waiting => {
 415                     VDEC_STATE.set_state(DecodingState::Normal);
 416                     return Ok(());
 417                 },
 418                 DecodingState::Prefetch => thread::yield_now(),
 419                 _ => return Err(()),
 420             };
 421         }
 422     }
 423
 424     pub fn is_yuv(&self) -> bool { self.do_yuv }
 425
 426     pub fn fill(&mut self, disp_queue: &mut DispQueue) {
 427         while !disp_queue.is_full() {
 428             let is_empty = disp_queue.is_empty();
 429             if let Ok((pic, time)) = self.vfrecv.try_recv() {
 430                 let buf = pic.get_vbuf().expect("video frame should be of u8 type");
 431                 self.do_yuv = buf.get_info().get_format().get_model().is_yuv();
 432                 let idx = disp_queue.end;
 433                 disp_queue.move_end();
 434                 let frm = &mut disp_queue.pool[idx];
 435                 if !self.do_yuv {
 436                     let sstride = buf.get_stride(0);
 437                     let src = buf.get_data();
 438                     frm.rgb_tex.with_lock(None, |buffer: &mut [u8], pitch: usize| {
 439                             let csize = sstride.min(pitch);
 440                             for (dst, src) in buffer.chunks_mut(pitch).zip(src.chunks(sstride)) {
 441                                 (&mut dst[..csize]).copy_from_slice(&src[..csize]);
 442                             }
 443                             true
 444                         }).expect("surface should be locked");
 445                 } else {
 446                     output_yuv(&mut frm.yuv_tex, &buf, disp_queue.width, disp_queue.height);
 447                 }
 448                 frm.valid = true;
 449                 frm.is_yuv = self.do_yuv;
 450                 frm.ts = time;
 451                 if is_empty {
 452                     disp_queue.first_ts = time;
 453                 }
 454                 disp_queue.last_ts = time;
 455             } else {
 456                 break;
 457             }
 458         }
 459     }
 460
 461     pub fn finish(self) {
 462         VDEC_STATE.set_state(DecodingState::Flush);
 463         for _ in 0..8 {
 464             let _ = self.vfrecv.try_recv();
 465         }
 466         let _ = self.vpsend.send(PktSendEvent::ImmediateEnd);
 467         self.vthread.join().unwrap();
 468     }
 469 }