projects / nihav.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
rv6: fix typo in transform
[nihav.git] / nihav-realmedia / src / codecs / rv60.rs
CommitLineData
5641dccf
KS		 1use nihav_core::formats::YUV420_FORMAT;
2use nihav_core::frame::*;
54d2b09a 3use nihav_core::codecs::{NADecoder, NADecoderSupport, DecoderError, DecoderResult, FrameSkipMode};
7d57ae2f 4use nihav_core::options::*;
b4d5b851 5use nihav_codec_support::codecs::{MV, ZERO_MV, IPBShuffler};
5641dccf
KS		 6use nihav_core::io::byteio::{MemoryReader,ByteReader};
7use nihav_core::io::bitreader::{BitReader,BitReaderMode};
8use nihav_core::io::intcode::*;
54d2b09a 9use std::str::FromStr;
52aad9fe
KS		 10
11use super::rv60codes::*;
12use super::rv60dsp::*;
13
14struct UniqueList<A> {
15 list: [A; 4],
16 fill: usize,
17 max_size: usize,
18}
19
20impl<A:Copy+Default+PartialEq> UniqueList<A> {
21 fn new(max_size: usize) -> Self {
22 Self { list: [A::default(); 4], fill: 0, max_size }
23 }
24 fn add(&mut self, cand: A) {
25 if self.fill == self.max_size { return; }
26 let mut unique = true;
61d3e294 27 for el in self.list.iter().take(self.fill) {
52aad9fe
KS		 28 if *el == cand {
29 unique = false;
30 break;
31 }
32 }
33 if unique {
34 self.list[self.fill] = cand;
35 self.fill += 1;
36 }
37 }
38}
39
40const RV60_FRAME_TYPES: [FrameType; 4] = [ FrameType::I, FrameType::P, FrameType::B, FrameType::Other ];
41const MAX_IMODE: u8 = 34;
42
43#[derive(Clone,Copy,Debug)]
3518aa85 44#[allow(dead_code)]
52aad9fe
KS		 45struct FrameHeader {
46 profile: u8,
47 ftype: FrameType,
48 qp: u8,
49 osvquant: u8,
50 ts: u32,
51 width: usize,
05294442 52 awidth: usize,
52aad9fe 53 height: usize,
05294442 54 aheight: usize,
52aad9fe
KS		 55 two_f_refs: bool,
56 qp_off_type: u8,
57 deblock: bool,
58 deblock_chroma: bool,
59}
60
61const RV60_CUSTOM_MSG_LENS: [u32; 4] = [ 2, 4, 16, 32 ];
62impl FrameHeader {
63 fn read(br: &mut BitReader) -> DecoderResult<Self> {
64 let marker = br.read(2)?;
65 validate!(marker == 3);
66 let profile = br.read(2)? as u8;
67 validate!(profile == 0);
68 let _someval = br.read(4)?;
69 let ftypeid = br.read(2)? as usize;
70 let ftype = RV60_FRAME_TYPES[ftypeid];
71 let qp = br.read(6)? as u8;
72 let marker = br.read(1)?;
73 validate!(marker == 0);
74 let toolset = br.read(2)?;
75 validate!(toolset == 0);
76 let osvquant = br.read(2)? as u8;
77 let _some_flag = br.read_bool()?;
78 let _some_val = br.read(2)?;
79 let ts = br.read(24)?;
80 let width = ((br.read(11)? as usize) + 1) * 4;
81 let height = ((br.read(11)? as usize) + 0) * 4;
82 validate!(height > 0);
05294442
KS		 83 let awidth = (width + 15) & !15;
84 let aheight = (height + 15) & !15;
52aad9fe
KS		 85 let _some_flag = br.read_bool()?;
86 let two_f_refs;
87 if ftype == FrameType::I {
88//byte17 = 0
89 two_f_refs = false;
90 } else {
91 let flag = br.read_bool()?;
92 if flag { // untested
93 br.skip(1)?;
94 br.skip(1)?;
95 br.skip(1)?;
96 }
97//byte17 = flag?
98 two_f_refs = br.read_bool()?;
99 }
100// if byte17 { dw40 = 2; dw3C = 2; } else { dw40 = 1; dw3C = 1; }
101 let _some_val = br.read_code(UintCodeType::Unary012)?; // luma_qp_diff?
102 let chroma_qp_diff = br.read(1)?;
103 validate!(chroma_qp_diff == 0);
104 let qp_off_type = br.read_code(UintCodeType::Unary012)? as u8;
105 let deblock = br.read_bool()?;
106 let deblock_chroma = deblock && !br.read_bool()?;
107 if br.read_bool()? {
108 let custom_msg_hdr_len = br.read(2)? as usize;
109 if custom_msg_hdr_len != 0 {
110 for i in 0..custom_msg_hdr_len {
111 br.skip(RV60_CUSTOM_MSG_LENS[i] * 8)?;
112 }
113 }
114 }
115
116 Ok(FrameHeader {
05294442
KS		 117 profile, ftype, qp, osvquant, ts, width, height, awidth, aheight,
118 two_f_refs, qp_off_type, deblock, deblock_chroma,
52aad9fe
KS		 119 })
120 }
121 fn parse_slice_sizes(&self, br: &mut BitReader, sizes: &mut Vec<usize>) -> DecoderResult<()> {
122 let nslices = self.get_height_cu();
123 let nbits = (br.read(5)? as u8) + 1;
124 validate!(nbits < 32);
125 let mut signs: Vec<bool> = Vec::with_capacity(nslices);
126 for _ in 0..nslices {
127 let sign = br.read_bool()?;
128 signs.push(sign);
129 }
130 validate!(signs[0]);
37952415 131 sizes.clear();
52aad9fe
KS		 132 let mut sum = 0;
133 let first_size = br.read(nbits)? as usize;
134 validate!(first_size > 0);
135 sum += first_size;
136 let mut lastsize = first_size;
137 sizes.push(first_size);
138 for i in 1..nslices {
139 let diff = br.read(nbits)? as isize;
140 let size;
141 if signs[i] {
142 let sum = (lastsize as isize).checked_add(diff);
143 validate!(sum.is_some());
144 size = sum.unwrap() as usize;
145 } else {
146 let sum = (lastsize as isize).checked_sub(diff);
147 validate!(sum.is_some());
148 size = sum.unwrap() as usize;
149 }
150 sizes.push(size);
151 sum += size;
152 lastsize = size;
153 }
154 br.align();
155if ((br.left() >> 3) as usize) != sum {
156println!(" left {} / {}", br.left() >> 3, sum);
157}
158 validate!((br.left() >> 3) >= (sum as isize));
159 Ok(())
160 }
161 fn read_line_qp_offset(&self, br: &mut BitReader) -> DecoderResult<i8> {
162 match self.qp_off_type {
163 0 => Ok(0),
164 1 => {
165 let val = br.read_code(UintCodeType::Unary012)?;
166 if val != 2 {
167 Ok(val as i8)
168 } else {
169 Ok(-1)
170 }
171 },
172 _ => {
173 if br.read(1)? == 0 {
174 Ok(0)
175 } else {
176 let val = br.read(2)? as i8;
177 if (val & 2) == 0 {
178 Ok(val + 1)
179 } else {
180 Ok(-((val & 1) + 1))
181 }
182 }
183 },
184 }
185 }
186 fn get_width_cu(&self) -> usize {
187 (self.width + 63) >> 6
188 }
189 fn get_height_cu(&self) -> usize {
190 (self.height + 63) >> 6
191 }
192 fn has_top_block(&self, xpos: usize, ypos: usize, dx: usize, dy: usize, size: usize) -> bool {
193 if (ypos + dy) == 0 { return false; }
194 let xpos2 = xpos + dx;
195 if (xpos2 + size) > self.width { return false; }
196 true
197 }
198 fn has_top_right_block(&self, xpos: usize, ypos: usize, dx: usize, dy: usize, size: usize) -> bool {
199 if (ypos + dy) == 0 { return false; }
200 let xpos2 = xpos + dx;
201 if (xpos2 + size * 2) > self.width { return false; }
202 let cxpos = ((xpos + dx) & 63) >> RV60_BLOCK_LOG2[size];
203 let cypos = ((ypos + dy) & 63) >> RV60_BLOCK_LOG2[size];
204 ((cypos as u8) & RV60_AVAIL_MASK[cxpos]) == 0
205 }
206 fn has_left_block(&self, xpos: usize, ypos: usize, dx: usize, dy: usize, size: usize) -> bool {
207 if (xpos + dx) == 0 { return false; }
208 let ypos2 = ypos + dy;
209 if (ypos2 + size) > self.height { return false; }
210 true
211 }
212 fn has_left_down_block(&self, xpos: usize, ypos: usize, dx: usize, dy: usize, size: usize) -> bool {
213 if (xpos + dx) == 0 { return false; }
214 let ypos2 = ypos + dy;
215 if (ypos2 + size * 2) > self.height { return false; }
216 let cxpos = (!(xpos + dx) & 63) >> RV60_BLOCK_LOG2[size];
217 let cypos = (!(ypos + dy) & 63) >> RV60_BLOCK_LOG2[size];
218 ((cypos as u8) & RV60_AVAIL_MASK[cxpos]) >= 1
219 }
220}
221
222const RV60_BLOCK_LOG2: [u8; 65] = [
223 0,
224 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
225 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
226 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
227 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6
228];
229const RV60_AVAIL_MASK: [u8; 64] = [
5536ee65 230 0, 1, 0, 3, 0, 1, 0, 7, 0, 1, 0, 3, 0, 1, 0, 0xF,
52aad9fe
KS		 231 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
232 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
233 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
234];
235
236#[derive(Clone,Copy,PartialEq,Debug)]
237enum CUType {
238 Intra,
239 InterMV,
240 Skip,
241 InterNoMV,
242}
243
244impl Default for CUType {
245 fn default() -> Self { CUType::Intra }
246}
247
248const RV60_CU_TYPES: [CUType; 4] = [ CUType::Intra, CUType::InterMV, CUType::Skip, CUType::InterNoMV ];
249
250#[derive(Clone,Copy,PartialEq,Debug)]
251enum PUType {
252 Full,
253 N2Hor,
254 N2Ver,
255 Quarters,
256 N4Hor,
257 N34Hor,
258 N4Ver,
259 N34Ver,
260}
261
262const RV60_PU_TYPES: [PUType; 8] = [
263 PUType::Full, PUType::N2Hor, PUType::N2Ver, PUType::Quarters,
d24468d9 264 PUType::N4Hor, PUType::N34Hor, PUType::N4Ver, PUType::N34Ver,
52aad9fe
KS		 265];
266
267impl PUType {
e07387c7
KS		 268 fn get_num_mvs(self) -> usize {
269 match self {
52aad9fe
KS		 270 PUType::Full => 1,
271 PUType::Quarters => 4,
272 _ => 2,
273 }
274 }
e07387c7 275 fn get_mv_size(self, part_no: usize, size: usize) -> (usize, usize) {
52aad9fe 276 let mv_size = size >> 2;
e07387c7 277 match self {
52aad9fe
KS		 278 PUType::Full => (mv_size, mv_size),
279 PUType::N2Hor => (mv_size, mv_size >> 1),
280 PUType::N2Ver => (mv_size >> 1, mv_size),
281 PUType::Quarters => (mv_size >> 1, mv_size >> 1),
282 PUType::N4Hor => {
283 if part_no == 0 {
e07387c7 284 (mv_size, mv_size >> 2)
52aad9fe 285 } else {
e07387c7 286 (mv_size, (3 * mv_size) >> 2)
52aad9fe
KS		 287 }
288 },
289 PUType::N34Hor => {
290 if part_no == 0 {
e07387c7 291 (mv_size, (3 * mv_size) >> 2)
52aad9fe 292 } else {
e07387c7 293 (mv_size, mv_size >> 2)
52aad9fe
KS		 294 }
295 },
296 PUType::N4Ver => {
297 if part_no == 0 {
e07387c7 298 ( mv_size >> 2, mv_size)
52aad9fe 299 } else {
e07387c7 300 ((3 * mv_size) >> 2, mv_size)
52aad9fe
KS		 301 }
302 },
303 PUType::N34Ver => {
304 if part_no == 0 {
e07387c7 305 ((3 * mv_size) >> 2, mv_size)
52aad9fe 306 } else {
e07387c7 307 ( mv_size >> 2, mv_size)
52aad9fe
KS		 308 }
309 },
310 }
311 }
e07387c7 312 fn has_hor_split(self) -> bool {
6f263099 313 matches!(self, PUType::N2Hor | PUType::N4Hor | PUType::N34Hor | PUType::Quarters)
52aad9fe 314 }
e07387c7 315 fn has_ver_split(self) -> bool {
6f263099 316 matches!(self, PUType::N2Ver | PUType::N4Ver | PUType::N34Ver | PUType::Quarters)
52aad9fe
KS		 317 }
318}
319
320impl Default for PUType {
321 fn default() -> Self { PUType::Full }
322}
323
324#[derive(Clone,Copy,Debug)]
325enum IntraMode {
326 Index(u8),
327 Mode(u8),
328 DC64,
329 Plane64,
330}
331
332#[derive(Clone,Copy,PartialEq,Debug)]
333enum TransformType {
334 None,
335 T4X4,
336 T8X8,
337 T16X16,
338}
339
340impl Default for TransformType {
341 fn default() -> Self { TransformType::None }
342}
343
344#[derive(Clone,Copy,PartialEq,Debug)]
345enum MVRef {
346 None,
347 Ref0,
348 Ref1,
349 BRef,
350 Ref0AndBRef,
351 Skip0,
352 Skip1,
353 Skip2,
354 Skip3,
355}
356
357const SKIP_MV_REF: [MVRef; 4] = [ MVRef::Skip0, MVRef::Skip1, MVRef::Skip2, MVRef::Skip3 ];
358
359impl MVRef {
e07387c7
KS		 360 fn get_skip_mv_num(self) -> usize {
361 match self {
52aad9fe
KS		 362 MVRef::Skip1 => 1,
363 MVRef::Skip2 => 2,
364 MVRef::Skip3 => 3,
365 _ => 0,
366 }
367 }
e07387c7 368 fn is_ref0(self) -> bool {
6f263099 369 matches!(self, MVRef::Ref0 | MVRef::Ref0AndBRef)
52aad9fe 370 }
e07387c7 371 fn is_fwd(self) -> bool {
6f263099 372 matches!(self, MVRef::Ref0 | MVRef::Ref1 | MVRef::Ref0AndBRef)
52aad9fe 373 }
e07387c7 374 fn is_bwd(self) -> bool {
6f263099 375 matches!(self, MVRef::BRef | MVRef::Ref0AndBRef)
52aad9fe
KS		 376 }
377}
378
379#[derive(Clone,Copy,PartialEq,Debug)]
380struct MVInfo {
381 f_mv: MV,
382 b_mv: MV,
383 mvref: MVRef,
384}
385
386impl MVInfo {
387 fn is_some(&self) -> bool { self.mvref != MVRef::None }
388 fn matches_fwd(&self, mvref: MVRef) -> bool {
389 (self.mvref == mvref) || (self.mvref.is_ref0() && mvref.is_ref0())
390 }
391 fn matches_bwd(&self, mvref: MVRef) -> bool {
392 self.mvref.is_bwd() && mvref.is_bwd()
393 }
394 fn is_deblock_cand(&self, other: &MVInfo) -> bool {
395 if self.mvref != other.mvref { return true; }
396 let mut mvdiff = 0;
397 if self.mvref.is_fwd() {
398 let diff = self.f_mv - other.f_mv;
399 mvdiff += diff.x.abs() + diff.y.abs();
400 }
401 if self.mvref.is_bwd() {
402 let diff = self.b_mv - other.b_mv;
403 mvdiff += diff.x.abs() + diff.y.abs();
404 }
405 mvdiff > 4
406 }
407}
408
409impl Default for MVInfo {
410 fn default() -> Self { Self { f_mv: ZERO_MV, b_mv: ZERO_MV, mvref: MVRef::None } }
411}
412
413#[derive(Clone,Copy,Debug)]
414struct CBHeader {
415 cu_type: CUType,
416 pu_type: PUType,
417 ttype: TransformType,
418 imode: [IntraMode; 4],
419 mv: [MVInfo; 4],
420}
421
422impl CBHeader {
423 fn read(br: &mut BitReader, ftype: FrameType, two_f_refs: bool, size: usize) -> DecoderResult<Self> {
424 let cu_type;
425 let pu_type;
426 let mut imode: [IntraMode; 4] = [IntraMode::Index(0); 4];
427 let mut mv: [MVInfo; 4] = [MVInfo::default(); 4];
428 if ftype == FrameType::I {
429 cu_type = CUType::Intra;
430 } else {
431 cu_type = RV60_CU_TYPES[br.read(2)? as usize];
432 }
433 match cu_type {
434 CUType::Intra => {
435 if (size == 8) && br.read_bool()? {
436 pu_type = PUType::Quarters;
437 } else {
438 pu_type = PUType::Full;
439 }
440 if pu_type == PUType::Quarters {
441 for i in 0..4 {
442 imode[i] = CBHeader::read_intra_mode(br)?;
443 }
444 } else if size <= 32 {
445 imode[0] = CBHeader::read_intra_mode(br)?;
446 } else {
447 if !br.read_bool()? {
448 imode[0] = IntraMode::DC64;
449 } else {
450 imode[0] = IntraMode::Plane64;
451 }
452 }
453 },
454 CUType::InterMV => {
455 let bits = if size == 8 { 2 } else { 3 };
456 pu_type = RV60_PU_TYPES[br.read(bits)? as usize];
457 CBHeader::read_mv_data(br, ftype, two_f_refs, size, pu_type, &mut mv)?;
458 },
459 _ => {
460 pu_type = PUType::Full;
461 let skip_mv_no = br.read_code(UintCodeType::LimitedUnary(3, 0))?;
462 mv[0].mvref = SKIP_MV_REF[skip_mv_no as usize];
463 },
464 };
465 let ttype;
466 if cu_type == CUType::Skip {
467 ttype = TransformType::None;
468 } else if size >= 32 {
469 ttype = TransformType::T16X16;
470 } else if size == 16 {
471 if (cu_type == CUType::Intra) || (pu_type == PUType::Full) {
472 ttype = TransformType::T16X16;
473 } else {
474 ttype = TransformType::T4X4;
475 }
476 } else {
477 if pu_type == PUType::Full {
478 ttype = TransformType::T8X8;
479 } else {
480 ttype = TransformType::T4X4;
481 }
482 }
483 Ok(Self {
484 cu_type, pu_type, ttype, imode, mv,
485 })
486 }
487 fn read_intra_mode(br: &mut BitReader) -> DecoderResult<IntraMode> {
488 if br.read_bool()? {
489 let idx = br.read_code(UintCodeType::Unary012)? as u8;
490 Ok(IntraMode::Index(idx))
491 } else {
492 let mode = br.read(5)? as u8;
493 Ok(IntraMode::Mode(mode))
494 }
495 }
496 fn read_mv_data(br: &mut BitReader, ftype: FrameType, two_f_refs: bool, size: usize, pu_type: PUType, mv: &mut [MVInfo; 4]) -> DecoderResult<()> {
497 let mv_count = pu_type.get_num_mvs();
498 for i in 0..mv_count {
499 mv[i] = CBHeader::read_mv_info(br, ftype, two_f_refs, size, pu_type)?;
500 }
501 Ok(())
502 }
503 fn read_mv_info(br: &mut BitReader, ftype: FrameType, two_f_refs: bool, size: usize, pu_type: PUType) -> DecoderResult<MVInfo> {
504 let mut f_mv = ZERO_MV;
505 let mut b_mv = ZERO_MV;
506 let mvref;
507 if ftype != FrameType::B {
508 if two_f_refs && br.read_bool()? {
509 mvref = MVRef::Ref1;
510 } else {
511 mvref = MVRef::Ref0;
512 }
513 f_mv = CBHeader::read_mv(br)?;
514 Ok(MVInfo { f_mv, b_mv: ZERO_MV, mvref })
515 } else {
516 if ((size <= 8) && ((size != 8) || (pu_type != PUType::Full))) || br.read_bool()? {
517 if !br.read_bool()? {
518 mvref = MVRef::Ref0;
519 f_mv = CBHeader::read_mv(br)?;
520 } else {
521 mvref = MVRef::BRef;
522 b_mv = CBHeader::read_mv(br)?;
523 }
524 } else {
525 mvref = MVRef::Ref0AndBRef;
526 f_mv = CBHeader::read_mv(br)?;
527 b_mv = CBHeader::read_mv(br)?;
528 }
529 Ok(MVInfo { f_mv, b_mv, mvref })
530 }
531 }
532 fn read_mv(br: &mut BitReader) -> DecoderResult<MV> {
533 let x = br.read_code_signed(IntCodeType::Gamma)? as i16;
534 let y = br.read_code_signed(IntCodeType::Gamma)? as i16;
535 Ok(MV { x, y })
536 }
537}
538
539#[derive(Clone,Copy,Default)]
540struct PUInfo {
541 cu_type: CUType,
542 ttype: TransformType,
543}
544
545impl PUInfo {
e07387c7 546 fn is_intra(self) -> bool { self.cu_type == CUType::Intra }
52aad9fe
KS		 547}
548
549const RV60_CANDIDATE_INTRA_ANGLES: [u8; 6] = [ 0, 1, 10, 26, 18, 2 ];
550
551#[derive(Clone,Copy,Default)]
552struct BlockInfo {
553 mv: MVInfo,
554 imode: u8,
555}
556
557struct DeblockInfo {
558 left_str: Vec<u8>,
559 top_str: Vec<u8>,
560 stride: usize,
561}
562
563impl DeblockInfo {
564 fn new() -> Self {
565 Self { left_str: Vec::new(), top_str: Vec::new(), stride: 0 }
566 }
567 fn reinit(&mut self, w: usize, h: usize) {
568 self.left_str.clear();
569 self.top_str.clear();
570 self.stride = w >> 2;
571 let size = self.stride * (h >> 2);
572 self.left_str.resize(size, 0);
573 self.top_str.resize(size, 0);
574 }
575 fn set_strength(&mut self, xpos: usize, ypos: usize, size: usize, q: u8, strength: u8) {
576 let pos = self.get_pos(xpos, ypos);
577 let dsize = size >> 2;
578 let dval = (q << 2) | strength;
579 for x in 0..dsize {
580 self.top_str[pos + x] = dval;
581 }
582 for y in 0..dsize {
583 self.left_str[pos + y * self.stride] = dval;
584 }
585 }
586 fn get_pos(&self, xpos: usize, ypos: usize) -> usize {
587 (xpos >> 2) + (ypos >> 2) * self.stride
588 }
589 fn get_top_strength(&self, pos: usize) -> u8 {
590 self.top_str[pos] & 3
591 }
592 fn get_left_strength(&self, pos: usize) -> u8 {
593 self.left_str[pos] & 3
594 }
595 fn set_top_strength(&mut self, pos: usize, str: u8) {
596 self.top_str[pos] |= str;
597 }
598 fn set_left_strength(&mut self, pos: usize, str: u8) {
599 self.left_str[pos] |= str;
600 }
601}
602
603struct RealVideo60Decoder {
2422d969 604 info: NACodecInfoRef,
52aad9fe
KS		 605 cbs: RV60Codebooks,
606 ipbs: IPBShuffler,
607 dsp: RV60DSP,
608 ipred: IntraPredContext,
54d2b09a 609 skip_mode: FrameSkipMode,
52aad9fe 610
3fc28ece 611 avg_buf: NAVideoBufferRef<u8>,
52aad9fe
KS		 612
613 y_coeffs: [i16; 16 * 16],
614 u_coeffs: [i16; 8 * 8],
615 v_coeffs: [i16; 8 * 8],
616 qp: u8,
617 sel_qp: u8,
618
619 cu_splits: Vec<bool>,
620 coded_blk: [bool; 64],
621 dblk: DeblockInfo,
622
623 pu_info: Vec<PUInfo>,
624 pu_stride: usize,
625 pu_pos: usize,
626
627 blk_info: Vec<BlockInfo>,
628 blk_stride: usize,
629 blk_pos: usize,
630
631 xpos: usize,
632 ypos: usize,
d32c444b
KS		 633
634 ts_scale: u64,
635 ref0_pts: u64,
636 ref1_pts: u64,
637 ref0_ts: u64,
638 ref1_ts: u64,
52aad9fe
KS		 639}
640
641impl RealVideo60Decoder {
642 fn new() -> Self {
643 let tmp_vinfo = NAVideoInfo::new(64, 64, false, YUV420_FORMAT);
b70cc006 644 let vt = alloc_video_buffer(tmp_vinfo, 4).unwrap();
52aad9fe
KS		 645 let vb = vt.get_vbuf();
646 let avg_buf = vb.unwrap();
647 RealVideo60Decoder{
2422d969 648 info: NACodecInfoRef::default(),
52aad9fe
KS		 649 cbs: RV60Codebooks::init(),
650 ipbs: IPBShuffler::new(),
651 ipred: IntraPredContext::new(),
54d2b09a 652 skip_mode: FrameSkipMode::default(),
52aad9fe 653 dsp: RV60DSP::new(),
e07387c7 654 avg_buf,
52aad9fe
KS		 655 y_coeffs: [0; 16 * 16],
656 u_coeffs: [0; 8 * 8],
657 v_coeffs: [0; 8 * 8],
658 qp: 0,
659 sel_qp: 0,
660 cu_splits: Vec::with_capacity(24),
661 coded_blk: [false; 64],
662 dblk: DeblockInfo::new(),
663 pu_info: Vec::new(),
664 pu_stride: 0,
665 pu_pos: 0,
666 blk_info: Vec::new(),
667 blk_stride: 0,
668 blk_pos: 0,
669 xpos: 0,
670 ypos: 0,
d32c444b
KS		 671
672 ts_scale: 1,
673 ref0_pts: 0,
674 ref1_pts: 0,
675 ref0_ts: 0,
676 ref1_ts: 0,
52aad9fe
KS		 677 }
678 }
cd830591 679 fn decode_cu_line(&mut self, buf: &mut NASimpleVideoFrame<u8>, hdr: &FrameHeader, src: &[u8], cu_y: usize) -> DecoderResult<()> {
fa90ccfb 680 let mut br = BitReader::new(src, BitReaderMode::BE);
52aad9fe
KS		 681 let cu_w = hdr.get_width_cu();
682 let dqp = hdr.read_line_qp_offset(&mut br)?;
683 let qps = (hdr.qp as i8) + dqp;
6f263099 684 validate!((0..32).contains(&qps));
52aad9fe
KS		 685 let qp = qps as u8;
686 self.qp = qp;
687 self.sel_qp = match hdr.osvquant {
688 0 => qp,
689 1 => {
690 if qp <= 25 {
691 qp + 5
692 } else {
693 qp
694 }
695 },
696 _ => {
697 if qp <= 18 {
698 qp + 10
699 } else if qp <= 25 {
700 qp + 5
701 } else {
702 qp
703 }
704 },
705 };
706
707 for cu_x in 0..cu_w {
708 self.cu_splits.clear();
709 self.coded_blk = [false; 64];
710 self.decode_cb_tree(buf, hdr, &mut br, cu_x << 6, cu_y << 6, 6)?;
711 if hdr.deblock {
712 self.cu_splits.reverse();
713 self.deblock_cb_tree(buf, hdr, cu_x << 6, cu_y << 6, 6);
714 }
715 }
716if br.left() >= 8 {
717println!(" left {} bits", br.left());
718}
719 Ok(())
720 }
b7c882c1 721 #[allow(clippy::cognitive_complexity)]
cd830591 722 fn decode_cb_tree(&mut self, buf: &mut NASimpleVideoFrame<u8>, hdr: &FrameHeader, br: &mut BitReader, xpos: usize, ypos: usize, log_size: u8) -> DecoderResult<()> {
05294442 723 if (xpos >= hdr.awidth) || (ypos >= hdr.aheight) { return Ok(()); }
52aad9fe
KS		 724
725 let size = 1 << log_size;
05294442 726 let split = (xpos + size > hdr.awidth) || (ypos + size > hdr.aheight) || (size > 8 && br.read_bool()?);
52aad9fe
KS		 727 self.cu_splits.push(split);
728 if split {
729 let hsize = size >> 1;
730 self.decode_cb_tree(buf, hdr, br, xpos, ypos, log_size - 1)?;
731 self.decode_cb_tree(buf, hdr, br, xpos + hsize, ypos, log_size - 1)?;
732 self.decode_cb_tree(buf, hdr, br, xpos, ypos + hsize, log_size - 1)?;
733 self.decode_cb_tree(buf, hdr, br, xpos + hsize, ypos + hsize, log_size - 1)?;
734 } else {
735 let cbh = CBHeader::read(br, hdr.ftype, hdr.two_f_refs, size)?;
736 self.pu_pos = (xpos >> 3) + (ypos >> 3) * self.pu_stride;
737 self.blk_pos = (xpos >> 2) + (ypos >> 2) * self.blk_stride;
738 self.xpos = xpos;
739 self.ypos = ypos;
740 self.reconstruct_info(hdr, &cbh, size)?;
741
742 let split_i4x4 = (cbh.cu_type == CUType::Intra) && (size == 8) && (cbh.pu_type == PUType::Quarters);
743 match cbh.cu_type {
744 CUType::Intra => {
745 let itype = self.blk_info[self.blk_pos].imode;
746 if !split_i4x4 {
cd830591 747 let dstride = buf.stride[0];
52aad9fe 748 let off = xpos + ypos * dstride;
cd830591 749 let dst = &mut buf.data;
52aad9fe
KS		 750 self.populate_ipred(hdr, dst, 0, dstride, 0, 0, size, true);
751 self.ipred.pred_angle(dst, off, dstride, size, itype as usize, true);
752 }
753 for comp in 1..3 {
cd830591
KS		 754 let dstride = buf.stride[comp];
755 let soff = buf.offset[comp];
52aad9fe 756 let off = soff + (xpos >> 1) + (ypos >> 1) * dstride;
6f263099 757 let dst = &mut buf.data;
52aad9fe 758 self.populate_ipred(hdr, dst, soff, dstride, 0, 0, size >> 1, false);
6f263099 759 self.ipred.pred_angle(dst, off, dstride, size >> 1, itype as usize, false);
52aad9fe
KS		 760 }
761 },
762 _ => {
763 let mut mv_x = xpos >> 2;
764 let mut mv_y = ypos >> 2;
765 let mut mv_pos = mv_x + mv_y * self.blk_stride;
766 for part_no in 0..cbh.pu_type.get_num_mvs() {
767 let (mv_w, mv_h) = cbh.pu_type.get_mv_size(part_no, size);
768 let mv = self.blk_info[mv_pos].mv;
769 let bw = mv_w << 2;
770 let bh = mv_h << 2;
771 let bx = mv_x << 2;
772 let by = mv_y << 2;
773 match mv.mvref {
774 MVRef::Ref0 => {
775 if hdr.ftype != FrameType::B {
776 if let Some(ref prevbuf) = self.ipbs.get_lastref() {
777 self.dsp.do_mc(buf, prevbuf, bx, by, bw, bh, mv.f_mv, false);
778 }
779 } else {
780 if let Some(ref prevbuf) = self.ipbs.get_b_fwdref() {
781 self.dsp.do_mc(buf, prevbuf, bx, by, bw, bh, mv.f_mv, false);
782 }
783 }
784 },
785 MVRef::Ref1 => {
786 if let Some(ref prevbuf) = self.ipbs.get_nextref() {
787 self.dsp.do_mc(buf, prevbuf, bx, by, bw, bh, mv.f_mv, false);
788 }
789 },
790 MVRef::BRef => {
791 validate!(hdr.ftype == FrameType::B);
792 if let Some(ref prevbuf) = self.ipbs.get_b_bwdref() {
793 self.dsp.do_mc(buf, prevbuf, bx, by, bw, bh, mv.b_mv, false);
794 }
795 },
796 MVRef::Ref0AndBRef => {
797 validate!(hdr.ftype == FrameType::B);
798 if let (Some(ref prevbuf), Some(ref nextbuf)) = (self.ipbs.get_b_fwdref(), self.ipbs.get_b_bwdref()) {
799 self.dsp.do_mc(buf, prevbuf, bx, by, bw, bh, mv.f_mv, false);
cd830591
KS		 800 {
801 let mut avg_buf = NASimpleVideoFrame::from_video_buf(&mut self.avg_buf).unwrap();
802 self.dsp.do_mc(&mut avg_buf, nextbuf, bx, by, bw, bh, mv.b_mv, true);
803 }
52aad9fe
KS		 804 self.dsp.do_avg(buf, &self.avg_buf, bx, by, bw, bh);
805 }
806 },
807 _ => unreachable!(),
808 };
809 if cbh.pu_type == PUType::Quarters {
810 if part_no != 1 {
811 mv_pos += mv_w;
812 mv_x += mv_w;
813 } else {
814 mv_pos += mv_h * self.blk_stride - mv_w;
815 mv_x -= mv_w;
816 mv_y += mv_h;
817 }
818 } else if cbh.pu_type.has_hor_split() {
819 mv_pos += mv_h * self.blk_stride;
820 mv_y += mv_h;
821 } else if cbh.pu_type.has_ver_split() {
822 mv_pos += mv_w;
823 mv_x += mv_w;
824 }
825 }
826 },
827 };
828 if cbh.ttype != TransformType::None {
829 self.y_coeffs = [0; 16 * 16];
830 self.u_coeffs = [0; 8 * 8];
831 self.v_coeffs = [0; 8 * 8];
832 }
833 let is_intra = cbh.cu_type == CUType::Intra;
834 let cb_pos = ((xpos & 63) >> 3) + ((ypos & 63) >> 3) * 8;
835 match cbh.ttype {
836 TransformType::T4X4 => {
837 let subset = if is_intra { 0 } else { 2 };
838 if size == 16 {
839 let cbp16;
840 if br.read_bool()? {
841 cbp16 = rv6_decode_cbp16(br, &self.cbs, subset, self.sel_qp)?;
842 } else {
843 cbp16 = 0;
844 }
845 if cbp16 != 0 {
846 self.coded_blk[cb_pos + 0] = true;
847 self.coded_blk[cb_pos + 1] = true;
848 self.coded_blk[cb_pos + 8] = true;
849 self.coded_blk[cb_pos + 9] = true;
850 rv6_decode_cu_4x4in16x16(br, &self.cbs, is_intra, self.qp, self.sel_qp, &mut self.y_coeffs, &mut self.u_coeffs, &mut self.v_coeffs, cbp16)?;
851 for y in 0..4 {
852 for x in 0..4 {
853 let i = x + y * 4;
854 if ((cbp16 >> i) & 1) != 0 {
855 self.dsp.transform4x4(&mut self.y_coeffs[i * 16..][..16]);
cd830591 856 let dstride = buf.stride[0];
52aad9fe 857 let off = xpos + x * 4 + (ypos + y * 4) * dstride;
6f263099
KS		 858 let dst = &mut buf.data;
859 self.dsp.add_block(dst, off, dstride, &self.y_coeffs[i*16..][..16], 4);
52aad9fe
KS		 860 }
861 }
862 }
863 for y in 0..2 {
864 for x in 0..2 {
865 let i = x + y * 2;
866 let xoff = (xpos >> 1) + x * 4;
867 let yoff = (ypos >> 1) + y * 4;
868 if ((cbp16 >> (16 + i)) & 1) != 0 {
869 self.dsp.transform4x4(&mut self.u_coeffs[i * 16..][..16]);
cd830591
KS		 870 let dstride = buf.stride[1];
871 let off = buf.offset[1] + xoff + yoff * dstride;
6f263099
KS		 872 let dst = &mut buf.data;
873 self.dsp.add_block(dst, off, dstride, &self.u_coeffs[i * 16..][..16], 4);
52aad9fe
KS		 874 }
875 if ((cbp16 >> (20 + i)) & 1) != 0 {
876 self.dsp.transform4x4(&mut self.v_coeffs[i * 16..][..16]);
cd830591
KS		 877 let dstride = buf.stride[2];
878 let off = buf.offset[2] + xoff + yoff * dstride;
6f263099
KS		 879 let dst = &mut buf.data;
880 self.dsp.add_block(dst, off, dstride, &self.v_coeffs[i * 16..][..16], 4);
52aad9fe
KS		 881 }
882 }
883 }
884 }
885 } else {
886 let cbp8 = rv6_decode_cbp8(br, &self.cbs, subset, self.sel_qp)?;
887 if cbp8 != 0 {
888 self.coded_blk[cb_pos] = true;
889 rv6_decode_cu_8x8(br, &self.cbs, is_intra, self.qp, self.sel_qp, &mut self.y_coeffs, &mut self.u_coeffs, &mut self.v_coeffs, cbp8, true)?;
890 }
891 for i in 0..4 {
892 let xoff = (i & 1) * 4;
893 let yoff = (i & 2) * 2;
894 if split_i4x4 {
cd830591 895 let dstride = buf.stride[0];
52aad9fe 896 let off = xpos + xoff + (ypos + yoff) * dstride;
6f263099 897 let dst = &mut buf.data;
52aad9fe
KS		 898 self.populate_ipred(hdr, dst, 0, dstride, xoff, yoff, 4, true);
899 let itype = self.blk_info[self.blk_pos + (i & 1) + (i >> 1) * self.blk_stride].imode;
6f263099 900 self.ipred.pred_angle(dst, off, dstride, 4, itype as usize, false);
52aad9fe
KS		 901 }
902 if ((cbp8 >> i) & 1) != 0 {
903 let blk = &mut self.y_coeffs[i * 16..][..16];
904 self.dsp.transform4x4(blk);
cd830591
KS		 905 let dstride = buf.stride[0];
906 let soff = buf.offset[0];
52aad9fe 907 let off = soff + xpos + xoff + (ypos + yoff) * dstride;
6f263099 908 self.dsp.add_block(buf.data, off, dstride, blk, 4);
52aad9fe
KS		 909 }
910 }
911 if ((cbp8 >> 4) & 1) != 0 {
912 self.dsp.transform4x4(&mut self.u_coeffs);
cd830591
KS		 913 let dstride = buf.stride[1];
914 let soff = buf.offset[1];
52aad9fe 915 let off = soff + (xpos >> 1) + (ypos >> 1) * dstride;
6f263099 916 self.dsp.add_block(buf.data, off, dstride, &self.u_coeffs, 4);
52aad9fe
KS		 917 }
918 if ((cbp8 >> 5) & 1) != 0 {
919 self.dsp.transform4x4(&mut self.v_coeffs);
cd830591
KS		 920 let dstride = buf.stride[2];
921 let soff = buf.offset[2];
52aad9fe 922 let off = soff + (xpos >> 1) + (ypos >> 1) * dstride;
6f263099 923 self.dsp.add_block(buf.data, off, dstride, &self.v_coeffs, 4);
52aad9fe
KS		 924 }
925 }
926 },
927 TransformType::T8X8 => {
928 let subset = if is_intra { 1 } else { 3 };
929 let cbp8 = rv6_decode_cbp8(br, &self.cbs, subset, self.sel_qp)?;
930 if cbp8 != 0 {
931 self.coded_blk[cb_pos] = true;
932 rv6_decode_cu_8x8(br, &self.cbs, is_intra, self.qp, self.sel_qp, &mut self.y_coeffs, &mut self.u_coeffs, &mut self.v_coeffs, cbp8, false)?;
933 if (cbp8 & 0xF) != 0 {
934 self.dsp.transform8x8(&mut self.y_coeffs);
cd830591 935 let dstride = buf.stride[0];
52aad9fe 936 let off = xpos + ypos * dstride;
6f263099 937 self.dsp.add_block(buf.data, off, dstride, &self.y_coeffs, 8);
52aad9fe
KS		 938 }
939 if ((cbp8 >> 4) & 1) != 0 {
940 self.dsp.transform4x4(&mut self.u_coeffs);
cd830591
KS		 941 let dstride = buf.stride[1];
942 let soff = buf.offset[1];
52aad9fe 943 let off = soff + (xpos >> 1) + (ypos >> 1) * dstride;
6f263099 944 self.dsp.add_block(buf.data, off, dstride, &self.u_coeffs, 4);
52aad9fe
KS		 945 }
946 if ((cbp8 >> 5) & 1) != 0 {
947 self.dsp.transform4x4(&mut self.v_coeffs);
cd830591
KS		 948 let dstride = buf.stride[2];
949 let soff = buf.offset[2];
52aad9fe 950 let off = soff + (xpos >> 1) + (ypos >> 1) * dstride;
6f263099 951 self.dsp.add_block(buf.data, off, dstride, &self.v_coeffs, 4);
52aad9fe
KS		 952 }
953 }
954 },
955 TransformType::T16X16 => {
956 let subset = if is_intra { 1 } else { 3 };
957 let num_clusters = size >> 4;
958 let cl_cbp = br.read((num_clusters * num_clusters) as u8)?;
959 for y in 0..num_clusters {
960 for x in 0..num_clusters {
961 if ((cl_cbp >> (x + y * num_clusters)) & 1) == 0 { continue; }
962 self.coded_blk[cb_pos + x * 2 + y * 2 * 8 + 0] = true;
963 self.coded_blk[cb_pos + x * 2 + y * 2 * 8 + 1] = true;
964 self.coded_blk[cb_pos + x * 2 + y * 2 * 8 + 8] = true;
965 self.coded_blk[cb_pos + x * 2 + y * 2 * 8 + 9] = true;
966 let super_cbp = rv6_decode_cbp16(br, &self.cbs, subset, self.sel_qp)?;
967 if super_cbp != 0 {
968 self.y_coeffs = [0; 16 * 16];
969 self.u_coeffs = [0; 8 * 8];
970 self.v_coeffs = [0; 8 * 8];
971 rv6_decode_cu_16x16(br, &self.cbs, is_intra, self.qp, self.sel_qp, &mut self.y_coeffs, &mut self.u_coeffs, &mut self.v_coeffs, super_cbp)?;
972 if (super_cbp & 0xFFFF) != 0 {
973 self.dsp.transform16x16(&mut self.y_coeffs);
cd830591 974 let dstride = buf.stride[0];
52aad9fe 975 let off = xpos + x * 16 + (ypos + y * 16) * dstride;
6f263099 976 self.dsp.add_block(buf.data, off, dstride, &self.y_coeffs, 16);
52aad9fe
KS		 977 }
978 if ((super_cbp >> 16) & 0xF) != 0 {
979 self.dsp.transform8x8(&mut self.u_coeffs);
cd830591
KS		 980 let dstride = buf.stride[1];
981 let soff = buf.offset[1];
52aad9fe 982 let off = soff + (xpos >> 1) + x * 8 + ((ypos >> 1) + y * 8) * dstride;
6f263099 983 self.dsp.add_block(buf.data, off, dstride, &self.u_coeffs, 8);
52aad9fe
KS		 984 }
985 if ((super_cbp >> 20) & 0xF) != 0 {
986 self.dsp.transform8x8(&mut self.v_coeffs);
cd830591
KS		 987 let dstride = buf.stride[2];
988 let soff = buf.offset[2];
52aad9fe 989 let off = soff + (xpos >> 1) + x * 8 + ((ypos >> 1) + y * 8) * dstride;
6f263099 990 self.dsp.add_block(buf.data, off, dstride, &self.v_coeffs, 8);
52aad9fe
KS		 991 }
992 }
993 }
994 }
995 },
996 _ => {},
997 };
998 }
999 Ok(())
1000 }
1001 fn reconstruct_info(&mut self, hdr: &FrameHeader, cbh: &CBHeader, size: usize) -> DecoderResult<()>{
1002 let mut pui = PUInfo::default();
1003 let pu_size = size >> 3;
1004 pui.cu_type = cbh.cu_type;
1005 pui.ttype = cbh.ttype;
1006 if (cbh.cu_type == CUType::Intra) && (cbh.pu_type == PUType::Quarters) { // very special case
1007 self.pu_info[self.pu_pos] = pui;
1008 for y in 0..2 {
1009 for x in 0..2 {
1010 let imode = self.reconstruct_intra(hdr, cbh, 4, x + y * 2);
1011 validate!(imode <= MAX_IMODE);
1012 self.blk_info[self.blk_pos + x + y * self.blk_stride].imode = imode;
1013 self.blk_info[self.blk_pos + x + y * self.blk_stride].mv = MVInfo::default();
1014 }
1015 }
1016 return Ok(());
1017 }
1018 match cbh.cu_type {
1019 CUType::Intra => {
1020 self.pu_info[self.pu_pos] = pui;
1021 let imode = self.reconstruct_intra(hdr, cbh, size, 0);
1022 validate!(imode <= MAX_IMODE);
1023 for y in 0..(size >> 2) {
1024 for x in 0..(size >> 2) {
1025 self.blk_info[self.blk_pos + x + y * self.blk_stride].imode = imode;
1026 }
1027 }
1028 },
1029 CUType::InterMV => {
1030 let mut mv_x = self.xpos >> 2;
1031 let mut mv_y = self.ypos >> 2;
1032 let mut mv_pos = self.blk_pos;
1033 let pu_type = cbh.pu_type;
1034 for part_no in 0..pu_type.get_num_mvs() {
1035 let (mv_w, mv_h) = pu_type.get_mv_size(part_no, size);
1036 let mv = self.predict_mv(hdr, mv_x, mv_y, mv_w, &cbh.mv[part_no]);
1037 for y in 0..mv_h {
1038 for x in 0..mv_w {
1039 self.blk_info[mv_pos + x + y * self.blk_stride].mv = mv;
1040 }
1041 }
1042 if pu_type == PUType::Quarters {
1043 if part_no != 1 {
1044 mv_pos += mv_w;
1045 mv_x += mv_w;
1046 } else {
1047 mv_pos += mv_h * self.blk_stride - mv_w;
1048 mv_x -= mv_w;
1049 mv_y += mv_h;
1050 }
1051 } else if pu_type.has_hor_split() {
1052 mv_pos += mv_h * self.blk_stride;
1053 mv_y += mv_h;
1054 } else if pu_type.has_ver_split() {
1055 mv_pos += mv_w;
1056 mv_x += mv_w;
1057 }
1058 }
1059 },
1060 _ => {
1061 let skip_idx = cbh.mv[0].mvref.get_skip_mv_num();
1062 let mut skip_cand: UniqueList<MVInfo> = UniqueList::new(4);
1063 self.fill_skip_cand(hdr, &mut skip_cand, size);
1064 let mv = skip_cand.list[skip_idx];
1065
1066 let mv_size = size >> 2;
1067 for y in 0..mv_size {
1068 for x in 0..mv_size {
1069 self.blk_info[self.blk_pos + x + y * self.blk_stride].mv = mv;
1070 }
1071 }
1072 },
1073 };
1074 for y in 0..pu_size {
1075 for x in 0..pu_size {
1076 self.pu_info[self.pu_pos + x + y * self.pu_stride] = pui;
1077 }
1078 }
1079 Ok(())
1080 }
1081 fn reconstruct_intra(&self, hdr: &FrameHeader, cbh: &CBHeader, size: usize, sub: usize) -> u8 {
1082 match cbh.imode[0] {
1083 IntraMode::DC64 => { return 1; },
1084 IntraMode::Plane64 => { return 0; },
1085 _ => {},
1086 };
1087 // form list of predictors
1088 let blk_pos = self.blk_pos + (sub & 1) + (sub >> 1) * self.blk_stride;
1089 let mut ipm_cand: UniqueList<u8> = UniqueList::new(3);
1090 if hdr.has_top_block(self.xpos, self.ypos, (sub & 1) * 4, 0, size) {
1091 let pu = &self.pu_info[self.pu_pos - self.pu_stride];
1092 if pu.is_intra() {
1093 ipm_cand.add(self.blk_info[self.blk_pos + (sub & 1) - self.blk_stride].imode);
1094 }
1095 }
1096 if hdr.has_left_block(self.xpos, self.ypos, 0, (sub & 2) * 2, size) {
1097 let pu = &self.pu_info[self.pu_pos - 1];
1098 if pu.is_intra() {
1099 ipm_cand.add(self.blk_info[blk_pos - 1 - (sub & 1)].imode);
1100 }
1101 }
1102 let tl_x = if (sub & 2) == 0 { self.xpos + (sub & 1) * 4 } else { self.xpos };
1103 let tl_y = self.ypos + (sub & 2) * 4;
1104 if (tl_x > 0) && (tl_y > 0) {
1105 let pu = match sub {
1106 0 => &self.pu_info[self.pu_pos - self.pu_stride - 1],
1107 1 => &self.pu_info[self.pu_pos - self.pu_stride],
1108 2 => &self.pu_info[self.pu_pos - 1],
1109 _ => &self.pu_info[self.pu_pos - 1],
1110 };
1111 if pu.is_intra() {
1112 if sub != 3 {
1113 ipm_cand.add(self.blk_info[blk_pos - self.blk_stride - 1].imode);
1114 } else {
1115 ipm_cand.add(self.blk_info[blk_pos - self.blk_stride - 2].imode);
1116 }
1117 }
1118 }
61d3e294 1119 for el in RV60_CANDIDATE_INTRA_ANGLES.iter() {
52aad9fe
KS		 1120 ipm_cand.add(*el);
1121 }
1122 // actually decode prediction mode
1123 match cbh.imode[sub] {
1124 IntraMode::Index(idx) => {
1125 ipm_cand.list[idx as usize]
1126 },
1127 IntraMode::Mode(mode) => {
1128 let mut imode = mode;
1129 let mut ipm_cs: [u8; 3] = [ipm_cand.list[0], ipm_cand.list[1], ipm_cand.list[2]];
1130 ipm_cs.sort();
61d3e294 1131 for ic in ipm_cs.iter() {
52aad9fe
KS		 1132 if imode >= *ic {
1133 imode += 1;
1134 }
1135 }
1136 imode
1137 },
1138 _ => unreachable!(),
1139 }
1140 }
1141 fn populate_ipred(&mut self, hdr: &FrameHeader, src: &[u8], soff: usize, stride: usize, xoff: usize, yoff: usize, size: usize, is_luma: bool) {
1142 let src_off = if is_luma {
1143 soff + self.xpos + xoff + (self.ypos + yoff) * stride
1144 } else {
1145 soff + (self.xpos >> 1) + (self.ypos >> 1) * stride
1146 };
1147 self.ipred = IntraPredContext::new();
1148 if (self.ypos + yoff) > 0 {
1149 self.ipred.has_t = true;
1150 for x in 0..size {
1151 self.ipred.t[x + 1] = src[src_off - stride + x];
1152 }
1153 if (is_luma && hdr.has_top_right_block(self.xpos, self.ypos, xoff, yoff, size)) ||
1154 (!is_luma && hdr.has_top_right_block(self.xpos, self.ypos, 0, 0, size << 1)) {
1155 self.ipred.has_tr = true;
1156 for x in size..size*2 {
1157 self.ipred.t[x + 1] = src[src_off - stride + x];
1158 }
1159 } else {
1160 for i in 0..size {
1161 self.ipred.t[size + i + 1] = self.ipred.t[size];
1162 }
1163 }
1164 if (self.xpos + xoff) > 0 {
1165 self.ipred.t[0] = src[src_off - stride - 1];
1166 }
1167 }
1168 if (self.xpos + xoff) > 0 {
1169 self.ipred.has_l = true;
1170 for y in 0..size {
1171 self.ipred.l[y + 1] = src[src_off - 1 + y * stride];
1172 }
1173 if (is_luma && hdr.has_left_down_block(self.xpos, self.ypos, xoff, yoff, size)) ||
1174 (!is_luma && hdr.has_left_down_block(self.xpos, self.ypos, 0, 0, size << 1)) {
1175 self.ipred.has_ld = true;
1176 for y in size..size*2 {
1177 self.ipred.l[y + 1] = src[src_off - 1 + y * stride];
1178 }
1179 } else {
1180 for i in 0..size {
1181 self.ipred.l[size + i + 1] = self.ipred.l[size];
1182 }
1183 }
1184 if (self.ypos + yoff) > 0 {
1185 self.ipred.l[0] = src[src_off - stride - 1];
1186 }
1187 }
1188 }
1189 fn predict_mv(&self, hdr: &FrameHeader, mv_x: usize, mv_y: usize, mv_w: usize, mvi: &MVInfo) -> MVInfo {
1190 let mv_pos = mv_x + mv_y * self.blk_stride;
1191 let f_mv: MV;
1192 let b_mv: MV;
1193 if mvi.mvref.is_fwd() {
1194 let mut mv_cand: [MV; 3] = [ZERO_MV; 3];
1195 let mut mv_cand_size: usize = 0;
1196 if mv_x > 0 {
1197 let ref_mv = &self.blk_info[mv_pos - 1].mv;
1198 if ref_mv.matches_fwd(mvi.mvref) {
1199 mv_cand[mv_cand_size] = ref_mv.f_mv;
1200 mv_cand_size += 1;
1201 }
1202 }
1203 if mv_y > 0 {
1204 let ref_mv = &self.blk_info[mv_pos - self.blk_stride].mv;
1205 if ref_mv.matches_fwd(mvi.mvref) {
1206 mv_cand[mv_cand_size] = ref_mv.f_mv;
1207 mv_cand_size += 1;
1208 }
1209 }
1210 if hdr.has_top_block(mv_x << 2, mv_y << 2, mv_w << 2, 0, 4) {
1211 let ref_mv = &self.blk_info[mv_pos - self.blk_stride + mv_w].mv;
1212 if ref_mv.matches_fwd(mvi.mvref) {
1213 mv_cand[mv_cand_size] = ref_mv.f_mv;
1214 mv_cand_size += 1;
1215 }
1216 }
1217 f_mv = match mv_cand_size {
1218 1 | 2 => {
1219 let x = mv_cand[0].x + mv_cand[1].x + mv_cand[2].x;
1220 let y = mv_cand[0].y + mv_cand[1].y + mv_cand[2].y;
1221 if mv_cand_size == 1 {
1222 MV { x, y }
1223 } else {
1224 MV { x: x >> 1, y: y >> 1 }
1225 }
1226 },
1227 3 => MV::pred(mv_cand[0], mv_cand[1], mv_cand[2]),
1228 _ => ZERO_MV,
1229 };
1230 } else {
1231 f_mv = ZERO_MV;
1232 }
1233 if mvi.mvref.is_bwd() {
1234 let mut mv_cand: [MV; 3] = [ZERO_MV; 3];
1235 let mut mv_cand_size: usize = 0;
1236 if mv_x > 0 {
1237 let ref_mv = &self.blk_info[mv_pos - 1].mv;
1238 if ref_mv.matches_bwd(mvi.mvref) {
1239 mv_cand[mv_cand_size] = ref_mv.b_mv;
1240 mv_cand_size += 1;
1241 }
1242 }
1243 if mv_y > 0 {
1244 let ref_mv = &self.blk_info[mv_pos - self.blk_stride].mv;
1245 if ref_mv.matches_bwd(mvi.mvref) {
1246 mv_cand[mv_cand_size] = ref_mv.b_mv;
1247 mv_cand_size += 1;
1248 }
1249 }
1250 if hdr.has_top_block(mv_x << 2, mv_y << 2, mv_w << 2, 0, 4) {
1251 let ref_mv = &self.blk_info[mv_pos - self.blk_stride + mv_w].mv;
1252 if ref_mv.matches_bwd(mvi.mvref) {
1253 mv_cand[mv_cand_size] = ref_mv.b_mv;
1254 mv_cand_size += 1;
1255 }
1256 }
1257 b_mv = match mv_cand_size {
1258 1 | 2 => {
1259 let x = mv_cand[0].x + mv_cand[1].x + mv_cand[2].x;
1260 let y = mv_cand[0].y + mv_cand[1].y + mv_cand[2].y;
1261 if mv_cand_size == 1 {
1262 MV { x, y }
1263 } else {
1264 MV { x: x >> 1, y: y >> 1 }
1265 }
1266 },
1267 3 => MV::pred(mv_cand[0], mv_cand[1], mv_cand[2]),
1268 _ => ZERO_MV,
1269 };
1270 } else {
1271 b_mv = ZERO_MV;
1272 }
d24468d9 1273
52aad9fe
KS		 1274 MVInfo { f_mv: mvi.f_mv + f_mv, b_mv: mvi.b_mv + b_mv, mvref: mvi.mvref }
1275 }
1276 fn fill_skip_cand(&mut self, hdr: &FrameHeader, skip_cand: &mut UniqueList<MVInfo>, size: usize) {
1277 let mv_size = size >> 2;
1278
1279 if self.xpos > 0 {
1280 let mv = &self.blk_info[self.blk_pos - 1].mv;
1281 if mv.is_some() {
1282 skip_cand.add(*mv);
1283 }
1284 }
1285 if self.ypos > 0 {
1286 let mv = &self.blk_info[self.blk_pos - self.blk_stride].mv;
1287 if mv.is_some() {
1288 skip_cand.add(*mv);
1289 }
1290 }
1291 if hdr.has_top_right_block(self.xpos, self.ypos, 0, 0, size) {
1292 let mv = &self.blk_info[self.blk_pos - self.blk_stride + mv_size].mv;
1293 if mv.is_some() {
1294 skip_cand.add(*mv);
1295 }
1296 }
1297 if hdr.has_left_down_block(self.xpos, self.ypos, 0, 0, size) {
1298 let mv = &self.blk_info[self.blk_pos + self.blk_stride * mv_size - 1].mv;
1299 if mv.is_some() {
1300 skip_cand.add(*mv);
1301 }
1302 }
1303 if hdr.has_left_block(self.xpos, self.ypos, 0, 0, size) {
1304 let mv = &self.blk_info[self.blk_pos + self.blk_stride * (mv_size - 1) - 1].mv;
1305 if mv.is_some() {
1306 skip_cand.add(*mv);
1307 }
1308 }
1309 if hdr.has_top_block(self.xpos, self.ypos, 0, 0, size) {
1310 let mv = &self.blk_info[self.blk_pos - self.blk_stride + mv_size - 1].mv;
1311 if mv.is_some() {
1312 skip_cand.add(*mv);
1313 }
1314 }
1315 if (self.xpos > 0) && (self.ypos > 0) {
1316 let mv = &self.blk_info[self.blk_pos - self.blk_stride - 1].mv;
1317 if mv.is_some() {
1318 skip_cand.add(*mv);
1319 }
1320 }
1321 for i in skip_cand.fill..4 {
1322 skip_cand.list[i] = MVInfo { f_mv: ZERO_MV, b_mv: ZERO_MV, mvref: MVRef::Ref0 };
1323 }
1324 }
cd830591 1325 fn deblock_cb_tree(&mut self, buf: &mut NASimpleVideoFrame<u8>, hdr: &FrameHeader, xpos: usize, ypos: usize, log_size: u8) {
52aad9fe
KS		 1326 if (xpos >= hdr.width) || (ypos >= hdr.height) { return; }
1327 let split = (log_size > 3) && self.cu_splits.pop().unwrap();
1328 if split {
1329 let hsize = 1 << (log_size - 1);
1330 self.deblock_cb_tree(buf, hdr, xpos, ypos, log_size - 1);
1331 self.deblock_cb_tree(buf, hdr, xpos + hsize, ypos, log_size - 1);
1332 self.deblock_cb_tree(buf, hdr, xpos, ypos + hsize, log_size - 1);
1333 self.deblock_cb_tree(buf, hdr, xpos + hsize, ypos + hsize, log_size - 1);
1334 } else {
1335 let pu_pos = (xpos >> 3) + (ypos >> 3) * self.pu_stride;
1336 let cu_type = self.pu_info[pu_pos].cu_type;
1337 let tsize = if self.pu_info[pu_pos].ttype == TransformType::T16X16 { 4 } else { 3 };
1338 let ntiles = 1 << (log_size - tsize);
1339 let dparams = RV60DeblockParams {
1340 deblock_chroma: hdr.deblock_chroma,
1341 width: hdr.width,
1342 height: hdr.height,
1343 dblkstride: self.dblk.stride,
1344 };
1345 for ty in 0..ntiles {
1346 for tx in 0..ntiles {
1347 let x = xpos + (tx << tsize);
1348 let y = ypos + (ty << tsize);
1349 let cb_pos = ((x & 63) >> 3) + ((y & 63) >> 3) * 8;
1350 if cu_type == CUType::Intra {
1351 self.dblk.set_strength(x, y, 1 << tsize, self.qp, 2);
1352 } else if (cu_type != CUType::Skip) && self.coded_blk[cb_pos] {
1353 self.dblk.set_strength(x, y, 1 << tsize, self.qp, 1);
1354 } else {
1355 self.dblk.set_strength(x, y, 1 << tsize, self.qp, 0);
1356 self.derive_deblock_strength(x, y, 1 << (tsize - 2));
1357 }
1358 self.dsp.do_deblock(&dparams, buf, x, y, 1 << tsize,
1359 self.dblk.top_str.as_slice(),
1360 self.dblk.left_str.as_slice(),
1361 self.dblk.get_pos(x, y));
1362 }
1363 }
1364 }
1365 }
1366 fn derive_deblock_strength(&mut self, xpos: usize, ypos: usize, size4: usize) {
1367 let blk_pos = (xpos >> 2) + (ypos >> 2) * self.blk_stride;
1368 let mut dblk_pos = self.dblk.get_pos(xpos, ypos);
1369 if ypos > 0 {
1370 let top_blk_pos = blk_pos - self.blk_stride;
1371 for i in 0..size4 {
1372 if self.dblk.get_top_strength(dblk_pos + i) == 0 {
1373 if self.blk_info[blk_pos + i].mv.is_deblock_cand(&self.blk_info[top_blk_pos + i].mv) {
1374 self.dblk.set_top_strength(dblk_pos + i, 1);
1375 }
1376 }
1377 }
1378 }
1379 if xpos > 0 {
1380 for i in 0..size4 {
1381 if self.dblk.get_left_strength(dblk_pos) == 0 {
1382 if self.blk_info[blk_pos + i].mv.is_deblock_cand(&self.blk_info[blk_pos + i - 1].mv) {
1383 self.dblk.set_left_strength(dblk_pos, 1);
1384 }
1385 }
1386 dblk_pos += self.dblk.stride;
1387 }
1388 }
1389 }
1390}
1391
1392impl NADecoder for RealVideo60Decoder {
3c69ce1b
KS		 1393 fn init(&mut self, supp: &mut NADecoderSupport, info: NACodecInfoRef) -> DecoderResult<()> {
1394 if let NACodecTypeInfo::Video(vinfo) = info.get_properties() {
52aad9fe
KS		 1395 let fmt = YUV420_FORMAT;
1396 let myinfo = NACodecTypeInfo::Video(NAVideoInfo::new(0, 0, false, fmt));
2422d969 1397 self.info = NACodecInfo::new_ref(info.get_name(), myinfo, info.get_extradata()).into_ref();
52aad9fe
KS		 1398
1399 let edata = info.get_extradata().unwrap();
1400 let src: &[u8] = &edata;
1401
1402 if src.len() < 8 { return Err(DecoderError::InvalidData); }
1403 let mut mr = MemoryReader::new_read(src);
1404 let mut br = ByteReader::new(&mut mr);
1405 let _flags = br.read_u32be()?;
1406 let version = br.read_u32be()?;
1407 let _unk = br.read_u16be()?;
1408 validate!((version >> 28) == 4);
1409 // then width and height again as 16be
1410
1411 //self.bd.width = vinfo.get_width();
1412 //self.bd.height = vinfo.get_height();
1413 //self.frmmgr.clear();
3c69ce1b
KS		 1414
1415 supp.pool_u8.set_dec_bufs(3);
1416 supp.pool_u8.prealloc_video(NAVideoInfo::new(vinfo.get_width(), vinfo.get_height(), false, fmt), 6)?;
1417
52aad9fe
KS		 1418 Ok(())
1419 } else {
1420println!("???");
1421 Err(DecoderError::InvalidData)
1422 }
1423 }
3c69ce1b 1424 fn decode(&mut self, supp: &mut NADecoderSupport, pkt: &NAPacket) -> DecoderResult<NAFrameRef> {
52aad9fe
KS		 1425 let src = pkt.get_buffer();
1426
1427 validate!(src.len() > 9);
1428 let hsize = (src[0] as usize) * 8 + 9;
fa90ccfb 1429 let mut br = BitReader::new(&src[hsize..], BitReaderMode::BE);
52aad9fe 1430 let hdr = FrameHeader::read(&mut br)?;
54d2b09a
KS		 1431 match self.skip_mode {
1432 FrameSkipMode::None => {},
1433 FrameSkipMode::KeyframesOnly => {
1434 if hdr.ftype == FrameType::B {
1435 let mut frm = NAFrame::new_from_pkt(pkt, self.info.clone(), NABufferType::None);
1436 frm.set_frame_type(FrameType::Skip);
1437 return Ok(frm.into_ref());
1438 }
1439 },
1440 FrameSkipMode::IntraOnly => {
1441 if hdr.ftype != FrameType::I {
1442 let mut frm = NAFrame::new_from_pkt(pkt, self.info.clone(), NABufferType::None);
1443 frm.set_frame_type(FrameType::Skip);
1444 return Ok(frm.into_ref());
1445 }
1446 },
1447 };
1448
52aad9fe
KS		 1449 let mut slices: Vec<usize> = Vec::new();
1450 hdr.parse_slice_sizes(&mut br, &mut slices)?;
6e24ec0b
KS		 1451 match hdr.ftype {
1452 FrameType::P => {
1453 if self.ipbs.get_lastref().is_none() {
1454 return Err(DecoderError::MissingReference);
1455 }
1456 },
1457 FrameType::B => {
1458 if self.ipbs.get_lastref().is_none() {
1459 return Err(DecoderError::MissingReference);
1460 }
1461 if self.ipbs.get_nextref().is_none() {
1462 return Err(DecoderError::MissingReference);
1463 }
1464 },
1465 _ => {},
1466 };
52aad9fe
KS		 1467
1468 let tmp_vinfo = NAVideoInfo::new(hdr.width, hdr.height, false, YUV420_FORMAT);
3c69ce1b
KS		 1469 let ret = supp.pool_u8.get_free();
1470 if ret.is_none() {
1471 return Err(DecoderError::AllocError);
1472 }
1473 let mut buf = ret.unwrap();
1474 if buf.get_info() != tmp_vinfo {
1475 self.ipbs.clear();
1476 supp.pool_u8.reset();
1477 supp.pool_u8.prealloc_video(tmp_vinfo, 6)?;
1478 let ret = supp.pool_u8.get_free();
1479 if ret.is_none() {
1480 return Err(DecoderError::AllocError);
1481 }
1482 buf = ret.unwrap();
1483 }
52aad9fe
KS		 1484
1485 let cu_w = hdr.get_width_cu();
1486 let cu_h = hdr.get_height_cu();
1487 self.pu_stride = cu_w << 3;
1488 self.pu_info.resize(self.pu_stride * (cu_h << 3), PUInfo::default());
1489 self.blk_stride = cu_w << 4;
37952415 1490 self.blk_info.clear();
52aad9fe
KS		 1491 self.blk_info.resize(self.blk_stride * (cu_h << 4), BlockInfo::default());
1492 if hdr.deblock {
05294442 1493 self.dblk.reinit(hdr.awidth, hdr.aheight);
52aad9fe
KS		 1494 }
1495 let mut off = hsize + ((br.tell() >> 3) as usize);
cd830591 1496 let mut dframe = NASimpleVideoFrame::from_video_buf(&mut buf).unwrap();
52aad9fe 1497 for (cu_y, size) in slices.into_iter().enumerate() {
cd830591 1498 self.decode_cu_line(&mut dframe, &hdr, &src[off..][..size], cu_y)?;
52aad9fe
KS		 1499 off += size;
1500 }
1501 if (hdr.ftype == FrameType::I) || (hdr.ftype == FrameType::P) {
3c69ce1b 1502 self.ipbs.add_frame(buf.clone());
52aad9fe
KS		 1503 }
1504
d32c444b
KS		 1505 if hdr.ftype != FrameType::B {
1506 self.ref0_pts = self.ref1_pts;
1507 self.ref1_pts = pkt.get_pts().unwrap_or(0);
1508 self.ref0_ts = self.ref1_ts;
1509 self.ref1_ts = hdr.ts as u64;
1510 if (self.ref1_pts > self.ref0_pts) && (self.ref1_ts > self.ref0_ts) {
1511 self.ts_scale = (self.ref1_pts - self.ref0_pts) / (self.ref1_ts - self.ref0_ts);
1512 }
1513 }
3c69ce1b 1514 let mut frm = NAFrame::new_from_pkt(pkt, self.info.clone(), NABufferType::Video(buf));
52aad9fe 1515 frm.set_keyframe(hdr.ftype == FrameType::I);
d32c444b
KS		 1516 if hdr.ftype == FrameType::B {
1517 let pts = self.ref0_pts + ((hdr.ts as u64) - self.ref0_ts) * self.ts_scale;
1518 frm.set_pts(Some(pts));
1519 }
52aad9fe 1520 frm.set_frame_type(hdr.ftype);
171860fc 1521 Ok(frm.into_ref())
52aad9fe 1522 }
f9be4e75
KS		 1523 fn flush(&mut self) {
1524 self.ipbs.clear();
1525 }
52aad9fe
KS		 1526}
1527
54d2b09a
KS		 1528const DECODER_OPTIONS: &[NAOptionDefinition] = &[
1529 NAOptionDefinition {
1530 name: FRAME_SKIP_OPTION, description: FRAME_SKIP_OPTION_DESC,
1531 opt_type: NAOptionDefinitionType::String(Some(&[
1532 FRAME_SKIP_OPTION_VAL_NONE,
1533 FRAME_SKIP_OPTION_VAL_KEYFRAME,
1534 FRAME_SKIP_OPTION_VAL_INTRA
1535 ])) },
1536];
1537
7d57ae2f 1538impl NAOptionHandler for RealVideo60Decoder {
54d2b09a
KS		 1539 fn get_supported_options(&self) -> &[NAOptionDefinition] { DECODER_OPTIONS }
1540 fn set_options(&mut self, options: &[NAOption]) {
1541 for option in options.iter() {
1542 for opt_def in DECODER_OPTIONS.iter() {
1543 if opt_def.check(option).is_ok() {
1544 match (option.name, &option.value) {
1545 (FRAME_SKIP_OPTION, NAValue::String(ref str)) => {
1546 if let Ok(smode) = FrameSkipMode::from_str(str) {
1547 self.skip_mode = smode;
1548 }
1549 },
1550 _ => {},
1551 }
1552 }
1553 }
1554 }
1555 }
1556 fn query_option_value(&self, name: &str) -> Option<NAValue> {
1557 match name {
1558 FRAME_SKIP_OPTION => Some(NAValue::String(self.skip_mode.to_string())),
1559 _ => None,
1560 }
1561 }
7d57ae2f
KS		 1562}
1563
08a1fab7 1564pub fn get_decoder() -> Box<dyn NADecoder + Send> {
52aad9fe
KS		 1565 Box::new(RealVideo60Decoder::new())
1566}
1567
1568#[cfg(test)]
1569mod test {
3167c45c
KS		 1570 use nihav_core::codecs::RegisteredDecoders;
1571 use nihav_core::demuxers::RegisteredDemuxers;
ce742854 1572 use nihav_codec_support::test::dec_video::*;
78fb6560 1573 use crate::realmedia_register_all_decoders;
e64739f8 1574 use crate::realmedia_register_all_demuxers;
52aad9fe
KS		 1575 #[test]
1576 fn test_rv60() {
3167c45c
KS		 1577 let mut dmx_reg = RegisteredDemuxers::new();
1578 realmedia_register_all_demuxers(&mut dmx_reg);
1579 let mut dec_reg = RegisteredDecoders::new();
78fb6560 1580 realmedia_register_all_decoders(&mut dec_reg);
3167c45c 1581
886cde48 1582 // sample from a private collection
c5123998
KS		 1583 test_decoding("realmedia", "realvideo6", "assets/RV/RV60.rmhd", Some(1000), &dmx_reg, &dec_reg,
1584 ExpectedTestResult::MD5Frames(vec![
1585 [0x2b1f1807, 0x09edef33, 0x0e6c78c1, 0x3b3c8179],
fc85bd90 1586 [0xc7d45c3b, 0x6a82ff3a, 0xaf49a7ea, 0x7cf9a533],
c5123998 1587 [0x2b1f1807, 0x09edef33, 0x0e6c78c1, 0x3b3c8179],
fc85bd90
KS		 1588 [0x3db0f7ea, 0xbbf24a80, 0x54c0dd7c, 0xbdea881a],
1589 [0x24134118, 0xeece4c59, 0x3f319c04, 0xd04951fd],
1590 [0xe5f1a7a5, 0x204ab47b, 0x678277b3, 0x179f3007],
1591 [0x61fb5e14, 0x47cce437, 0xaeeed91f, 0x03f727aa],
1592 [0x46c71f20, 0x8e6ee603, 0xb68965ee, 0xf5bf3c45],
1593 [0xf87589a4, 0xd9cc9120, 0xff27a8e6, 0xc1cc1dd5],
1594 [0x12ea3288, 0x810b766b, 0x9a83ac11, 0x88f9996a],
1595 [0xe1a8020f, 0x972fadbb, 0x771f0f7e, 0x7a3a3e41],
1596 [0xed041308, 0x3112b04e, 0xcb39b23b, 0x5f73798c],
1597 [0xb0a2db76, 0x56dd7f97, 0x87e4f6d4, 0xe69ecfd8]]));
52aad9fe
KS		 1598 }
1599}
NihAV is a research multimedia framework