From: Kostya Shishkov Date: Wed, 20 Nov 2019 18:19:16 +0000 (+0100) Subject: core/test: add better decoder testing system X-Git-Url: https://git.nihav.org/?a=commitdiff_plain;h=eca0802bec4df1354e249d68305926fa495a1678;p=nihav.git core/test: add better decoder testing system --- diff --git a/nihav-core/src/test/dec_video.rs b/nihav-core/src/test/dec_video.rs index b27eee1..9ccefd5 100644 --- a/nihav-core/src/test/dec_video.rs +++ b/nihav-core/src/test/dec_video.rs @@ -6,6 +6,8 @@ use crate::demuxers::*; //use crate::io::byteio::*; use crate::scale::*; use super::wavwriter::WavWriter; +use super::md5::MD5; +pub use super::ExpectedTestResult; const OUTPUT_PREFIX: &str = "assets/test_out"; @@ -283,3 +285,164 @@ pub fn test_decode_audio(demuxer: &str, name: &str, limit: Option, audio_pf } } } + +fn frame_checksum(md5: &mut MD5, frm: NAFrameRef) { + match frm.get_buffer() { + NABufferType::Video(ref vb) => { + md5.update_hash(vb.get_data()); + }, + NABufferType::Video16(ref vb) => { + let mut samp = [0u8; 2]; + let data = vb.get_data(); + for el in data.iter() { + samp[0] = (*el >> 8) as u8; + samp[1] = (*el >> 0) as u8; + md5.update_hash(&samp); + } + }, + NABufferType::Video32(ref vb) => { + let mut samp = [0u8; 4]; + let data = vb.get_data(); + for el in data.iter() { + samp[0] = (*el >> 24) as u8; + samp[1] = (*el >> 16) as u8; + samp[2] = (*el >> 8) as u8; + samp[3] = (*el >> 0) as u8; + md5.update_hash(&samp); + } + }, + NABufferType::VideoPacked(ref vb) => { + md5.update_hash(vb.get_data()); + }, + NABufferType::AudioU8(ref ab) => { + md5.update_hash(ab.get_data()); + }, + NABufferType::AudioI16(ref ab) => { + let mut samp = [0u8; 2]; + let data = ab.get_data(); + for el in data.iter() { + samp[0] = (*el >> 8) as u8; + samp[1] = (*el >> 0) as u8; + md5.update_hash(&samp); + } + }, + NABufferType::AudioI32(ref ab) => { + let mut samp = [0u8; 4]; + let data = ab.get_data(); + for el in data.iter() { + samp[0] = (*el >> 24) as u8; + samp[1] = (*el >> 16) as u8; + samp[2] = (*el >> 8) as u8; + samp[3] = (*el >> 0) as u8; + md5.update_hash(&samp); + } + }, + NABufferType::AudioF32(ref ab) => { + let mut samp = [0u8; 4]; + let data = ab.get_data(); + for el in data.iter() { + let bits = el.to_bits(); + samp[0] = (bits >> 24) as u8; + samp[1] = (bits >> 16) as u8; + samp[2] = (bits >> 8) as u8; + samp[3] = (bits >> 0) as u8; + md5.update_hash(&samp); + } + }, + NABufferType::AudioPacked(ref ab) => { + md5.update_hash(ab.get_data()); + }, + NABufferType::Data(ref db) => { + md5.update_hash(db.as_ref()); + }, + NABufferType::None => {}, + }; +} + +pub fn test_decoding(demuxer: &str, dec_name: &str, filename: &str, limit: Option, + dmx_reg: &RegisteredDemuxers, dec_reg: &RegisteredDecoders, + test: ExpectedTestResult) { + let dmx_f = dmx_reg.find_demuxer(demuxer).unwrap(); + let mut file = File::open(filename).unwrap(); + let mut fr = FileReader::new_read(&mut file); + let mut br = ByteReader::new(&mut fr); + let mut dmx = create_demuxer(dmx_f, &mut br).unwrap(); + + let mut decs: Vec, Box)>> = Vec::new(); + let mut found = false; + for i in 0..dmx.get_num_streams() { + let s = dmx.get_stream(i).unwrap(); + let info = s.get_info(); +println!("stream {} codec {} / {}", i, info.get_name(), dec_name); + if !found && (info.get_name() == dec_name) { + let decfunc = dec_reg.find_decoder(info.get_name()); + if let Some(df) = decfunc { + let mut dec = (df)(); + let mut dsupp = Box::new(NADecoderSupport::new()); + dec.init(&mut dsupp, info).unwrap(); + decs.push(Some((dsupp, dec))); + found = true; + } else { + decs.push(None); + } + } else { + decs.push(None); + } + } + + let mut md5 = MD5::new(); + let mut frameiter = if let ExpectedTestResult::MD5Frames(ref vec) = test { + Some(vec.iter()) + } else { + None + }; + loop { + let pktres = dmx.get_frame(); + if let Err(e) = pktres { + if e == DemuxerError::EOF { break; } + panic!("error"); + } + let pkt = pktres.unwrap(); + let streamno = pkt.get_stream().get_id() as usize; + if let Some((ref mut dsupp, ref mut dec)) = decs[streamno] { + if limit.is_some() && pkt.get_pts().is_some() && pkt.get_pts().unwrap() > limit.unwrap() { + break; + } + let frm = dec.decode(dsupp, &pkt).unwrap(); + match &test { + ExpectedTestResult::Decodes => {}, + ExpectedTestResult::MD5(_) => { frame_checksum(&mut md5, frm); }, + ExpectedTestResult::MD5Frames(_) => { + md5 = MD5::new(); + frame_checksum(&mut md5, frm); + md5.finish(); + if let Some(ref mut iter) = frameiter { + let ret = iter.next(); + if ret.is_none() { break; } + let ref_hash = ret.unwrap(); + let mut hash = [0u32; 4]; + md5.get_hash(&mut hash); +println!("frame pts {:?} hash {}", pkt.get_pts(), md5); + assert_eq!(&hash, ref_hash); + } + }, + ExpectedTestResult::GenerateMD5Frames => { + md5 = MD5::new(); + frame_checksum(&mut md5, frm); + md5.finish(); +println!("frame pts {:?} hash {}", pkt.get_pts(), md5); + }, + }; + } + } + if let ExpectedTestResult::MD5(ref ref_hash) = test { + md5.finish(); + let mut hash = [0u32; 4]; + md5.get_hash(&mut hash); +println!("full hash {}", md5); + assert_eq!(&hash, ref_hash); + } + if let ExpectedTestResult::GenerateMD5Frames = test { + panic!("generated hashes"); + } +} diff --git a/nihav-core/src/test/md5.rs b/nihav-core/src/test/md5.rs new file mode 100644 index 0000000..af9a155 --- /dev/null +++ b/nihav-core/src/test/md5.rs @@ -0,0 +1,172 @@ +use std::fmt; + +const MD5_SHIFTS: [u8; 64] = [ + 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, + 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, + 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, + 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21 +]; + +const MD5_K: [u32; 64] = [ + 0xD76AA478, 0xE8C7B756, 0x242070DB, 0xC1BDCEEE, 0xF57C0FAF, 0x4787C62A, 0xA8304613, 0xFD469501, + 0x698098D8, 0x8B44F7AF, 0xFFFF5BB1, 0x895CD7BE, 0x6B901122, 0xFD987193, 0xA679438E, 0x49B40821, + 0xF61E2562, 0xC040B340, 0x265E5A51, 0xE9B6C7AA, 0xD62F105D, 0x02441453, 0xD8A1E681, 0xE7D3FBC8, + 0x21E1CDE6, 0xC33707D6, 0xF4D50D87, 0x455A14ED, 0xA9E3E905, 0xFCEFA3F8, 0x676F02D9, 0x8D2A4C8A, + 0xFFFA3942, 0x8771F681, 0x6D9D6122, 0xFDE5380C, 0xA4BEEA44, 0x4BDECFA9, 0xF6BB4B60, 0xBEBFBC70, + 0x289B7EC6, 0xEAA127FA, 0xD4EF3085, 0x04881D05, 0xD9D4D039, 0xE6DB99E5, 0x1FA27CF8, 0xC4AC5665, + 0xF4292244, 0x432AFF97, 0xAB9423A7, 0xFC93A039, 0x655B59C3, 0x8F0CCC92, 0xFFEFF47D, 0x85845DD1, + 0x6FA87E4F, 0xFE2CE6E0, 0xA3014314, 0x4E0811A1, 0xF7537E82, 0xBD3AF235, 0x2AD7D2BB, 0xEB86D391 +]; + +const INITIAL_MD5_HASH: [u32; 4] = [ 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476 ]; + +fn box0(b: u32, c: u32, d: u32) -> u32 { (b & c) | (!b & d) } +fn box1(b: u32, c: u32, d: u32) -> u32 { (d & b) | (!d & c) } +fn box2(b: u32, c: u32, d: u32) -> u32 { b ^ c ^ d } +fn box3(b: u32, c: u32, d: u32) -> u32 { c ^ (b | !d) } + +#[derive(Clone)] +pub struct MD5 { + pub hash: [u32; 4], + inwords: [u32; 16], + buf: [u8; 64], + pos: usize, + count: usize, +} + +impl PartialEq for MD5 { + fn eq(&self, other: &Self) -> bool { + self.hash == other.hash + } +} + +impl Eq for MD5 { } + +impl fmt::Display for MD5 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:08x}{:08x}{:08x}{:08x}", self.hash[0].swap_bytes(), self.hash[1].swap_bytes(), self.hash[2].swap_bytes(), self.hash[3].swap_bytes()) + } +} + +macro_rules! round { + ($a: ident, $b: ident, $c: ident, $d: ident, $box: ident, $k: expr, $inval: expr) => { + let f = $box($b, $c, $d).wrapping_add($a).wrapping_add(MD5_K[$k]).wrapping_add($inval); + $a = $d; + $d = $c; + $c = $b; + $b = $b.wrapping_add(f.rotate_left(MD5_SHIFTS[$k].into())); + } +} + +#[allow(dead_code)] +impl MD5 { + pub fn new() -> Self { + Self { + hash: INITIAL_MD5_HASH, + inwords: [0; 16], + buf: [0; 64], + pos: 0, + count: 0, + } + } + fn calc_one_block(&mut self) { + let mut a = self.hash[0]; + let mut b = self.hash[1]; + let mut c = self.hash[2]; + let mut d = self.hash[3]; + + for (out, src) in self.inwords.iter_mut().zip(self.buf.chunks_exact(4)) { + *out = (u32::from(src[0]) << 0) | + (u32::from(src[1]) << 8) | + (u32::from(src[2]) << 16) | + (u32::from(src[3]) << 24); + } + + for k in 0..16 { round!(a, b, c, d, box0, k, self.inwords[k]); } + for k in 16..32 { round!(a, b, c, d, box1, k, self.inwords[(5 * k + 1) & 0xF]); } + for k in 32..48 { round!(a, b, c, d, box2, k, self.inwords[(3 * k + 5) & 0xF]); } + for k in 48..64 { round!(a, b, c, d, box3, k, self.inwords[(7 * k) & 0xF]); } + + self.hash[0] = self.hash[0].wrapping_add(a); + self.hash[1] = self.hash[1].wrapping_add(b); + self.hash[2] = self.hash[2].wrapping_add(c); + self.hash[3] = self.hash[3].wrapping_add(d); + + self.pos = 0; + } + pub fn update_hash(&mut self, src: &[u8]) { + for byte in src.iter() { + self.buf[self.pos] = *byte; + self.pos += 1; + if self.pos == 64 { + self.calc_one_block(); + } + } + self.count += src.len(); + } + pub fn finish(&mut self) { + self.buf[self.pos] = 0x80; + self.pos += 1; + if self.pos > 48 { + while self.pos < 64 { + self.buf[self.pos] = 0x00; + self.pos += 1; + } + self.calc_one_block(); + } + while self.pos < 64 { + self.buf[self.pos] = 0x00; + self.pos += 1; + } + for i in 0..8 { + self.buf[56 + i] = ((self.count * 8) >> (i * 8)) as u8; + } + self.calc_one_block(); + } + pub fn get_hash(&self, dst: &mut [u32; 4]) { + for (dst, src) in dst.iter_mut().zip(self.hash.iter()) { + *dst = src.swap_bytes(); + } + } + pub fn get_hash_bytes(&self, dst: &mut [u8; 4]) { + for (dst, src) in dst.chunks_exact_mut(4).zip(self.hash.iter()) { + dst[0] = (*src >> 0) as u8; + dst[1] = (*src >> 8) as u8; + dst[2] = (*src >> 16) as u8; + dst[3] = (*src >> 24) as u8; + } + } + pub fn calculate_hash(src: &[u8], hash: &mut [u32; 4]) { + let mut md5 = Self::new(); + md5.update_hash(src); + md5.finish(); + md5.get_hash(hash); + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_md5() { + let mut hash = [0u32; 4]; + + MD5::calculate_hash(&[], &mut hash); + assert_eq!(hash, [ 0xD41D8CD9, 0x8F00B204, 0xE9800998, 0xECF8427E ]); + + MD5::calculate_hash(b"abc", &mut hash); + assert_eq!(hash, [ 0x90015098, 0x3CD24FB0, 0xD6963F7D, 0x28E17F72 ]); + + MD5::calculate_hash(b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", &mut hash); + assert_eq!(hash, [ 0x8215EF07, 0x96A20BCA, 0xAAE116D3, 0x876C664A ]); + + let mut md5 = MD5::new(); + for _ in 0..1000000 { + md5.update_hash(b"a"); + } + md5.finish(); + md5.get_hash(&mut hash); + assert_eq!(hash, [ 0x7707D6AE, 0x4E027C70, 0xEEA2A935, 0xC2296F21 ]); + } +} diff --git a/nihav-core/src/test/mod.rs b/nihav-core/src/test/mod.rs index aaabd51..15f13eb 100644 --- a/nihav-core/src/test/mod.rs +++ b/nihav-core/src/test/mod.rs @@ -1,2 +1,11 @@ pub mod dec_video; pub mod wavwriter; + +mod md5; // for internal checksums only + +pub enum ExpectedTestResult { + Decodes, + MD5([u32; 4]), + MD5Frames(Vec<[u32; 4]>), + GenerateMD5Frames, +}