X-Git-Url: https://git.nihav.org/?a=blobdiff_plain;f=nihav-itu%2Fsrc%2Fcodecs%2Fh264%2Fdsp%2Fmod.rs;h=a2a58a45dbf8a14b702598d06834eafa28413d56;hb=HEAD;hp=19d260a3c1831fbc220f16079bb1ab7950be94e6;hpb=22de733b14e3ef23eabaaa9f8b2e98a59188c5cf;p=nihav.git diff --git a/nihav-itu/src/codecs/h264/dsp/mod.rs b/nihav-itu/src/codecs/h264/dsp/mod.rs index 19d260a..a2a58a4 100644 --- a/nihav-itu/src/codecs/h264/dsp/mod.rs +++ b/nihav-itu/src/codecs/h264/dsp/mod.rs @@ -1,15 +1,7 @@ -use nihav_core::frame::*; -use nihav_codec_support::codecs::blockdsp::*; -use nihav_codec_support::codecs::MV; - -#[cfg(not(debug_assertions))] -mod release; -#[cfg(not(debug_assertions))] -use release::*; -#[cfg(debug_assertions)] -mod debug; -#[cfg(debug_assertions)] -use debug::*; +mod mc; +pub use mc::{H264MC, McBlock}; +#[cfg(target_arch="x86_64")] +use std::arch::asm; pub const CHROMA_QUANTS: [u8; 52] = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, @@ -138,12 +130,12 @@ pub fn idct_luma_dc(blk: &mut [i16; 16], qp: u8) { for i in 0..4 { transform!(luma_dc; blk[i], blk[i + 4], blk[i + 8], blk[i + 12]); } - for row in blk.chunks_mut(4) { + for row in blk.chunks_exact_mut(4) { transform!(luma_dc; row[0], row[1], row[2], row[3]); } } -pub fn idct(blk: &mut [i16; 16], qp: u8, quant_dc: bool) { +pub fn idct_skip_dc(blk: &mut [i16; 16], qp: u8) { const BLK_INDEX: [usize; 16] = [ 0, 2, 0, 2, 2, 1, 2, 1, @@ -152,11 +144,30 @@ pub fn idct(blk: &mut [i16; 16], qp: u8, quant_dc: bool) { ]; let qidx = (qp % 6) as usize; let shift = qp / 6; - let start = if quant_dc { 0 } else { 1 }; - for (el, &idx) in blk.iter_mut().zip(BLK_INDEX.iter()).skip(start) { + for (el, &idx) in blk.iter_mut().zip(BLK_INDEX.iter()).skip(1) { *el = (*el * LEVEL_SCALE[idx][qidx]) << shift; } - for row in blk.chunks_mut(4) { + for row in blk.chunks_exact_mut(4) { + transform!(row[0], row[1], row[2], row[3], 0); + } + for i in 0..4 { + transform!(blk[i], blk[i + 4], blk[i + 8], blk[i + 12], 6); + } +} + +pub fn idct(blk: &mut [i16; 16], qp: u8) { + const BLK_INDEX: [usize; 16] = [ + 0, 2, 0, 2, + 2, 1, 2, 1, + 0, 2, 0, 2, + 2, 1, 2, 1 + ]; + let qidx = (qp % 6) as usize; + let shift = qp / 6; + for (el, &idx) in blk.iter_mut().zip(BLK_INDEX.iter()) { + *el = (*el * LEVEL_SCALE[idx][qidx]) << shift; + } + for row in blk.chunks_exact_mut(4) { transform!(row[0], row[1], row[2], row[3], 0); } for i in 0..4 { @@ -236,7 +247,7 @@ pub fn idct8x8(blk: &mut [i16; 64], qp: u8) { *dst = i32::from(src).wrapping_mul(i32::from(qmat[idx])).wrapping_add(bias) >> shift; } } - for row in tmp.chunks_mut(8) { + for row in tmp.chunks_exact_mut(8) { transform!(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7]); } for col in 0..8 { @@ -250,7 +261,7 @@ pub fn idct8x8(blk: &mut [i16; 64], qp: u8) { pub fn add_coeffs(dst: &mut [u8], offset: usize, stride: usize, coeffs: &[i16]) { let out = &mut dst[offset..][..stride * 3 + 4]; - for (line, src) in out.chunks_mut(stride).take(4).zip(coeffs.chunks(4)) { + for (line, src) in out.chunks_mut(stride).take(4).zip(coeffs.chunks_exact(4)) { for (dst, src) in line.iter_mut().take(4).zip(src.iter()) { *dst = (i32::from(*dst) + i32::from(*src)).max(0).min(255) as u8; } @@ -259,22 +270,13 @@ pub fn add_coeffs(dst: &mut [u8], offset: usize, stride: usize, coeffs: &[i16]) pub fn add_coeffs8(dst: &mut [u8], offset: usize, stride: usize, coeffs: &[i16; 64]) { let out = &mut dst[offset..]; - for (line, src) in out.chunks_mut(stride).take(8).zip(coeffs.chunks(8)) { + for (line, src) in out.chunks_mut(stride).take(8).zip(coeffs.chunks_exact(8)) { for (dst, src) in line.iter_mut().take(8).zip(src.iter()) { *dst = (i32::from(*dst) + i32::from(*src)).max(0).min(255) as u8; } } } -pub fn avg(dst: &mut [u8], dstride: usize, - src: &[u8], sstride: usize, bw: usize, bh: usize) { - for (dline, sline) in dst.chunks_mut(dstride).zip(src.chunks(sstride)).take(bh) { - for (dst, src) in dline.iter_mut().zip(sline.iter()).take(bw) { - *dst = ((u16::from(*dst) + u16::from(*src) + 1) >> 1) as u8; - } - } -} - fn clip8(val: i16) -> u8 { val.max(0).min(255) as u8 } fn ipred_dc128(buf: &mut [u8], stride: usize, bsize: usize) { @@ -368,7 +370,7 @@ fn ipred_4x4_diag_down_left(buf: &mut [u8], stride: usize, top: &[u8], _left: &[ fn ipred_4x4_diag_down_right(buf: &mut [u8], stride: usize, top: &[u8], left: &[u8], _tr: &[u8]) { let mut t: [u16; 5] = [0; 5]; t[0] = u16::from(left[0]); - load(&mut t[1..], &top); + load(&mut t[1..], top); let mut l: [u16; 5] = [0; 5]; load(&mut l, left); let dst = buf; @@ -386,7 +388,7 @@ fn ipred_4x4_diag_down_right(buf: &mut [u8], stride: usize, top: &[u8], left: &[ fn ipred_4x4_ver_right(buf: &mut [u8], stride: usize, top: &[u8], left: &[u8], _tr: &[u8]) { let mut t: [u16; 5] = [0; 5]; t[0] = u16::from(left[0]); - load(&mut t[1..], &top); + load(&mut t[1..], top); let mut l: [u16; 5] = [0; 5]; load(&mut l, left); let dst = buf; @@ -414,7 +416,7 @@ fn ipred_4x4_ver_right(buf: &mut [u8], stride: usize, top: &[u8], left: &[u8], _ } fn ipred_4x4_ver_left(buf: &mut [u8], stride: usize, top: &[u8], _left: &[u8], tr: &[u8]) { let mut t: [u16; 8] = [0; 8]; - load(&mut t[..4], &top); + load(&mut t[..4], top); load(&mut t[4..], tr); let dst = buf; @@ -444,7 +446,7 @@ fn ipred_4x4_ver_left(buf: &mut [u8], stride: usize, top: &[u8], _left: &[u8], t fn ipred_4x4_hor_down(buf: &mut [u8], stride: usize, top: &[u8], left: &[u8], _tr: &[u8]) { let mut t: [u16; 5] = [0; 5]; t[0] = u16::from(left[0]); - load(&mut t[1..], &top); + load(&mut t[1..], top); let mut l: [u16; 5] = [0; 5]; load(&mut l, left); let dst = buf; @@ -758,7 +760,7 @@ fn ipred_8x8_dc(buf: &mut [u8], stride: usize, top: &[u8], left: &[u8]) { let mut l = [0; 8]; load(&mut l, &left[1..]); let mut t = [0; 8]; - load(&mut t, &top); + load(&mut t, top); let dc0 = ((t[0] + t[1] + t[2] + t[3] + l[0] + l[1] + l[2] + l[3] + 4) >> 3) as u8; let sum1 = t[4] + t[5] + t[6] + t[7]; @@ -907,82 +909,6 @@ pub const IPRED_FUNCS16X16: [IPred8x8Func; 7] = [ ipred_16x16_left_dc, ipred_16x16_top_dc, ipred_16x16_dc128 ]; -fn clip_u8(val: i16) -> u8 { val.max(0).min(255) as u8 } - -pub fn do_mc(frm: &mut NASimpleVideoFrame, refpic: NAVideoBufferRef, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { - let mode = ((mv.x & 3) + (mv.y & 3) * 4) as usize; - copy_block(frm, refpic.clone(), 0, xpos, ypos, mv.x >> 2, mv.y >> 2, w, h, 2, 3, mode, H264_LUMA_INTERP); - - let (cw, ch) = refpic.get_dimensions(1); - let mvx = mv.x >> 3; - let mvy = mv.y >> 3; - let dx = (mv.x & 7) as u16; - let dy = (mv.y & 7) as u16; - let mut ebuf = [0u8; 18 * 9]; - let src_x = ((xpos >> 1) as isize) + (mvx as isize); - let src_y = ((ypos >> 1) as isize) + (mvy as isize); - let suoff = refpic.get_offset(1); - let svoff = refpic.get_offset(2); - let sustride = refpic.get_stride(1); - let svstride = refpic.get_stride(2); - let src = refpic.get_data(); - let cbw = w / 2; - let cbh = h / 2; - let (csrc, cstride) = if (src_x < 0) || (src_x + (cbw as isize) + 1 > (cw as isize)) || (src_y < 0) || (src_y + (cbh as isize) + 1 > (ch as isize)) { - edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf, 18, 1, 4); - edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf[9..], 18, 2, 4); - ([&ebuf, &ebuf[9..]], [18, 18]) - } else { - ([&src[suoff + (src_x as usize) + (src_y as usize) * sustride..], - &src[svoff + (src_x as usize) + (src_y as usize) * svstride..]], - [sustride, svstride]) - }; - for chroma in 1..3 { - let off = frm.offset[chroma] + xpos / 2 + (ypos / 2) * frm.stride[chroma]; - chroma_interp(&mut frm.data[off..], frm.stride[chroma], csrc[chroma - 1], cstride[chroma - 1], dx, dy, cbw, cbh); - } -} - -pub fn gray_block(frm: &mut NASimpleVideoFrame, x: usize, y: usize, w: usize, h: usize) { - let yoff = frm.offset[0] + x + y * frm.stride[0]; - let coff = [frm.offset[1] + x / 2 + y / 2 * frm.stride[1], - frm.offset[2] + x / 2 + y / 2 * frm.stride[2]]; - if w == 16 && h == 16 { - IPRED_FUNCS16X16[IPRED8_DC128](&mut frm.data[yoff..], frm.stride[0], &[], &[]); - for chroma in 1..2 { - IPRED_FUNCS8X8_CHROMA[IPRED8_DC128](&mut frm.data[coff[chroma - 1]..], frm.stride[chroma], &[], &[]); - } - } else if w == 8 && h == 8 { - IPRED_FUNCS8X8_CHROMA[IPRED8_DC128](&mut frm.data[yoff..], frm.stride[0], &[], &[]); - for chroma in 1..2 { - IPRED_FUNCS4X4[IPRED4_DC128](&mut frm.data[coff[chroma - 1]..], frm.stride[chroma], &[128; 4], &[128; 4], &[128; 4]); - } - } else { - for row in frm.data[yoff..].chunks_mut(frm.stride[0]).take(h) { - for el in row[..w].iter_mut() { - *el = 128; - } - } - for chroma in 0..2 { - for row in frm.data[coff[chroma]..].chunks_mut(frm.stride[chroma + 1]).take(h / 2) { - for el in row[..w / 2].iter_mut() { - *el = 128; - } - } - } - } -} - -pub fn do_mc_avg(frm: &mut NASimpleVideoFrame, refpic: NAVideoBufferRef, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV, avg_buf: &mut NAVideoBufferRef) { - let mut afrm = NASimpleVideoFrame::from_video_buf(avg_buf).unwrap(); - let amv = MV { x: mv.x + (xpos as i16) * 4, y: mv.y + (ypos as i16) * 4 }; - do_mc(&mut afrm, refpic, 0, 0, w, h, amv); - for comp in 0..3 { - let shift = if comp == 0 { 0 } else { 1 }; - avg(&mut frm.data[frm.offset[comp] + (xpos >> shift) + (ypos >> shift) * frm.stride[comp]..], frm.stride[comp], &afrm.data[afrm.offset[comp]..], afrm.stride[comp], w >> shift, h >> shift); - } -} - macro_rules! loop_filter { (lumaedge; $buf: expr, $off: expr, $step: expr, $alpha: expr, $beta: expr) => { let p2 = i16::from($buf[$off - $step * 3]); @@ -1058,32 +984,122 @@ fn check_filter(buf: &[u8], off: usize, step: usize, alpha: i16, beta: i16) -> b (p0 - q0).abs() < alpha && (p1 - p0).abs() < beta && (q1 - q0).abs() < beta } +#[cfg(not(target_arch="x86_64"))] +fn check_filter4(buf: &[u8], mut off: usize, step: usize, stride: usize, alpha: i16, beta: i16) -> [bool; 4] { + let mut flags = [false; 4]; + for flag in flags.iter_mut() { + let p1 = i16::from(buf[off - step * 2]); + let p0 = i16::from(buf[off - step]); + let q0 = i16::from(buf[off]); + let q1 = i16::from(buf[off + step]); + *flag = (p0 - q0).abs() < alpha && (p1 - p0).abs() < beta && (q1 - q0).abs() < beta; + off += stride; + } + flags +} + +#[cfg(target_arch="x86_64")] +fn check_filter4(buf: &[u8], off: usize, step: usize, stride: usize, alpha: i16, beta: i16) -> [bool; 4] { + unsafe { + let mut flags = [false; 4]; + let src = buf[off - step * 2..].as_ptr(); + let load_stride = step.max(stride); + let fptr = flags.as_mut_ptr(); + let tflag = u32::from(step == 1); + asm! { + // load block + "pxor xmm4, xmm4", + "movd xmm0, dword ptr [{src}]", + "lea {tmp}, [{src} + {stride} * 2]", + "movd xmm1, dword ptr [{src} + {stride}]", + "movd xmm2, dword ptr [{tmp}]", + "movd xmm3, dword ptr [{tmp} + {stride}]", + "punpcklbw xmm0, xmm4", + "punpcklbw xmm1, xmm4", + "punpcklbw xmm2, xmm4", + "punpcklbw xmm3, xmm4", + + // transpose block if necessary so it's always processed by rows + "test {tflag:e}, {tflag:e}", + "jz 1f", + "punpcklwd xmm0, xmm1", + "movhlps xmm4, xmm0", + "punpcklwd xmm2, xmm3", + "movhlps xmm1, xmm2", + "punpckldq xmm0, xmm2", + "punpckldq xmm4, xmm1", + "movhlps xmm1, xmm0", + "movhlps xmm3, xmm4", + "movaps xmm2, xmm4", + "1:", + + // calculate deltas and flags + "movd xmm4, {alpha:r}", + "movd xmm5, {beta:r}", + "psubw xmm0, xmm1", + "psubw xmm1, xmm2", + "psubw xmm3, xmm2", + "pshuflw xmm4, xmm4, 0", + "pshuflw xmm5, xmm5, 0", + "pabsw xmm0, xmm0", // |p1 - p0| + "pabsw xmm1, xmm1", // |p0 - q0| + "pabsw xmm2, xmm3", // |q1 - q0| + "movaps xmm3, xmm5", + "pcmpgtw xmm4, xmm1", + "pcmpgtw xmm5, xmm0", + "pcmpgtw xmm3, xmm2", + "pand xmm4, xmm5", + "pand xmm4, xmm3", + "packsswb xmm4, xmm4", + "movd [{flags}], xmm4", + tmp = out(reg) _, + src = in(reg) src, + stride = in(reg) load_stride, + alpha = in(reg) alpha, + beta = in(reg) beta, + flags = in(reg) fptr, + tflag = in(reg) tflag, + out("xmm0") _, + out("xmm1") _, + out("xmm2") _, + out("xmm3") _, + out("xmm4") _, + out("xmm5") _, + } + flags + } +} + pub fn loop_filter_lumaedge_v(dst: &mut [u8], mut off: usize, stride: usize, alpha: i16, beta: i16) { - for _ in 0..4 { - if check_filter(dst, off, 1, alpha, beta) { + let flags = check_filter4(dst, off, 1, stride, alpha, beta); + for &flag in flags.iter() { + if flag { loop_filter!(lumaedge; dst, off, 1, alpha, beta); } off += stride; } } pub fn loop_filter_lumaedge_h(dst: &mut [u8], off: usize, stride: usize, alpha: i16, beta: i16) { - for x in 0..4 { - if check_filter(dst, off + x, stride, alpha, beta) { + let flags = check_filter4(dst, off, stride, 1, alpha, beta); + for (x, &flag) in flags.iter().enumerate() { + if flag { loop_filter!(lumaedge; dst, off + x, stride, alpha, beta); } } } pub fn loop_filter_lumanormal_v(dst: &mut [u8], mut off: usize, stride: usize, alpha: i16, beta: i16, tc0: i16) { - for _ in 0..4 { - if check_filter(dst, off, 1, alpha, beta) { + let flags = check_filter4(dst, off, 1, stride, alpha, beta); + for &flag in flags.iter() { + if flag { loop_filter!(lumanormal; dst, off, 1, tc0, beta); } off += stride; } } pub fn loop_filter_lumanormal_h(dst: &mut [u8], off: usize, stride: usize, alpha: i16, beta: i16, tc0: i16) { - for x in 0..4 { - if check_filter(dst, off + x, stride, alpha, beta) { + let flags = check_filter4(dst, off, stride, 1, alpha, beta); + for (x, &flag) in flags.iter().enumerate() { + if flag { loop_filter!(lumanormal; dst, off + x, stride, tc0, beta); } }