X-Git-Url: https://git.nihav.org/?a=blobdiff_plain;f=nihav-itu%2Fsrc%2Fcodecs%2Fh264%2Fdsp%2Fmc%2Fmod.rs;h=615563ee78113f1e4a629da2361bd96d1f6ef83d;hb=fe64781def821c3900abf44bdfbb38f3b3d21345;hp=aa275aea5e6f5406173f20f70342c579dbaec046;hpb=306c604f0d9888a7eadbd34a70520ba0d819d4f3;p=nihav.git diff --git a/nihav-itu/src/codecs/h264/dsp/mc/mod.rs b/nihav-itu/src/codecs/h264/dsp/mc/mod.rs index aa275ae..615563e 100644 --- a/nihav-itu/src/codecs/h264/dsp/mc/mod.rs +++ b/nihav-itu/src/codecs/h264/dsp/mc/mod.rs @@ -1,15 +1,32 @@ use nihav_core::frame::*; use nihav_codec_support::codecs::MV; -use nihav_codec_support::codecs::blockdsp::*; +use super::super::SimpleFrame; -#[cfg(not(debug_assertions))] -mod release; -#[cfg(not(debug_assertions))] -use release::*; -#[cfg(debug_assertions)] -mod debug; -#[cfg(debug_assertions)] -use debug::*; +macro_rules! module_selector { + ($( ($cond:meta, $module:ident) ),*) => { + module_selector!(list; r#false; $(($cond, $module)),*); + }; + (list; $nocond:meta; ($ccar:meta, $carmod:ident), $(($condcdr:meta, $cdrmod:ident)),*) => { + module_selector!(single; $nocond; $ccar; $carmod); + module_selector!(list; any($nocond, $ccar); $(($condcdr, $cdrmod)),*); + }; + (list; $nocond:meta; ($yescond:meta, $module:ident)) => { + module_selector!(single; $nocond; $yescond; $module); + }; + (list; $_:meta; ) => {}; + (single; $nocond:meta; $yescond:meta; $module:ident) => { + #[cfg(all(not($nocond), $yescond))] + mod $module; + #[cfg(all(not($nocond), $yescond))] + use $module::*; + }; +} + +module_selector! ( + (all(feature = "simd", target_arch = "x86_64"), x86), + (debug_assertions, debug), + (not(debug_assertions), release) +); type MCFunc = fn (dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, h: usize); @@ -19,11 +36,29 @@ trait RegisterSIMD { fn register_simd(&mut self); } +#[repr(align(16))] +pub struct McBlock { + pub y: [u8; 16 * 16], + pub u: [u8; 16 * 16], + pub v: [u8; 16 * 16], +} + +impl McBlock { + pub fn new() -> Self { + unsafe { + let blk = std::mem::MaybeUninit::uninit(); + blk.assume_init() + } + } +} + +#[allow(clippy::type_complexity)] pub struct H264MC { avg_buf: NAVideoBufferRef, pub put_block_weighted: [fn (dst: &mut [u8], stride: usize, src: &[u8], h: usize, wparams: [i8; 3]); 4], pub put_block_weighted2: [fn (dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], h: usize, wparams: [i8; 5]); 4], pub chroma_interp: [fn (dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, dx: u16, dy: u16, h: usize); 3], + avg: [fn (dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize); 4], width: usize, height: usize, @@ -36,6 +71,7 @@ impl H264MC { put_block_weighted: [put_blk_w_2, put_blk_w_4, put_blk_w_8, put_blk_w_16], put_block_weighted2: [put_blk_w2_2, put_blk_w2_4, put_blk_w2_8, put_blk_w2_16], chroma_interp: [chroma_interp_2, chroma_interp_4, chroma_interp_8], + avg: [avg_2, avg_4, avg_8, avg_16], width: 0, height: 0, }; obj.register_simd(); @@ -45,7 +81,7 @@ impl H264MC { self.width = width; self.height = height; } - pub fn do_mc(&mut self, frm: &mut NASimpleVideoFrame, refpic: NAVideoBufferRef, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { + pub fn do_mc(&mut self, frm: &mut NASimpleVideoFrame, refpic: &SimpleFrame, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { let mut ebuf = [0u8; 22 * 22]; let mvx = mv.x >> 2; let mvy = mv.y >> 2; @@ -53,16 +89,16 @@ impl H264MC { let pre = if mode != 0 { 2isize } else { 0 }; let post = if mode != 0 { 3isize } else { 0 }; let (yw, yh) = (self.width, self.height); - let src = refpic.get_data(); - let systride = refpic.get_stride(0); + let src = refpic.data; + let systride = refpic.stride[0]; let src_x = (xpos as isize) + (mvx as isize); let src_y = (ypos as isize) + (mvy as isize); let (ysrc, ystride) = if (src_x - pre < 0) || (src_x + (w as isize) + post > (yw as isize)) || (src_y - pre < 0) || (src_y + (h as isize) + post > (yh as isize)) { let add = (pre + post) as usize; - edge_emu(&refpic, src_x - pre, src_y - pre, w + add, h + add, &mut ebuf, 22, 0, 0); - (ebuf.as_slice(), 22) + edge_emu_sf(refpic, src_x - pre, src_y - pre, yw, yh, w + add, h + add, &mut ebuf, 22, 0); + (&ebuf[..], 22) } else { - (&src[refpic.get_offset(0) + ((src_x - pre) as usize) + ((src_y - pre) as usize) * systride..], systride) + (&src[refpic.offset[0] + ((src_x - pre) as usize) + ((src_y - pre) as usize) * systride..], systride) }; let wmode = match w { 4 => 0, @@ -78,15 +114,17 @@ impl H264MC { let dy = (mv.y & 7) as u16; let src_x = ((xpos >> 1) as isize) + (mvx as isize); let src_y = ((ypos >> 1) as isize) + (mvy as isize); - let suoff = refpic.get_offset(1); - let svoff = refpic.get_offset(2); - let sustride = refpic.get_stride(1); - let svstride = refpic.get_stride(2); + let suoff = refpic.offset[1]; + let svoff = refpic.offset[2]; + let sustride = refpic.stride[1]; + let svstride = refpic.stride[2]; let cbw = w / 2; let cbh = h / 2; let (csrc, cstride) = if (src_x < 0) || (src_x + (cbw as isize) + 1 > (cw as isize)) || (src_y < 0) || (src_y + (cbh as isize) + 1 > (ch as isize)) { - edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf, 18, 1, 4); - edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf[9..], 18, 2, 4); + let aw = (cw + 7) & !7; + let ah = (ch + 7) & !7; + edge_emu_sf(refpic, src_x, src_y, aw, ah, cbw+1, cbh+1, &mut ebuf, 18, 1); + edge_emu_sf(refpic, src_x, src_y, aw, ah, cbw+1, cbh+1, &mut ebuf[9..], 18, 2); ([&ebuf, &ebuf[9..]], [18, 18]) } else { ([&src[suoff + (src_x as usize) + (src_y as usize) * sustride..], @@ -99,7 +137,7 @@ impl H264MC { } } - pub fn mc_blocks(&mut self, ydst: &mut [u8], udst: &mut [u8], vdst: &mut [u8], refpic: NAVideoBufferRef, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { + pub fn mc_blocks(&mut self, dst: &mut McBlock, refpic: &SimpleFrame, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { let mode = ((mv.x & 3) + (mv.y & 3) * 4) as usize; let pre = if mode != 0 { 2 } else { 0 }; @@ -119,16 +157,15 @@ impl H264MC { if (sx - pre < 0) || (sx + (w as isize) + post > (width as isize)) || (sy - pre < 0) || (sy + (h as isize) + post > (height as isize)) { let edge = (pre + post) as usize; - edge_emu(&refpic, sx - pre, sy - pre, w + edge, h + edge, - &mut ebuf, EBUF_STRIDE, 0, 0); - (H264_LUMA_INTERP[wmode][mode])(ydst, 16, &ebuf, EBUF_STRIDE, h); + edge_emu_sf(refpic, sx - pre, sy - pre, width, height, w + edge, h + edge, + &mut ebuf, EBUF_STRIDE, 0); + (H264_LUMA_INTERP[wmode][mode])(&mut dst.y, 16, &ebuf, EBUF_STRIDE, h); } else { - let sstride = refpic.get_stride(0); - let soff = refpic.get_offset(0); - let sdta = refpic.get_data(); - let sbuf: &[u8] = sdta.as_slice(); + let sstride = refpic.stride[0]; + let soff = refpic.offset[0]; + let sbuf = refpic.data; let saddr = soff + ((sx - pre) as usize) + ((sy - pre) as usize) * sstride; - (H264_LUMA_INTERP[wmode][mode])(ydst, 16, &sbuf[saddr..], sstride, h); + (H264_LUMA_INTERP[wmode][mode])(&mut dst.y, 16, &sbuf[saddr..], sstride, h); } let (cw, ch) = (self.width >> 1, self.height >> 1); @@ -138,45 +175,59 @@ impl H264MC { let dy = (mv.y & 7) as u16; let src_x = ((xpos >> 1) as isize) + (mvx as isize); let src_y = ((ypos >> 1) as isize) + (mvy as isize); - let suoff = refpic.get_offset(1); - let svoff = refpic.get_offset(2); - let sustride = refpic.get_stride(1); - let svstride = refpic.get_stride(2); - let src = refpic.get_data(); + let suoff = refpic.offset[1]; + let svoff = refpic.offset[2]; + let sustride = refpic.stride[1]; + let svstride = refpic.stride[2]; + let src = refpic.data; let cbw = w / 2; let cbh = h / 2; let (csrc, cstride) = if (src_x < 0) || (src_x + (cbw as isize) + 1 > (cw as isize)) || (src_y < 0) || (src_y + (cbh as isize) + 1 > (ch as isize)) { - edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf, 18, 1, 4); - edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf[9..], 18, 2, 4); + let aw = (cw + 7) & !7; + let ah = (ch + 7) & !7; + edge_emu_sf(refpic, src_x, src_y, aw, ah, cbw+1, cbh+1, &mut ebuf, 18, 1); + edge_emu_sf(refpic, src_x, src_y, aw, ah, cbw+1, cbh+1, &mut ebuf[9..], 18, 2); ([&ebuf, &ebuf[9..]], [18, 18]) } else { ([&src[suoff + (src_x as usize) + (src_y as usize) * sustride..], &src[svoff + (src_x as usize) + (src_y as usize) * svstride..]], [sustride, svstride]) }; - (self.chroma_interp[wmode])(udst, 16, csrc[0], cstride[0], dx, dy, cbh); - (self.chroma_interp[wmode])(vdst, 16, csrc[1], cstride[1], dx, dy, cbh); - } - - pub fn avg(&mut self, dst: &mut [u8], dstride: usize, bw: usize, bh: usize, comp: usize) { - let afrm = NASimpleVideoFrame::from_video_buf(&mut self.avg_buf).unwrap(); - let src = &afrm.data[afrm.offset[comp]..]; - let sstride = afrm.stride[comp]; - for (dline, sline) in dst.chunks_mut(dstride).zip(src.chunks(sstride)).take(bh) { - for (dst, src) in dline.iter_mut().zip(sline.iter()).take(bw) { - *dst = ((u16::from(*dst) + u16::from(*src) + 1) >> 1) as u8; - } - } + (self.chroma_interp[wmode])(&mut dst.u, 16, csrc[0], cstride[0], dx, dy, cbh); + (self.chroma_interp[wmode])(&mut dst.v, 16, csrc[1], cstride[1], dx, dy, cbh); } - pub fn do_mc_avg(&mut self, frm: &mut NASimpleVideoFrame, refpic: NAVideoBufferRef, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { + pub fn do_mc_avg(&mut self, frm: &mut NASimpleVideoFrame, refpic: &SimpleFrame, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { let mut abuf = self.avg_buf.clone(); - let mut afrm = NASimpleVideoFrame::from_video_buf(&mut abuf).unwrap(); + let stride_y = abuf.get_stride(0); + let stride_c = abuf.get_stride(1); + let off_y = abuf.get_offset(0); + let off_u = abuf.get_offset(1); + let off_v = abuf.get_offset(2); + let data = abuf.get_data_mut().unwrap(); + let mut afrm = NASimpleVideoFrame { + width: [64, 32, 32, 0], + height: [64, 32, 32, 0], + flip: false, + stride: [stride_y, stride_c, stride_c, 0], + offset: [off_y, off_u, off_v, 0], + components: 3, + data, + }; let amv = MV { x: mv.x + (xpos as i16) * 4, y: mv.y + (ypos as i16) * 4 }; self.do_mc(&mut afrm, refpic, 0, 0, w, h, amv); + let wsize = match w { + 2 => 0, + 4 => 1, + 8 => 2, + _ => 3, + }; + let src = self.avg_buf.get_data(); for comp in 0..3 { let shift = if comp == 0 { 0 } else { 1 }; - self.avg(&mut frm.data[frm.offset[comp] + (xpos >> shift) + (ypos >> shift) * frm.stride[comp]..], frm.stride[comp], w >> shift, h >> shift, comp); + let sstride = self.avg_buf.get_stride(comp); + let soff = self.avg_buf.get_offset(comp); + (self.avg[wsize - shift])(&mut frm.data[frm.offset[comp] + (xpos >> shift) + (ypos >> shift) * frm.stride[comp]..], frm.stride[comp], &src[soff..], sstride, h >> shift); } } @@ -199,13 +250,68 @@ impl H264MC { } } +fn edge_emu_sf(src: &SimpleFrame, xpos: isize, ypos: isize, w: usize, h: usize, bw: usize, bh: usize, dst: &mut [u8], dstride: usize, comp: usize) { + let stride = src.stride[comp]; + let offs = src.offset[comp]; + let framebuf = src.data; + + for y in 0..bh { + let srcy; + if (y as isize) + ypos < 0 { srcy = 0; } + else if (y as isize) + ypos >= (h as isize) { srcy = h - 1; } + else { srcy = ((y as isize) + ypos) as usize; } + + for x in 0..bw { + let srcx; + if (x as isize) + xpos < 0 { srcx = 0; } + else if (x as isize) + xpos >= (w as isize) { srcx = w - 1; } + else { srcx = ((x as isize) + xpos) as usize; } + dst[x + y * dstride] = framebuf[offs + srcx + srcy * stride]; + } + } +} + +fn avg(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bw: usize, bh: usize) { + for (dline, sline) in dst.chunks_mut(dstride).zip(src.chunks(sstride)).take(bh) { + for (dst, src) in dline.iter_mut().zip(sline.iter()).take(bw) { + *dst = ((u16::from(*dst) + u16::from(*src) + 1) >> 1) as u8; + } + } +} + +fn avg_2(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize) { + let _ = src[sstride + 1]; + let _ = dst[dstride + 1]; + dst[0] = ((u16::from(dst[0]) + u16::from(src[0]) + 1) >> 1) as u8; + dst[1] = ((u16::from(dst[1]) + u16::from(src[1]) + 1) >> 1) as u8; + dst[dstride] = ((u16::from(dst[dstride]) + u16::from(src[sstride]) + 1) >> 1) as u8; + dst[dstride + 1] = ((u16::from(dst[dstride + 1]) + u16::from(src[sstride + 1]) + 1) >> 1) as u8; + if bh == 4 { + let _ = src[sstride * 3 + 1]; + let _ = dst[dstride * 3 + 1]; + dst[dstride * 2] = ((u16::from(dst[dstride * 2]) + u16::from(src[sstride * 2]) + 1) >> 1) as u8; + dst[dstride * 2 + 1] = ((u16::from(dst[dstride * 2 + 1]) + u16::from(src[sstride * 2 + 1]) + 1) >> 1) as u8; + dst[dstride * 3] = ((u16::from(dst[dstride * 3]) + u16::from(src[sstride * 3]) + 1) >> 1) as u8; + dst[dstride * 3 + 1] = ((u16::from(dst[dstride * 3 + 1]) + u16::from(src[sstride * 3 + 1]) + 1) >> 1) as u8; + } +} +fn avg_4(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize) { + avg(dst, dstride, src, sstride, 4, bh); +} +fn avg_8(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize) { + avg(dst, dstride, src, sstride, 8, bh); +} +fn avg_16(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize) { + avg(dst, dstride, src, sstride, 16, bh); +} + fn put_block_weighted(dst: &mut [u8], stride: usize, src: &[u8], w: usize, h: usize, wparams: [i8; 3]) { let weight = i16::from(wparams[0]); let offset = i16::from(wparams[1]); let wshift = wparams[2] as u8; let bias = (1 << wshift) >> 1; - for (drow, srow) in dst.chunks_mut(stride).zip(src.chunks(16)).take(h) { + for (drow, srow) in dst.chunks_mut(stride).zip(src.chunks_exact(16)).take(h) { for (dst, &src) in drow[..w].iter_mut().zip(srow.iter()) { *dst = clip_u8(((i16::from(src) * weight + bias) >> wshift) + offset); } @@ -234,7 +340,7 @@ fn put_block_weighted2(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], let offset = (offset0 + offset1 + 1) >> 1; let bias = (1 << wshift) >> 1; - for (drow, (srow0, srow1)) in dst.chunks_mut(stride).zip(src0.chunks(16).zip(src1.chunks(16))).take(h) { + for (drow, (srow0, srow1)) in dst.chunks_mut(stride).zip(src0.chunks_exact(16).zip(src1.chunks_exact(16))).take(h) { for (dst, (&src0, &src1)) in drow[..w].iter_mut().zip(srow0.iter().zip(srow1.iter())) { *dst = clip_u8(((i16::from(src0) * weight0 + i16::from(src1) * weight1 + bias) >> wshift) + offset); } @@ -242,7 +348,30 @@ fn put_block_weighted2(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], } fn put_blk_w2_2(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], h: usize, wparams: [i8; 5]) { - put_block_weighted2(dst, stride, src0, src1, 2, h, wparams); + let weight0 = i16::from(wparams[0]); + let offset0 = i16::from(wparams[1]); + let weight1 = i16::from(wparams[2]); + let offset1 = i16::from(wparams[3]); + let wshift = (wparams[4] as u8) + 1; + let offset = (offset0 + offset1 + 1) >> 1; + let bias = (1 << wshift) >> 1; + + let _ = src0[16 + 1]; + let _ = src1[16 + 1]; + let _ = dst[stride + 1]; + dst[0] = clip_u8(((i16::from(src0[ 0]) * weight0 + i16::from(src1[ 0]) * weight1 + bias) >> wshift) + offset); + dst[1] = clip_u8(((i16::from(src0[ 1]) * weight0 + i16::from(src1[ 1]) * weight1 + bias) >> wshift) + offset); + dst[stride] = clip_u8(((i16::from(src0[16]) * weight0 + i16::from(src1[16]) * weight1 + bias) >> wshift) + offset); + dst[stride + 1] = clip_u8(((i16::from(src0[17]) * weight0 + i16::from(src1[17]) * weight1 + bias) >> wshift) + offset); + if h == 4 { + let _ = src0[16 * 3 + 1]; + let _ = src1[16 * 3 + 1]; + let _ = dst[stride * 3 + 1]; + dst[stride * 2] = clip_u8(((i16::from(src0[32]) * weight0 + i16::from(src1[32]) * weight1 + bias) >> wshift) + offset); + dst[stride * 2 + 1] = clip_u8(((i16::from(src0[33]) * weight0 + i16::from(src1[33]) * weight1 + bias) >> wshift) + offset); + dst[stride * 3] = clip_u8(((i16::from(src0[48]) * weight0 + i16::from(src1[48]) * weight1 + bias) >> wshift) + offset); + dst[stride * 3 + 1] = clip_u8(((i16::from(src0[49]) * weight0 + i16::from(src1[49]) * weight1 + bias) >> wshift) + offset); + } } fn put_blk_w2_4(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], h: usize, wparams: [i8; 5]) { put_block_weighted2(dst, stride, src0, src1, 4, h, wparams);