-use nihav_core::frame::*;
-use nihav_codec_support::codecs::blockdsp::*;
-use nihav_codec_support::codecs::MV;
-
-#[cfg(not(debug_assertions))]
-mod release;
-#[cfg(not(debug_assertions))]
-use release::*;
-#[cfg(debug_assertions)]
-mod debug;
-#[cfg(debug_assertions)]
-use debug::*;
+mod mc;
+pub use mc::{H264MC, McBlock};
+#[cfg(target_arch="x86_64")]
+use std::arch::asm;
pub const CHROMA_QUANTS: [u8; 52] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
}
}
-pub fn avg(dst: &mut [u8], dstride: usize,
- src: &[u8], sstride: usize, bw: usize, bh: usize) {
- for (dline, sline) in dst.chunks_mut(dstride).zip(src.chunks(sstride)).take(bh) {
- for (dst, src) in dline.iter_mut().zip(sline.iter()).take(bw) {
- *dst = ((u16::from(*dst) + u16::from(*src) + 1) >> 1) as u8;
- }
- }
-}
-
fn clip8(val: i16) -> u8 { val.max(0).min(255) as u8 }
fn ipred_dc128(buf: &mut [u8], stride: usize, bsize: usize) {
fn ipred_4x4_diag_down_right(buf: &mut [u8], stride: usize, top: &[u8], left: &[u8], _tr: &[u8]) {
let mut t: [u16; 5] = [0; 5];
t[0] = u16::from(left[0]);
- load(&mut t[1..], &top);
+ load(&mut t[1..], top);
let mut l: [u16; 5] = [0; 5];
load(&mut l, left);
let dst = buf;
fn ipred_4x4_ver_right(buf: &mut [u8], stride: usize, top: &[u8], left: &[u8], _tr: &[u8]) {
let mut t: [u16; 5] = [0; 5];
t[0] = u16::from(left[0]);
- load(&mut t[1..], &top);
+ load(&mut t[1..], top);
let mut l: [u16; 5] = [0; 5];
load(&mut l, left);
let dst = buf;
}
fn ipred_4x4_ver_left(buf: &mut [u8], stride: usize, top: &[u8], _left: &[u8], tr: &[u8]) {
let mut t: [u16; 8] = [0; 8];
- load(&mut t[..4], &top);
+ load(&mut t[..4], top);
load(&mut t[4..], tr);
let dst = buf;
fn ipred_4x4_hor_down(buf: &mut [u8], stride: usize, top: &[u8], left: &[u8], _tr: &[u8]) {
let mut t: [u16; 5] = [0; 5];
t[0] = u16::from(left[0]);
- load(&mut t[1..], &top);
+ load(&mut t[1..], top);
let mut l: [u16; 5] = [0; 5];
load(&mut l, left);
let dst = buf;
let mut l = [0; 8];
load(&mut l, &left[1..]);
let mut t = [0; 8];
- load(&mut t, &top);
+ load(&mut t, top);
let dc0 = ((t[0] + t[1] + t[2] + t[3] + l[0] + l[1] + l[2] + l[3] + 4) >> 3) as u8;
let sum1 = t[4] + t[5] + t[6] + t[7];
ipred_16x16_left_dc, ipred_16x16_top_dc, ipred_16x16_dc128
];
-fn clip_u8(val: i16) -> u8 { val.max(0).min(255) as u8 }
-
-pub fn do_mc(frm: &mut NASimpleVideoFrame<u8>, refpic: NAVideoBufferRef<u8>, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) {
- let mode = ((mv.x & 3) + (mv.y & 3) * 4) as usize;
- copy_block(frm, refpic.clone(), 0, xpos, ypos, mv.x >> 2, mv.y >> 2, w, h, 2, 3, mode, H264_LUMA_INTERP);
-
- let (cw, ch) = refpic.get_dimensions(1);
- let mvx = mv.x >> 3;
- let mvy = mv.y >> 3;
- let dx = (mv.x & 7) as u16;
- let dy = (mv.y & 7) as u16;
- let mut ebuf = [0u8; 18 * 9];
- let src_x = ((xpos >> 1) as isize) + (mvx as isize);
- let src_y = ((ypos >> 1) as isize) + (mvy as isize);
- let suoff = refpic.get_offset(1);
- let svoff = refpic.get_offset(2);
- let sustride = refpic.get_stride(1);
- let svstride = refpic.get_stride(2);
- let src = refpic.get_data();
- let cbw = w / 2;
- let cbh = h / 2;
- let (csrc, cstride) = if (src_x < 0) || (src_x + (cbw as isize) + 1 > (cw as isize)) || (src_y < 0) || (src_y + (cbh as isize) + 1 > (ch as isize)) {
- edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf, 18, 1, 4);
- edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf[9..], 18, 2, 4);
- ([&ebuf, &ebuf[9..]], [18, 18])
- } else {
- ([&src[suoff + (src_x as usize) + (src_y as usize) * sustride..],
- &src[svoff + (src_x as usize) + (src_y as usize) * svstride..]],
- [sustride, svstride])
- };
- for chroma in 1..3 {
- let off = frm.offset[chroma] + xpos / 2 + (ypos / 2) * frm.stride[chroma];
- chroma_interp(&mut frm.data[off..], frm.stride[chroma], csrc[chroma - 1], cstride[chroma - 1], dx, dy, cbw, cbh);
- }
-}
-
-pub fn mc_blocks(ydst: &mut [u8], udst: &mut [u8], vdst: &mut [u8], refpic: NAVideoBufferRef<u8>, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) {
- let mode = ((mv.x & 3) + (mv.y & 3) * 4) as usize;
-
- let pre = if mode != 0 { 2 } else { 0 };
- let post = if mode != 0 { 3 } else { 0 };
- let (width, height) = refpic.get_dimensions(0);
- let sx = (xpos as isize) + ((mv.x >> 2) as isize);
- let sy = (ypos as isize) + ((mv.y >> 2) as isize);
-
- const EBUF_STRIDE: usize = 32;
- let mut ebuf = [0u8; EBUF_STRIDE * (16 + 2 + 3)];
-
- if (sx - pre < 0) || (sx + (w as isize) + post > (width as isize)) ||
- (sy - pre < 0) || (sy + (h as isize) + post > (height as isize)) {
- let edge = (pre + post) as usize;
- edge_emu(&refpic, sx - pre, sy - pre, w + edge, h + edge,
- &mut ebuf, EBUF_STRIDE, 0, 0);
- (H264_LUMA_INTERP[mode])(ydst, 16, &ebuf, EBUF_STRIDE, w, h);
- } else {
- let sstride = refpic.get_stride(0);
- let soff = refpic.get_offset(0);
- let sdta = refpic.get_data();
- let sbuf: &[u8] = sdta.as_slice();
- let saddr = soff + ((sx - pre) as usize) + ((sy - pre) as usize) * sstride;
- (H264_LUMA_INTERP[mode])(ydst, 16, &sbuf[saddr..], sstride, w, h);
- }
-
- let (cw, ch) = refpic.get_dimensions(1);
- let mvx = mv.x >> 3;
- let mvy = mv.y >> 3;
- let dx = (mv.x & 7) as u16;
- let dy = (mv.y & 7) as u16;
- let src_x = ((xpos >> 1) as isize) + (mvx as isize);
- let src_y = ((ypos >> 1) as isize) + (mvy as isize);
- let suoff = refpic.get_offset(1);
- let svoff = refpic.get_offset(2);
- let sustride = refpic.get_stride(1);
- let svstride = refpic.get_stride(2);
- let src = refpic.get_data();
- let cbw = w / 2;
- let cbh = h / 2;
- let (csrc, cstride) = if (src_x < 0) || (src_x + (cbw as isize) + 1 > (cw as isize)) || (src_y < 0) || (src_y + (cbh as isize) + 1 > (ch as isize)) {
- edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf, 18, 1, 4);
- edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf[9..], 18, 2, 4);
- ([&ebuf, &ebuf[9..]], [18, 18])
- } else {
- ([&src[suoff + (src_x as usize) + (src_y as usize) * sustride..],
- &src[svoff + (src_x as usize) + (src_y as usize) * svstride..]],
- [sustride, svstride])
- };
- chroma_interp(udst, 16, csrc[0], cstride[0], dx, dy, cbw, cbh);
- chroma_interp(vdst, 16, csrc[1], cstride[1], dx, dy, cbw, cbh);
-}
-
-pub fn gray_block(frm: &mut NASimpleVideoFrame<u8>, x: usize, y: usize, w: usize, h: usize) {
- let yoff = frm.offset[0] + x + y * frm.stride[0];
- let coff = [frm.offset[1] + x / 2 + y / 2 * frm.stride[1],
- frm.offset[2] + x / 2 + y / 2 * frm.stride[2]];
- if w == 16 && h == 16 {
- IPRED_FUNCS16X16[IPRED8_DC128](&mut frm.data[yoff..], frm.stride[0], &[], &[]);
- for chroma in 1..2 {
- IPRED_FUNCS8X8_CHROMA[IPRED8_DC128](&mut frm.data[coff[chroma - 1]..], frm.stride[chroma], &[], &[]);
- }
- } else if w == 8 && h == 8 {
- IPRED_FUNCS8X8_CHROMA[IPRED8_DC128](&mut frm.data[yoff..], frm.stride[0], &[], &[]);
- for chroma in 1..2 {
- IPRED_FUNCS4X4[IPRED4_DC128](&mut frm.data[coff[chroma - 1]..], frm.stride[chroma], &[128; 4], &[128; 4], &[128; 4]);
- }
- } else {
- for row in frm.data[yoff..].chunks_mut(frm.stride[0]).take(h) {
- for el in row[..w].iter_mut() {
- *el = 128;
- }
- }
- for chroma in 0..2 {
- for row in frm.data[coff[chroma]..].chunks_mut(frm.stride[chroma + 1]).take(h / 2) {
- for el in row[..w / 2].iter_mut() {
- *el = 128;
- }
- }
- }
- }
-}
-
-pub fn do_mc_avg(frm: &mut NASimpleVideoFrame<u8>, refpic: NAVideoBufferRef<u8>, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV, avg_buf: &mut NAVideoBufferRef<u8>) {
- let mut afrm = NASimpleVideoFrame::from_video_buf(avg_buf).unwrap();
- let amv = MV { x: mv.x + (xpos as i16) * 4, y: mv.y + (ypos as i16) * 4 };
- do_mc(&mut afrm, refpic, 0, 0, w, h, amv);
- for comp in 0..3 {
- let shift = if comp == 0 { 0 } else { 1 };
- avg(&mut frm.data[frm.offset[comp] + (xpos >> shift) + (ypos >> shift) * frm.stride[comp]..], frm.stride[comp], &afrm.data[afrm.offset[comp]..], afrm.stride[comp], w >> shift, h >> shift);
- }
-}
-
-pub fn put_block_weighted(dst: &mut [u8], stride: usize, src: &[u8], w: usize, h: usize, wparams: [i8; 3]) {
- let weight = i16::from(wparams[0]);
- let offset = i16::from(wparams[1]);
- let wshift = wparams[2] as u8;
- let bias = (1 << wshift) >> 1;
-
- for (drow, srow) in dst.chunks_mut(stride).zip(src.chunks(16)).take(h) {
- for (dst, &src) in drow[..w].iter_mut().zip(srow.iter()) {
- *dst = clip8(((i16::from(src) * weight + bias) >> wshift) + offset);
- }
- }
-}
-
-pub fn put_block_weighted2(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], w: usize, h: usize, wparams: [i8; 5]) {
- let weight0 = i16::from(wparams[0]);
- let offset0 = i16::from(wparams[1]);
- let weight1 = i16::from(wparams[2]);
- let offset1 = i16::from(wparams[3]);
- let wshift = (wparams[4] as u8) + 1;
- let offset = (offset0 + offset1 + 1) >> 1;
- let bias = (1 << wshift) >> 1;
-
- for (drow, (srow0, srow1)) in dst.chunks_mut(stride).zip(src0.chunks(16).zip(src1.chunks(16))).take(h) {
- for (dst, (&src0, &src1)) in drow[..w].iter_mut().zip(srow0.iter().zip(srow1.iter())) {
- *dst = clip8(((i16::from(src0) * weight0 + i16::from(src1) * weight1 + bias) >> wshift) + offset);
- }
- }
-}
-
macro_rules! loop_filter {
(lumaedge; $buf: expr, $off: expr, $step: expr, $alpha: expr, $beta: expr) => {
let p2 = i16::from($buf[$off - $step * 3]);
(p0 - q0).abs() < alpha && (p1 - p0).abs() < beta && (q1 - q0).abs() < beta
}
+#[cfg(not(target_arch="x86_64"))]
+fn check_filter4(buf: &[u8], mut off: usize, step: usize, stride: usize, alpha: i16, beta: i16) -> [bool; 4] {
+ let mut flags = [false; 4];
+ for flag in flags.iter_mut() {
+ let p1 = i16::from(buf[off - step * 2]);
+ let p0 = i16::from(buf[off - step]);
+ let q0 = i16::from(buf[off]);
+ let q1 = i16::from(buf[off + step]);
+ *flag = (p0 - q0).abs() < alpha && (p1 - p0).abs() < beta && (q1 - q0).abs() < beta;
+ off += stride;
+ }
+ flags
+}
+
+#[cfg(target_arch="x86_64")]
+fn check_filter4(buf: &[u8], off: usize, step: usize, stride: usize, alpha: i16, beta: i16) -> [bool; 4] {
+ unsafe {
+ let mut flags = [false; 4];
+ let src = buf[off - step * 2..].as_ptr();
+ let load_stride = step.max(stride);
+ let fptr = flags.as_mut_ptr();
+ let tflag = u32::from(step == 1);
+ asm! {
+ // load block
+ "pxor xmm4, xmm4",
+ "movd xmm0, dword ptr [{src}]",
+ "lea {tmp}, [{src} + {stride} * 2]",
+ "movd xmm1, dword ptr [{src} + {stride}]",
+ "movd xmm2, dword ptr [{tmp}]",
+ "movd xmm3, dword ptr [{tmp} + {stride}]",
+ "punpcklbw xmm0, xmm4",
+ "punpcklbw xmm1, xmm4",
+ "punpcklbw xmm2, xmm4",
+ "punpcklbw xmm3, xmm4",
+
+ // transpose block if necessary so it's always processed by rows
+ "test {tflag:e}, {tflag:e}",
+ "jz 1f",
+ "punpcklwd xmm0, xmm1",
+ "movhlps xmm4, xmm0",
+ "punpcklwd xmm2, xmm3",
+ "movhlps xmm1, xmm2",
+ "punpckldq xmm0, xmm2",
+ "punpckldq xmm4, xmm1",
+ "movhlps xmm1, xmm0",
+ "movhlps xmm3, xmm4",
+ "movaps xmm2, xmm4",
+ "1:",
+
+ // calculate deltas and flags
+ "movd xmm4, {alpha:r}",
+ "movd xmm5, {beta:r}",
+ "psubw xmm0, xmm1",
+ "psubw xmm1, xmm2",
+ "psubw xmm3, xmm2",
+ "pshuflw xmm4, xmm4, 0",
+ "pshuflw xmm5, xmm5, 0",
+ "pabsw xmm0, xmm0", // |p1 - p0|
+ "pabsw xmm1, xmm1", // |p0 - q0|
+ "pabsw xmm2, xmm3", // |q1 - q0|
+ "movaps xmm3, xmm5",
+ "pcmpgtw xmm4, xmm1",
+ "pcmpgtw xmm5, xmm0",
+ "pcmpgtw xmm3, xmm2",
+ "pand xmm4, xmm5",
+ "pand xmm4, xmm3",
+ "packsswb xmm4, xmm4",
+ "movd [{flags}], xmm4",
+ tmp = out(reg) _,
+ src = in(reg) src,
+ stride = in(reg) load_stride,
+ alpha = in(reg) alpha,
+ beta = in(reg) beta,
+ flags = in(reg) fptr,
+ tflag = in(reg) tflag,
+ out("xmm0") _,
+ out("xmm1") _,
+ out("xmm2") _,
+ out("xmm3") _,
+ out("xmm4") _,
+ out("xmm5") _,
+ }
+ flags
+ }
+}
+
pub fn loop_filter_lumaedge_v(dst: &mut [u8], mut off: usize, stride: usize, alpha: i16, beta: i16) {
- for _ in 0..4 {
- if check_filter(dst, off, 1, alpha, beta) {
+ let flags = check_filter4(dst, off, 1, stride, alpha, beta);
+ for &flag in flags.iter() {
+ if flag {
loop_filter!(lumaedge; dst, off, 1, alpha, beta);
}
off += stride;
}
}
pub fn loop_filter_lumaedge_h(dst: &mut [u8], off: usize, stride: usize, alpha: i16, beta: i16) {
- for x in 0..4 {
- if check_filter(dst, off + x, stride, alpha, beta) {
+ let flags = check_filter4(dst, off, stride, 1, alpha, beta);
+ for (x, &flag) in flags.iter().enumerate() {
+ if flag {
loop_filter!(lumaedge; dst, off + x, stride, alpha, beta);
}
}
}
pub fn loop_filter_lumanormal_v(dst: &mut [u8], mut off: usize, stride: usize, alpha: i16, beta: i16, tc0: i16) {
- for _ in 0..4 {
- if check_filter(dst, off, 1, alpha, beta) {
+ let flags = check_filter4(dst, off, 1, stride, alpha, beta);
+ for &flag in flags.iter() {
+ if flag {
loop_filter!(lumanormal; dst, off, 1, tc0, beta);
}
off += stride;
}
}
pub fn loop_filter_lumanormal_h(dst: &mut [u8], off: usize, stride: usize, alpha: i16, beta: i16, tc0: i16) {
- for x in 0..4 {
- if check_filter(dst, off + x, stride, alpha, beta) {
+ let flags = check_filter4(dst, off, stride, 1, alpha, beta);
+ for (x, &flag) in flags.iter().enumerate() {
+ if flag {
loop_filter!(lumanormal; dst, off + x, stride, tc0, beta);
}
}
projects / nihav.git / blobdiff