| 1 | use nihav_core::frame::*; |
| 2 | use nihav_codec_support::codecs::MV; |
| 3 | use nihav_codec_support::codecs::blockdsp::*; |
| 4 | |
| 5 | macro_rules! module_selector { |
| 6 | ($( ($cond:meta, $module:ident) ),*) => { |
| 7 | module_selector!(list; r#false; $(($cond, $module)),*); |
| 8 | }; |
| 9 | (list; $nocond:meta; ($ccar:meta, $carmod:ident), $(($condcdr:meta, $cdrmod:ident)),*) => { |
| 10 | module_selector!(single; $nocond; $ccar; $carmod); |
| 11 | module_selector!(list; any($nocond, $ccar); $(($condcdr, $cdrmod)),*); |
| 12 | }; |
| 13 | (list; $nocond:meta; ($yescond:meta, $module:ident)) => { |
| 14 | module_selector!(single; $nocond; $yescond; $module); |
| 15 | }; |
| 16 | (list; $_:meta; ) => {}; |
| 17 | (single; $nocond:meta; $yescond:meta; $module:ident) => { |
| 18 | #[cfg(all(not($nocond), $yescond))] |
| 19 | mod $module; |
| 20 | #[cfg(all(not($nocond), $yescond))] |
| 21 | use $module::*; |
| 22 | }; |
| 23 | } |
| 24 | |
| 25 | module_selector! ( |
| 26 | (all(feature = "simd", target_arch = "x86_64"), x86), |
| 27 | (debug_assertions, debug), |
| 28 | (not(debug_assertions), release) |
| 29 | ); |
| 30 | |
| 31 | type MCFunc = fn (dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, h: usize); |
| 32 | |
| 33 | fn clip_u8(val: i16) -> u8 { val.max(0).min(255) as u8 } |
| 34 | |
| 35 | trait RegisterSIMD { |
| 36 | fn register_simd(&mut self); |
| 37 | } |
| 38 | |
| 39 | #[repr(align(16))] |
| 40 | pub struct McBlock { |
| 41 | pub y: [u8; 16 * 16], |
| 42 | pub u: [u8; 16 * 16], |
| 43 | pub v: [u8; 16 * 16], |
| 44 | } |
| 45 | |
| 46 | impl McBlock { |
| 47 | pub fn new() -> Self { |
| 48 | unsafe { |
| 49 | let blk = std::mem::MaybeUninit::uninit(); |
| 50 | blk.assume_init() |
| 51 | } |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | #[allow(clippy::type_complexity)] |
| 56 | pub struct H264MC { |
| 57 | avg_buf: NAVideoBufferRef<u8>, |
| 58 | pub put_block_weighted: [fn (dst: &mut [u8], stride: usize, src: &[u8], h: usize, wparams: [i8; 3]); 4], |
| 59 | pub put_block_weighted2: [fn (dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], h: usize, wparams: [i8; 5]); 4], |
| 60 | pub chroma_interp: [fn (dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, dx: u16, dy: u16, h: usize); 3], |
| 61 | avg: [fn (dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize); 4], |
| 62 | |
| 63 | width: usize, |
| 64 | height: usize, |
| 65 | } |
| 66 | |
| 67 | impl H264MC { |
| 68 | pub fn new(avg_buf: NAVideoBufferRef<u8>) -> Self { |
| 69 | let mut obj = Self { |
| 70 | avg_buf, |
| 71 | put_block_weighted: [put_blk_w_2, put_blk_w_4, put_blk_w_8, put_blk_w_16], |
| 72 | put_block_weighted2: [put_blk_w2_2, put_blk_w2_4, put_blk_w2_8, put_blk_w2_16], |
| 73 | chroma_interp: [chroma_interp_2, chroma_interp_4, chroma_interp_8], |
| 74 | avg: [avg_2, avg_4, avg_8, avg_16], |
| 75 | width: 0, height: 0, |
| 76 | }; |
| 77 | obj.register_simd(); |
| 78 | obj |
| 79 | } |
| 80 | pub fn set_dimensions(&mut self, width: usize, height: usize) { |
| 81 | self.width = width; |
| 82 | self.height = height; |
| 83 | } |
| 84 | pub fn do_mc(&mut self, frm: &mut NASimpleVideoFrame<u8>, refpic: NAVideoBufferRef<u8>, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { |
| 85 | let mut ebuf = [0u8; 22 * 22]; |
| 86 | let mvx = mv.x >> 2; |
| 87 | let mvy = mv.y >> 2; |
| 88 | let mode = ((mv.x & 3) + (mv.y & 3) * 4) as usize; |
| 89 | let pre = if mode != 0 { 2isize } else { 0 }; |
| 90 | let post = if mode != 0 { 3isize } else { 0 }; |
| 91 | let (yw, yh) = (self.width, self.height); |
| 92 | let src = refpic.get_data(); |
| 93 | let systride = refpic.get_stride(0); |
| 94 | let src_x = (xpos as isize) + (mvx as isize); |
| 95 | let src_y = (ypos as isize) + (mvy as isize); |
| 96 | let (ysrc, ystride) = if (src_x - pre < 0) || (src_x + (w as isize) + post > (yw as isize)) || (src_y - pre < 0) || (src_y + (h as isize) + post > (yh as isize)) { |
| 97 | let add = (pre + post) as usize; |
| 98 | edge_emu(&refpic, src_x - pre, src_y - pre, w + add, h + add, &mut ebuf, 22, 0, 0); |
| 99 | (&ebuf[..], 22) |
| 100 | } else { |
| 101 | (&src[refpic.get_offset(0) + ((src_x - pre) as usize) + ((src_y - pre) as usize) * systride..], systride) |
| 102 | }; |
| 103 | let wmode = match w { |
| 104 | 4 => 0, |
| 105 | 8 => 1, |
| 106 | _ => 2, |
| 107 | }; |
| 108 | (H264_LUMA_INTERP[wmode][mode])(&mut frm.data[frm.offset[0] + xpos + ypos * frm.stride[0]..], frm.stride[0], ysrc, ystride, h); |
| 109 | |
| 110 | let (cw, ch) = (self.width >> 1, self.height >> 1); |
| 111 | let mvx = mv.x >> 3; |
| 112 | let mvy = mv.y >> 3; |
| 113 | let dx = (mv.x & 7) as u16; |
| 114 | let dy = (mv.y & 7) as u16; |
| 115 | let src_x = ((xpos >> 1) as isize) + (mvx as isize); |
| 116 | let src_y = ((ypos >> 1) as isize) + (mvy as isize); |
| 117 | let suoff = refpic.get_offset(1); |
| 118 | let svoff = refpic.get_offset(2); |
| 119 | let sustride = refpic.get_stride(1); |
| 120 | let svstride = refpic.get_stride(2); |
| 121 | let cbw = w / 2; |
| 122 | let cbh = h / 2; |
| 123 | let (csrc, cstride) = if (src_x < 0) || (src_x + (cbw as isize) + 1 > (cw as isize)) || (src_y < 0) || (src_y + (cbh as isize) + 1 > (ch as isize)) { |
| 124 | edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf, 18, 1, 4); |
| 125 | edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf[9..], 18, 2, 4); |
| 126 | ([&ebuf, &ebuf[9..]], [18, 18]) |
| 127 | } else { |
| 128 | ([&src[suoff + (src_x as usize) + (src_y as usize) * sustride..], |
| 129 | &src[svoff + (src_x as usize) + (src_y as usize) * svstride..]], |
| 130 | [sustride, svstride]) |
| 131 | }; |
| 132 | for chroma in 1..3 { |
| 133 | let off = frm.offset[chroma] + xpos / 2 + (ypos / 2) * frm.stride[chroma]; |
| 134 | (self.chroma_interp[wmode])(&mut frm.data[off..], frm.stride[chroma], csrc[chroma - 1], cstride[chroma - 1], dx, dy, cbh); |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | pub fn mc_blocks(&mut self, dst: &mut McBlock, refpic: NAVideoBufferRef<u8>, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { |
| 139 | let mode = ((mv.x & 3) + (mv.y & 3) * 4) as usize; |
| 140 | |
| 141 | let pre = if mode != 0 { 2 } else { 0 }; |
| 142 | let post = if mode != 0 { 3 } else { 0 }; |
| 143 | let (width, height) = (self.width, self.height); |
| 144 | let sx = (xpos as isize) + ((mv.x >> 2) as isize); |
| 145 | let sy = (ypos as isize) + ((mv.y >> 2) as isize); |
| 146 | |
| 147 | const EBUF_STRIDE: usize = 32; |
| 148 | let mut ebuf = [0u8; EBUF_STRIDE * (16 + 2 + 3)]; |
| 149 | |
| 150 | let wmode = match w { |
| 151 | 4 => 0, |
| 152 | 8 => 1, |
| 153 | _ => 2, |
| 154 | }; |
| 155 | if (sx - pre < 0) || (sx + (w as isize) + post > (width as isize)) || |
| 156 | (sy - pre < 0) || (sy + (h as isize) + post > (height as isize)) { |
| 157 | let edge = (pre + post) as usize; |
| 158 | edge_emu(&refpic, sx - pre, sy - pre, w + edge, h + edge, |
| 159 | &mut ebuf, EBUF_STRIDE, 0, 0); |
| 160 | (H264_LUMA_INTERP[wmode][mode])(&mut dst.y, 16, &ebuf, EBUF_STRIDE, h); |
| 161 | } else { |
| 162 | let sstride = refpic.get_stride(0); |
| 163 | let soff = refpic.get_offset(0); |
| 164 | let sdta = refpic.get_data(); |
| 165 | let sbuf: &[u8] = sdta.as_slice(); |
| 166 | let saddr = soff + ((sx - pre) as usize) + ((sy - pre) as usize) * sstride; |
| 167 | (H264_LUMA_INTERP[wmode][mode])(&mut dst.y, 16, &sbuf[saddr..], sstride, h); |
| 168 | } |
| 169 | |
| 170 | let (cw, ch) = (self.width >> 1, self.height >> 1); |
| 171 | let mvx = mv.x >> 3; |
| 172 | let mvy = mv.y >> 3; |
| 173 | let dx = (mv.x & 7) as u16; |
| 174 | let dy = (mv.y & 7) as u16; |
| 175 | let src_x = ((xpos >> 1) as isize) + (mvx as isize); |
| 176 | let src_y = ((ypos >> 1) as isize) + (mvy as isize); |
| 177 | let suoff = refpic.get_offset(1); |
| 178 | let svoff = refpic.get_offset(2); |
| 179 | let sustride = refpic.get_stride(1); |
| 180 | let svstride = refpic.get_stride(2); |
| 181 | let src = refpic.get_data(); |
| 182 | let cbw = w / 2; |
| 183 | let cbh = h / 2; |
| 184 | let (csrc, cstride) = if (src_x < 0) || (src_x + (cbw as isize) + 1 > (cw as isize)) || (src_y < 0) || (src_y + (cbh as isize) + 1 > (ch as isize)) { |
| 185 | edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf, 18, 1, 4); |
| 186 | edge_emu(&refpic, src_x, src_y, cbw+1, cbh+1, &mut ebuf[9..], 18, 2, 4); |
| 187 | ([&ebuf, &ebuf[9..]], [18, 18]) |
| 188 | } else { |
| 189 | ([&src[suoff + (src_x as usize) + (src_y as usize) * sustride..], |
| 190 | &src[svoff + (src_x as usize) + (src_y as usize) * svstride..]], |
| 191 | [sustride, svstride]) |
| 192 | }; |
| 193 | (self.chroma_interp[wmode])(&mut dst.u, 16, csrc[0], cstride[0], dx, dy, cbh); |
| 194 | (self.chroma_interp[wmode])(&mut dst.v, 16, csrc[1], cstride[1], dx, dy, cbh); |
| 195 | } |
| 196 | |
| 197 | pub fn do_mc_avg(&mut self, frm: &mut NASimpleVideoFrame<u8>, refpic: NAVideoBufferRef<u8>, xpos: usize, ypos: usize, w: usize, h: usize, mv: MV) { |
| 198 | let mut abuf = self.avg_buf.clone(); |
| 199 | let mut afrm = NASimpleVideoFrame::from_video_buf(&mut abuf).unwrap(); |
| 200 | let amv = MV { x: mv.x + (xpos as i16) * 4, y: mv.y + (ypos as i16) * 4 }; |
| 201 | self.do_mc(&mut afrm, refpic, 0, 0, w, h, amv); |
| 202 | let wsize = match w { |
| 203 | 2 => 0, |
| 204 | 4 => 1, |
| 205 | 8 => 2, |
| 206 | _ => 3, |
| 207 | }; |
| 208 | let src = self.avg_buf.get_data(); |
| 209 | for comp in 0..3 { |
| 210 | let shift = if comp == 0 { 0 } else { 1 }; |
| 211 | let sstride = self.avg_buf.get_stride(comp); |
| 212 | let soff = self.avg_buf.get_offset(comp); |
| 213 | (self.avg[wsize - shift])(&mut frm.data[frm.offset[comp] + (xpos >> shift) + (ypos >> shift) * frm.stride[comp]..], frm.stride[comp], &src[soff..], sstride, h >> shift); |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | pub fn gray_block(&mut self, frm: &mut NASimpleVideoFrame<u8>, x: usize, y: usize, w: usize, h: usize) { |
| 218 | let yoff = frm.offset[0] + x + y * frm.stride[0]; |
| 219 | let coff = [frm.offset[1] + x / 2 + y / 2 * frm.stride[1], |
| 220 | frm.offset[2] + x / 2 + y / 2 * frm.stride[2]]; |
| 221 | for row in frm.data[yoff..].chunks_mut(frm.stride[0]).take(h) { |
| 222 | for el in row[..w].iter_mut() { |
| 223 | *el = 128; |
| 224 | } |
| 225 | } |
| 226 | for chroma in 0..2 { |
| 227 | for row in frm.data[coff[chroma]..].chunks_mut(frm.stride[chroma + 1]).take(h / 2) { |
| 228 | for el in row[..w / 2].iter_mut() { |
| 229 | *el = 128; |
| 230 | } |
| 231 | } |
| 232 | } |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | fn avg(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bw: usize, bh: usize) { |
| 237 | for (dline, sline) in dst.chunks_mut(dstride).zip(src.chunks(sstride)).take(bh) { |
| 238 | for (dst, src) in dline.iter_mut().zip(sline.iter()).take(bw) { |
| 239 | *dst = ((u16::from(*dst) + u16::from(*src) + 1) >> 1) as u8; |
| 240 | } |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | fn avg_2(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize) { |
| 245 | let _ = src[sstride + 1]; |
| 246 | let _ = dst[dstride + 1]; |
| 247 | dst[0] = ((u16::from(dst[0]) + u16::from(src[0]) + 1) >> 1) as u8; |
| 248 | dst[1] = ((u16::from(dst[1]) + u16::from(src[1]) + 1) >> 1) as u8; |
| 249 | dst[dstride] = ((u16::from(dst[dstride]) + u16::from(src[sstride]) + 1) >> 1) as u8; |
| 250 | dst[dstride + 1] = ((u16::from(dst[dstride + 1]) + u16::from(src[sstride + 1]) + 1) >> 1) as u8; |
| 251 | if bh == 4 { |
| 252 | let _ = src[sstride * 3 + 1]; |
| 253 | let _ = dst[dstride * 3 + 1]; |
| 254 | dst[dstride * 2] = ((u16::from(dst[dstride * 2]) + u16::from(src[sstride * 2]) + 1) >> 1) as u8; |
| 255 | dst[dstride * 2 + 1] = ((u16::from(dst[dstride * 2 + 1]) + u16::from(src[sstride * 2 + 1]) + 1) >> 1) as u8; |
| 256 | dst[dstride * 3] = ((u16::from(dst[dstride * 3]) + u16::from(src[sstride * 3]) + 1) >> 1) as u8; |
| 257 | dst[dstride * 3 + 1] = ((u16::from(dst[dstride * 3 + 1]) + u16::from(src[sstride * 3 + 1]) + 1) >> 1) as u8; |
| 258 | } |
| 259 | } |
| 260 | fn avg_4(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize) { |
| 261 | avg(dst, dstride, src, sstride, 4, bh); |
| 262 | } |
| 263 | fn avg_8(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize) { |
| 264 | avg(dst, dstride, src, sstride, 8, bh); |
| 265 | } |
| 266 | fn avg_16(dst: &mut [u8], dstride: usize, src: &[u8], sstride: usize, bh: usize) { |
| 267 | avg(dst, dstride, src, sstride, 16, bh); |
| 268 | } |
| 269 | |
| 270 | fn put_block_weighted(dst: &mut [u8], stride: usize, src: &[u8], w: usize, h: usize, wparams: [i8; 3]) { |
| 271 | let weight = i16::from(wparams[0]); |
| 272 | let offset = i16::from(wparams[1]); |
| 273 | let wshift = wparams[2] as u8; |
| 274 | let bias = (1 << wshift) >> 1; |
| 275 | |
| 276 | for (drow, srow) in dst.chunks_mut(stride).zip(src.chunks_exact(16)).take(h) { |
| 277 | for (dst, &src) in drow[..w].iter_mut().zip(srow.iter()) { |
| 278 | *dst = clip_u8(((i16::from(src) * weight + bias) >> wshift) + offset); |
| 279 | } |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | fn put_blk_w_2(dst: &mut [u8], stride: usize, src: &[u8], h: usize, wparams: [i8; 3]) { |
| 284 | put_block_weighted(dst, stride, src, 2, h, wparams); |
| 285 | } |
| 286 | fn put_blk_w_4(dst: &mut [u8], stride: usize, src: &[u8], h: usize, wparams: [i8; 3]) { |
| 287 | put_block_weighted(dst, stride, src, 4, h, wparams); |
| 288 | } |
| 289 | fn put_blk_w_8(dst: &mut [u8], stride: usize, src: &[u8], h: usize, wparams: [i8; 3]) { |
| 290 | put_block_weighted(dst, stride, src, 8, h, wparams); |
| 291 | } |
| 292 | fn put_blk_w_16(dst: &mut [u8], stride: usize, src: &[u8], h: usize, wparams: [i8; 3]) { |
| 293 | put_block_weighted(dst, stride, src, 16, h, wparams); |
| 294 | } |
| 295 | |
| 296 | fn put_block_weighted2(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], w: usize, h: usize, wparams: [i8; 5]) { |
| 297 | let weight0 = i16::from(wparams[0]); |
| 298 | let offset0 = i16::from(wparams[1]); |
| 299 | let weight1 = i16::from(wparams[2]); |
| 300 | let offset1 = i16::from(wparams[3]); |
| 301 | let wshift = (wparams[4] as u8) + 1; |
| 302 | let offset = (offset0 + offset1 + 1) >> 1; |
| 303 | let bias = (1 << wshift) >> 1; |
| 304 | |
| 305 | for (drow, (srow0, srow1)) in dst.chunks_mut(stride).zip(src0.chunks_exact(16).zip(src1.chunks_exact(16))).take(h) { |
| 306 | for (dst, (&src0, &src1)) in drow[..w].iter_mut().zip(srow0.iter().zip(srow1.iter())) { |
| 307 | *dst = clip_u8(((i16::from(src0) * weight0 + i16::from(src1) * weight1 + bias) >> wshift) + offset); |
| 308 | } |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | fn put_blk_w2_2(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], h: usize, wparams: [i8; 5]) { |
| 313 | let weight0 = i16::from(wparams[0]); |
| 314 | let offset0 = i16::from(wparams[1]); |
| 315 | let weight1 = i16::from(wparams[2]); |
| 316 | let offset1 = i16::from(wparams[3]); |
| 317 | let wshift = (wparams[4] as u8) + 1; |
| 318 | let offset = (offset0 + offset1 + 1) >> 1; |
| 319 | let bias = (1 << wshift) >> 1; |
| 320 | |
| 321 | let _ = src0[16 + 1]; |
| 322 | let _ = src1[16 + 1]; |
| 323 | let _ = dst[stride + 1]; |
| 324 | dst[0] = clip_u8(((i16::from(src0[ 0]) * weight0 + i16::from(src1[ 0]) * weight1 + bias) >> wshift) + offset); |
| 325 | dst[1] = clip_u8(((i16::from(src0[ 1]) * weight0 + i16::from(src1[ 1]) * weight1 + bias) >> wshift) + offset); |
| 326 | dst[stride] = clip_u8(((i16::from(src0[16]) * weight0 + i16::from(src1[16]) * weight1 + bias) >> wshift) + offset); |
| 327 | dst[stride + 1] = clip_u8(((i16::from(src0[17]) * weight0 + i16::from(src1[17]) * weight1 + bias) >> wshift) + offset); |
| 328 | if h == 4 { |
| 329 | let _ = src0[16 * 3 + 1]; |
| 330 | let _ = src1[16 * 3 + 1]; |
| 331 | let _ = dst[stride * 3 + 1]; |
| 332 | dst[stride * 2] = clip_u8(((i16::from(src0[32]) * weight0 + i16::from(src1[32]) * weight1 + bias) >> wshift) + offset); |
| 333 | dst[stride * 2 + 1] = clip_u8(((i16::from(src0[33]) * weight0 + i16::from(src1[33]) * weight1 + bias) >> wshift) + offset); |
| 334 | dst[stride * 3] = clip_u8(((i16::from(src0[48]) * weight0 + i16::from(src1[48]) * weight1 + bias) >> wshift) + offset); |
| 335 | dst[stride * 3 + 1] = clip_u8(((i16::from(src0[49]) * weight0 + i16::from(src1[49]) * weight1 + bias) >> wshift) + offset); |
| 336 | } |
| 337 | } |
| 338 | fn put_blk_w2_4(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], h: usize, wparams: [i8; 5]) { |
| 339 | put_block_weighted2(dst, stride, src0, src1, 4, h, wparams); |
| 340 | } |
| 341 | fn put_blk_w2_8(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], h: usize, wparams: [i8; 5]) { |
| 342 | put_block_weighted2(dst, stride, src0, src1, 8, h, wparams); |
| 343 | } |
| 344 | fn put_blk_w2_16(dst: &mut [u8], stride: usize, src0: &[u8], src1: &[u8], h: usize, wparams: [i8; 5]) { |
| 345 | put_block_weighted2(dst, stride, src0, src1, 16, h, wparams); |
| 346 | } |