core/io: mark FullCodebookDesc and ShortCodebookDesc as Copy
[nihav.git] / nihav-core / src / io / codebook.rs
1 //! Codebook support for bitstream reader.
2 //!
3 //! Codebook is a set of unique bit strings and values assigned to them.
4 //! Since there are many ways to define codebook, this implementation employs [`CodebookDescReader`] trait to provide codebook generator with the codes.
5 //! Please also pay attention to the codebook creation mode: if bitstream reader reads bits starting from most significant bit first then you should use [`MSB`] mode and [`LSB`] mode otherwise.
6 //!
7 //! # Examples
8 //!
9 //! Create a codebook from arrays with codeword descriptions:
10 //! ```
11 //! use nihav_core::io::codebook::{ShortCodebookDesc, ShortCodebookDescReader, Codebook, CodebookMode};
12 //!
13 //! let cb_desc: Vec<ShortCodebookDesc> = vec!(
14 //! ShortCodebookDesc { code: 0b00, bits: 2 },
15 //! ShortCodebookDesc { code: 0, bits: 0 },
16 //! ShortCodebookDesc { code: 0b01, bits: 2 },
17 //! ShortCodebookDesc { code: 0b1, bits: 1 });
18 //! let mut cr = ShortCodebookDescReader::new(cb_desc);
19 //! let cb = Codebook::new(&mut cr, CodebookMode::LSB).unwrap();
20 //! ```
21 //!
22 //! Create a codebook using more flexible [`TableCodebookDescReader`] approach.
23 //! This will create a codebook for the following set: `1` -> -2, `01` -> -1, `001` -> 0, `0001` -> 1, `00001` -> 2.
24 //! ```
25 //! use nihav_core::io::codebook::{TableCodebookDescReader, Codebook, CodebookMode};
26 //!
27 //! fn map_cb_index(index: usize) -> i16 { (index as i16) - 2 }
28 //! const CB_BITS: [u8; 5] = [ 1, 2, 3, 4, 5 ];
29 //! const CB_CODES: [u8; 5] = [ 1, 1, 1, 1, 1 ];
30 //!
31 //! let mut tcr = TableCodebookDescReader::new(&CB_CODES, &CB_BITS, map_cb_index);
32 //! let cb = Codebook::new(&mut tcr, CodebookMode::MSB).unwrap();
33 //! ```
34 //!
35 //! Read value using a codebook:
36 //! ```no_run
37 //! use nihav_core::io::bitreader::BitReader;
38 //! use nihav_core::io::codebook::{Codebook, CodebookReader, CodebookMode};
39 //! # use nihav_core::io::codebook::{ShortCodebookDesc, ShortCodebookDescReader, CodebookDescReader, CodebookResult};
40 //!
41 //! # fn foo(br: &mut BitReader) -> CodebookResult<()> {
42 //! # let mut cr = ShortCodebookDescReader::new(vec![ShortCodebookDesc { code: 0b00, bits: 2 }]);
43 //! let cb = Codebook::new(&mut cr, CodebookMode::MSB).unwrap();
44 //! let value = br.read_cb(&cb)?;
45 //! # Ok(())
46 //! # }
47 //! ```
48 //!
49 //! [`MSB`]: ./enum.CodebookMode.html#variant.MSB
50 //! [`LSB`]: ./enum.CodebookMode.html#variant.LSB
51 //! [`CodebookDescReader`]: ./trait.CodebookDescReader.html
52 //! [`TableCodebookDescReader`]: ./struct.TableCodebookDescReader.html
53
54 use std::collections::HashMap;
55 use std::cmp::{max, min};
56 use super::bitreader::BitReader;
57
58 /// A list specifying general codebook operations errors.
59 #[derive(Debug)]
60 pub enum CodebookError {
61 /// Codebook description contains errors.
62 InvalidCodebook,
63 /// Could not allocate memory for codebook.
64 MemoryError,
65 /// Bitstream contains a sequence not present in codebook.
66 InvalidCode,
67 }
68
69 /// Codebook operation modes.
70 #[derive(Debug, Copy, Clone)]
71 pub enum CodebookMode {
72 /// Codes in the codebook should be read most significant bit first.
73 MSB,
74 /// Codes in the codebook should be read least significant bit first.
75 LSB,
76 }
77
78 /// A specialised `Result` type for codebook operations.
79 pub type CodebookResult<T> = Result<T, CodebookError>;
80
81 /// Codebook description for `(code bits, code length, code value)` triplet.
82 ///
83 /// This should be used to create a list of codeword definitions for [`FullCodebookDescReader`].
84 ///
85 /// [`FullCodebookDescReader`]: ./struct.FullCodebookDescReader.html
86 #[derive(Clone,Copy)]
87 pub struct FullCodebookDesc<S> {
88 /// Codeword bits.
89 pub code: u32,
90 /// Codeword length.
91 pub bits: u8,
92 /// Codeword value (symbol).
93 pub sym: S,
94 }
95
96 /// Codebook description for `(code bits, code length)` pair with array index being used as codeword value.
97 ///
98 /// This should be used to create a list of codeword definitions for [`ShortCodebookDescReader`].
99 ///
100 /// [`ShortCodebookDescReader`]: ./struct.ShortCodebookDescReader.html
101 #[derive(Clone,Copy)]
102 pub struct ShortCodebookDesc {
103 /// Codeword bits.
104 pub code: u32,
105 /// Codeword length.
106 pub bits: u8,
107 }
108
109 /// The interface for providing a list of codeword definitions to the codebook creator.
110 ///
111 /// The structure implementing this trait should be able to provide the total number of defined codewords and their bits and values. [`ShortCodebookDescReader`] or [`TableCodebookDescReader`] are some examples of such implementation.
112 /// Codeword definitions with zero length are ignored (those may be used to create sparse codebook definitions though).
113 ///
114 /// [`ShortCodebookDescReader`]: ./struct.ShortCodebookDescReader.html
115 /// [`TableCodebookDescReader`]: ./struct.TableCodebookDescReader.html
116 pub trait CodebookDescReader<S> {
117 /// Returns the codeword length for the provided index.
118 fn bits(&mut self, idx: usize) -> u8;
119 /// Returns the codeword bits for the provided index.
120 fn code(&mut self, idx: usize) -> u32;
121 /// Returns the codeword value (aka codeword symbol) for the provided index.
122 fn sym (&mut self, idx: usize) -> S;
123 /// Returns the total number of defined codewords.
124 fn len (&mut self) -> usize;
125 }
126
127 /// The codebook structure for code reading.
128 #[allow(dead_code)]
129 pub struct Codebook<S> {
130 table: Vec<u32>,
131 syms: Vec<S>,
132 lut_bits: u8,
133 }
134
135 /// Trait allowing bitreader to use codebook for decoding bit sequences.
136 pub trait CodebookReader<S> {
137 /// Reads the codeword from the bitstream and returns its value (or [`InvalidCode`] on error).
138 ///
139 /// [`InvalidCode`]: ./enum.CodebookError.html#variant.InvalidCode
140 fn read_cb(&mut self, cb: &Codebook<S>) -> CodebookResult<S>;
141 }
142
143 const TABLE_FILL_VALUE: u32 = 0x7F;
144 const MAX_LUT_BITS: u8 = 10;
145
146 fn fill_lut_msb(table: &mut Vec<u32>, off: usize,
147 code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) {
148 if !esc {
149 let fill_len = lut_bits - bits;
150 let fill_size = 1 << fill_len;
151 let fill_code = code << (lut_bits - bits);
152 let lut_value = (symidx << 8) | u32::from(bits);
153 for j in 0..fill_size {
154 let idx = (fill_code + j) as usize;
155 table[idx + off] = lut_value;
156 }
157 } else {
158 let idx = (code as usize) + off;
159 table[idx] = (symidx << 8) | 0x80 | u32::from(bits);
160 }
161 }
162
163 fn fill_lut_lsb(table: &mut Vec<u32>, off: usize,
164 code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) {
165 if !esc {
166 let fill_len = lut_bits - bits;
167 let fill_size = 1 << fill_len;
168 let fill_code = code;
169 let step = lut_bits - fill_len;
170 for j in 0..fill_size {
171 let idx = (fill_code + (j << step)) as usize;
172 table[idx + off] = (symidx << 8) | u32::from(bits);
173 }
174 } else {
175 let idx = (code as usize) + off;
176 table[idx] = (symidx << 8) | 0x80 | u32::from(bits);
177 }
178 }
179
180 fn fill_lut(table: &mut Vec<u32>, mode: CodebookMode,
181 off: usize, code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) -> bool {
182 match mode {
183 CodebookMode::MSB => fill_lut_msb(table, off, code, bits, lut_bits, symidx, esc),
184 CodebookMode::LSB => fill_lut_lsb(table, off, code, bits, lut_bits, symidx, esc),
185 };
186 bits > lut_bits
187 }
188
189 fn resize_table(table: &mut Vec<u32>, bits: u8) -> CodebookResult<u32> {
190 let add_size = (1 << bits) as usize;
191 table.reserve(add_size);
192 let cur_off = table.len() as u32;
193 let new_size = table.len() + add_size;
194 if table.capacity() < new_size { return Err(CodebookError::MemoryError); }
195 table.resize(new_size, TABLE_FILL_VALUE);
196 Ok(cur_off)
197 }
198
199
200 fn extract_lut_part(code: u32, bits: u8, lut_bits: u8, mode: CodebookMode) -> u32 {
201 match mode {
202 CodebookMode::MSB => code >> (bits - lut_bits),
203 CodebookMode::LSB => code & ((1 << lut_bits) - 1),
204 }
205 }
206
207 fn extract_esc_part(code: u32, bits: u8, lut_bits: u8, mode: CodebookMode) -> u32 {
208 match mode {
209 CodebookMode::MSB => code & ((1 << (bits - lut_bits)) - 1),
210 CodebookMode::LSB => code >> lut_bits,
211 }
212 }
213
214 #[derive(Clone,Copy)]
215 struct Code {
216 code: u32,
217 bits: u8,
218 idx: usize,
219 }
220
221 struct CodeBucket {
222 maxlen: u8,
223 offset: usize,
224 codes: Vec<Code>,
225 }
226
227 impl CodeBucket {
228 fn new() -> Self {
229 CodeBucket { maxlen: 0, offset: 0, codes: Vec::new() }
230 }
231 fn add_code(&mut self, c: Code) {
232 if c.bits > self.maxlen { self.maxlen = c.bits; }
233 self.codes.push(c);
234 }
235 }
236
237 type EscapeCodes = HashMap<u32, CodeBucket>;
238
239 fn add_esc_code(cc: &mut EscapeCodes, key: u32, code: u32, bits: u8, idx: usize) {
240 cc.entry(key).or_insert_with(CodeBucket::new);
241 let b = cc.get_mut(&key);
242 if let Some(bucket) = b {
243 bucket.add_code(Code {code, bits, idx });
244 } else { panic!("no bucket when expected!"); }
245 }
246
247 fn build_esc_lut(table: &mut Vec<u32>,
248 mode: CodebookMode,
249 bucket: &CodeBucket) -> CodebookResult<()> {
250 let mut escape_list: EscapeCodes = HashMap::new();
251 let maxlen = if bucket.maxlen > MAX_LUT_BITS { MAX_LUT_BITS } else { bucket.maxlen };
252
253 for code in &bucket.codes {
254 let bits = code.bits;
255 if code.bits <= MAX_LUT_BITS {
256 fill_lut(table, mode, bucket.offset, code.code, bits,
257 maxlen, code.idx as u32, false);
258 } else {
259 let ckey = extract_lut_part(code.code, bits, MAX_LUT_BITS, mode);
260 let cval = extract_esc_part(code.code, bits, MAX_LUT_BITS, mode);
261 add_esc_code(&mut escape_list, ckey, cval, bits - MAX_LUT_BITS, code.idx);
262 }
263 }
264
265 let cur_offset = bucket.offset;
266 for (ckey, sec_bucket) in &mut escape_list {
267 let key = *ckey as u32;
268 let maxlen = min(sec_bucket.maxlen, MAX_LUT_BITS);
269 let new_off = resize_table(table, maxlen)?;
270 fill_lut(table, mode, cur_offset, key, maxlen,
271 MAX_LUT_BITS, new_off, true);
272 sec_bucket.offset = new_off as usize;
273 }
274
275 for sec_bucket in escape_list.values() {
276 build_esc_lut(table, mode, sec_bucket)?;
277 }
278
279 Ok(())
280 }
281
282 impl<S: Copy> Codebook<S> {
283
284 /// Constructs a new `Codebook` instance using provided codebook description and mode.
285 pub fn new(cb: &mut CodebookDescReader<S>, mode: CodebookMode) -> CodebookResult<Self> {
286 let mut maxbits = 0;
287 let mut nnz = 0;
288 let mut escape_list: EscapeCodes = HashMap::new();
289
290 let mut symidx: usize = 0;
291 for i in 0..cb.len() {
292 let bits = cb.bits(i);
293 if bits > 0 {
294 nnz += 1;
295 if cb.code(i) >= (1 << bits) {
296 return Err(CodebookError::InvalidCodebook);
297 }
298 }
299 maxbits = max(bits, maxbits);
300 if bits > MAX_LUT_BITS {
301 let code = cb.code(i);
302 let ckey = extract_lut_part(code, bits, MAX_LUT_BITS, mode);
303 let cval = extract_esc_part(code, bits, MAX_LUT_BITS, mode);
304 add_esc_code(&mut escape_list, ckey, cval, bits - MAX_LUT_BITS, symidx);
305 }
306 if bits > 0 { symidx += 1; }
307 }
308 if maxbits == 0 { return Err(CodebookError::InvalidCodebook); }
309
310 if maxbits > MAX_LUT_BITS { maxbits = MAX_LUT_BITS; }
311
312 let tab_len = 1 << maxbits;
313 let mut table: Vec<u32> = Vec::with_capacity(tab_len);
314 let mut syms: Vec<S> = Vec::with_capacity(nnz);
315 if table.capacity() < tab_len { return Err(CodebookError::MemoryError); }
316 if syms.capacity() < nnz { return Err(CodebookError::MemoryError); }
317 table.resize(tab_len, TABLE_FILL_VALUE);
318
319 let mut symidx: u32 = 0;
320 for i in 0..cb.len() {
321 let bits = cb.bits(i);
322 let code = cb.code(i);
323 if bits == 0 { continue; }
324 if bits <= MAX_LUT_BITS {
325 fill_lut(&mut table, mode, 0, code, bits, maxbits, symidx, false);
326 } else {
327 let ckey = extract_lut_part(code, bits, MAX_LUT_BITS, mode) as usize;
328 if table[ckey] == TABLE_FILL_VALUE {
329 let key = ckey as u32;
330 if let Some(bucket) = escape_list.get_mut(&key) {
331 let maxlen = min(bucket.maxlen, MAX_LUT_BITS);
332 let new_off = resize_table(&mut table, maxlen)?;
333 fill_lut(&mut table, mode, 0, key, maxlen, MAX_LUT_BITS, new_off, true);
334 bucket.offset = new_off as usize;
335 }
336 }
337 }
338 symidx += 1;
339 }
340
341 for bucket in escape_list.values() {
342 build_esc_lut(&mut table, mode, &bucket)?;
343 }
344
345 for i in 0..cb.len() {
346 if cb.bits(i) > 0 {
347 syms.push(cb.sym(i));
348 }
349 }
350
351 Ok(Codebook { table, syms, lut_bits: maxbits })
352 }
353 }
354
355 impl<'a, S: Copy> CodebookReader<S> for BitReader<'a> {
356 #[allow(unused_variables)]
357 fn read_cb(&mut self, cb: &Codebook<S>) -> CodebookResult<S> {
358 let mut esc = true;
359 let mut idx = 0;
360 let mut lut_bits = cb.lut_bits;
361 while esc {
362 let lut_idx = (self.peek(lut_bits) as usize) + (idx as usize);
363 if cb.table[lut_idx] == TABLE_FILL_VALUE { return Err(CodebookError::InvalidCode); }
364 let bits = cb.table[lut_idx] & 0x7F;
365 esc = (cb.table[lut_idx] & 0x80) != 0;
366 idx = (cb.table[lut_idx] >> 8) as usize;
367 if (bits as isize) > self.left() {
368 return Err(CodebookError::InvalidCode);
369 }
370 let skip_bits = if esc { u32::from(lut_bits) } else { bits };
371 self.skip(skip_bits as u32).unwrap();
372 lut_bits = bits as u8;
373 }
374 Ok(cb.syms[idx])
375 }
376 }
377
378 /// Codebook description that stores a list of codewords and their values.
379 pub struct FullCodebookDescReader<S> {
380 data: Vec<FullCodebookDesc<S>>,
381 }
382
383 impl<S> FullCodebookDescReader<S> {
384 /// Constructs a new `FullCodebookDescReader` instance.
385 pub fn new(data: Vec<FullCodebookDesc<S>>) -> Self {
386 FullCodebookDescReader { data }
387 }
388 }
389
390 impl<S: Copy> CodebookDescReader<S> for FullCodebookDescReader<S> {
391 fn bits(&mut self, idx: usize) -> u8 { self.data[idx].bits }
392 fn code(&mut self, idx: usize) -> u32 { self.data[idx].code }
393 fn sym (&mut self, idx: usize) -> S { self.data[idx].sym }
394 fn len(&mut self) -> usize { self.data.len() }
395 }
396
397 /// Codebook description that stores a list of codewords and their value is equal to the index.
398 pub struct ShortCodebookDescReader {
399 data: Vec<ShortCodebookDesc>,
400 }
401
402 impl ShortCodebookDescReader {
403 /// Constructs a new `ShortCodebookDescReader` instance.
404 pub fn new(data: Vec<ShortCodebookDesc<>>) -> Self {
405 ShortCodebookDescReader { data }
406 }
407 }
408
409 impl CodebookDescReader<u32> for ShortCodebookDescReader {
410 fn bits(&mut self, idx: usize) -> u8 { self.data[idx].bits }
411 fn code(&mut self, idx: usize) -> u32 { self.data[idx].code }
412 fn sym (&mut self, idx: usize) -> u32 { idx as u32 }
413 fn len(&mut self) -> usize { self.data.len() }
414 }
415
416 /// Flexible codebook description that uses two separate arrays for codeword bits and lengths and a function that maps codeword index into its symbol.
417 pub struct TableCodebookDescReader<'a, CodeType:'static, IndexType:'static> {
418 bits: &'a [u8],
419 codes: &'a [CodeType],
420 idx_map: fn(usize) -> IndexType,
421 }
422
423 impl<'a, CodeType, IndexType> TableCodebookDescReader<'a, CodeType, IndexType> {
424 /// Constructs a new `TableCodebookDescReader` instance.
425 pub fn new(codes: &'a [CodeType], bits: &'a [u8], idx_map: fn(usize) -> IndexType) -> Self {
426 Self { bits, codes, idx_map }
427 }
428 }
429 impl<'a, CodeType: Copy+Into<u32>, IndexType> CodebookDescReader<IndexType> for TableCodebookDescReader<'a, CodeType, IndexType>
430 {
431 fn bits(&mut self, idx: usize) -> u8 { self.bits[idx] }
432 fn code(&mut self, idx: usize) -> u32 { self.codes[idx].into() }
433 fn sym (&mut self, idx: usize) -> IndexType { (self.idx_map)(idx) }
434 fn len(&mut self) -> usize { self.bits.len() }
435 }
436
437 #[cfg(test)]
438 mod test {
439 use super::*;
440 use crate::io::bitreader::*;
441
442 #[test]
443 fn test_cb() {
444 const BITS: [u8; 2] = [0b01011011, 0b10111100];
445 let cb_desc: Vec<FullCodebookDesc<i8>> = vec!(
446 FullCodebookDesc { code: 0b0, bits: 1, sym: 16 },
447 FullCodebookDesc { code: 0b10, bits: 2, sym: -3 },
448 FullCodebookDesc { code: 0b110, bits: 3, sym: 42 },
449 FullCodebookDesc { code: 0b1110, bits: 4, sym: -42 }
450 );
451 let buf = &BITS;
452 let mut br = BitReader::new(buf, BitReaderMode::BE);
453 let mut cfr = FullCodebookDescReader::new(cb_desc);
454 let cb = Codebook::new(&mut cfr, CodebookMode::MSB).unwrap();
455 assert_eq!(br.read_cb(&cb).unwrap(), 16);
456 assert_eq!(br.read_cb(&cb).unwrap(), -3);
457 assert_eq!(br.read_cb(&cb).unwrap(), 42);
458 assert_eq!(br.read_cb(&cb).unwrap(), -42);
459 let ret = br.read_cb(&cb);
460 if let Err(e) = ret {
461 assert_eq!(e as i32, CodebookError::InvalidCode as i32);
462 } else {
463 assert_eq!(0, 1);
464 }
465
466 let scb_desc: Vec<ShortCodebookDesc> = vec!(
467 ShortCodebookDesc { code: 0b0, bits: 1 },
468 ShortCodebookDesc { code: 0, bits: 0 },
469 ShortCodebookDesc { code: 0b10, bits: 2 },
470 ShortCodebookDesc { code: 0, bits: 0 },
471 ShortCodebookDesc { code: 0, bits: 0 },
472 ShortCodebookDesc { code: 0b110, bits: 3 },
473 ShortCodebookDesc { code: 0, bits: 0 },
474 ShortCodebookDesc { code: 0b11100, bits: 5 },
475 ShortCodebookDesc { code: 0b11101, bits: 5 },
476 ShortCodebookDesc { code: 0b1111010, bits: 7 },
477 ShortCodebookDesc { code: 0b1111011, bits: 7 },
478 ShortCodebookDesc { code: 0b1111110, bits: 7 },
479 ShortCodebookDesc { code: 0b11111111, bits: 8 }
480 );
481 let mut br2 = BitReader::new(buf, BitReaderMode::BE);
482 let mut cfr = ShortCodebookDescReader::new(scb_desc);
483 let cb = Codebook::new(&mut cfr, CodebookMode::MSB).unwrap();
484 assert_eq!(br2.read_cb(&cb).unwrap(), 0);
485 assert_eq!(br2.read_cb(&cb).unwrap(), 2);
486 assert_eq!(br2.read_cb(&cb).unwrap(), 5);
487 assert_eq!(br2.read_cb(&cb).unwrap(), 8);
488
489 assert_eq!(reverse_bits(0b0000_0101_1011_1011_1101_1111_0111_1111, 32),
490 0b1111_1110_1111_1011_1101_1101_1010_0000);
491
492 const BITS_LE: [u8; 3] = [0b11101111, 0b01110010, 0b01];
493 let buf = &BITS_LE;
494 let scble_desc: Vec<ShortCodebookDesc> = vec!(
495 ShortCodebookDesc { code: 0b00, bits: 2 },
496 ShortCodebookDesc { code: 0, bits: 0 },
497 ShortCodebookDesc { code: 0b01, bits: 2 },
498 ShortCodebookDesc { code: 0, bits: 0 },
499 ShortCodebookDesc { code: 0, bits: 0 },
500 ShortCodebookDesc { code: 0b011, bits: 3 },
501 ShortCodebookDesc { code: 0, bits: 0 },
502 ShortCodebookDesc { code: 0b10111, bits: 5 },
503 ShortCodebookDesc { code: 0b00111, bits: 5 },
504 ShortCodebookDesc { code: 0b0101111, bits: 7 },
505 ShortCodebookDesc { code: 0b0111111, bits: 7 },
506 ShortCodebookDesc { code: 0b1011101111, bits: 10 }
507 );
508 let mut brl = BitReader::new(buf, BitReaderMode::LE);
509 let mut cfr = ShortCodebookDescReader::new(scble_desc);
510 let cb = Codebook::new(&mut cfr, CodebookMode::LSB).unwrap();
511 assert_eq!(brl.read_cb(&cb).unwrap(), 11);
512 assert_eq!(brl.read_cb(&cb).unwrap(), 0);
513 assert_eq!(brl.read_cb(&cb).unwrap(), 7);
514 assert_eq!(brl.read_cb(&cb).unwrap(), 0);
515 }
516 }