1 //! Codebook support for bitstream reader.
3 //! Codebook is a set of unique bit strings and values assigned to them.
4 //! Since there are many ways to define codebook, this implementation employs [`CodebookDescReader`] trait to provide codebook generator with the codes.
5 //! Please also pay attention to the codebook creation mode: if bitstream reader reads bits starting from most significant bit first then you should use [`MSB`] mode and [`LSB`] mode otherwise.
9 //! Create a codebook from arrays with codeword descriptions:
11 //! use nihav_core::io::codebook::{ShortCodebookDesc, ShortCodebookDescReader, Codebook, CodebookMode};
13 //! let cb_desc: Vec<ShortCodebookDesc> = vec!(
14 //! ShortCodebookDesc { code: 0b00, bits: 2 },
15 //! ShortCodebookDesc { code: 0, bits: 0 },
16 //! ShortCodebookDesc { code: 0b01, bits: 2 },
17 //! ShortCodebookDesc { code: 0b1, bits: 1 });
18 //! let mut cr = ShortCodebookDescReader::new(cb_desc);
19 //! let cb = Codebook::new(&mut cr, CodebookMode::MSB).unwrap();
22 //! Create a codebook using more flexible [`TableCodebookDescReader`] approach.
23 //! This will create a codebook for the following set: `1` -> -2, `01` -> -1, `001` -> 0, `0001` -> 1, `00001` -> 2.
25 //! use nihav_core::io::codebook::{TableCodebookDescReader, Codebook, CodebookMode};
27 //! fn map_cb_index(index: usize) -> i16 { (index as i16) - 2 }
28 //! const CB_BITS: [u8; 5] = [ 1, 2, 3, 4, 5 ];
29 //! const CB_CODES: [u8; 5] = [ 1, 1, 1, 1, 1 ];
31 //! let mut tcr = TableCodebookDescReader::new(&CB_CODES, &CB_BITS, map_cb_index);
32 //! let cb = Codebook::new(&mut tcr, CodebookMode::MSB).unwrap();
35 //! Read value using a codebook:
37 //! use nihav_core::io::bitreader::BitReader;
38 //! use nihav_core::io::codebook::{Codebook, CodebookReader, CodebookMode};
39 //! # use nihav_core::io::codebook::{ShortCodebookDesc, ShortCodebookDescReader, CodebookDescReader, CodebookResult};
41 //! # fn foo(br: &mut BitReader) -> CodebookResult<()> {
42 //! # let mut cr = ShortCodebookDescReader::new(vec![ShortCodebookDesc { code: 0b00, bits: 2 }]);
43 //! let cb = Codebook::new(&mut cr, CodebookMode::MSB).unwrap();
44 //! let value = br.read_cb(&cb)?;
49 //! [`MSB`]: ./enum.CodebookMode.html#variant.MSB
50 //! [`LSB`]: ./enum.CodebookMode.html#variant.LSB
51 //! [`CodebookDescReader`]: ./trait.CodebookDescReader.html
52 //! [`TableCodebookDescReader`]: ./struct.TableCodebookDescReader.html
54 use std::collections::HashMap;
55 use std::cmp::{max, min};
56 use super::bitreader::BitReader;
58 /// A list specifying general codebook operations errors.
60 pub enum CodebookError {
61 /// Codebook description contains errors.
63 /// Could not allocate memory for codebook.
65 /// Bitstream contains a sequence not present in codebook.
69 /// Codebook operation modes.
70 #[derive(Debug, Copy, Clone)]
71 pub enum CodebookMode {
72 /// Codes in the codebook should be read most significant bit first.
74 /// Codes in the codebook should be read least significant bit first.
78 /// A specialised `Result` type for codebook operations.
79 pub type CodebookResult<T> = Result<T, CodebookError>;
81 /// Codebook description for `(code bits, code length, code value)` triplet.
83 /// This should be used to create a list of codeword definitions for [`FullCodebookDescReader`].
85 /// [`FullCodebookDescReader`]: ./struct.FullCodebookDescReader.html
87 pub struct FullCodebookDesc<S> {
92 /// Codeword value (symbol).
96 /// Codebook description for `(code bits, code length)` pair with array index being used as codeword value.
98 /// This should be used to create a list of codeword definitions for [`ShortCodebookDescReader`].
100 /// [`ShortCodebookDescReader`]: ./struct.ShortCodebookDescReader.html
101 #[derive(Clone,Copy)]
102 pub struct ShortCodebookDesc {
109 /// The interface for providing a list of codeword definitions to the codebook creator.
111 /// The structure implementing this trait should be able to provide the total number of defined codewords and their bits and values. [`ShortCodebookDescReader`] or [`TableCodebookDescReader`] are some examples of such implementation.
112 /// Codeword definitions with zero length are ignored (those may be used to create sparse codebook definitions though).
114 /// [`ShortCodebookDescReader`]: ./struct.ShortCodebookDescReader.html
115 /// [`TableCodebookDescReader`]: ./struct.TableCodebookDescReader.html
116 #[allow(clippy::len_without_is_empty)]
117 pub trait CodebookDescReader<S> {
118 /// Returns the codeword length for the provided index.
119 fn bits(&mut self, idx: usize) -> u8;
120 /// Returns the codeword bits for the provided index.
121 fn code(&mut self, idx: usize) -> u32;
122 /// Returns the codeword value (aka codeword symbol) for the provided index.
123 fn sym (&mut self, idx: usize) -> S;
124 /// Returns the total number of defined codewords.
125 fn len (&mut self) -> usize;
128 /// The codebook structure for code reading.
130 pub struct Codebook<S> {
136 /// Trait allowing bitreader to use codebook for decoding bit sequences.
137 pub trait CodebookReader<S> {
138 /// Reads the codeword from the bitstream and returns its value (or [`InvalidCode`] on error).
140 /// [`InvalidCode`]: ./enum.CodebookError.html#variant.InvalidCode
141 fn read_cb(&mut self, cb: &Codebook<S>) -> CodebookResult<S>;
144 pub const TABLE_FILL_VALUE: u32 = 0x7F;
145 const MAX_LUT_BITS: u8 = 10;
147 fn fill_lut_msb(table: &mut Vec<u32>, off: usize,
148 code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) -> CodebookResult<()> {
150 let fill_len = lut_bits - bits;
151 let fill_size = 1 << fill_len;
152 let fill_code = code << (lut_bits - bits);
153 let lut_value = (symidx << 8) | u32::from(bits);
154 for j in 0..fill_size {
155 let idx = (fill_code + j) as usize;
156 if table[idx + off] != TABLE_FILL_VALUE { return Err(CodebookError::InvalidCodebook); }
157 table[idx + off] = lut_value;
160 let idx = (code as usize) + off;
161 if table[idx] != TABLE_FILL_VALUE { return Err(CodebookError::InvalidCodebook); }
162 table[idx] = (symidx << 8) | 0x80 | u32::from(bits);
167 fn fill_lut_lsb(table: &mut Vec<u32>, off: usize,
168 code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) -> CodebookResult<()> {
170 let fill_len = lut_bits - bits;
171 let fill_size = 1 << fill_len;
172 let fill_code = code;
173 let step = lut_bits - fill_len;
174 for j in 0..fill_size {
175 let idx = (fill_code + (j << step)) as usize;
176 if table[idx + off] != TABLE_FILL_VALUE { return Err(CodebookError::InvalidCodebook); }
177 table[idx + off] = (symidx << 8) | u32::from(bits);
180 let idx = (code as usize) + off;
181 if table[idx] != TABLE_FILL_VALUE { return Err(CodebookError::InvalidCodebook); }
182 table[idx] = (symidx << 8) | 0x80 | u32::from(bits);
187 fn fill_lut(table: &mut Vec<u32>, mode: CodebookMode,
188 off: usize, code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) -> CodebookResult<bool> {
190 CodebookMode::MSB => fill_lut_msb(table, off, code, bits, lut_bits, symidx, esc)?,
191 CodebookMode::LSB => fill_lut_lsb(table, off, code, bits, lut_bits, symidx, esc)?,
196 fn resize_table(table: &mut Vec<u32>, bits: u8) -> CodebookResult<u32> {
197 let add_size = (1 << bits) as usize;
198 table.reserve(add_size);
199 let cur_off = table.len() as u32;
200 let new_size = table.len() + add_size;
201 if table.capacity() < new_size { return Err(CodebookError::MemoryError); }
202 table.resize(new_size, TABLE_FILL_VALUE);
207 fn extract_lut_part(code: u32, bits: u8, lut_bits: u8, mode: CodebookMode) -> u32 {
209 CodebookMode::MSB => code >> (bits - lut_bits),
210 CodebookMode::LSB => code & ((1 << lut_bits) - 1),
214 fn extract_esc_part(code: u32, bits: u8, lut_bits: u8, mode: CodebookMode) -> u32 {
216 CodebookMode::MSB => code & ((1 << (bits - lut_bits)) - 1),
217 CodebookMode::LSB => code >> lut_bits,
221 #[derive(Clone,Copy)]
236 CodeBucket { maxlen: 0, offset: 0, codes: Vec::new() }
238 fn add_code(&mut self, c: Code) {
239 if c.bits > self.maxlen { self.maxlen = c.bits; }
244 type EscapeCodes = HashMap<u32, CodeBucket>;
246 fn add_esc_code(cc: &mut EscapeCodes, key: u32, code: u32, bits: u8, idx: usize) {
247 cc.entry(key).or_insert_with(CodeBucket::new);
248 let b = cc.get_mut(&key);
249 if let Some(bucket) = b {
250 bucket.add_code(Code {code, bits, idx });
251 } else { panic!("no bucket when expected!"); }
254 fn build_esc_lut(table: &mut Vec<u32>,
256 bucket: &CodeBucket) -> CodebookResult<()> {
257 let mut escape_list: EscapeCodes = HashMap::new();
258 let maxlen = if bucket.maxlen > MAX_LUT_BITS { MAX_LUT_BITS } else { bucket.maxlen };
260 for code in &bucket.codes {
261 let bits = code.bits;
262 if code.bits <= MAX_LUT_BITS {
263 fill_lut(table, mode, bucket.offset, code.code, bits,
264 maxlen, code.idx as u32, false)?;
266 let ckey = extract_lut_part(code.code, bits, MAX_LUT_BITS, mode);
267 let cval = extract_esc_part(code.code, bits, MAX_LUT_BITS, mode);
268 add_esc_code(&mut escape_list, ckey, cval, bits - MAX_LUT_BITS, code.idx);
272 let cur_offset = bucket.offset;
273 for (ckey, sec_bucket) in &mut escape_list {
274 let key = *ckey as u32;
275 let maxlen = min(sec_bucket.maxlen, MAX_LUT_BITS);
276 let new_off = resize_table(table, maxlen)?;
277 fill_lut(table, mode, cur_offset, key, maxlen,
278 MAX_LUT_BITS, new_off, true)?;
279 sec_bucket.offset = new_off as usize;
282 for sec_bucket in escape_list.values() {
283 build_esc_lut(table, mode, sec_bucket)?;
289 impl<S: Copy> Codebook<S> {
291 /// Constructs a new `Codebook` instance using provided codebook description and mode.
292 pub fn new(cb: &mut dyn CodebookDescReader<S>, mode: CodebookMode) -> CodebookResult<Self> {
295 let mut escape_list: EscapeCodes = HashMap::new();
297 let mut symidx: usize = 0;
298 for i in 0..cb.len() {
299 let bits = cb.bits(i);
302 if cb.code(i) >= (1 << bits) {
303 return Err(CodebookError::InvalidCodebook);
306 maxbits = max(bits, maxbits);
307 if bits > MAX_LUT_BITS {
308 let code = cb.code(i);
309 let ckey = extract_lut_part(code, bits, MAX_LUT_BITS, mode);
310 let cval = extract_esc_part(code, bits, MAX_LUT_BITS, mode);
311 add_esc_code(&mut escape_list, ckey, cval, bits - MAX_LUT_BITS, symidx);
313 if bits > 0 { symidx += 1; }
315 if maxbits == 0 { return Err(CodebookError::InvalidCodebook); }
317 if maxbits > MAX_LUT_BITS { maxbits = MAX_LUT_BITS; }
319 let tab_len = 1 << maxbits;
320 let mut table: Vec<u32> = Vec::with_capacity(tab_len);
321 let mut syms: Vec<S> = Vec::with_capacity(nnz);
322 if table.capacity() < tab_len { return Err(CodebookError::MemoryError); }
323 if syms.capacity() < nnz { return Err(CodebookError::MemoryError); }
324 table.resize(tab_len, TABLE_FILL_VALUE);
326 let mut symidx: u32 = 0;
327 for i in 0..cb.len() {
328 let bits = cb.bits(i);
329 let code = cb.code(i);
330 if bits == 0 { continue; }
331 if bits <= MAX_LUT_BITS {
332 fill_lut(&mut table, mode, 0, code, bits, maxbits, symidx, false)?;
334 let ckey = extract_lut_part(code, bits, MAX_LUT_BITS, mode) as usize;
335 if table[ckey] == TABLE_FILL_VALUE {
336 let key = ckey as u32;
337 if let Some(bucket) = escape_list.get_mut(&key) {
338 let maxlen = min(bucket.maxlen, MAX_LUT_BITS);
339 let new_off = resize_table(&mut table, maxlen)?;
340 fill_lut(&mut table, mode, 0, key, maxlen, MAX_LUT_BITS, new_off, true)?;
341 bucket.offset = new_off as usize;
348 for bucket in escape_list.values() {
349 build_esc_lut(&mut table, mode, &bucket)?;
352 for i in 0..cb.len() {
354 syms.push(cb.sym(i));
358 Ok(Codebook { table, syms, lut_bits: maxbits })
362 impl<'a, S: Copy> CodebookReader<S> for BitReader<'a> {
363 #[allow(unused_variables)]
364 fn read_cb(&mut self, cb: &Codebook<S>) -> CodebookResult<S> {
367 let mut lut_bits = cb.lut_bits;
369 let lut_idx = (self.peek(lut_bits) as usize) + (idx as usize);
370 if cb.table[lut_idx] == TABLE_FILL_VALUE { return Err(CodebookError::InvalidCode); }
371 let bits = cb.table[lut_idx] & 0x7F;
372 esc = (cb.table[lut_idx] & 0x80) != 0;
373 idx = (cb.table[lut_idx] >> 8) as usize;
374 let skip_bits = if esc { u32::from(lut_bits) } else { bits };
375 if (skip_bits as isize) > self.left() {
376 return Err(CodebookError::InvalidCode);
378 self.skip(skip_bits as u32).unwrap();
379 lut_bits = bits as u8;
385 /// Codebook description that stores a list of codewords and their values.
386 pub struct FullCodebookDescReader<S> {
387 data: Vec<FullCodebookDesc<S>>,
390 impl<S> FullCodebookDescReader<S> {
391 /// Constructs a new `FullCodebookDescReader` instance.
392 pub fn new(data: Vec<FullCodebookDesc<S>>) -> Self {
393 FullCodebookDescReader { data }
397 impl<S: Copy> CodebookDescReader<S> for FullCodebookDescReader<S> {
398 fn bits(&mut self, idx: usize) -> u8 { self.data[idx].bits }
399 fn code(&mut self, idx: usize) -> u32 { self.data[idx].code }
400 fn sym (&mut self, idx: usize) -> S { self.data[idx].sym }
401 fn len(&mut self) -> usize { self.data.len() }
404 /// Codebook description that stores a list of codewords and their value is equal to the index.
405 pub struct ShortCodebookDescReader {
406 data: Vec<ShortCodebookDesc>,
409 impl ShortCodebookDescReader {
410 /// Constructs a new `ShortCodebookDescReader` instance.
411 pub fn new(data: Vec<ShortCodebookDesc<>>) -> Self {
412 ShortCodebookDescReader { data }
416 impl CodebookDescReader<u32> for ShortCodebookDescReader {
417 fn bits(&mut self, idx: usize) -> u8 { self.data[idx].bits }
418 fn code(&mut self, idx: usize) -> u32 { self.data[idx].code }
419 fn sym (&mut self, idx: usize) -> u32 { idx as u32 }
420 fn len(&mut self) -> usize { self.data.len() }
423 /// Flexible codebook description that uses two separate arrays for codeword bits and lengths and a function that maps codeword index into its symbol.
424 pub struct TableCodebookDescReader<'a, CodeType:'static, IndexType:'static> {
426 codes: &'a [CodeType],
427 idx_map: fn(usize) -> IndexType,
430 impl<'a, CodeType, IndexType> TableCodebookDescReader<'a, CodeType, IndexType> {
431 /// Constructs a new `TableCodebookDescReader` instance.
432 pub fn new(codes: &'a [CodeType], bits: &'a [u8], idx_map: fn(usize) -> IndexType) -> Self {
433 Self { bits, codes, idx_map }
436 impl<'a, CodeType: Copy+Into<u32>, IndexType> CodebookDescReader<IndexType> for TableCodebookDescReader<'a, CodeType, IndexType>
438 fn bits(&mut self, idx: usize) -> u8 { self.bits[idx] }
439 fn code(&mut self, idx: usize) -> u32 { self.codes[idx].into() }
440 fn sym (&mut self, idx: usize) -> IndexType { (self.idx_map)(idx) }
441 fn len(&mut self) -> usize { self.bits.len() }
447 use crate::io::bitreader::*;
451 const BITS: [u8; 2] = [0b01011011, 0b10111100];
452 let cb_desc: Vec<FullCodebookDesc<i8>> = vec!(
453 FullCodebookDesc { code: 0b0, bits: 1, sym: 16 },
454 FullCodebookDesc { code: 0b10, bits: 2, sym: -3 },
455 FullCodebookDesc { code: 0b110, bits: 3, sym: 42 },
456 FullCodebookDesc { code: 0b1110, bits: 4, sym: -42 }
459 let mut br = BitReader::new(buf, BitReaderMode::BE);
460 let mut cfr = FullCodebookDescReader::new(cb_desc);
461 let cb = Codebook::new(&mut cfr, CodebookMode::MSB).unwrap();
462 assert_eq!(br.read_cb(&cb).unwrap(), 16);
463 assert_eq!(br.read_cb(&cb).unwrap(), -3);
464 assert_eq!(br.read_cb(&cb).unwrap(), 42);
465 assert_eq!(br.read_cb(&cb).unwrap(), -42);
466 let ret = br.read_cb(&cb);
467 if let Err(e) = ret {
468 assert_eq!(e as i32, CodebookError::InvalidCode as i32);
473 let scb_desc: Vec<ShortCodebookDesc> = vec!(
474 ShortCodebookDesc { code: 0b0, bits: 1 },
475 ShortCodebookDesc { code: 0, bits: 0 },
476 ShortCodebookDesc { code: 0b10, bits: 2 },
477 ShortCodebookDesc { code: 0, bits: 0 },
478 ShortCodebookDesc { code: 0, bits: 0 },
479 ShortCodebookDesc { code: 0b110, bits: 3 },
480 ShortCodebookDesc { code: 0, bits: 0 },
481 ShortCodebookDesc { code: 0b11100, bits: 5 },
482 ShortCodebookDesc { code: 0b11101, bits: 5 },
483 ShortCodebookDesc { code: 0b1111010, bits: 7 },
484 ShortCodebookDesc { code: 0b1111011, bits: 7 },
485 ShortCodebookDesc { code: 0b1111110, bits: 7 },
486 ShortCodebookDesc { code: 0b11111111, bits: 8 }
488 let mut br2 = BitReader::new(buf, BitReaderMode::BE);
489 let mut cfr = ShortCodebookDescReader::new(scb_desc);
490 let cb = Codebook::new(&mut cfr, CodebookMode::MSB).unwrap();
491 assert_eq!(br2.read_cb(&cb).unwrap(), 0);
492 assert_eq!(br2.read_cb(&cb).unwrap(), 2);
493 assert_eq!(br2.read_cb(&cb).unwrap(), 5);
494 assert_eq!(br2.read_cb(&cb).unwrap(), 8);
496 assert_eq!(reverse_bits(0b0000_0101_1011_1011_1101_1111_0111_1111, 32),
497 0b1111_1110_1111_1011_1101_1101_1010_0000);
499 const BITS_LE: [u8; 3] = [0b11101111, 0b01110010, 0b01];
501 let scble_desc: Vec<ShortCodebookDesc> = vec!(
502 ShortCodebookDesc { code: 0b00, bits: 2 },
503 ShortCodebookDesc { code: 0, bits: 0 },
504 ShortCodebookDesc { code: 0b01, bits: 2 },
505 ShortCodebookDesc { code: 0, bits: 0 },
506 ShortCodebookDesc { code: 0, bits: 0 },
507 ShortCodebookDesc { code: 0b011, bits: 3 },
508 ShortCodebookDesc { code: 0, bits: 0 },
509 ShortCodebookDesc { code: 0b10111, bits: 5 },
510 ShortCodebookDesc { code: 0b00111, bits: 5 },
511 ShortCodebookDesc { code: 0b0101111, bits: 7 },
512 ShortCodebookDesc { code: 0b0111111, bits: 7 },
513 ShortCodebookDesc { code: 0b1011101111, bits: 10 }
515 let mut brl = BitReader::new(buf, BitReaderMode::LE);
516 let mut cfr = ShortCodebookDescReader::new(scble_desc);
517 let cb = Codebook::new(&mut cfr, CodebookMode::LSB).unwrap();
518 assert_eq!(brl.read_cb(&cb).unwrap(), 11);
519 assert_eq!(brl.read_cb(&cb).unwrap(), 0);
520 assert_eq!(brl.read_cb(&cb).unwrap(), 7);
521 assert_eq!(brl.read_cb(&cb).unwrap(), 0);