core/io: make Codebook fields and TABLE_FILL_VALUE public
[nihav.git] / nihav-core / src / io / codebook.rs
1 //! Codebook support for bitstream reader.
2 //!
3 //! Codebook is a set of unique bit strings and values assigned to them.
4 //! Since there are many ways to define codebook, this implementation employs [`CodebookDescReader`] trait to provide codebook generator with the codes.
5 //! Please also pay attention to the codebook creation mode: if bitstream reader reads bits starting from most significant bit first then you should use [`MSB`] mode and [`LSB`] mode otherwise.
6 //!
7 //! # Examples
8 //!
9 //! Create a codebook from arrays with codeword descriptions:
10 //! ```
11 //! use nihav_core::io::codebook::{ShortCodebookDesc, ShortCodebookDescReader, Codebook, CodebookMode};
12 //!
13 //! let cb_desc: Vec<ShortCodebookDesc> = vec!(
14 //! ShortCodebookDesc { code: 0b00, bits: 2 },
15 //! ShortCodebookDesc { code: 0, bits: 0 },
16 //! ShortCodebookDesc { code: 0b01, bits: 2 },
17 //! ShortCodebookDesc { code: 0b1, bits: 1 });
18 //! let mut cr = ShortCodebookDescReader::new(cb_desc);
19 //! let cb = Codebook::new(&mut cr, CodebookMode::MSB).unwrap();
20 //! ```
21 //!
22 //! Create a codebook using more flexible [`TableCodebookDescReader`] approach.
23 //! This will create a codebook for the following set: `1` -> -2, `01` -> -1, `001` -> 0, `0001` -> 1, `00001` -> 2.
24 //! ```
25 //! use nihav_core::io::codebook::{TableCodebookDescReader, Codebook, CodebookMode};
26 //!
27 //! fn map_cb_index(index: usize) -> i16 { (index as i16) - 2 }
28 //! const CB_BITS: [u8; 5] = [ 1, 2, 3, 4, 5 ];
29 //! const CB_CODES: [u8; 5] = [ 1, 1, 1, 1, 1 ];
30 //!
31 //! let mut tcr = TableCodebookDescReader::new(&CB_CODES, &CB_BITS, map_cb_index);
32 //! let cb = Codebook::new(&mut tcr, CodebookMode::MSB).unwrap();
33 //! ```
34 //!
35 //! Read value using a codebook:
36 //! ```no_run
37 //! use nihav_core::io::bitreader::BitReader;
38 //! use nihav_core::io::codebook::{Codebook, CodebookReader, CodebookMode};
39 //! # use nihav_core::io::codebook::{ShortCodebookDesc, ShortCodebookDescReader, CodebookDescReader, CodebookResult};
40 //!
41 //! # fn foo(br: &mut BitReader) -> CodebookResult<()> {
42 //! # let mut cr = ShortCodebookDescReader::new(vec![ShortCodebookDesc { code: 0b00, bits: 2 }]);
43 //! let cb = Codebook::new(&mut cr, CodebookMode::MSB).unwrap();
44 //! let value = br.read_cb(&cb)?;
45 //! # Ok(())
46 //! # }
47 //! ```
48 //!
49 //! [`MSB`]: ./enum.CodebookMode.html#variant.MSB
50 //! [`LSB`]: ./enum.CodebookMode.html#variant.LSB
51 //! [`CodebookDescReader`]: ./trait.CodebookDescReader.html
52 //! [`TableCodebookDescReader`]: ./struct.TableCodebookDescReader.html
53
54 use std::collections::HashMap;
55 use std::cmp::{max, min};
56 use super::bitreader::BitReader;
57
58 /// A list specifying general codebook operations errors.
59 #[derive(Debug)]
60 pub enum CodebookError {
61 /// Codebook description contains errors.
62 InvalidCodebook,
63 /// Could not allocate memory for codebook.
64 MemoryError,
65 /// Bitstream contains a sequence not present in codebook.
66 InvalidCode,
67 }
68
69 /// Codebook operation modes.
70 #[derive(Debug, Copy, Clone)]
71 pub enum CodebookMode {
72 /// Codes in the codebook should be read most significant bit first.
73 MSB,
74 /// Codes in the codebook should be read least significant bit first.
75 LSB,
76 }
77
78 /// A specialised `Result` type for codebook operations.
79 pub type CodebookResult<T> = Result<T, CodebookError>;
80
81 /// Codebook description for `(code bits, code length, code value)` triplet.
82 ///
83 /// This should be used to create a list of codeword definitions for [`FullCodebookDescReader`].
84 ///
85 /// [`FullCodebookDescReader`]: ./struct.FullCodebookDescReader.html
86 #[derive(Clone,Copy)]
87 pub struct FullCodebookDesc<S> {
88 /// Codeword bits.
89 pub code: u32,
90 /// Codeword length.
91 pub bits: u8,
92 /// Codeword value (symbol).
93 pub sym: S,
94 }
95
96 /// Codebook description for `(code bits, code length)` pair with array index being used as codeword value.
97 ///
98 /// This should be used to create a list of codeword definitions for [`ShortCodebookDescReader`].
99 ///
100 /// [`ShortCodebookDescReader`]: ./struct.ShortCodebookDescReader.html
101 #[derive(Clone,Copy)]
102 pub struct ShortCodebookDesc {
103 /// Codeword bits.
104 pub code: u32,
105 /// Codeword length.
106 pub bits: u8,
107 }
108
109 /// The interface for providing a list of codeword definitions to the codebook creator.
110 ///
111 /// The structure implementing this trait should be able to provide the total number of defined codewords and their bits and values. [`ShortCodebookDescReader`] or [`TableCodebookDescReader`] are some examples of such implementation.
112 /// Codeword definitions with zero length are ignored (those may be used to create sparse codebook definitions though).
113 ///
114 /// [`ShortCodebookDescReader`]: ./struct.ShortCodebookDescReader.html
115 /// [`TableCodebookDescReader`]: ./struct.TableCodebookDescReader.html
116 pub trait CodebookDescReader<S> {
117 /// Returns the codeword length for the provided index.
118 fn bits(&mut self, idx: usize) -> u8;
119 /// Returns the codeword bits for the provided index.
120 fn code(&mut self, idx: usize) -> u32;
121 /// Returns the codeword value (aka codeword symbol) for the provided index.
122 fn sym (&mut self, idx: usize) -> S;
123 /// Returns the total number of defined codewords.
124 fn len (&mut self) -> usize;
125 }
126
127 /// The codebook structure for code reading.
128 #[allow(dead_code)]
129 pub struct Codebook<S> {
130 pub table: Vec<u32>,
131 pub syms: Vec<S>,
132 pub lut_bits: u8,
133 }
134
135 /// Trait allowing bitreader to use codebook for decoding bit sequences.
136 pub trait CodebookReader<S> {
137 /// Reads the codeword from the bitstream and returns its value (or [`InvalidCode`] on error).
138 ///
139 /// [`InvalidCode`]: ./enum.CodebookError.html#variant.InvalidCode
140 fn read_cb(&mut self, cb: &Codebook<S>) -> CodebookResult<S>;
141 }
142
143 pub const TABLE_FILL_VALUE: u32 = 0x7F;
144 const MAX_LUT_BITS: u8 = 10;
145
146 fn fill_lut_msb(table: &mut Vec<u32>, off: usize,
147 code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) -> CodebookResult<()> {
148 if !esc {
149 let fill_len = lut_bits - bits;
150 let fill_size = 1 << fill_len;
151 let fill_code = code << (lut_bits - bits);
152 let lut_value = (symidx << 8) | u32::from(bits);
153 for j in 0..fill_size {
154 let idx = (fill_code + j) as usize;
155 if table[idx + off] != TABLE_FILL_VALUE { return Err(CodebookError::InvalidCodebook); }
156 table[idx + off] = lut_value;
157 }
158 } else {
159 let idx = (code as usize) + off;
160 if table[idx] != TABLE_FILL_VALUE { return Err(CodebookError::InvalidCodebook); }
161 table[idx] = (symidx << 8) | 0x80 | u32::from(bits);
162 }
163 Ok(())
164 }
165
166 fn fill_lut_lsb(table: &mut Vec<u32>, off: usize,
167 code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) -> CodebookResult<()> {
168 if !esc {
169 let fill_len = lut_bits - bits;
170 let fill_size = 1 << fill_len;
171 let fill_code = code;
172 let step = lut_bits - fill_len;
173 for j in 0..fill_size {
174 let idx = (fill_code + (j << step)) as usize;
175 if table[idx + off] != TABLE_FILL_VALUE { return Err(CodebookError::InvalidCodebook); }
176 table[idx + off] = (symidx << 8) | u32::from(bits);
177 }
178 } else {
179 let idx = (code as usize) + off;
180 if table[idx] != TABLE_FILL_VALUE { return Err(CodebookError::InvalidCodebook); }
181 table[idx] = (symidx << 8) | 0x80 | u32::from(bits);
182 }
183 Ok(())
184 }
185
186 fn fill_lut(table: &mut Vec<u32>, mode: CodebookMode,
187 off: usize, code: u32, bits: u8, lut_bits: u8, symidx: u32, esc: bool) -> CodebookResult<bool> {
188 match mode {
189 CodebookMode::MSB => fill_lut_msb(table, off, code, bits, lut_bits, symidx, esc)?,
190 CodebookMode::LSB => fill_lut_lsb(table, off, code, bits, lut_bits, symidx, esc)?,
191 };
192 Ok(bits > lut_bits)
193 }
194
195 fn resize_table(table: &mut Vec<u32>, bits: u8) -> CodebookResult<u32> {
196 let add_size = (1 << bits) as usize;
197 table.reserve(add_size);
198 let cur_off = table.len() as u32;
199 let new_size = table.len() + add_size;
200 if table.capacity() < new_size { return Err(CodebookError::MemoryError); }
201 table.resize(new_size, TABLE_FILL_VALUE);
202 Ok(cur_off)
203 }
204
205
206 fn extract_lut_part(code: u32, bits: u8, lut_bits: u8, mode: CodebookMode) -> u32 {
207 match mode {
208 CodebookMode::MSB => code >> (bits - lut_bits),
209 CodebookMode::LSB => code & ((1 << lut_bits) - 1),
210 }
211 }
212
213 fn extract_esc_part(code: u32, bits: u8, lut_bits: u8, mode: CodebookMode) -> u32 {
214 match mode {
215 CodebookMode::MSB => code & ((1 << (bits - lut_bits)) - 1),
216 CodebookMode::LSB => code >> lut_bits,
217 }
218 }
219
220 #[derive(Clone,Copy)]
221 struct Code {
222 code: u32,
223 bits: u8,
224 idx: usize,
225 }
226
227 struct CodeBucket {
228 maxlen: u8,
229 offset: usize,
230 codes: Vec<Code>,
231 }
232
233 impl CodeBucket {
234 fn new() -> Self {
235 CodeBucket { maxlen: 0, offset: 0, codes: Vec::new() }
236 }
237 fn add_code(&mut self, c: Code) {
238 if c.bits > self.maxlen { self.maxlen = c.bits; }
239 self.codes.push(c);
240 }
241 }
242
243 type EscapeCodes = HashMap<u32, CodeBucket>;
244
245 fn add_esc_code(cc: &mut EscapeCodes, key: u32, code: u32, bits: u8, idx: usize) {
246 cc.entry(key).or_insert_with(CodeBucket::new);
247 let b = cc.get_mut(&key);
248 if let Some(bucket) = b {
249 bucket.add_code(Code {code, bits, idx });
250 } else { panic!("no bucket when expected!"); }
251 }
252
253 fn build_esc_lut(table: &mut Vec<u32>,
254 mode: CodebookMode,
255 bucket: &CodeBucket) -> CodebookResult<()> {
256 let mut escape_list: EscapeCodes = HashMap::new();
257 let maxlen = if bucket.maxlen > MAX_LUT_BITS { MAX_LUT_BITS } else { bucket.maxlen };
258
259 for code in &bucket.codes {
260 let bits = code.bits;
261 if code.bits <= MAX_LUT_BITS {
262 fill_lut(table, mode, bucket.offset, code.code, bits,
263 maxlen, code.idx as u32, false)?;
264 } else {
265 let ckey = extract_lut_part(code.code, bits, MAX_LUT_BITS, mode);
266 let cval = extract_esc_part(code.code, bits, MAX_LUT_BITS, mode);
267 add_esc_code(&mut escape_list, ckey, cval, bits - MAX_LUT_BITS, code.idx);
268 }
269 }
270
271 let cur_offset = bucket.offset;
272 for (ckey, sec_bucket) in &mut escape_list {
273 let key = *ckey as u32;
274 let maxlen = min(sec_bucket.maxlen, MAX_LUT_BITS);
275 let new_off = resize_table(table, maxlen)?;
276 fill_lut(table, mode, cur_offset, key, maxlen,
277 MAX_LUT_BITS, new_off, true)?;
278 sec_bucket.offset = new_off as usize;
279 }
280
281 for sec_bucket in escape_list.values() {
282 build_esc_lut(table, mode, sec_bucket)?;
283 }
284
285 Ok(())
286 }
287
288 impl<S: Copy> Codebook<S> {
289
290 /// Constructs a new `Codebook` instance using provided codebook description and mode.
291 pub fn new(cb: &mut CodebookDescReader<S>, mode: CodebookMode) -> CodebookResult<Self> {
292 let mut maxbits = 0;
293 let mut nnz = 0;
294 let mut escape_list: EscapeCodes = HashMap::new();
295
296 let mut symidx: usize = 0;
297 for i in 0..cb.len() {
298 let bits = cb.bits(i);
299 if bits > 0 {
300 nnz += 1;
301 if cb.code(i) >= (1 << bits) {
302 return Err(CodebookError::InvalidCodebook);
303 }
304 }
305 maxbits = max(bits, maxbits);
306 if bits > MAX_LUT_BITS {
307 let code = cb.code(i);
308 let ckey = extract_lut_part(code, bits, MAX_LUT_BITS, mode);
309 let cval = extract_esc_part(code, bits, MAX_LUT_BITS, mode);
310 add_esc_code(&mut escape_list, ckey, cval, bits - MAX_LUT_BITS, symidx);
311 }
312 if bits > 0 { symidx += 1; }
313 }
314 if maxbits == 0 { return Err(CodebookError::InvalidCodebook); }
315
316 if maxbits > MAX_LUT_BITS { maxbits = MAX_LUT_BITS; }
317
318 let tab_len = 1 << maxbits;
319 let mut table: Vec<u32> = Vec::with_capacity(tab_len);
320 let mut syms: Vec<S> = Vec::with_capacity(nnz);
321 if table.capacity() < tab_len { return Err(CodebookError::MemoryError); }
322 if syms.capacity() < nnz { return Err(CodebookError::MemoryError); }
323 table.resize(tab_len, TABLE_FILL_VALUE);
324
325 let mut symidx: u32 = 0;
326 for i in 0..cb.len() {
327 let bits = cb.bits(i);
328 let code = cb.code(i);
329 if bits == 0 { continue; }
330 if bits <= MAX_LUT_BITS {
331 fill_lut(&mut table, mode, 0, code, bits, maxbits, symidx, false)?;
332 } else {
333 let ckey = extract_lut_part(code, bits, MAX_LUT_BITS, mode) as usize;
334 if table[ckey] == TABLE_FILL_VALUE {
335 let key = ckey as u32;
336 if let Some(bucket) = escape_list.get_mut(&key) {
337 let maxlen = min(bucket.maxlen, MAX_LUT_BITS);
338 let new_off = resize_table(&mut table, maxlen)?;
339 fill_lut(&mut table, mode, 0, key, maxlen, MAX_LUT_BITS, new_off, true)?;
340 bucket.offset = new_off as usize;
341 }
342 }
343 }
344 symidx += 1;
345 }
346
347 for bucket in escape_list.values() {
348 build_esc_lut(&mut table, mode, &bucket)?;
349 }
350
351 for i in 0..cb.len() {
352 if cb.bits(i) > 0 {
353 syms.push(cb.sym(i));
354 }
355 }
356
357 Ok(Codebook { table, syms, lut_bits: maxbits })
358 }
359 }
360
361 impl<'a, S: Copy> CodebookReader<S> for BitReader<'a> {
362 #[allow(unused_variables)]
363 fn read_cb(&mut self, cb: &Codebook<S>) -> CodebookResult<S> {
364 let mut esc = true;
365 let mut idx = 0;
366 let mut lut_bits = cb.lut_bits;
367 while esc {
368 let lut_idx = (self.peek(lut_bits) as usize) + (idx as usize);
369 if cb.table[lut_idx] == TABLE_FILL_VALUE { return Err(CodebookError::InvalidCode); }
370 let bits = cb.table[lut_idx] & 0x7F;
371 esc = (cb.table[lut_idx] & 0x80) != 0;
372 idx = (cb.table[lut_idx] >> 8) as usize;
373 if (bits as isize) > self.left() {
374 return Err(CodebookError::InvalidCode);
375 }
376 let skip_bits = if esc { u32::from(lut_bits) } else { bits };
377 self.skip(skip_bits as u32).unwrap();
378 lut_bits = bits as u8;
379 }
380 Ok(cb.syms[idx])
381 }
382 }
383
384 /// Codebook description that stores a list of codewords and their values.
385 pub struct FullCodebookDescReader<S> {
386 data: Vec<FullCodebookDesc<S>>,
387 }
388
389 impl<S> FullCodebookDescReader<S> {
390 /// Constructs a new `FullCodebookDescReader` instance.
391 pub fn new(data: Vec<FullCodebookDesc<S>>) -> Self {
392 FullCodebookDescReader { data }
393 }
394 }
395
396 impl<S: Copy> CodebookDescReader<S> for FullCodebookDescReader<S> {
397 fn bits(&mut self, idx: usize) -> u8 { self.data[idx].bits }
398 fn code(&mut self, idx: usize) -> u32 { self.data[idx].code }
399 fn sym (&mut self, idx: usize) -> S { self.data[idx].sym }
400 fn len(&mut self) -> usize { self.data.len() }
401 }
402
403 /// Codebook description that stores a list of codewords and their value is equal to the index.
404 pub struct ShortCodebookDescReader {
405 data: Vec<ShortCodebookDesc>,
406 }
407
408 impl ShortCodebookDescReader {
409 /// Constructs a new `ShortCodebookDescReader` instance.
410 pub fn new(data: Vec<ShortCodebookDesc<>>) -> Self {
411 ShortCodebookDescReader { data }
412 }
413 }
414
415 impl CodebookDescReader<u32> for ShortCodebookDescReader {
416 fn bits(&mut self, idx: usize) -> u8 { self.data[idx].bits }
417 fn code(&mut self, idx: usize) -> u32 { self.data[idx].code }
418 fn sym (&mut self, idx: usize) -> u32 { idx as u32 }
419 fn len(&mut self) -> usize { self.data.len() }
420 }
421
422 /// Flexible codebook description that uses two separate arrays for codeword bits and lengths and a function that maps codeword index into its symbol.
423 pub struct TableCodebookDescReader<'a, CodeType:'static, IndexType:'static> {
424 bits: &'a [u8],
425 codes: &'a [CodeType],
426 idx_map: fn(usize) -> IndexType,
427 }
428
429 impl<'a, CodeType, IndexType> TableCodebookDescReader<'a, CodeType, IndexType> {
430 /// Constructs a new `TableCodebookDescReader` instance.
431 pub fn new(codes: &'a [CodeType], bits: &'a [u8], idx_map: fn(usize) -> IndexType) -> Self {
432 Self { bits, codes, idx_map }
433 }
434 }
435 impl<'a, CodeType: Copy+Into<u32>, IndexType> CodebookDescReader<IndexType> for TableCodebookDescReader<'a, CodeType, IndexType>
436 {
437 fn bits(&mut self, idx: usize) -> u8 { self.bits[idx] }
438 fn code(&mut self, idx: usize) -> u32 { self.codes[idx].into() }
439 fn sym (&mut self, idx: usize) -> IndexType { (self.idx_map)(idx) }
440 fn len(&mut self) -> usize { self.bits.len() }
441 }
442
443 #[cfg(test)]
444 mod test {
445 use super::*;
446 use crate::io::bitreader::*;
447
448 #[test]
449 fn test_cb() {
450 const BITS: [u8; 2] = [0b01011011, 0b10111100];
451 let cb_desc: Vec<FullCodebookDesc<i8>> = vec!(
452 FullCodebookDesc { code: 0b0, bits: 1, sym: 16 },
453 FullCodebookDesc { code: 0b10, bits: 2, sym: -3 },
454 FullCodebookDesc { code: 0b110, bits: 3, sym: 42 },
455 FullCodebookDesc { code: 0b1110, bits: 4, sym: -42 }
456 );
457 let buf = &BITS;
458 let mut br = BitReader::new(buf, BitReaderMode::BE);
459 let mut cfr = FullCodebookDescReader::new(cb_desc);
460 let cb = Codebook::new(&mut cfr, CodebookMode::MSB).unwrap();
461 assert_eq!(br.read_cb(&cb).unwrap(), 16);
462 assert_eq!(br.read_cb(&cb).unwrap(), -3);
463 assert_eq!(br.read_cb(&cb).unwrap(), 42);
464 assert_eq!(br.read_cb(&cb).unwrap(), -42);
465 let ret = br.read_cb(&cb);
466 if let Err(e) = ret {
467 assert_eq!(e as i32, CodebookError::InvalidCode as i32);
468 } else {
469 assert_eq!(0, 1);
470 }
471
472 let scb_desc: Vec<ShortCodebookDesc> = vec!(
473 ShortCodebookDesc { code: 0b0, bits: 1 },
474 ShortCodebookDesc { code: 0, bits: 0 },
475 ShortCodebookDesc { code: 0b10, bits: 2 },
476 ShortCodebookDesc { code: 0, bits: 0 },
477 ShortCodebookDesc { code: 0, bits: 0 },
478 ShortCodebookDesc { code: 0b110, bits: 3 },
479 ShortCodebookDesc { code: 0, bits: 0 },
480 ShortCodebookDesc { code: 0b11100, bits: 5 },
481 ShortCodebookDesc { code: 0b11101, bits: 5 },
482 ShortCodebookDesc { code: 0b1111010, bits: 7 },
483 ShortCodebookDesc { code: 0b1111011, bits: 7 },
484 ShortCodebookDesc { code: 0b1111110, bits: 7 },
485 ShortCodebookDesc { code: 0b11111111, bits: 8 }
486 );
487 let mut br2 = BitReader::new(buf, BitReaderMode::BE);
488 let mut cfr = ShortCodebookDescReader::new(scb_desc);
489 let cb = Codebook::new(&mut cfr, CodebookMode::MSB).unwrap();
490 assert_eq!(br2.read_cb(&cb).unwrap(), 0);
491 assert_eq!(br2.read_cb(&cb).unwrap(), 2);
492 assert_eq!(br2.read_cb(&cb).unwrap(), 5);
493 assert_eq!(br2.read_cb(&cb).unwrap(), 8);
494
495 assert_eq!(reverse_bits(0b0000_0101_1011_1011_1101_1111_0111_1111, 32),
496 0b1111_1110_1111_1011_1101_1101_1010_0000);
497
498 const BITS_LE: [u8; 3] = [0b11101111, 0b01110010, 0b01];
499 let buf = &BITS_LE;
500 let scble_desc: Vec<ShortCodebookDesc> = vec!(
501 ShortCodebookDesc { code: 0b00, bits: 2 },
502 ShortCodebookDesc { code: 0, bits: 0 },
503 ShortCodebookDesc { code: 0b01, bits: 2 },
504 ShortCodebookDesc { code: 0, bits: 0 },
505 ShortCodebookDesc { code: 0, bits: 0 },
506 ShortCodebookDesc { code: 0b011, bits: 3 },
507 ShortCodebookDesc { code: 0, bits: 0 },
508 ShortCodebookDesc { code: 0b10111, bits: 5 },
509 ShortCodebookDesc { code: 0b00111, bits: 5 },
510 ShortCodebookDesc { code: 0b0101111, bits: 7 },
511 ShortCodebookDesc { code: 0b0111111, bits: 7 },
512 ShortCodebookDesc { code: 0b1011101111, bits: 10 }
513 );
514 let mut brl = BitReader::new(buf, BitReaderMode::LE);
515 let mut cfr = ShortCodebookDescReader::new(scble_desc);
516 let cb = Codebook::new(&mut cfr, CodebookMode::LSB).unwrap();
517 assert_eq!(brl.read_cb(&cb).unwrap(), 11);
518 assert_eq!(brl.read_cb(&cb).unwrap(), 0);
519 assert_eq!(brl.read_cb(&cb).unwrap(), 7);
520 assert_eq!(brl.read_cb(&cb).unwrap(), 0);
521 }
522 }