chess/formats/
cbv.rs

1//! Chessbase archive file format
2
3// https://github.com/antoyo/uncbv
4// https://github.com/antoyo/uncbv/issues/3
5// https://github.com/antoyo/uncbv/issues/3
6
7// Note: It's called cbV for archiVe (or also version
8
9// TODO: We should have a constructor that looks like a map
10// and offers Index-based access to the packed files (as references).
11// The packed file then has a `PackedFile::unpack` method.
12//
13// Implement this with `fs::File::seek` operations and streaming
14// processing, using the `indexmap` crate.
15
16use core::ffi::c_str::CStr;
17use std::path::Path;
18
19use encoding_rs::WINDOWS_1252;
20
21use super::{Bits, ByteInput as Input, prelude::*};
22
23mod huffman;
24pub use huffman::Tree;
25
26/////////////
27// Parsing //
28/////////////
29
30/// Vector of entries
31pub type Header = Vec<Entry>;
32
33// 8 byte header
34// - 2B magic number tag
35// - 2B number of files
36// - 1B length of metadata entry
37// - 3B unknown
38//
39// Metadata for each file (all same length e.g. 173 bytes)
40// - offset   0: C-string filename (Windows1252 encoding?)
41// - offset 132..136: 4B packed size
42// - offset 136..140: 4B unpacked size
43/// Parse the header out of a Chessbase archive
44pub fn header(input: &mut Input<'_>) -> ModalResult<Vec<Entry>> {
45    let (_, file_count, entry_len, _) = (magic_tag, le_u16, u8, take(3u8)).parse_next(input)?;
46    let header: Vec<Entry> =
47        repeat(..=file_count as usize, cut_err(take(entry_len).and_then(entry)))
48            .parse_next(input)?;
49    Ok(header)
50}
51
52/// Name and packed/unpacked lengths of an archive entry
53#[derive(Debug)]
54pub struct Entry {
55    /// Name of the file
56    ///
57    /// This can contain embedded Windows-style backslash paths,
58    /// e.g., `D85 Gruenfeld Defence-163.html\42289499p0.jpg`
59    pub name: String,
60    /// The length of the packed data, in bytes
61    pub packed: usize,
62    /// The length of the (original) data, in bytes
63    pub len: usize,
64}
65
66fn entry(input: &mut Input<'_>) -> ModalResult<Entry> {
67    // TODO: pass along errors
68    let name = take(132usize).map(cstr).parse_next(input)?;
69
70    // offsets 132 and 136
71    let (packed, len) = (le_i32.map(as_usize), le_i32.map(as_usize)).parse_next(input)?;
72
73    let _ = take(1usize).parse_next(input)?;
74
75    // offsets 141 and 145
76    // let last_modified_date = (le_u32, le_u32).parse_next(input)?;
77    // let (mdate, mtime) = (le_u32, le_u32).parse_next(input)?;
78    // println!("{} and {:08}", mdate, mtime);
79
80    Ok(Entry { name, packed, len })
81}
82
83/// Packed content of a file
84pub struct PackedFile<'a> {
85    pub name: String,
86    pub data: &'a [u8],
87    pub len: usize,
88}
89
90// This is the way to pass arguments:
91// return a Parser that is implemented in terms of the arguments
92pub fn packed_file<'a>(entry: &Entry) -> impl FnMut(&mut Input<'a>) -> ModalResult<PackedFile<'a>> {
93    move |input: &mut Input<'_>| {
94        take(entry.packed)
95            .map(|data: &[u8]| PackedFile { name: entry.name.clone(), data, len: entry.len })
96            .parse_next(input)
97    }
98}
99
100///////////////
101// Unpacking //
102///////////////
103
104pub fn unpack_cbv_to_disk(input: &mut Input<'_>) -> ModalResult<Header> {
105    unpack_cbv_to(Path::new(".")).parse_next(input)
106}
107
108pub fn unpack_cbv_to<'a>(directory: &'a Path) -> impl FnMut(&mut Input<'a>) -> ModalResult<Header> {
109    move |input: &mut Input<'_>| {
110        let header = header.parse_next(input)?;
111        for file in header.iter() {
112            let mut packed: PackedFile = packed_file(file).parse_next(input)?;
113            let unpacked = unpack_file.parse_next(&mut packed.data)?;
114            assert_eq!(file.len, unpacked.len());
115            let path = directory.join(&file.name);
116            if let Some(parent) = path.parent() {
117                std::fs::create_dir_all(parent).expect("can create output directory");
118            }
119            std::fs::write(path, &unpacked).expect("can write");
120        }
121        Ok(header)
122    }
123}
124
125// #[cfg(test)]
126// fn unpack_cbv_dummy(input: &mut Input<'_>) -> ModalResult<Header> {
127//     let header = header.parse_next(input)?;
128//     for file in header.iter() {
129//         let mut packed: PackedFile = packed_file(file).parse_next(input)?;
130//         let unpacked = unpack_file.parse_next(&mut packed.data)?;
131//         assert_eq!(file.len, unpacked.len());
132//     }
133//     Ok(header)
134// }
135
136pub fn unpack_file(input: &mut Input<'_>) -> ModalResult<Vec<u8>> {
137    let mut data = Vec::new();
138    while !input.is_empty() {
139        // 1. Read out a block
140        let (packing, mut block) = read_block.parse_next(input)?;
141
142        // 2. Remove Huffman encoding
143        let block = if packing.huffman {
144            unpack_huffman(&mut block).expect("correct Huffman")
145        } else {
146            block.to_vec()
147        };
148
149        // 3. Remove RLE/backref encoding
150        let mut block = if packing.rle {
151            unpack_rle_backref(&mut block.as_slice()).expect("correct RLE/backref")
152        } else {
153            block
154        };
155        data.append(&mut block);
156    }
157    Ok(data)
158}
159
160/// Encodings applied to a block of a packed file
161#[derive(Clone, Copy, Debug)]
162pub struct Packing {
163    /// Huffman-tree encoding
164    pub huffman: bool,
165    /// Mix of run-time length encoding and backward references
166    pub rle: bool,
167}
168
169pub fn read_block<'a>(input: &mut Input<'a>) -> ModalResult<(Packing, &'a [u8])> {
170    // let (file_count, entry_len) =
171    //     seq!(_: magic_tag, le_u16, u8, _: take(3usize)).parse_next(input)?;
172    let (block_size, _) = (le_u16, take(2usize)).parse_next(input)?;
173    let (packing, block) = take(block_size as usize).parse_next(input)?.split_first().unwrap();
174    assert!(*packing < 4);
175    let rle = packing & 1 != 0;
176    let huffman = packing & 2 != 0;
177    Ok((Packing { huffman, rle }, block))
178}
179
180// https://reverseengineering.stackexchange.com/questions/8593/unknown-decompression-algorithm/8601
181pub fn unpack_huffman(input: &mut Input<'_>) -> winnow::Result<Vec<u8>> {
182    use core::array::from_fn;
183
184    // 1. unpacked length
185    let len = be_u16(input)? as usize;
186
187    // 2. Huffman tree
188    let mut bits = Bits::from(*input);
189    let values: [(usize, u16); 256] = from_fn(|_| huffman_node(&mut bits).unwrap());
190
191    // 3. Decode block with tree
192    let tree = Tree::load(values);
193    let unpacked = tree.unpack(bits, len).unwrap();
194
195    assert_eq!(len, unpacked.len());
196    Ok(unpacked)
197}
198
199fn huffman_node(bits: &mut Bits) -> Option<(usize, u16)> {
200    let len = bits.unsigned(4)?;
201    let bits = bits.unsigned(len)? as u16;
202    Some((len, bits))
203}
204
205// This has run-length encoding and backward references
206pub fn unpack_rle_backref(input: &mut Input<'_>) -> winnow::Result<Vec<u8>> {
207    use winnow::binary::{le_u16, u8};
208
209    let mut result = vec![];
210    while !input.is_empty() {
211        let indicator = le_u16(input)?.to_be_bytes();
212        let indicator = Bits::from(indicator.as_slice());
213
214        for encoded in indicator {
215            if input.is_empty() {
216                return Ok(result);
217            }
218
219            if !encoded {
220                result.push(u8(input)?);
221                continue;
222            }
223
224            let (high, low) = u8(input).map(nibbles)?;
225
226            match high {
227                // run-length encoding, 3..=18
228                0 => {
229                    result.append(&mut vec![u8(input)?; 3 + low]);
230                }
231                // run-length encoding, 19..=4114
232                1 => {
233                    let size = 19 + low + ((u8(input)? as usize) << 4);
234                    result.append(&mut vec![u8(input)?; size]);
235                }
236                // backward reference
237                _ => {
238                    // 3..=4098
239                    let start = result.len() - (3 + low + ((u8(input)? as usize) << 4));
240                    let end = start
241                        + if high > 2 {
242                            // 3..=15
243                            high as usize
244                        } else {
245                            // 16..=271
246                            16 + u8(input)? as usize
247                        };
248                    result.extend_from_within(start..end);
249                }
250            }
251        }
252    }
253
254    Ok(result)
255}
256
257fn nibbles(x: u8) -> (u8, usize) {
258    (x >> 4, x as usize & 0xF)
259}
260
261////////////////////////////
262// Implementation details //
263////////////////////////////
264
265fn magic_tag(input: &mut Input<'_>) -> ModalResult<[u8; 2]> {
266    const MAGIC_NUMBER: [u8; 2] = [0x08, 0x00];
267    MAGIC_NUMBER.as_slice().map(|_| MAGIC_NUMBER).parse_next(input)
268}
269
270fn as_usize(x: i32) -> usize {
271    x as usize
272}
273
274pub fn cstr(input: &[u8]) -> String {
275    let name = CStr::from_bytes_until_nul(input).expect("valid name");
276    let (cow, encoding_used, had_errors) = WINDOWS_1252.decode(name.to_bytes());
277    assert_eq!(encoding_used, WINDOWS_1252);
278    assert!(!had_errors);
279    cow.to_string()
280}
281
282///////////
283// Tests //
284///////////
285
286#[test]
287fn example() {
288    const EXAMPLE: &[u8] = include_bytes!("../../examples/twic1616.cbv");
289
290    // let header = unpack_cbv_dummy.parse(EXAMPLE).unwrap();
291    let header = unpack_cbv_to_disk.parse(EXAMPLE).unwrap();
292    println!("{header:?}");
293}
chess/formats/cbv.rs

chess/formats/
cbv.rs
