Skip to content

refactor: shrink HPACK encode tables #949

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
seanmonstar wants to merge 1 commit into
base: master
Choose a base branch
from
+69 −271
Open

refactor: shrink HPACK encode tables #949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions src/hpack/huffman/decode.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
//!
//! These are decoded by walking the generated `DECODE_TABLE` one input byte at a time.

use crate::hpack::huffman::table::{DECODE_TABLE, ENCODE_TABLE};
use crate::hpack::huffman::table::{DECODE_TABLE, ENCODE_CODES, ENCODE_CODE_LENGTHS};
use crate::hpack::DecoderError;

use bytes::BytesMut;
Expand Down Expand Up @@ -44,7 +44,8 @@ const fn build_fast_table() -> [u32; 1 << FAST_BITS] {
// First fill in every index that starts with one whole code
let mut a = 0;
while a < 256 {
let (len1, code1) = ENCODE_TABLE[a];
let len1 = ENCODE_CODE_LENGTHS[a] as usize;
let code1 = ENCODE_CODES[a] as u64;
if len1 <= FAST_BITS {
let rem = FAST_BITS - len1;
let base = (code1 as usize) << rem;
Expand All @@ -61,13 +62,15 @@ const fn build_fast_table() -> [u32; 1 << FAST_BITS] {
// Overwrite the indices where a second whole code fit right after the first
let mut a = 0;
while a < 256 {
let (len1, code1) = ENCODE_TABLE[a];
let len1 = ENCODE_CODE_LENGTHS[a] as usize;
let code1 = ENCODE_CODES[a] as u64;
// A second code needs at least 5 more bits (the shortest code)
if len1 + 5 <= FAST_BITS {
let rem = FAST_BITS - len1;
let mut b = 0;
while b < 256 {
let (len2, code2) = ENCODE_TABLE[b];
let len2 = ENCODE_CODE_LENGTHS[b] as usize;
let code2 = ENCODE_CODES[b] as u64;
if len2 <= rem {
let rem2 = rem - len2;
let base = ((code1 as usize) << rem) | ((code2 as usize) << rem2);
Expand Down Expand Up @@ -222,9 +225,10 @@ mod test {
// out on purpose: encoded EOS is an error per RFC 7541 §5.2.
fn reference_decode(src: &[u8]) -> Result<BytesMut, DecoderError> {
// Each symbol's code as a string of '0' and '1', e.g. "11111000".
let codes: Vec<String> = ENCODE_TABLE[..256]
let codes: Vec<String> = ENCODE_CODES
.iter()
.map(|&(len, code)| format!("{code:0len$b}"))
.zip(ENCODE_CODE_LENGTHS.iter())
.map(|(&code, &len)| format!("{code:0len$b}", len = len as usize))
.collect();
let bits: String = src.iter().map(|byte| format!("{byte:08b}")).collect();

Expand Down
5 changes: 3 additions & 2 deletions src/hpack/huffman/encode.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::hpack::huffman::table::ENCODE_TABLE;
use crate::hpack::huffman::table::{ENCODE_CODES, ENCODE_CODE_LENGTHS};

use bytes::{BufMut, BytesMut};

Expand All @@ -7,7 +7,8 @@ pub fn encode(src: &[u8], dst: &mut BytesMut) {
let mut bits_len = 0;

for &b in src {
let (nbits, code) = ENCODE_TABLE[b as usize];
let nbits = ENCODE_CODE_LENGTHS[b as usize] as usize;
let code = ENCODE_CODES[b as usize] as u64;

bits = (bits << nbits) | code;
bits_len += nbits;
Expand Down
301 changes: 42 additions & 259 deletions src/hpack/huffman/table.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,266 +1,49 @@
// !!! DO NOT EDIT !!! Generated by util/genhuff/src/main.rs

// (num-bits, bits)
pub const ENCODE_TABLE: [(usize, u64); 257] = [
(13, 0x1ff8),
(23, 0x7fffd8),
(28, 0xfffffe2),
(28, 0xfffffe3),
(28, 0xfffffe4),
(28, 0xfffffe5),
(28, 0xfffffe6),
(28, 0xfffffe7),
(28, 0xfffffe8),
(24, 0xffffea),
(30, 0x3ffffffc),
(28, 0xfffffe9),
(28, 0xfffffea),
(30, 0x3ffffffd),
(28, 0xfffffeb),
(28, 0xfffffec),
(28, 0xfffffed),
(28, 0xfffffee),
(28, 0xfffffef),
(28, 0xffffff0),
(28, 0xffffff1),
(28, 0xffffff2),
(30, 0x3ffffffe),
(28, 0xffffff3),
(28, 0xffffff4),
(28, 0xffffff5),
(28, 0xffffff6),
(28, 0xffffff7),
(28, 0xffffff8),
(28, 0xffffff9),
(28, 0xffffffa),
(28, 0xffffffb),
(6, 0x14),
(10, 0x3f8),
(10, 0x3f9),
(12, 0xffa),
(13, 0x1ff9),
(6, 0x15),
(8, 0xf8),
(11, 0x7fa),
(10, 0x3fa),
(10, 0x3fb),
(8, 0xf9),
(11, 0x7fb),
(8, 0xfa),
(6, 0x16),
(6, 0x17),
(6, 0x18),
(5, 0x0),
(5, 0x1),
(5, 0x2),
(6, 0x19),
(6, 0x1a),
(6, 0x1b),
(6, 0x1c),
(6, 0x1d),
(6, 0x1e),
(6, 0x1f),
(7, 0x5c),
(8, 0xfb),
(15, 0x7ffc),
(6, 0x20),
(12, 0xffb),
(10, 0x3fc),
(13, 0x1ffa),
(6, 0x21),
(7, 0x5d),
(7, 0x5e),
(7, 0x5f),
(7, 0x60),
(7, 0x61),
(7, 0x62),
(7, 0x63),
(7, 0x64),
(7, 0x65),
(7, 0x66),
(7, 0x67),
(7, 0x68),
(7, 0x69),
(7, 0x6a),
(7, 0x6b),
(7, 0x6c),
(7, 0x6d),
(7, 0x6e),
(7, 0x6f),
(7, 0x70),
(7, 0x71),
(7, 0x72),
(8, 0xfc),
(7, 0x73),
(8, 0xfd),
(13, 0x1ffb),
(19, 0x7fff0),
(13, 0x1ffc),
(14, 0x3ffc),
(6, 0x22),
(15, 0x7ffd),
(5, 0x3),
(6, 0x23),
(5, 0x4),
(6, 0x24),
(5, 0x5),
(6, 0x25),
(6, 0x26),
(6, 0x27),
(5, 0x6),
(7, 0x74),
(7, 0x75),
(6, 0x28),
(6, 0x29),
(6, 0x2a),
(5, 0x7),
(6, 0x2b),
(7, 0x76),
(6, 0x2c),
(5, 0x8),
(5, 0x9),
(6, 0x2d),
(7, 0x77),
(7, 0x78),
(7, 0x79),
(7, 0x7a),
(7, 0x7b),
(15, 0x7ffe),
(11, 0x7fc),
(14, 0x3ffd),
(13, 0x1ffd),
(28, 0xffffffc),
(20, 0xfffe6),
(22, 0x3fffd2),
(20, 0xfffe7),
(20, 0xfffe8),
(22, 0x3fffd3),
(22, 0x3fffd4),
(22, 0x3fffd5),
(23, 0x7fffd9),
(22, 0x3fffd6),
(23, 0x7fffda),
(23, 0x7fffdb),
(23, 0x7fffdc),
(23, 0x7fffdd),
(23, 0x7fffde),
(24, 0xffffeb),
(23, 0x7fffdf),
(24, 0xffffec),
(24, 0xffffed),
(22, 0x3fffd7),
(23, 0x7fffe0),
(24, 0xffffee),
(23, 0x7fffe1),
(23, 0x7fffe2),
(23, 0x7fffe3),
(23, 0x7fffe4),
(21, 0x1fffdc),
(22, 0x3fffd8),
(23, 0x7fffe5),
(22, 0x3fffd9),
(23, 0x7fffe6),
(23, 0x7fffe7),
(24, 0xffffef),
(22, 0x3fffda),
(21, 0x1fffdd),
(20, 0xfffe9),
(22, 0x3fffdb),
(22, 0x3fffdc),
(23, 0x7fffe8),
(23, 0x7fffe9),
(21, 0x1fffde),
(23, 0x7fffea),
(22, 0x3fffdd),
(22, 0x3fffde),
(24, 0xfffff0),
(21, 0x1fffdf),
(22, 0x3fffdf),
(23, 0x7fffeb),
(23, 0x7fffec),
(21, 0x1fffe0),
(21, 0x1fffe1),
(22, 0x3fffe0),
(21, 0x1fffe2),
(23, 0x7fffed),
(22, 0x3fffe1),
(23, 0x7fffee),
(23, 0x7fffef),
(20, 0xfffea),
(22, 0x3fffe2),
(22, 0x3fffe3),
(22, 0x3fffe4),
(23, 0x7ffff0),
(22, 0x3fffe5),
(22, 0x3fffe6),
(23, 0x7ffff1),
(26, 0x3ffffe0),
(26, 0x3ffffe1),
(20, 0xfffeb),
(19, 0x7fff1),
(22, 0x3fffe7),
(23, 0x7ffff2),
(22, 0x3fffe8),
(25, 0x1ffffec),
(26, 0x3ffffe2),
(26, 0x3ffffe3),
(26, 0x3ffffe4),
(27, 0x7ffffde),
(27, 0x7ffffdf),
(26, 0x3ffffe5),
(24, 0xfffff1),
(25, 0x1ffffed),
(19, 0x7fff2),
(21, 0x1fffe3),
(26, 0x3ffffe6),
(27, 0x7ffffe0),
(27, 0x7ffffe1),
(26, 0x3ffffe7),
(27, 0x7ffffe2),
(24, 0xfffff2),
(21, 0x1fffe4),
(21, 0x1fffe5),
(26, 0x3ffffe8),
(26, 0x3ffffe9),
(28, 0xffffffd),
(27, 0x7ffffe3),
(27, 0x7ffffe4),
(27, 0x7ffffe5),
(20, 0xfffec),
(24, 0xfffff3),
(20, 0xfffed),
(21, 0x1fffe6),
(22, 0x3fffe9),
(21, 0x1fffe7),
(21, 0x1fffe8),
(23, 0x7ffff3),
(22, 0x3fffea),
(22, 0x3fffeb),
(25, 0x1ffffee),
(25, 0x1ffffef),
(24, 0xfffff4),
(24, 0xfffff5),
(26, 0x3ffffea),
(23, 0x7ffff4),
(26, 0x3ffffeb),
(27, 0x7ffffe6),
(26, 0x3ffffec),
(26, 0x3ffffed),
(27, 0x7ffffe7),
(27, 0x7ffffe8),
(27, 0x7ffffe9),
(27, 0x7ffffea),
(27, 0x7ffffeb),
(28, 0xffffffe),
(27, 0x7ffffec),
(27, 0x7ffffed),
(27, 0x7ffffee),
(27, 0x7ffffef),
(27, 0x7fffff0),
(26, 0x3ffffee),
(30, 0x3fffffff),
pub const ENCODE_CODES: [u32; 256] = [
0x1ff8, 0x7fffd8, 0xfffffe2, 0xfffffe3, 0xfffffe4, 0xfffffe5, 0xfffffe6, 0xfffffe7, 0xfffffe8,
0xffffea, 0x3ffffffc, 0xfffffe9, 0xfffffea, 0x3ffffffd, 0xfffffeb, 0xfffffec, 0xfffffed,
0xfffffee, 0xfffffef, 0xffffff0, 0xffffff1, 0xffffff2, 0x3ffffffe, 0xffffff3, 0xffffff4,
0xffffff5, 0xffffff6, 0xffffff7, 0xffffff8, 0xffffff9, 0xffffffa, 0xffffffb, 0x14, 0x3f8,
0x3f9, 0xffa, 0x1ff9, 0x15, 0xf8, 0x7fa, 0x3fa, 0x3fb, 0xf9, 0x7fb, 0xfa, 0x16, 0x17, 0x18,
0x0, 0x1, 0x2, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x5c, 0xfb, 0x7ffc, 0x20, 0xffb,
0x3fc, 0x1ffa, 0x21, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xfc, 0x73, 0xfd, 0x1ffb, 0x7fff0,
0x1ffc, 0x3ffc, 0x22, 0x7ffd, 0x3, 0x23, 0x4, 0x24, 0x5, 0x25, 0x26, 0x27, 0x6, 0x74, 0x75,
0x28, 0x29, 0x2a, 0x7, 0x2b, 0x76, 0x2c, 0x8, 0x9, 0x2d, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7ffe,
0x7fc, 0x3ffd, 0x1ffd, 0xffffffc, 0xfffe6, 0x3fffd2, 0xfffe7, 0xfffe8, 0x3fffd3, 0x3fffd4,
0x3fffd5, 0x7fffd9, 0x3fffd6, 0x7fffda, 0x7fffdb, 0x7fffdc, 0x7fffdd, 0x7fffde, 0xffffeb,
0x7fffdf, 0xffffec, 0xffffed, 0x3fffd7, 0x7fffe0, 0xffffee, 0x7fffe1, 0x7fffe2, 0x7fffe3,
0x7fffe4, 0x1fffdc, 0x3fffd8, 0x7fffe5, 0x3fffd9, 0x7fffe6, 0x7fffe7, 0xffffef, 0x3fffda,
0x1fffdd, 0xfffe9, 0x3fffdb, 0x3fffdc, 0x7fffe8, 0x7fffe9, 0x1fffde, 0x7fffea, 0x3fffdd,
0x3fffde, 0xfffff0, 0x1fffdf, 0x3fffdf, 0x7fffeb, 0x7fffec, 0x1fffe0, 0x1fffe1, 0x3fffe0,
0x1fffe2, 0x7fffed, 0x3fffe1, 0x7fffee, 0x7fffef, 0xfffea, 0x3fffe2, 0x3fffe3, 0x3fffe4,
0x7ffff0, 0x3fffe5, 0x3fffe6, 0x7ffff1, 0x3ffffe0, 0x3ffffe1, 0xfffeb, 0x7fff1, 0x3fffe7,
0x7ffff2, 0x3fffe8, 0x1ffffec, 0x3ffffe2, 0x3ffffe3, 0x3ffffe4, 0x7ffffde, 0x7ffffdf,
0x3ffffe5, 0xfffff1, 0x1ffffed, 0x7fff2, 0x1fffe3, 0x3ffffe6, 0x7ffffe0, 0x7ffffe1, 0x3ffffe7,
0x7ffffe2, 0xfffff2, 0x1fffe4, 0x1fffe5, 0x3ffffe8, 0x3ffffe9, 0xffffffd, 0x7ffffe3, 0x7ffffe4,
0x7ffffe5, 0xfffec, 0xfffff3, 0xfffed, 0x1fffe6, 0x3fffe9, 0x1fffe7, 0x1fffe8, 0x7ffff3,
0x3fffea, 0x3fffeb, 0x1ffffee, 0x1ffffef, 0xfffff4, 0xfffff5, 0x3ffffea, 0x7ffff4, 0x3ffffeb,
0x7ffffe6, 0x3ffffec, 0x3ffffed, 0x7ffffe7, 0x7ffffe8, 0x7ffffe9, 0x7ffffea, 0x7ffffeb,
0xffffffe, 0x7ffffec, 0x7ffffed, 0x7ffffee, 0x7ffffef, 0x7fffff0, 0x3ffffee,
];

pub const ENCODE_CODE_LENGTHS: [u8; 256] = [
13, 23, 28, 28, 28, 28, 28, 28, 28, 24, 30, 28, 28, 30, 28, 28, 28, 28, 28, 28, 28, 28, 30, 28,
28, 28, 28, 28, 28, 28, 28, 28, 6, 10, 10, 12, 13, 6, 8, 11, 10, 10, 8, 11, 8, 6, 6, 6, 5, 5,
5, 6, 6, 6, 6, 6, 6, 6, 7, 8, 15, 6, 12, 10, 13, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7, 8, 7, 8, 13, 19, 13, 14, 6, 15, 5, 6, 5, 6, 5, 6, 6, 6, 5, 7, 7, 6, 6,
6, 5, 6, 7, 6, 5, 5, 6, 7, 7, 7, 7, 7, 15, 11, 14, 13, 28, 20, 22, 20, 20, 22, 22, 22, 23, 22,
23, 23, 23, 23, 23, 24, 23, 24, 24, 22, 23, 24, 23, 23, 23, 23, 21, 22, 23, 22, 23, 23, 24, 22,
21, 20, 22, 22, 23, 23, 21, 23, 22, 22, 24, 21, 22, 23, 23, 21, 21, 22, 21, 23, 22, 23, 23, 20,
22, 22, 22, 23, 22, 22, 23, 26, 26, 20, 19, 22, 23, 22, 25, 26, 26, 26, 27, 27, 26, 24, 25, 19,
21, 26, 27, 27, 26, 27, 24, 21, 21, 26, 26, 28, 27, 27, 27, 20, 24, 20, 21, 22, 21, 21, 23, 22,
22, 25, 25, 24, 24, 26, 23, 26, 27, 26, 26, 27, 27, 27, 27, 27, 28, 27, 27, 27, 27, 27, 26,
];

const _: [(); 1024] = [(); std::mem::size_of::<[u32; 256]>()];
const _: [(); 256] = [(); std::mem::size_of::<[u8; 256]>()];

// A branch has bit 15 set and its table index in bits 8..=14.
// A leaf stores the number of consumed bits in its high byte and the decoded byte in its low byte.
pub const DECODE_TABLE: [u16; 15 * 256] = [
Expand Down
Loading