LEB128编码算法

环境

C++17

概念

leb128编码的本质是std::vector，每个字节用7位来存储数据，1位来做解码标记，每次溢出就再拓展8位，故一个32位长的数据需要5个字节来存储而不是4个。

代码

#include 
using namespace std;

#define panda_bit_utils_clz __builtin_clz      // NOLINT(cppcoreguidelines-macro-usage)
#define panda_bit_utils_clzll __builtin_clzll  // NOLINT(cppcoreguidelines-macro-usage)
#define UNLIKELY(exp) (__builtin_expect((exp) != 0, false))  // NOLINT(cppcoreguidelines-macro-usage)

constexpr size_t PAYLOAD_WIDTH = 7;
constexpr size_t PAYLOAD_MASK = 0x7f;
constexpr size_t EXTENSION_BIT = 0x80;
constexpr size_t SIGN_BIT = 0x40;

template <typename T>
constexpr int Clz(T x)
{
    constexpr size_t RADIX = 2;
    static_assert(std::is_integral<T>::value, "T must be integral");
    static_assert(std::is_unsigned<T>::value, "T must be unsigned");
    static_assert(std::numeric_limits<T>::radix == RADIX, "Unexpected radix!");
    static_assert(sizeof(T) == sizeof(uint64_t) || sizeof(T) <= sizeof(uint32_t), "Unsupported sizeof(T)");
    assert(x != 0U);

    if (sizeof(T) == sizeof(uint64_t)) {
        return panda_bit_utils_clzll(x);
    }
    return panda_bit_utils_clz(x) - (std::numeric_limits<uint32_t>::digits - std::numeric_limits<T>::digits);
}

// How many bits (minimally) does it take to store the constant 'value'? i.e. 1 for 1, 2 for 2 and 3, 3 for 4 and 5 etc.
template <typename T>
constexpr size_t MinimumBitsToStore(T value)
{
    static_assert(sizeof(T) == sizeof(uint64_t) || sizeof(T) <= sizeof(uint32_t), "Unsupported sizeof(T)");
    if constexpr (std::is_enum_v<T>) {  // NOLINT
        using UnderlyingType = std::make_unsigned_t<std::underlying_type_t<T>>;
        auto uvalue = static_cast<UnderlyingType>(value);
        if (uvalue == 0) {
            uvalue = 1;
        }
        return std::numeric_limits<UnderlyingType>::digits - Clz(static_cast<UnderlyingType>(uvalue));
    } else {  // NOLINT
        constexpr size_t RADIX = 2;
        static_assert(std::is_integral_v<T>, "T must be integral");
        static_assert(std::is_unsigned_v<T>, "T must be unsigned");
        static_assert(std::numeric_limits<T>::radix == RADIX, "Unexpected radix!");
        if (value == 0) {
            return 0;
        }
        return std::numeric_limits<T>::digits - Clz(value);
    }
}

template <class T>
inline std::tuple<T, size_t, bool> DecodeUnsigned(const uint8_t *data)
{
    static_assert(std::is_unsigned_v<T>, "T must be unsigned");

    T result = 0;

    constexpr size_t BITWIDTH = std::numeric_limits<T>::digits;
    constexpr size_t N = (BITWIDTH + PAYLOAD_WIDTH - 1) / PAYLOAD_WIDTH;

    for (size_t i = 0; i < N; i++) {
        // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
        uint8_t byte = data[i] & PAYLOAD_MASK;
        size_t shift = i * PAYLOAD_WIDTH;
        result |= static_cast<T>(byte) << shift;
        // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
        if ((data[i] & EXTENSION_BIT) == 0) {
            bool is_full = MinimumBitsToStore(byte) <= (BITWIDTH - shift);
            return {result, i + 1, is_full};
        }
    }

    return {result, N, false};
}

template <>
inline std::tuple<uint32_t, size_t, bool> DecodeUnsigned<uint32_t>(const uint8_t *data)
{
    constexpr size_t LEB128_BYTE2_SHIFT = 7U;
    constexpr size_t LEB128_BYTE3_SHIFT = 14U;
    constexpr size_t LEB128_BYTE4_SHIFT = 21U;
    constexpr size_t LEB128_BYTE5_SHIFT = 28U;
    constexpr size_t LEB128_BYTE5_BIT_SIZE = 4;

    bool valid = true;
    const uint8_t *p = data;
    // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
    uint32_t result = *(p++);
    if (UNLIKELY(result > PAYLOAD_MASK)) {
        // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
        uint32_t byte = *(p++);
        result = (result & PAYLOAD_MASK) | ((byte & PAYLOAD_MASK) << LEB128_BYTE2_SHIFT);
        if (byte > PAYLOAD_MASK) {
            // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
            byte = *(p++);
            result |= (byte & PAYLOAD_MASK) << LEB128_BYTE3_SHIFT;
            if (byte > PAYLOAD_MASK) {
                // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
                byte = *(p++);
                result |= (byte & PAYLOAD_MASK) << LEB128_BYTE4_SHIFT;
                if (byte > PAYLOAD_MASK) {
                    // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
                    byte = *(p++);
                    valid = byte < (1U << LEB128_BYTE5_BIT_SIZE);
                    result |= byte << LEB128_BYTE5_SHIFT;
                }
            }
        }
    }
    return {result, p - data, valid};
}

template <class T>
inline std::tuple<T, size_t, bool> DecodeSigned(const uint8_t *data)
{
    static_assert(std::is_signed_v<T>, "T must be signed");

    T result = 0;

    using unsigned_type = std::make_unsigned_t<T>;

    constexpr size_t BITWIDTH = std::numeric_limits<unsigned_type>::digits;
    constexpr size_t N = (BITWIDTH + PAYLOAD_WIDTH - 1) / PAYLOAD_WIDTH;

    for (size_t i = 0; i < N; i++) {
        // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
        uint8_t byte = data[i];
        size_t shift = i * PAYLOAD_WIDTH;
        // NOLINTNEXTLINE(hicpp-signed-bitwise)
        result |= static_cast<unsigned_type>(byte & PAYLOAD_MASK) << shift;

        if ((byte & EXTENSION_BIT) == 0) {
            shift = BITWIDTH - shift;
            // NOLINTNEXTLINE(hicpp-signed-bitwise)
            int8_t signed_extended = static_cast<int8_t>(byte << 1) >> 1;
            // NOLINTNEXTLINE(hicpp-signed-bitwise)
            uint8_t masked = (signed_extended ^ (signed_extended >> PAYLOAD_WIDTH)) | 1;
            bool is_full = MinimumBitsToStore(masked) <= shift;
            if (shift > PAYLOAD_WIDTH) {
                shift -= PAYLOAD_WIDTH;
                // NOLINTNEXTLINE(hicpp-signed-bitwise)
                result = static_cast<T>(result << shift) >> shift;
            }
            return {result, i + 1, is_full};
        }
    }

    return {result, N, false};
}

template <class T>
inline size_t EncodeUnsigned(T data, uint8_t *out)
{
    size_t i = 0;
    uint8_t byte = data & PAYLOAD_MASK;
    data >>= PAYLOAD_WIDTH;

    while (data != 0) {
        // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
        out[i++] = byte | EXTENSION_BIT;
        byte = data & PAYLOAD_MASK;
        data >>= PAYLOAD_WIDTH;
    }

    out[i++] = byte;  // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
    return i;
}

template <class T>
inline size_t EncodeSigned(T data, uint8_t *out)
{
    static_assert(std::is_signed_v<T>, "T must be signed");

    size_t i = 0;
    bool more = true;

    while (more) {
        auto byte = static_cast<uint8_t>(static_cast<size_t>(data) & PAYLOAD_MASK);
        // NOLINTNEXTLINE(hicpp-signed-bitwise)
        data >>= PAYLOAD_WIDTH;
        more = !((data == 0 && (byte & SIGN_BIT) == 0) || (data == -1 && (byte & SIGN_BIT) != 0));
        if (more) {
            byte |= EXTENSION_BIT;
        }
        out[i++] = byte;  // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
    }

    return i;
}

template <class T>
inline size_t UnsignedEncodingSize(T data)
{
    return (MinimumBitsToStore(data | 1U) + PAYLOAD_WIDTH - 1) / PAYLOAD_WIDTH;
}

template <class T>
inline size_t SignedEncodingSize(T data)
{
    // NOLINTNEXTLINE(hicpp-signed-bitwise)
    data = data ^ (data >> (std::numeric_limits<T>::digits - 1));
    using unsigned_type = std::make_unsigned_t<T>;
    auto udata = static_cast<unsigned_type>(data);
    return MinimumBitsToStore(static_cast<unsigned_type>(udata | 1U)) / PAYLOAD_WIDTH + 1;
}

int main() {
    int32_t num = -8848;
    size_t sz = SignedEncodingSize(num);
    cout << "num size = " << sz << endl;
    vector<uint8_t> data(sz);
    size_t n = EncodeSigned(num, data.data());
    auto [value, size, is_full] = DecodeSigned<int32_t>(data.data());
    cout << "value = " << value << ", size = " << size << endl;
    return 0;
}