Quantization.hpp

Go to the documentation of this file.

 
#pragma once
#include <cstdint>
#include <bitset>
namespace Foundation::Math {
    float quantizeFP32(float v, int32_t N);
 
    uint16_t quantizeFP16(float v);
 
    float dequantizeFP16(uint16_t h);
 
    /* [0,1] range -> [0, 1 << NBits) \in N */
    inline uint32_t quantizeUnorm(float v, int32_t N) {
        const auto scale = static_cast<float>((1 << N) - 1);
 
        v = (v >= 0) ? v : 0;
        v = (v <= 1) ? v : 1;
 
        return static_cast<int>(v * scale + 0.5f);
    }
 
    /* [0, 1 << NBits) \in N -> [0, 1] range */
    inline float dequantizeUnorm(int32_t q, int32_t Nbits) {
        return q / static_cast<float>((1 << Nbits) - 1);
    }
 
    inline int32_t quantizeSnorm(float v, int32_t N) {
        const auto scale = static_cast<float>((1 << (N - 1)) - 1);
 
        float round = (v >= 0 ? 0.5f : -0.5f);
 
        v = (v >= -1) ? v : -1;
        v = (v <= +1) ? v : +1;
 
        return static_cast<int>(v * scale + round);
    }
 
    // [-(1<< (Nbits - 1)) - 1, (1 << (Nbits - 1))] \in N -> [-1, 1]
    inline float dequantizeSnorm(int32_t q, int32_t Nbits) {
        return q / static_cast<float>((1 << (Nbits - 1)) - 1);
    }
 
    // [-1, 1] range -> [0, 1 << NBits) \in N
    inline uint32_t quantizeSnormShifted(float v, int32_t Nbits) {
        return quantizeSnorm(v, Nbits) + (1 << (Nbits - 1));
    }
 
    // [0, 1 << NBits) \in N -> [-1, 1] range
    inline float dequantizeSnormShifted(uint32_t q, int32_t Nbits) {
        return dequantizeSnorm(q - (1 << (Nbits - 1)), Nbits);
    }
}