UTF8Decoder.cpp

#include "UTF8Decoder.hpp"

// Simple UTF-8 DFA decoder based on Bjoern Hoehrmann's implementation
static const uint8_t utf8d[] = {
  // The first part of the table maps bytes to character classes that
  // to reduce the size of the transition table and create bitmasks.
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
  8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
  10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
  
  // The second part is a transition table that maps a combination
  // of a state of the automaton and a character class to a state.
  0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
  12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
  12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
  12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
  12,36,12,12,12,12,12,12,12,12,12,12,
};

#define UTF8_ACCEPT 0
#define UTF8_REJECT 12

UTF8Decoder::UTF8Decoder() : m_state(UTF8_ACCEPT), m_codepoint(0) {
}

char32_t UTF8Decoder::decode(uint8_t byte) {
    uint32_t type = utf8d[byte];
    
    m_codepoint = (m_state != UTF8_ACCEPT) 
        ? (byte & 0x3fu) | (m_codepoint << 6)
        : (0xff >> type) & (byte);
        
    m_state = utf8d[256 + m_state + type];
    
    if (m_state == UTF8_ACCEPT) {
        return m_codepoint;
    } else if (m_state == UTF8_REJECT) {
        reset();
        return 0xFFFD; // Replacement character for invalid UTF-8
    }
    
    return 0; // Incomplete, need more bytes
}

void UTF8Decoder::reset() {
    m_state = UTF8_ACCEPT;
    m_codepoint = 0;
}