generic_codecvt.hpp

//
//  Copyright (c) 2015 Artyom Beilis (Tonkikh)
//
//  Distributed under the Boost Software License, Version 1.0. (See
//  accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//
#ifndef BOOST_LOCALE_GENERIC_CODECVT_HPP
#define BOOST_LOCALE_GENERIC_CODECVT_HPP

#include <boost/locale/utf.hpp>
#include <boost/cstdint.hpp>
#include <boost/static_assert.hpp>
#include <locale>

namespace boost {
namespace locale {

#ifndef BOOST_LOCALE_DOXYGEN
//
// Make sure that mbstate can keep 16 bit of UTF-16 sequence
//
BOOST_STATIC_ASSERT(sizeof(std::mbstate_t)>=2);
#endif

#if defined(_MSC_VER) && _MSC_VER < 1700
// up to MSVC 11 (2012) do_length is non-standard it counts wide characters instead of narrow and does not change mbstate
#define BOOST_LOCALE_DO_LENGTH_MBSTATE_CONST
#endif

class generic_codecvt_base {
public:
    enum initial_convertion_state {
        to_unicode_state, 
        from_unicode_state 
    };
};

template<typename CharType,typename CodecvtImpl,int CharSize=sizeof(CharType)>
class generic_codecvt;

template<typename CharType,typename CodecvtImpl>
class generic_codecvt<CharType,CodecvtImpl,2> : public std::codecvt<CharType,char,std::mbstate_t>, public generic_codecvt_base
{
public:

    typedef CharType uchar;

    generic_codecvt(size_t refs = 0) : 
        std::codecvt<CharType,char,std::mbstate_t>(refs)
    {
    }
    CodecvtImpl const &implementation() const
    {
        return *static_cast<CodecvtImpl const *>(this);
    }

protected:


    virtual std::codecvt_base::result do_unshift(std::mbstate_t &s,char *from,char * /*to*/,char *&next) const
    {
        boost::uint16_t &state = *reinterpret_cast<boost::uint16_t *>(&s);
#ifdef DEBUG_CODECVT            
        std::cout << "Entering unshift " << std::hex << state << std::dec << std::endl;
#endif            
        if(state != 0)
            return std::codecvt_base::error;
        next=from;
        return std::codecvt_base::ok;
    }
    virtual int do_encoding() const throw()
    {
        return 0;
    }
    virtual int do_max_length() const throw()
    {
        return implementation().max_encoding_length();
    }
    virtual bool do_always_noconv() const throw()
    {
        return false;
    }

    virtual int
    do_length(  std::mbstate_t 
    #ifdef BOOST_LOCALE_DO_LENGTH_MBSTATE_CONST
            const   
    #endif
            &std_state,
            char const *from,
            char const *from_end,
            size_t max) const
    {
        #ifndef BOOST_LOCALE_DO_LENGTH_MBSTATE_CONST
        char const *save_from = from;
        boost::uint16_t &state = *reinterpret_cast<boost::uint16_t *>(&std_state);
        #else
        size_t save_max = max;
        boost::uint16_t state = *reinterpret_cast<boost::uint16_t const *>(&std_state);
        #endif

        typedef typename CodecvtImpl::state_type state_type;
        state_type cvt_state = implementation().initial_state(generic_codecvt_base::to_unicode_state);
        while(max > 0 && from < from_end){
            char const *prev_from = from;
            boost::uint32_t ch=implementation().to_unicode(cvt_state,from,from_end);
            if(ch==boost::locale::utf::incomplete || ch==boost::locale::utf::illegal) {
                from = prev_from;
                break;
            }
            max --;
            if(ch > 0xFFFF) {
                if(state == 0) {
                    from = prev_from;
                    state = 1;
                }
                else {
                    state = 0;
                }
            }        
        }
        #ifndef BOOST_LOCALE_DO_LENGTH_MBSTATE_CONST
        return from - save_from;
        #else
        return save_max - max;
        #endif
    }

    
    virtual std::codecvt_base::result 
    do_in(  std::mbstate_t &std_state,
            char const *from,
            char const *from_end,
            char const *&from_next,
            uchar *to,
            uchar *to_end,
            uchar *&to_next) const
    {
        std::codecvt_base::result r=std::codecvt_base::ok;
        
        // mbstate_t is POD type and should be initialized to 0 (i.a. state = stateT())
        // according to standard. We use it to keep a flag 0/1 for surrogate pair writing
        //
        // if 0 no code above >0xFFFF observed, of 1 a code above 0xFFFF observerd
        // and first pair is written, but no input consumed
        boost::uint16_t &state = *reinterpret_cast<boost::uint16_t *>(&std_state);
        typedef typename CodecvtImpl::state_type state_type;
        state_type cvt_state = implementation().initial_state(generic_codecvt_base::to_unicode_state);
        while(to < to_end && from < from_end)
        {
#ifdef DEBUG_CODECVT            
            std::cout << "Entering IN--------------" << std::endl;
            std::cout << "State " << std::hex << state <<std::endl;
            std::cout << "Left in " << std::dec << from_end - from << " out " << to_end -to << std::endl;
#endif           
            char const *from_saved = from;
            
            uint32_t ch=implementation().to_unicode(cvt_state,from,from_end);
            
            if(ch==boost::locale::utf::illegal) {
                from = from_saved;
                r=std::codecvt_base::error;
                break;
            }
            if(ch==boost::locale::utf::incomplete) {
                from = from_saved;
                r=std::codecvt_base::partial;
                break;
            }
            // Normal codepoints go direcly to stream
            if(ch <= 0xFFFF) {
                *to++=ch;
            }
            else {
                // for  other codepoints we do following
                //
                // 1. We can't consume our input as we may find ourselfs
                //    in state where all input consumed but not all output written,i.e. only
                //    1st pair is written
                // 2. We only write first pair and mark this in the state, we also revert back
                //    the from pointer in order to make sure this codepoint would be read
                //    once again and then we would consume our input together with writing
                //    second surrogate pair
                ch-=0x10000;
                boost::uint16_t vh = ch >> 10;
                boost::uint16_t vl = ch & 0x3FF;
                boost::uint16_t w1 = vh + 0xD800;
                boost::uint16_t w2 = vl + 0xDC00;
                if(state == 0) {
                    from = from_saved;
                    *to++ = w1;
                    state = 1;
                }
                else {
                    *to++ = w2;
                    state = 0;
                }
            }
        }
        from_next=from;
        to_next=to;
        if(r == std::codecvt_base::ok && (from!=from_end || state!=0))
            r = std::codecvt_base::partial;
#ifdef DEBUG_CODECVT            
        std::cout << "Returning ";
        switch(r) {
        case std::codecvt_base::ok:
            std::cout << "ok" << std::endl;
            break;
        case std::codecvt_base::partial:
            std::cout << "partial" << std::endl;
            break;
        case std::codecvt_base::error:
            std::cout << "error" << std::endl;
            break;
        default:
            std::cout << "other" << std::endl;
            break;
        }
        std::cout << "State " << std::hex << state <<std::endl;
        std::cout << "Left in " << std::dec << from_end - from << " out " << to_end -to << std::endl;
#endif            
        return r;
    }
    
    virtual std::codecvt_base::result 
    do_out( std::mbstate_t &std_state,
            uchar const *from,
            uchar const *from_end,
            uchar const *&from_next,
            char *to,
            char *to_end,
            char *&to_next) const
    {
        std::codecvt_base::result r=std::codecvt_base::ok;
        // mbstate_t is POD type and should be initialized to 0 (i.a. state = stateT())
        // according to standard. We assume that sizeof(mbstate_t) >=2 in order
        // to be able to store first observerd surrogate pair
        //
        // State: state!=0 - a first surrogate pair was observerd (state = first pair),
        // we expect the second one to come and then zero the state
        boost::uint16_t &state = *reinterpret_cast<boost::uint16_t *>(&std_state);
        typedef typename CodecvtImpl::state_type state_type;
        state_type cvt_state = implementation().initial_state(generic_codecvt_base::from_unicode_state);
        while(to < to_end && from < from_end)
        {
#ifdef DEBUG_CODECVT            
        std::cout << "Entering OUT --------------" << std::endl;
        std::cout << "State " << std::hex << state <<std::endl;
        std::cout << "Left in " << std::dec << from_end - from << " out " << to_end -to << std::endl;
#endif            
            boost::uint32_t ch=0;
            if(state != 0) {
                // if the state idecates that 1st surrogate pair was written
                // we should make sure that the second one that comes is actually
                // second surrogate
                boost::uint16_t w1 = state;
                boost::uint16_t w2 = *from; 
                // we don't forward from as writing may fail to incomplete or
                // partial conversion
                if(0xDC00 <= w2 && w2<=0xDFFF) {
                    boost::uint16_t vh = w1 - 0xD800;
                    boost::uint16_t vl = w2 - 0xDC00;
                    ch=((uint32_t(vh) << 10)  | vl) + 0x10000;
                }
                else {
                    // Invalid surrogate
                    r=std::codecvt_base::error;
                    break;
                }
            }
            else {
                ch = *from;
                if(0xD800 <= ch && ch<=0xDBFF) {
                    // if this is a first surrogate pair we put
                    // it into the state and consume it, note we don't
                    // go forward as it should be illegal so we increase
                    // the from pointer manually
                    state = ch;
                    from++;
                    continue;
                }
                else if(0xDC00 <= ch && ch<=0xDFFF) {
                    // if we observe second surrogate pair and 
                    // first only may be expected we should break from the loop with error
                    // as it is illegal input
                    r=std::codecvt_base::error;
                    break;
                }
            }
            if(!boost::locale::utf::is_valid_codepoint(ch)) {
                r=std::codecvt_base::error;
                break;
            }
            boost::uint32_t len = implementation().from_unicode(cvt_state,ch,to,to_end);
            if(len == boost::locale::utf::incomplete) {
                r=std::codecvt_base::partial;
                break;
            }
            else if(len == boost::locale::utf::illegal) {
                r=std::codecvt_base::error;
                break;
            }
            else
                    to+= len;
            state = 0;
            from++;
        }
        from_next=from;
        to_next=to;
        if(r==std::codecvt_base::ok && from!=from_end)
            r = std::codecvt_base::partial;
#ifdef DEBUG_CODECVT            
        std::cout << "Returning ";
        switch(r) {
        case std::codecvt_base::ok:
            std::cout << "ok" << std::endl;
            break;
        case std::codecvt_base::partial:
            std::cout << "partial" << std::endl;
            break;
        case std::codecvt_base::error:
            std::cout << "error" << std::endl;
            break;
        default:
            std::cout << "other" << std::endl;
            break;
        }
        std::cout << "State " << std::hex << state <<std::endl;
        std::cout << "Left in " << std::dec << from_end - from << " out " << to_end -to << std::endl;
#endif            
        return r;
    }
    
};

template<typename CharType,typename CodecvtImpl>
class generic_codecvt<CharType,CodecvtImpl,4> : public std::codecvt<CharType,char,std::mbstate_t>, public generic_codecvt_base
{
public:
    typedef CharType uchar;

    generic_codecvt(size_t refs = 0) : 
        std::codecvt<CharType,char,std::mbstate_t>(refs)
    {
    }
    
    CodecvtImpl const &implementation() const
    {
        return *static_cast<CodecvtImpl const *>(this);
    }
    
protected:

    virtual std::codecvt_base::result do_unshift(std::mbstate_t &/*s*/,char *from,char * /*to*/,char *&next) const
    {
        next=from;
        return std::codecvt_base::ok;
    }
    virtual int do_encoding() const throw()
    {
        return 0;
    }
    virtual int do_max_length() const throw()
    {
        return implementation().max_encoding_length();
    }
    virtual bool do_always_noconv() const throw()
    {
        return false;
    }

    virtual int
    do_length(  std::mbstate_t 
    #ifdef BOOST_LOCALE_DO_LENGTH_MBSTATE_CONST
            const   
    #endif    
            &/*state*/,
            char const *from,
            char const *from_end,
            size_t max) const
    {
        #ifndef BOOST_LOCALE_DO_LENGTH_MBSTATE_CONST 
        char const *start_from = from;
        #else
        size_t save_max = max;
        #endif
        typedef typename CodecvtImpl::state_type state_type;
        state_type cvt_state = implementation().initial_state(generic_codecvt_base::to_unicode_state);
        while(max > 0 && from < from_end){
            char const *save_from = from;
            boost::uint32_t ch=implementation().to_unicode(cvt_state,from,from_end);
            if(ch==boost::locale::utf::incomplete || ch==boost::locale::utf::illegal) {
                from = save_from;
                break;
            }
            max--;
        }
        #ifndef BOOST_LOCALE_DO_LENGTH_MBSTATE_CONST 
        return from - start_from;
        #else
        return save_max - max;
        #endif
    }

    
    virtual std::codecvt_base::result 
    do_in(  std::mbstate_t &/*state*/,
            char const *from,
            char const *from_end,
            char const *&from_next,
            uchar *to,
            uchar *to_end,
            uchar *&to_next) const
    {
        std::codecvt_base::result r=std::codecvt_base::ok;
        
        // mbstate_t is POD type and should be initialized to 0 (i.a. state = stateT())
        // according to standard. We use it to keep a flag 0/1 for surrogate pair writing
        //
        // if 0 no code above >0xFFFF observed, of 1 a code above 0xFFFF observerd
        // and first pair is written, but no input consumed
        typedef typename CodecvtImpl::state_type state_type;
        state_type cvt_state = implementation().initial_state(generic_codecvt_base::to_unicode_state);
        while(to < to_end && from < from_end)
        {
#ifdef DEBUG_CODECVT            
            std::cout << "Entering IN--------------" << std::endl;
            std::cout << "State " << std::hex << state <<std::endl;
            std::cout << "Left in " << std::dec << from_end - from << " out " << to_end -to << std::endl;
#endif           
            char const *from_saved = from;
            
            uint32_t ch=implementation().to_unicode(cvt_state,from,from_end);
            
            if(ch==boost::locale::utf::illegal) {
                r=std::codecvt_base::error;
                from = from_saved;
                break;
            }
            if(ch==boost::locale::utf::incomplete) {
                r=std::codecvt_base::partial;
                from=from_saved;
                break;
            }
            *to++=ch;
        }
        from_next=from;
        to_next=to;
        if(r == std::codecvt_base::ok && from!=from_end)
            r = std::codecvt_base::partial;
#ifdef DEBUG_CODECVT            
        std::cout << "Returning ";
        switch(r) {
        case std::codecvt_base::ok:
            std::cout << "ok" << std::endl;
            break;
        case std::codecvt_base::partial:
            std::cout << "partial" << std::endl;
            break;
        case std::codecvt_base::error:
            std::cout << "error" << std::endl;
            break;
        default:
            std::cout << "other" << std::endl;
            break;
        }
        std::cout << "State " << std::hex << state <<std::endl;
        std::cout << "Left in " << std::dec << from_end - from << " out " << to_end -to << std::endl;
#endif            
        return r;
    }
    
    virtual std::codecvt_base::result 
    do_out( std::mbstate_t &/*std_state*/,
            uchar const *from,
            uchar const *from_end,
            uchar const *&from_next,
            char *to,
            char *to_end,
            char *&to_next) const
    {
        std::codecvt_base::result r=std::codecvt_base::ok;
        typedef typename CodecvtImpl::state_type state_type;
        state_type cvt_state = implementation().initial_state(generic_codecvt_base::from_unicode_state);
        while(to < to_end && from < from_end)
        {
#ifdef DEBUG_CODECVT            
        std::cout << "Entering OUT --------------" << std::endl;
        std::cout << "State " << std::hex << state <<std::endl;
        std::cout << "Left in " << std::dec << from_end - from << " out " << to_end -to << std::endl;
#endif            
            boost::uint32_t ch=0;
            ch = *from;
            if(!boost::locale::utf::is_valid_codepoint(ch)) {
                r=std::codecvt_base::error;
                break;
            }
            boost::uint32_t len = implementation().from_unicode(cvt_state,ch,to,to_end);
            if(len == boost::locale::utf::incomplete) {
                r=std::codecvt_base::partial;
                break;
            }
            else if(len == boost::locale::utf::illegal) {
                r=std::codecvt_base::error;
                break;
            }
            to+=len;
            from++;
        }
        from_next=from;
        to_next=to;
        if(r==std::codecvt_base::ok && from!=from_end)
            r = std::codecvt_base::partial;
#ifdef DEBUG_CODECVT            
        std::cout << "Returning ";
        switch(r) {
        case std::codecvt_base::ok:
            std::cout << "ok" << std::endl;
            break;
        case std::codecvt_base::partial:
            std::cout << "partial" << std::endl;
            break;
        case std::codecvt_base::error:
            std::cout << "error" << std::endl;
            break;
        default:
            std::cout << "other" << std::endl;
            break;
        }
        std::cout << "State " << std::hex << state <<std::endl;
        std::cout << "Left in " << std::dec << from_end - from << " out " << to_end -to << std::endl;
#endif            
        return r;
    }
};


template<typename CharType,typename CodecvtImpl>
class generic_codecvt<CharType,CodecvtImpl,1> : public std::codecvt<CharType,char,std::mbstate_t>, public generic_codecvt_base
{
public:
    typedef CharType uchar;

    CodecvtImpl const &implementation() const
    {
        return *static_cast<CodecvtImpl const *>(this);
    }

    generic_codecvt(size_t refs = 0) :  std::codecvt<char,char,std::mbstate_t>(refs)
    {
    }
};

} // locale
} // namespace boost

#endif
// vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4