+2000-10-19 Benjamin Kosnik <bkoz@purist.soma.redhat.com>
+
+ * testsuite/22_locale/codecvt_unicode_char.cc (test01): Adjust
+ creation of state_type for unicode_codecvt to take into account
+ the byte order markings. Add distinct tests for UCS-2BE and UCS-2LE.
+ * testsuite/22_locale/codecvt_unicode_wchar_t.cc (test01): Same.
+ * include/bits/codecvt.h (__enc_traits): Add support for encodings
+ that need a byte order marker. Needed for correct unicode support.
+
+ * src/locale.cc: Remove explicit qualification std::.
+ (locale::locale(const char* __name)): Revert, as named locale
+ support not finished.
+ * src/localename.cc (locale::_Impl:: _Impl(size_t __numfacets,
+ size_t __refs, bool __has_name = false, string __name): Move
+ default argument...
+ * include/bits/localefwd.h: Here.
+
2000-10-18 Chip Salzenberg <chip@valinux.com>
* libio/libio.h (_IO_USER_LOCK): Define.
-2000-10-18 Benjamin Kosnik <bkoz@purist.soma.redhat.com>
+2000-10-18 Benjamin Kosnik <bkoz@purist.soma.redhat.com>
* libsupc++/Makefile.am (exception): Change exception.cc to
exception_support.cc.
// Max size of charset encoding name
static const int _S_max_size = 32;
// Name of internal character set encoding.
- char _M_intc_enc[_S_max_size];
+ char _M_int_enc[_S_max_size];
// Name of external character set encoding.
- char _M_extc_enc[_S_max_size];
+ char _M_ext_enc[_S_max_size];
// Conversion descriptor between external encoding to internal encoding.
__desc_type _M_in_desc;
// Conversion descriptor between internal encoding to external encoding.
__desc_type _M_out_desc;
+ // Details the byte-order marker for the external encoding, if necessary.
+ int _M_ext_bom;
+
+ // Details the byte-order marker for the internal encoding, if necessary.
+ int _M_int_bom;
+
public:
- __enc_traits() : _M_in_desc(0), _M_out_desc(0)
+ __enc_traits()
+ : _M_in_desc(0), _M_out_desc(0), _M_ext_bom(0), _M_int_bom(0)
{
// __intc_end = whatever we are using internally, which is
// UCS4 (linux)
// UCS2 == UNICODE (microsoft, java, aix, whatever...)
// XXX Currently don't know how to get this data from target system...
- strcpy(_M_intc_enc, "UCS4");
+ strcpy(_M_int_enc, "UCS4");
// __extc_end = external codeset in current locale
- strcpy(_M_extc_enc, nl_langinfo(CODESET));
+ strcpy(_M_ext_enc, nl_langinfo(CODESET));
}
- __enc_traits(const char* __int, const char* __ext)
- : _M_in_desc(0), _M_out_desc(0)
+ __enc_traits(const char* __int, const char* __ext, int __ibom = 0,
+ int __ebom = 0)
+ : _M_in_desc(0), _M_out_desc(0), _M_ext_bom(0), _M_int_bom(0)
{
- strncpy(_M_intc_enc, __int, _S_max_size);
- strncpy(_M_extc_enc, __ext, _S_max_size);
+ strncpy(_M_int_enc, __int, _S_max_size);
+ strncpy(_M_ext_enc, __ext, _S_max_size);
}
// 21.1.2 traits typedefs
// CopyConstructible types (20.1.3)
__enc_traits(const __enc_traits& __obj)
{
- strncpy(_M_intc_enc, __obj._M_intc_enc, _S_max_size);
- strncpy(_M_extc_enc, __obj._M_extc_enc, _S_max_size);
+ strncpy(_M_int_enc, __obj._M_int_enc, _S_max_size);
+ strncpy(_M_ext_enc, __obj._M_ext_enc, _S_max_size);
+ _M_ext_bom = __obj._M_ext_bom;
+ _M_int_bom = __obj._M_int_bom;
}
~__enc_traits()
void
_M_init()
{
- _M_in_desc = iconv_open(_M_intc_enc, _M_extc_enc);
- _M_out_desc = iconv_open(_M_extc_enc, _M_intc_enc);
+ _M_in_desc = iconv_open(_M_int_enc, _M_ext_enc);
+ _M_out_desc = iconv_open(_M_ext_enc, _M_int_enc);
if (_M_out_desc == iconv_t(-1) || _M_in_desc == iconv_t(-1))
{
// XXX Extended error checking.
const char*
_M_get_internal_enc()
- { return _M_intc_enc; }
+ { return _M_int_enc; }
const char*
_M_get_external_enc()
- { return _M_extc_enc; }
+ { return _M_ext_enc; }
+
+ int
+ _M_get_external_bom()
+ { return _M_ext_bom; }
+
+ int
+ _M_get_internal_bom()
+ { return _M_int_bom; }
};
#endif //_GLIBCPP_USE_WCHAR_T
// Argument list for iconv specifies a byte sequence. Thus,
// all to/from arrays must be brutally casted to char*.
- char* __cfrom = reinterpret_cast<char*>(const_cast<intern_type*>(__from));
char* __cto = reinterpret_cast<char*>(__to);
- size_t __conv = iconv(*__desc, &__cfrom, &__flen, &__cto, &__tlen);
-
+ char* __cfrom;
+ size_t __conv;
+
+ // Some encodings need a byte order marker as the first item
+ // in the byte stream, to designate endian-ness. The default
+ // value for the byte order marker is NULL, so if this is
+ // the case, it's not necessary and we can just go on our
+ // merry way.
+ int __int_bom = __state._M_get_internal_bom();
+ if (__int_bom)
+ {
+ size_t __size = __from_end - __from;
+ intern_type __cfixed[__size + 1];
+ __cfixed[0] = static_cast<intern_type>(__int_bom);
+ char_traits<intern_type>::copy(__cfixed + 1, __from, __size);
+ __cfrom = reinterpret_cast<char*>(__cfixed);
+ __conv = iconv(*__desc, &__cfrom, &__flen, &__cto, &__tlen);
+ }
+ else
+ {
+ intern_type* __cfixed = const_cast<intern_type*>(__from);
+ __cfrom = reinterpret_cast<char*>(__cfixed);
+ __conv = iconv(*__desc, &__cfrom, &__flen, &__cto, &__tlen);
+ }
+
if (__conv != size_t(-1))
{
__from_next = reinterpret_cast<const intern_type*>(__cfrom);
// Argument list for iconv specifies a byte sequence. Thus,
// all to/from arrays must be brutally casted to char*.
- char* __cfrom = reinterpret_cast<char*>(const_cast<extern_type*>(__from));
char* __cto = reinterpret_cast<char*>(__to);
- size_t __conv = iconv(*__desc, &__cfrom, &__flen, &__cto, &__tlen);
+ char* __cfrom;
+ size_t __conv;
+
+ // Some encodings need a byte order marker as the first item
+ // in the byte stream, to designate endian-ness. The default
+ // value for the byte order marker is NULL, so if this is
+ // the case, it's not necessary and we can just go on our
+ // merry way.
+ int __ext_bom = __state._M_get_external_bom();
+ if (__ext_bom)
+ {
+ size_t __size = __from_end - __from;
+ extern_type __cfixed[__size + 1];
+ __cfixed[0] = static_cast<extern_type>(__ext_bom);
+ char_traits<extern_type>::copy(__cfixed + 1, __from, __size);
+ __cfrom = reinterpret_cast<char*>(__cfixed);
+ __conv = iconv(*__desc, &__cfrom, &__flen, &__cto, &__tlen);
+ }
+ else
+ {
+ extern_type* __cfixed = const_cast<extern_type*>(__from);
+ __cfrom = reinterpret_cast<char*>(__cfixed);
+ __conv = iconv(*__desc, &__cfrom, &__flen, &__cto, &__tlen);
+ }
+
if (__conv != size_t(-1))
{
_Impl(const _Impl&, size_t __refs);
_Impl(const _Impl&, const string&, category, size_t __refs);
- _Impl(size_t __facets, size_t __refs, bool __has_name, string __name);
+ _Impl(size_t __facets, size_t __refs, bool __has_name,
+ string __name = "*");
~_Impl() throw();
void
#ifdef _GLIBCPP_USE_WCHAR_T
&std::ctype<wchar_t>::id,
#endif
- &std::codecvt<char, char, mbstate_t>::id,
+ &codecvt<char, char, mbstate_t>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::codecvt<wchar_t, char, mbstate_t>::id,
+ &codecvt<wchar_t, char, mbstate_t>::id,
#endif
0
};
const locale::id* const
locale::_Impl::_S_id_monetary[] =
{
- &std::moneypunct<char, false>::id,
+ &moneypunct<char, false>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::moneypunct<wchar_t, false>::id,
+ &moneypunct<wchar_t, false>::id,
#endif
&std::moneypunct<char,true >::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::moneypunct<wchar_t,true >::id,
+ &moneypunct<wchar_t,true >::id,
#endif
- &std::money_get<char>::id,
+ &money_get<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::money_get<wchar_t>::id,
+ &money_get<wchar_t>::id,
#endif
- &std::money_put<char>::id,
+ &money_put<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::money_put<wchar_t>::id,
+ &money_put<wchar_t>::id,
#endif
0
};
const locale::id* const
locale::_Impl::_S_id_numeric[] =
{
- &std::numpunct<char>::id,
+ &numpunct<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::numpunct<wchar_t>::id,
+ &numpunct<wchar_t>::id,
#endif
- &std::num_get<char>::id,
+ &num_get<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::num_get<wchar_t>::id,
+ &num_get<wchar_t>::id,
#endif
- &std::num_put<char>::id,
+ &num_put<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::num_put<wchar_t>::id,
+ &num_put<wchar_t>::id,
#endif
0
};
const locale::id* const
locale::_Impl::_S_id_time[] =
{
- &std::time_get<char>::id,
+ &time_get<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::time_get<wchar_t>::id,
+ &time_get<wchar_t>::id,
#endif
- &std::time_put<char>::id,
+ &time_put<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::time_put<wchar_t>::id,
+ &time_put<wchar_t>::id,
#endif
0
};
const locale::id* const
locale::_Impl::_S_id_messages[] =
{
- &std::time_get<char>::id,
+ &time_get<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::time_get<wchar_t>::id,
+ &time_get<wchar_t>::id,
#endif
- &std::time_put<char>::id,
+ &time_put<char>::id,
#ifdef _GLIBCPP_USE_WCHAR_T
- &std::time_put<wchar_t>::id,
+ &time_put<wchar_t>::id,
#endif
0
};
const locale::id* const* const
locale::_Impl::_S_facet_categories[] =
{
- // order must match the decl order in class locale.
+ // Order must match the decl order in class locale.
locale::_Impl::_S_id_collate,
locale::_Impl::_S_id_ctype,
locale::_Impl::_S_id_monetary,
(_M_impl = _S_classic)->_M_add_reference();
// Might throw:
else
- _M_impl = new _Impl(_S_facets_num, 1, true, __name);
+ // XXX Named locale support not finished.
+ // _M_impl = new _Impl(_S_facets_num, 1, true, __name);
+ _M_impl = new _Impl(*_S_classic, __name, all, 1);
}
else
throw runtime_error("attempt to create named locale from NULL name");
try {
// 26 Standard facets, 2 references.
// One reference for _M_classic, one for _M_global
- // XXX _S_classic = _S_global = new _Impl(26, 2);
_S_classic = new _Impl(_S_facets_num, 2, true, "C");
_S_global = _S_classic;
// including the standard "C" locale.
locale::_Impl::
_Impl(size_t __numfacets, size_t __refs, bool __has_name = false,
- string __name = "*")
+ string __name)
: _M_references(__refs - 1), _M_facets(0), _M_category_names(0),
_M_has_name(__has_name), _M_name(__name)
- {
+ {
try
{ _M_facets = new __vec_facet(__numfacets, NULL); }
catch(...)
it shows that the other byte-order is used (25856 == 0x6500).
*/
-#if 0
-void
-create_internal_literal(unsigned short* i_lit)
-{
- i_lit[00] = 25088; //b
- i_lit[01] = 27648; //l
- i_lit[02] = 24832; //a
- i_lit[03] = 25344; //c
- i_lit[04] = 27392; //k
- i_lit[05] = 8192;
- i_lit[06] = 28672; //p
- i_lit[07] = 25856; //e
- i_lit[08] = 24832; //a
- i_lit[09] = 29148; //r
- i_lit[10] = 27648; //l
- i_lit[11] = 8192;
- i_lit[12] = 27136; //j
- i_lit[13] = 24832;
- i_lit[14] = 29440;
- i_lit[15] = 27904;
- i_lit[16] = 26880;
- i_lit[17] = 28160;
- i_lit[18] = 25856; //e
- i_lit[19] = 8192;
- i_lit[20] = 29696; //t
- i_lit[21] = 25856; //e
- i_lit[22] = 24832; //a
- i_lit[23] = 2560;
-}
-#endif
void
initialize_state(__enc_traits& state)
// Partial specialization using __enc_traits.
// codecvt<unicode_t, char, __enc_traits>
+// UNICODE - UCS2 (big endian)
void test01()
{
typedef codecvt_base::result result;
const ext_type* e_lit = "black pearl jasmine tea";
int size = strlen(e_lit);
- int_type i_lit_base[24] =
- { 25088, 27648, 24832, 25344, 27392, 8192, 28672, 25856, 24832, 29184,
- 27648, 8192, 27136, 24832, 29440, 27904, 26880, 28160, 25856, 8192, 29696,
- 25856, 24832, 2560
+ int_type i_lit_base[25] =
+ {
+ 0x6200, 0x6c00, 0x6100, 0x6300, 0x6b00, 0x2000, 0x7000, 0x6500, 0x6100,
+ 0x7200, 0x6c00, 0x2000, 0x6a00, 0x6100, 0x7300, 0x6d00, 0x6900, 0x6e00,
+ 0x6500, 0x2000, 0x7400, 0x6500, 0x6100, 0xa000
+ };
+ const int_type* i_lit = i_lit_base;
+
+ const ext_type* efrom_next;
+ const int_type* ifrom_next;
+ ext_type* e_arr = new ext_type[size + 1];
+ ext_type* eto_next;
+ int_type* i_arr = new int_type[size + 1];
+ int_type* ito_next;
+
+ // construct a locale object with the specialized facet.
+ locale loc(locale::classic(), new unicode_codecvt);
+ // sanity check the constructed locale has the specialized facet.
+ VERIFY( has_facet<unicode_codecvt>(loc) );
+ const unicode_codecvt& cvt = use_facet<unicode_codecvt>(loc);
+
+ // in
+ unicode_codecvt::state_type state01("UCS-2BE", "ISO-8859-15", 0xfeff, 0);
+ initialize_state(state01);
+ // internal encoding is bigger because of bom
+ result r1 = cvt.in(state01, e_lit, e_lit + size, efrom_next,
+ i_arr, i_arr + size + 1, ito_next);
+ VERIFY( r1 == codecvt_base::ok );
+ VERIFY( !int_traits::compare(i_arr, i_lit, size) );
+ VERIFY( efrom_next == e_lit + size );
+ VERIFY( ito_next == i_arr + size );
+
+ // out
+ unicode_codecvt::state_type state02("UCS-2BE", "ISO-8859-15", 0xfeff, 0);
+ initialize_state(state02);
+ result r2 = cvt.out(state02, i_lit, i_lit + size, ifrom_next,
+ e_arr, e_arr + size, eto_next);
+ VERIFY( r2 == codecvt_base::ok );
+ VERIFY( !ext_traits::compare(e_arr, e_lit, size) );
+ VERIFY( ifrom_next == i_lit + size );
+ VERIFY( eto_next == e_arr + size );
+
+ // unshift
+ ext_traits::copy(e_arr, e_lit, size);
+ unicode_codecvt::state_type state03("UCS-2BE", "ISO-8859-15", 0xfeff, 0);
+ initialize_state(state03);
+ result r3 = cvt.unshift(state03, e_arr, e_arr + size, eto_next);
+ VERIFY( r3 == codecvt_base::noconv );
+ VERIFY( !ext_traits::compare(e_arr, e_lit, size) );
+ VERIFY( eto_next == e_arr );
+
+ int i = cvt.encoding();
+ VERIFY( i == 0 );
+
+ VERIFY( !cvt.always_noconv() );
+
+ unicode_codecvt::state_type state04("UCS-2BE", "ISO-8859-15", 0xfeff, 0);
+ initialize_state(state04);
+ int j = cvt.length(state03, e_lit, e_lit + size, 5);
+ VERIFY( j == 5 );
+
+ int k = cvt.max_length();
+ VERIFY( k == 1 );
+
+ delete [] e_arr;
+ delete [] i_arr;
+}
+
+// Partial specialization using __enc_traits.
+// codecvt<unicode_t, char, __enc_traits>
+// UNICODE - UCS2 (little endian)
+void test02()
+{
+ typedef codecvt_base::result result;
+ typedef unsigned short unicode_t;
+ typedef unicode_t int_type;
+ typedef char ext_type;
+ typedef __enc_traits enc_type;
+ typedef codecvt<int_type, ext_type, enc_type> unicode_codecvt;
+ typedef char_traits<int_type> int_traits;
+ typedef char_traits<ext_type> ext_traits;
+
+ bool test = true;
+ const ext_type* e_lit = "black pearl jasmine tea";
+ int size = strlen(e_lit);
+
+ int_type i_lit_base[25] =
+ {
+ 0x0062, 0x006c, 0x0061, 0x0063, 0x006b, 0x0020, 0x0070, 0x0065, 0x0061,
+ 0x0072, 0x006c, 0x0020, 0x006a, 0x0061, 0x0073, 0x006d, 0x0069, 0x006e,
+ 0x0065, 0x0020, 0x0074, 0x0065, 0x0061, 0x00a0
};
const int_type* i_lit = i_lit_base;
const unicode_codecvt& cvt = use_facet<unicode_codecvt>(loc);
// in
- unicode_codecvt::state_type state01("UNICODE", "ISO-8859-15");
+ unicode_codecvt::state_type state01("UCS-2LE", "ISO-8859-15", 0xfeff, 0);
initialize_state(state01);
+ // internal encoding is bigger because of bom
result r1 = cvt.in(state01, e_lit, e_lit + size, efrom_next,
- i_arr, i_arr + size, ito_next);
+ i_arr, i_arr + size + 1, ito_next);
VERIFY( r1 == codecvt_base::ok );
VERIFY( !int_traits::compare(i_arr, i_lit, size) );
VERIFY( efrom_next == e_lit + size );
VERIFY( ito_next == i_arr + size );
// out
- unicode_codecvt::state_type state02("UNICODE", "ISO-8859-15");
+ unicode_codecvt::state_type state02("UCS-2LE", "ISO-8859-15", 0xfeff, 0);
initialize_state(state02);
result r2 = cvt.out(state02, i_lit, i_lit + size, ifrom_next,
e_arr, e_arr + size, eto_next);
// unshift
ext_traits::copy(e_arr, e_lit, size);
- unicode_codecvt::state_type state03("UNICODE", "ISO-8859-15");
+ unicode_codecvt::state_type state03("UCS-2LE", "ISO-8859-15", 0xfeff, 0);
initialize_state(state03);
result r3 = cvt.unshift(state03, e_arr, e_arr + size, eto_next);
VERIFY( r3 == codecvt_base::noconv );
VERIFY( !cvt.always_noconv() );
- unicode_codecvt::state_type state04("UNICODE", "ISO-8859-15");
+ unicode_codecvt::state_type state04("UCS-2LE", "ISO-8859-15", 0xfeff, 0);
initialize_state(state04);
int j = cvt.length(state03, e_lit, e_lit + size, 5);
VERIFY( j == 5 );
int main ()
{
test01();
+ test02();
return 0;
}
const ext_type* e_lit = e_lit_base;
int_type i_lit_base[24] =
- { 25088, 27648, 24832, 25344, 27392, 8192,
- 28672, 25856, 24832, 29184, 27648, 8192,
- 27136, 24832, 29440, 27904, 26880, 28160,
- 25856, 8192, 29696, 25856, 24832, 2560
+ {
+ 0x6200, 0x6c00, 0x6100, 0x6300, 0x6b00, 0x2000, 0x7000, 0x6500, 0x6100,
+ 0x7200, 0x6c00, 0x2000, 0x6a00, 0x6100, 0x7300, 0x6d00, 0x6900, 0x6e00,
+ 0x6500, 0x2000, 0x7400, 0x6500, 0x6100, 0xa000
};
const int_type* i_lit = i_lit_base;
const unicode_codecvt& cvt = use_facet<unicode_codecvt>(loc);
// in
- unicode_codecvt::state_type state01("UNICODE", "UCS4");
+ unicode_codecvt::state_type state01("UCS-2BE", "UCS4", 0xfeff, 0);
initialize_state(state01);
result r1 = cvt.in(state01, e_lit, e_lit + size, efrom_next,
- i_arr, i_arr + size, ito_next);
+ i_arr, i_arr + size + 1, ito_next);
VERIFY( r1 == codecvt_base::ok );
VERIFY( !int_traits::compare(i_arr, i_lit, size) );
VERIFY( efrom_next == e_lit + size );
VERIFY( ito_next == i_arr + size );
// out
- unicode_codecvt::state_type state02("UNICODE", "UCS4");
+ unicode_codecvt::state_type state02("UCS-2BE", "UCS4", 0xfeff, 0);
initialize_state(state02);
result r2 = cvt.out(state02, i_lit, i_lit + size, ifrom_next,
e_arr, e_arr + size, eto_next);
// unshift
ext_traits::copy(e_arr, e_lit, size);
- unicode_codecvt::state_type state03("UNICODE", "UCS4");
+ unicode_codecvt::state_type state03("UCS-2BE", "UCS4", 0xfeff, 0);
initialize_state(state03);
result r3 = cvt.unshift(state03, e_arr, e_arr + size, eto_next);
VERIFY( r3 == codecvt_base::noconv );
VERIFY( !cvt.always_noconv() );
- unicode_codecvt::state_type state04("UNICODE", "UCS4");
+ unicode_codecvt::state_type state04("UCS-2BE", "UCS4", 0xfeff, 0);
initialize_state(state04);
int j = cvt.length(state03, e_lit, e_lit + size, 5);
VERIFY( j == 5 );
projects / gcc.git / commitdiff