#undef DECODE_DEBUG
#include "decoder.h"
using namespace khtml;
#include "htmlhashes.h"
#include <qregexp.h>
#include <qtextcodec.h>
#include <kglobal.h>
#include <kcharsets.h>
#include <ctype.h>
#include <kdebug.h>
#include <klocale.h>
class KanjiCode
{
public:
enum Type {ASCII, JIS, EUC, SJIS, UNICODE, UTF8 };
static enum Type judge(const char *str, int length);
static const int ESC;
static const int _SS2_;
static const unsigned char kanji_map_sjis[];
static int ISkanji(int code)
{
if (code >= 0x100)
return 0;
return (kanji_map_sjis[code & 0xff] & 1);
}
static int ISkana(int code)
{
if (code >= 0x100)
return 0;
return (kanji_map_sjis[code & 0xff] & 2);
}
};
const int KanjiCode::ESC = 0x1b;
const int KanjiCode::_SS2_ = 0x8e;
const unsigned char KanjiCode::kanji_map_sjis[] =
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0
};
enum KanjiCode::Type KanjiCode::judge(const char *str, int size)
{
enum Type code;
int i;
int bfr = FALSE;
int bfk = 0;
int sjis = 0;
int euc = 0;
const unsigned char *ptr = (const unsigned char *) str;
code = ASCII;
i = 0;
while (i < size) {
if (ptr[i] == ESC && (size - i >= 3)) {
if ((ptr[i + 1] == '$' && ptr[i + 2] == 'B')
|| (ptr[i + 1] == '(' && ptr[i + 2] == 'B')) {
code = JIS;
goto breakBreak;
} else if ((ptr[i + 1] == '$' && ptr[i + 2] == '@')
|| (ptr[i + 1] == '(' && ptr[i + 2] == 'J')) {
code = JIS;
goto breakBreak;
} else if (ptr[i + 1] == '(' && ptr[i + 2] == 'I') {
code = JIS;
i += 3;
} else if (ptr[i + 1] == ')' && ptr[i + 2] == 'I') {
code = JIS;
i += 3;
} else {
i++;
}
bfr = FALSE;
bfk = 0;
} else {
if (ptr[i] < 0x20) {
bfr = FALSE;
bfk = 0;
if ((i >= 2) && (ptr[i - 2] == 0x81)
&& (0x41 <= ptr[i - 1] && ptr[i - 1] <= 0x49)) {
code = SJIS;
sjis += 100;
} else if ((i >= 2) && (ptr[i - 2] == 0xa1)
&& (0xa2 <= ptr[i - 1] && ptr[i - 1] <= 0xaa)) {
code = EUC;
euc += 100;
} else if ((i >= 2) && (ptr[i - 2] == 0x82) && (0xa0 <= ptr[i - 1])) {
sjis += 40;
} else if ((i >= 2) && (ptr[i - 2] == 0xa4) && (0xa0 <= ptr[i - 1])) {
euc += 40;
}
} else {
if ((size - i > 1) && (ptr[i] == 0x82) && (0xa0 <= ptr[i + 1])) {
sjis++;
} else if ((size - i > 1) && (ptr[i] == 0x83)
&& (0x40 <= ptr[i + 1] && ptr[i + 1] <= 0x9f)) {
sjis++;
} else if ((size - i > 1) && (ptr[i] == 0xa4) && (0xa0 <= ptr[i + 1])) {
euc++;
} else if ((size - i > 1) && (ptr[i] == 0xa5) && (0xa0 <= ptr[i + 1])) {
euc++;
}
if (bfr) {
if ((i >= 1) && (0x40 <= ptr[i] && ptr[i] <= 0xa0) && ISkanji(ptr[i - 1])) {
code = SJIS;
goto breakBreak;
} else if ((i >= 1) && (0x81 <= ptr[i - 1] && ptr[i - 1] <= 0x9f) && ((0x40 <= ptr[i] && ptr[i] < 0x7e) || (0x7e < ptr[i] && ptr[i] <= 0xfc))) {
code = SJIS;
goto breakBreak;
} else if ((i >= 1) && (0xfd <= ptr[i] && ptr[i] <= 0xfe) && (0xa1 <= ptr[i - 1] && ptr[i - 1] <= 0xfe)) {
code = EUC;
goto breakBreak;
} else if ((i >= 1) && (0xfd <= ptr[i - 1] && ptr[i - 1] <= 0xfe) && (0xa1 <= ptr[i] && ptr[i] <= 0xfe)) {
code = EUC;
goto breakBreak;
} else if ((i >= 1) && (ptr[i] < 0xa0 || 0xdf < ptr[i]) && (0x8e == ptr[i - 1])) {
code = SJIS;
goto breakBreak;
} else if (ptr[i] <= 0x7f) {
code = SJIS;
goto breakBreak;
} else {
if (0xa1 <= ptr[i] && ptr[i] <= 0xa6) {
euc++;
} else if (0xa1 <= ptr[i] && ptr[i] <= 0xdf) {
;
} else if (0xa1 <= ptr[i] && ptr[i] <= 0xfe) {
euc++;
} else if (0x8e == ptr[i]) {
euc++;
} else if (0x20 <= ptr[i] && ptr[i] <= 0x7f) {
sjis++;
}
bfr = FALSE;
bfk = 0;
}
} else if (0x8e == ptr[i]) {
if (size - i <= 1) {
;
} else if (0xa1 <= ptr[i + 1] && ptr[i + 1] <= 0xdf) {
if (bfk == 1) {
euc += 100;
}
bfk++;
i++;
} else {
code = SJIS;
goto breakBreak;
}
} else if (0x81 <= ptr[i] && ptr[i] <= 0x9f) {
code = SJIS;
if ((size - i >= 1)
&& ((0x40 <= ptr[i + 1] && ptr[i + 1] <= 0x7e)
|| (0x80 <= ptr[i + 1] && ptr[i + 1] <= 0xfc))) {
goto breakBreak;
}
} else if (0xfd <= ptr[i] && ptr[i] <= 0xfe) {
code = EUC;
if ((size - i >= 1)
&& (0xa1 <= ptr[i + 1] && ptr[i + 1] <= 0xfe)) {
goto breakBreak;
}
} else if (ptr[i] <= 0x7f) {
;
} else {
bfr = TRUE;
bfk = 0;
}
}
i++;
}
}
if (code == ASCII) {
if (sjis > euc) {
code = SJIS;
} else if (sjis < euc) {
code = EUC;
}
}
breakBreak:
return (code);
}
Decoder::Decoder()
{
_refCount = 1;
m_codec = QTextCodec::codecForName("iso8859-1");
m_decoder = m_codec->makeDecoder();
enc = 0;
m_type = DefaultEncoding;
body = false;
beginning = true;
visualRTL = false;
}
Decoder::~Decoder()
{
assert(_refCount == 0);
delete m_decoder;
}
void Decoder::setEncoding(const char *_encoding, EncodingType type)
{
#ifdef DECODE_DEBUG
kdDebug(6005) << "setEncoding " << _encoding << " " << force << endl;
#endif
enc = _encoding;
#ifdef DECODE_DEBUG
kdDebug(6005) << "old encoding is:" << m_codec->name() << endl;
#endif
enc = enc.lower();
#ifdef DECODE_DEBUG
kdDebug(6005) << "requesting:" << enc << endl;
#endif
if(enc.isNull() || enc.isEmpty())
return;
#if APPLE_CHANGES
QTextCodec *codec = (type == EncodingFromMetaTag || type == EncodingFromXMLHeader)
? QTextCodec::codecForNameEightBitOnly(enc)
: QTextCodec::codecForName(enc);
if (codec) {
enc = codec->name();
visualRTL = codec->usesVisualOrdering();
}
#else
if(enc == "visual") enc = "iso8859-8";
bool b;
QTextCodec *codec = KGlobal::charsets()->codecForName(enc, b);
if (!b)
codec = 0;
if (codec && codec->mibEnum() == 11) {
if( !(enc == "iso-8859-8-i" || enc == "iso_8859-8-i"
|| enc == "csiso88598i" || enc == "logical") )
visualRTL = true;
}
#endif
if( codec ) { m_codec = codec;
m_type = type;
delete m_decoder;
m_decoder = m_codec->makeDecoder();
}
#ifdef DECODE_DEBUG
kdDebug(6005) << "Decoder::encoding used is" << m_codec->name() << endl;
#endif
}
const char *Decoder::encoding() const
{
return enc;
}
static void skipComment(const char *&ptr, const char *pEnd)
{
const char *p = ptr;
if (*p == '>') {
p++;
} else {
while (p != pEnd) {
if (*p == '-') {
if (p[1] == '-' && p[2] == '>') {
p += 3;
break;
}
if (p[1] == '-' && p[2] == '!' && p[3] == '>') {
p += 4;
break;
}
}
p++;
}
}
ptr = p;
}
static int findXMLEncoding(const QCString &str, int &encodingLength)
{
int len = str.length();
int pos = str.find("encoding");
if (pos == -1)
return -1;
pos += 8;
while (str[pos] <= ' ' && pos != len)
++pos;
if (str[pos] != '=')
return -1;
++pos;
while (str[pos] <= ' ' && pos != len)
++pos;
char quoteMark = str[pos];
if (quoteMark != '"' && quoteMark != '\'')
return -1;
++pos;
int end = pos;
while (str[end] != quoteMark)
++end;
if (end == len)
return -1;
encodingLength = end - pos;
return pos;
}
QString Decoder::decode(const char *data, int len)
{
int bufferLength = buffer.length();
const int maximumBOMLength = 3;
if (beginning && bufferLength + len >= maximumBOMLength) {
if (m_type != UserChosenEncoding) {
const uchar *udata = (const uchar *)data;
uchar c1 = bufferLength >= 1 ? (uchar)buffer[0] : *udata++;
uchar c2 = bufferLength >= 2 ? (uchar)buffer[1] : *udata++;
assert(bufferLength < 3);
uchar c3 = *udata;
const char *autoDetectedEncoding;
if ((c1 == 0xFE && c2 == 0xFF) || (c1 == 0xFF && c2 == 0xFE)) {
autoDetectedEncoding = "ISO-10646-UCS-2";
} else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
autoDetectedEncoding = "UTF-8";
} else {
autoDetectedEncoding = 0;
}
if (autoDetectedEncoding != 0) {
m_type = AutoDetectedEncoding;
m_codec = QTextCodec::codecForName(autoDetectedEncoding);
assert(m_codec);
enc = m_codec->name();
delete m_decoder;
m_decoder = m_codec->makeDecoder();
}
}
beginning = false;
}
bool lookForMetaTag = m_type == DefaultEncoding && !body;
if (lookForMetaTag) {
#ifdef DECODE_DEBUG
kdDebug(6005) << "looking for charset definition" << endl;
#endif
{ #if APPLE_CHANGES
buffer.append(data, len);
#else
if(m_codec->mibEnum() != 1000) {
char *d = const_cast<char *>(data);
int i = len - 1;
while(i >= 0) {
if(*(d+i) == 0) *(d+i) = ' ';
i--;
}
}
buffer += QCString(data, len+1);
#endif
#if APPLE_CHANGES
const char *ptr = buffer.latin1();
const char *pEnd = ptr + buffer.length();
#else
const char *ptr = buffer.data();
const char *pEnd = buffer.length();
#endif
while(ptr != pEnd)
{
if(*ptr == '<') {
bool end = false;
ptr++;
if (ptr[0] == '!' && ptr[1] == '-' && ptr[2] == '-') {
ptr += 3;
skipComment(ptr, pEnd);
continue;
}
if (ptr[0] == '?' && ptr[1] == 'x' && ptr[2] == 'm' && ptr[3] == 'l') {
const char *end = ptr;
while (*end != '>' && *end != '\0') end++;
if (*end == '\0')
break;
QCString str(ptr, end - ptr);
int len;
int pos = findXMLEncoding(str, len);
if (pos != -1) {
setEncoding(str.mid(pos, len), EncodingFromXMLHeader);
if (m_type == EncodingFromXMLHeader)
goto found;
}
}
if(*ptr == '/') ptr++, end=true;
char tmp[20];
int len = 0;
while (
((*ptr >= 'a') && (*ptr <= 'z') ||
(*ptr >= 'A') && (*ptr <= 'Z') ||
(*ptr >= '0') && (*ptr <= '9'))
&& len < 19 )
{
tmp[len] = tolower( *ptr );
ptr++;
len++;
}
tmp[len] = 0;
int id = khtml::getTagID(tmp, len);
if(end) id += ID_CLOSE_TAG;
switch( id ) {
case ID_META:
{
const char * end = ptr;
while(*end != '>' && *end != '\0') end++;
if ( *end == '\0' ) break;
QCString str( ptr, (end-ptr)+1);
str = str.lower();
int pos = 0;
while( pos < ( int ) str.length() ) {
if( (pos = str.find("charset", pos, false)) == -1) break;
pos += 7;
while( pos < (int)str.length() && str[pos] <= ' ' ) pos++;
if ( pos == ( int )str.length()) break;
if ( str[pos++] != '=' ) continue;
while ( pos < ( int )str.length() &&
( str[pos] <= ' ' ) || str[pos] == '=' || str[pos] == '"' || str[pos] == '\'')
pos++;
if ( pos == ( int )str.length() ) break;
uint endpos = pos;
while( endpos < str.length() &&
(str[endpos] != ' ' && str[endpos] != '"' && str[endpos] != '\''
&& str[endpos] != ';' && str[endpos] != '>') )
endpos++;
#ifdef DECODE_DEBUG
kdDebug( 6005 ) << "Decoder: found charset: " << str.mid(pos, endpos-pos) << endl;
#endif
setEncoding(str.mid(pos, endpos-pos), EncodingFromMetaTag);
if( m_type == EncodingFromMetaTag ) goto found;
if ( endpos >= str.length() || str[endpos] == '/' || str[endpos] == '>' ) break;
pos = endpos + 1;
}
}
case ID_SCRIPT:
case (ID_SCRIPT+ID_CLOSE_TAG):
case ID_NOSCRIPT:
case (ID_NOSCRIPT+ID_CLOSE_TAG):
case ID_STYLE:
case (ID_STYLE+ID_CLOSE_TAG):
case ID_LINK:
case (ID_LINK+ID_CLOSE_TAG):
case ID_OBJECT:
case (ID_OBJECT+ID_CLOSE_TAG):
case ID_TITLE:
case (ID_TITLE+ID_CLOSE_TAG):
case ID_BASE:
case (ID_BASE+ID_CLOSE_TAG):
case ID_HTML:
case ID_HEAD:
case 0:
case (0 + ID_CLOSE_TAG ):
break;
default:
body = true;
#ifdef DECODE_DEBUG
kdDebug( 6005 ) << "Decoder: no charset found. Id=" << id << endl;
#endif
goto found;
}
}
else
ptr++;
}
return QString::null;
}
}
found:
#if APPLE_CHANGES
if (m_type != UserChosenEncoding && m_type != AutoDetectedEncoding && m_codec && m_codec->isJapanese())
#else
if (m_type == DefaultEncoding && KGlobal::locale()->languageList()[0] == "ja")
#endif
{
#ifdef DECODE_DEBUG
kdDebug( 6005 ) << "Decoder: use auto-detect (" << strlen(data) << ")" << endl;
#endif
const char *autoDetectedEncoding;
switch ( KanjiCode::judge( data, len ) ) {
case KanjiCode::JIS:
autoDetectedEncoding = "jis7";
break;
case KanjiCode::EUC:
autoDetectedEncoding = "eucjp";
break;
case KanjiCode::SJIS:
autoDetectedEncoding = "sjis";
break;
default:
autoDetectedEncoding = NULL;
break;
}
#ifdef DECODE_DEBUG
kdDebug( 6005 ) << "Decoder: auto detect encoding is "
<< (autoDetectedEncoding ? autoDetectedEncoding : "NULL") << endl;
#endif
if (autoDetectedEncoding != 0) {
setEncoding(autoDetectedEncoding, AutoDetectedEncoding);
}
}
if (!m_codec)
{
if(enc.isEmpty()) enc = "iso8859-1";
m_codec = QTextCodec::codecForName(enc);
if(!m_codec) {
enc = "iso8859-1";
m_codec = QTextCodec::codecForName(enc);
}
delete m_decoder;
m_decoder = m_codec->makeDecoder();
}
QString out;
#if APPLE_CHANGES
if (!buffer.isEmpty()) {
if (!lookForMetaTag)
buffer.append(data, len);
out = m_decoder->toUnicode(buffer.latin1(), buffer.length());
buffer.truncate(0);
} else {
out = m_decoder->toUnicode(data, len);
}
#else
if(!buffer.isEmpty() && enc != "ISO-10646-UCS-2") {
out = m_decoder->toUnicode(buffer.latin1(), buffer.length());
buffer = "";
} else {
if(m_codec->mibEnum() != 1000) {
char *d = const_cast<char *>(data);
int i = len - 1;
while(i >= 0) {
if(*(d+i) == 0) *(d+i) = ' ';
i--;
}
}
out = m_decoder->toUnicode(data, len);
}
if (out.isNull()) {
fprintf(stderr, "ERROR: decoded string is null\n");
} else if (out.length() == 0) {
fprintf(stderr, "ERROR: decoded string length == 0\n");
}
else if(out[out.length()-1] == QChar::null)
out.truncate(out.length() - 1);
#endif
return out;
}
QString Decoder::flush() const
{
#if APPLE_CHANGES
return m_decoder->toUnicode(buffer.latin1(), buffer.length(), true);
#else
return m_decoder->toUnicode(buffer.latin1(), buffer.length());
#endif
}
#undef DECODE_DEBUG