r22584 by jghali - #15350: xtg import broken for multibyte encoded input files
scribus-commit
scribus-commit at lists.scribus.net
Sun Jun 17 23:14:13 UTC 2018
Author: jghali
Date: Sun Jun 17 23:14:13 2018
New Revision: 22584
URL: http://scribus.net/websvn/listing.php?repname=Scribus&sc=1&rev=22584
Log:
#15350: xtg import broken for multibyte encoded input files
Modified:
trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp
trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h
Modified: trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp
URL: http://scribus.net/websvn/diff.php?repname=Scribus&rev=22584&path=/trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp
==============================================================================
--- trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp (original)
+++ trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp Sun Jun 17 23:14:13 2018
@@ -26,10 +26,12 @@
#include "xtgscanner.h"
-#include <QString>
-#include <QList>
#include <QApplication>
#include <QDebug>
+#include <QList>
+#include <QString>
+#include <QTextCodec>
+#include <QTextDecoder>
#include "fonts/scface.h"
#include "scribusdoc.h"
@@ -47,10 +49,12 @@
newlineFlag(false),
xflag(false),
inDef(false),
+ m_bufferIndex(0),
top(0),
define(0),
m_isBold(false),
- m_isItalic(false)
+ m_isItalic(false),
+ m_decoder(nullptr)
{
doc = item->doc();
initTagMode();
@@ -59,9 +63,17 @@
initLanguages();
prevMode = textMode;
styleEffects = ScStyle_None;
- m_codec = QTextCodec::codecForName("cp1252");
- if (!m_codec)
- m_codec = QTextCodec::codecForLocale();
+
+ QTextCodec* codec = QTextCodec::codecForName("cp1252");
+ if (!codec)
+ codec = QTextCodec::codecForLocale();
+ m_decoder = new QTextDecoder(codec, QTextCodec::IgnoreHeader);
+}
+
+XtgScanner::~XtgScanner()
+{
+ if (m_decoder)
+ delete m_decoder;
}
bool XtgScanner::open(const QString& fileName)
@@ -69,10 +81,21 @@
bool forceUTF8 = false;
input_Buffer.clear();
+ decoded_text.clear();
+ if (m_decoder)
+ {
+ delete m_decoder;
+ m_decoder = nullptr;
+ }
+
newlineFlag = false;
xflag = false;
inDef = false;
+ m_bufferIndex = 0;
top = 0;
+ textToAppend.clear();
+ token.clear();
+ sfcName.clear();
define = 0;
m_isBold = false;
m_isItalic = false;
@@ -102,12 +125,16 @@
forceUTF8 = true;
}
}
- prevMode = textMode;
+ Mode = prevMode = textMode;
styleEffects = ScStyle_None;
- m_codec = QTextCodec::codecForName(forceUTF8 ? "UTF-8" : "cp1252");
- if (!m_codec)
- m_codec = QTextCodec::codecForLocale();
-
+
+ QTextCodec* codec = QTextCodec::codecForName(forceUTF8 ? "UTF-8" : "cp1252");
+ if (!codec)
+ codec = QTextCodec::codecForLocale();
+ m_decoder = new QTextDecoder(codec, QTextCodec::IgnoreHeader);
+
+ if (input_Buffer.size() > 0)
+ decoded_text.reserve(input_Buffer.size());
return (input_Buffer.size() > 0);
}
@@ -149,7 +176,6 @@
languages.insert(8,"Spanish");
languages.insert(7,"Swedish");
languages.insert(62,"Ukranian");
-
}
/** Initialise the tagModeHash with the tokens returned in tagMode and its corresponding function pointers */
@@ -908,11 +934,16 @@
encTest = "windows-949";
else if (enc == 20)
encTest = "KSC_5601";
- m_codec = QTextCodec::codecForName(encTest);
- if (!m_codec)
- m_codec = QTextCodec::codecForName("cp1252");
- if (!m_codec)
- m_codec = QTextCodec::codecForLocale();
+
+ QTextCodec* codec = QTextCodec::codecForName(encTest);
+ if (!codec)
+ codec = QTextCodec::codecForName("cp1252");
+ if (!codec)
+ codec = QTextCodec::codecForLocale();
+
+ if (m_decoder)
+ delete m_decoder;
+ m_decoder = new QTextDecoder(codec, QTextCodec::IgnoreHeader);
}
/** Functions corresponding to tokens in textMode
@@ -1203,7 +1234,7 @@
top++; // to ensure that ] is avoided
QString pStyle = CommonStrings::DefaultParagraphStyle;
ParagraphStyle newStyle;
- if (s1 != "")
+ if (s1.length() > 0)
{
if (doc->paragraphStyles().contains(m_item->itemName() + "_" + s1))
newStyle.setParent(m_item->itemName() + "_" + s1);
@@ -1219,7 +1250,7 @@
else
newStyle.setName((sfcName));
newStyle.setLineSpacingMode(ParagraphStyle::AutomaticLineSpacing);
- if (s3 != "")
+ if (s3.length() > 0)
{
if (doc->charStyles().contains(m_item->itemName() + "_" + s3))
newStyle.charStyle().setParent(m_item->itemName() + "_" + s3);
@@ -1441,12 +1472,12 @@
}
else
{ // find the name and return it as a tag
- while( 1 )
+ while (1)
{
temp = lookAhead();
if (temp == ':' || temp == '=' )
break;
- if (top >= input_Buffer.length())
+ if (m_bufferIndex >= input_Buffer.length())
break;
token.append(nextSymbol());
}
@@ -1470,54 +1501,62 @@
return prevMode;
}
+bool XtgScanner::decodeText(int index)
+{
+ if (decoded_text.length() > index)
+ return true;
+ if (m_bufferIndex >= input_Buffer.length())
+ return false;
+
+ QString txt;
+ const char* bufferData = input_Buffer.data() + m_bufferIndex;
+
+ while (m_bufferIndex < input_Buffer.length() &&
+ decoded_text.length() <= index)
+ {
+ txt = m_decoder->toUnicode(bufferData, 1);
+ if (!txt.isEmpty())
+ decoded_text += txt;
+ ++bufferData;
+ ++m_bufferIndex;
+ }
+
+ return (decoded_text.length() > index);
+}
+
QChar XtgScanner::lookAhead(int adj)
{
- char ch = 0;
- if (top < input_Buffer.length())
- {
- ch = input_Buffer.at(top + adj);
- QByteArray ba;
- ba.append(ch);
- QString m_txt = m_codec->toUnicode(ba);
- if (!m_txt.isEmpty())
- return m_txt.at(0);
- else
- return QChar(0);
- }
- return QChar(0);
+ if (!decodeText(top + adj))
+ return QChar(0);
+ if (top + adj >= decoded_text.length())
+ return QChar(0);
+ return decoded_text.at(top + adj);
}
QChar XtgScanner::nextSymbol()
{
- char ch = 0;
- if (top < input_Buffer.length())
- {
- ch = input_Buffer.at(top++);
- QByteArray ba;
- ba.append(ch);
- QString m_txt = m_codec->toUnicode(ba);
- if (!m_txt.isEmpty())
- return m_txt.at(0);
- else
- return QChar(0);
- }
- return QChar(0);
+ if (!decodeText(top))
+ return QChar(0);
+ if (top >= decoded_text.length())
+ return QChar(0);
+ QChar newChar = decoded_text.at(top);
+ ++top;
+ return newChar;
}
void XtgScanner::flushText()
{
- if (!textToAppend.isEmpty())
- {
- textToAppend.replace(QChar(10), SpecialChars::LINEBREAK);
- textToAppend.replace(QChar(12), SpecialChars::FRAMEBREAK);
- textToAppend.replace(QChar(30), SpecialChars::NBHYPHEN);
- textToAppend.replace(QChar(160), SpecialChars::NBSPACE);
- int posC = m_item->itemText.length();
- m_item->itemText.insertChars(posC, textToAppend);
- m_item->itemText.applyStyle(posC, currentParagraphStyle);
- m_item->itemText.applyCharStyle(posC, textToAppend.length(), currentCharStyle);
- textToAppend="";
- }
+ if (textToAppend.isEmpty())
+ return;
+ textToAppend.replace(QChar(10), SpecialChars::LINEBREAK);
+ textToAppend.replace(QChar(12), SpecialChars::FRAMEBREAK);
+ textToAppend.replace(QChar(30), SpecialChars::NBHYPHEN);
+ textToAppend.replace(QChar(160), SpecialChars::NBSPACE);
+ int posC = m_item->itemText.length();
+ m_item->itemText.insertChars(posC, textToAppend);
+ m_item->itemText.applyStyle(posC, currentParagraphStyle);
+ m_item->itemText.applyCharStyle(posC, textToAppend.length(), currentCharStyle);
+ textToAppend.clear();
}
bool XtgScanner::styleStatus(QStringList &name,QString &sfcname)
@@ -1599,7 +1638,7 @@
// }
// currentCharStyle = writer->getCurrentCharStyle();
}
- if (top >= input_Buffer.length())
+ if (m_bufferIndex >= input_Buffer.length())
break;
}
if (!textToAppend.isEmpty())
@@ -1617,7 +1656,3 @@
}
// qDebug()<<"Unsupported : "<<unSupported;
}
-
-XtgScanner::~XtgScanner()
-{
-}
Modified: trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h
URL: http://scribus.net/websvn/diff.php?repname=Scribus&rev=22584&path=/trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h
==============================================================================
--- trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h (original)
+++ trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h Sun Jun 17 23:14:13 2018
@@ -36,7 +36,9 @@
#include <QHash>
#include <QString>
#include <QSet>
-#include <QTextCodec>
+
+class QTextCodec;
+class QTextDecoder;
/**
Enum datatype for determining the Scanner mode
@@ -76,9 +78,12 @@
bool xflag;
bool inDef;
/**
- \variable Input Buffer to which properly encoded file is loaded
+ \variable Input buffer to which properly encoded file is loaded
*/
QByteArray input_Buffer;
+ int m_bufferIndex;
+
+ QString decoded_text;
int top;
ScribusDoc* doc;
@@ -113,7 +118,12 @@
bool m_isBold;
bool m_isItalic;
- QTextCodec *m_codec;
+ QTextDecoder *m_decoder;
+
+ /**
+ \brief Decode text from input buffer until specified index
+ */
+ bool decodeText(int index);
public:
XtgScanner(PageItem* item, bool textOnly, bool prefix, bool append);
More information about the scribus-commit
mailing list