r22583 by jghali - xtg importer: improve handling of UTF-16 encoded files

scribus-commit scribus-commit at lists.scribus.net
Sun Jun 17 02:05:57 UTC 2018


Author: jghali
Date: Sun Jun 17 02:05:56 2018
New Revision: 22583

URL: http://scribus.net/websvn/listing.php?repname=Scribus&sc=1&rev=22583
Log:
xtg importer: improve handling of UTF-16 encoded files

Modified:
    trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.cpp
    trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.h
    trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp
    trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h

Modified: trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.cpp
URL: http://scribus.net/websvn/diff.php?repname=Scribus&rev=22583&path=/trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.cpp
==============================================================================
--- trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.cpp	(original)
+++ trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.cpp	Sun Jun 17 02:05:56 2018
@@ -29,7 +29,7 @@
 
 QString FileFormatName()
 {
-    return QObject::tr("Quark XPress Tags Document");
+	return QObject::tr("Quark XPress Tags Document");
 }
 
 QStringList FileExtensions()
@@ -39,19 +39,30 @@
 
 void GetText2(QString filename, QString encoding, bool textOnly, bool prefix, bool append, PageItem *textItem)
 {
-	XtgIm* xtgim = new XtgIm(filename, textItem, textOnly, prefix, append);
+	XtgIm* xtgim = new XtgIm(textItem, textOnly, prefix, append);
+	xtgim->import(filename);
 	delete xtgim;
 }
 
 /********************************Class XtgIm***********************************/
 
-XtgIm::XtgIm(QString fileName, PageItem *textItem, bool textOnly, bool prefix, bool append)
+XtgIm::XtgIm(PageItem *textItem, bool textOnly, bool prefix, bool append)
 {
-	XtgScanner *scanner = new XtgScanner(fileName, textItem, textOnly, prefix, append);
-	scanner->xtgParse();
-	delete scanner;
+	m_scanner = new XtgScanner(textItem, textOnly, prefix, append);
 }
 
 XtgIm::~XtgIm()
 {
+	if (m_scanner)
+		delete m_scanner;
 }
+
+bool XtgIm::import(QString fileName)
+{
+	if (!m_scanner)
+		return false;
+	if (!m_scanner->open(fileName))
+		return false;
+	m_scanner->xtgParse();
+	return true;
+}

Modified: trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.h
URL: http://scribus.net/websvn/diff.php?repname=Scribus&rev=22583&path=/trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.h
==============================================================================
--- trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.h	(original)
+++ trunk/Scribus/scribus/plugins/gettext/xtgim/xtgim.h	Sun Jun 17 02:05:56 2018
@@ -34,6 +34,7 @@
 #include <QString>
 
 class StyleManager;
+class XtgScanner;
 
 extern "C" PLUGIN_API void GetText2(QString filename, QString encoding, bool textOnly, bool prefix, bool append, PageItem *textItem);
 extern "C" PLUGIN_API QString FileFormatName();
@@ -42,7 +43,12 @@
 class XtgIm
 {
 public:
-	XtgIm(QString fileName, PageItem *textItem, bool textOnly, bool prefix, bool append);
+	XtgIm(PageItem *textItem, bool textOnly, bool prefix, bool append);
 	~XtgIm();
+
+	bool import(QString fileName);
+
+protected:
+	XtgScanner* m_scanner;
 };
 #endif			/* XTGIM_H */

Modified: trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp
URL: http://scribus.net/websvn/diff.php?repname=Scribus&rev=22583&path=/trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp
==============================================================================
--- trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp	(original)
+++ trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.cpp	Sun Jun 17 02:05:56 2018
@@ -39,29 +39,19 @@
 #include "util.h"
 
 
-XtgScanner::XtgScanner(QString filename, PageItem *item, bool textOnly, bool prefix, bool append)
-	: m_item(item),
-	  importTextOnly(textOnly),
-	  m_prefixName(prefix),
-	  m_append(append),
-	  newlineFlag(false),
-	  xflag(false),
-	  inDef(false),
-	  top(0),
-	  define(0),
-	  m_isBold(false),
-	  m_isItalic(false)
-{
-	loadRawBytes(filename, input_Buffer);
-	if ((input_Buffer[0] == '\xFF') && (input_Buffer[1] == '\xFE'))
-	{
-		QByteArray tmpBuf;
-		for (int a = 2; a < input_Buffer.count(); a += 2)
-		{
-			tmpBuf.append(input_Buffer[a]);
-		}
-		input_Buffer = tmpBuf;
-	}
+XtgScanner::XtgScanner(PageItem *item, bool textOnly, bool prefix, bool append)
+    : m_item(item),
+    importTextOnly(textOnly),
+    m_prefixName(prefix),
+    m_append(append),
+    newlineFlag(false),
+    xflag(false),
+    inDef(false),
+    top(0),
+    define(0),
+    m_isBold(false),
+    m_isItalic(false)
+{
 	doc = item->doc();
 	initTagMode();
 	initTextMode();
@@ -72,6 +62,53 @@
 	m_codec = QTextCodec::codecForName("cp1252");
 	if (!m_codec)
 		m_codec = QTextCodec::codecForLocale();
+}
+
+bool XtgScanner::open(const QString& fileName)
+{
+	bool forceUTF8 = false;
+
+	input_Buffer.clear();
+	newlineFlag = false;
+	xflag = false;
+	inDef = false;
+	top = 0;
+	define = 0;
+	m_isBold = false;
+	m_isItalic = false;
+
+	if (!loadRawBytes(fileName, input_Buffer))
+		return false;
+	if (input_Buffer.size() >= 2)
+	{
+		if ((input_Buffer[0] == '\xFF') && (input_Buffer[1] == '\xFE'))
+		{
+			QTextCodec* utf8Codec  = QTextCodec::codecForName("UTF-8");
+			QTextCodec* utf16Codec = QTextCodec::codecForName("UTF-16LE");
+			if (!utf8Codec || !utf16Codec)
+				return false;
+			QString text = utf16Codec->toUnicode(input_Buffer.constData(), input_Buffer.length());
+			input_Buffer = utf8Codec->fromUnicode(text);
+			forceUTF8 = true;
+		}
+		else if ((input_Buffer[0] == '\xFE') && (input_Buffer[1] == '\xFF'))
+		{
+			QTextCodec* utf8Codec = QTextCodec::codecForName("UTF-8");
+			QTextCodec* utf16Codec = QTextCodec::codecForName("UTF-16BE");
+			if (!utf8Codec || !utf16Codec)
+				return false;
+			QString text = utf16Codec->toUnicode(input_Buffer.constData(), input_Buffer.length());
+			input_Buffer = utf8Codec->fromUnicode(text);
+			forceUTF8  = true;
+		}
+	}
+	prevMode = textMode;
+	styleEffects = ScStyle_None;
+	m_codec = QTextCodec::codecForName(forceUTF8 ? "UTF-8" : "cp1252");
+	if (!m_codec)
+		m_codec = QTextCodec::codecForLocale();
+
+	return (input_Buffer.size() > 0);
 }
 
 /** Initialise a QHash which maps the values of n## to corresponding language strings

Modified: trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h
URL: http://scribus.net/websvn/diff.php?repname=Scribus&rev=22583&path=/trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h
==============================================================================
--- trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h	(original)
+++ trunk/Scribus/scribus/plugins/gettext/xtgim/xtgscanner.h	Sun Jun 17 02:05:56 2018
@@ -38,7 +38,6 @@
 #include <QSet>
 #include <QTextCodec>
 
-
 /**
 Enum datatype for determining the Scanner mode 
 */
@@ -111,12 +110,13 @@
 		2	Paragraph Stylesheet Definition
 	 */
 	int define;
-	QTextCodec *m_codec;
 	bool m_isBold;
 	bool m_isItalic;
+
+	QTextCodec *m_codec;
 	
 public:
-	XtgScanner(QString filename, PageItem* item, bool textOnly, bool prefix, bool append);
+	XtgScanner(PageItem* item, bool textOnly, bool prefix, bool append);
 	~XtgScanner();
 
 	/**
@@ -128,9 +128,15 @@
 	void initNameMode();
 	void initLanguages();
 	/**
-	\brief parse function which will parse the inputBuffer and append it into the PageItem
+	\brief Open file and initialize inputBuffer for parsing
+	*/
+	bool open(const QString& fileName);
+
+	/**
+	\brief Parse function which will parse the inputBuffer and append it into the PageItem
 	*/
 	void xtgParse();
+
 	/**
 	\brief This function will return the character in Buffer to which top is now pointing to 
 	*/




More information about the scribus-commit mailing list