Import data from mtgjson's allsets.xml

* add multiverse id to cards.xml
* remove pictures urls from cards.xml
TBD:
* fix/clean oracle’s gui
* add support in cockatrice

cockatrice/src/carddatabase.cpp

@@ -406,12 +406,27 @@ QXmlStreamWriter &operator<<(QXmlStreamWriter &xml, const CardInfo *info)
     xml.writeTextElement("name", info->getName());
 
     const SetList &sets = info->getSets();
+    QString tmpString;
+    QString tmpSet;
     for (int i = 0; i < sets.size(); i++) {
         xml.writeStartElement("set");
-        xml.writeAttribute("picURL", info->getPicURL(sets[i]->getShortName()));
-        xml.writeAttribute("picURLHq", info->getPicURLHq(sets[i]->getShortName()));
-        xml.writeAttribute("picURLSt", info->getPicURLSt(sets[i]->getShortName()));
-        xml.writeCharacters(sets[i]->getShortName());
+
+        tmpSet=sets[i]->getShortName();
+        xml.writeAttribute("muId", QString::number(info->getMuId(tmpSet)));
+
+        tmpString = info->getPicURL(tmpSet);
+        if(!tmpString.isEmpty())
+            xml.writeAttribute("picURL", tmpString);
+
+        tmpString = info->getPicURLHq(tmpSet);
+        if(!tmpString.isEmpty())
+            xml.writeAttribute("picURLHq", tmpString);
+
+        tmpString = info->getPicURLSt(tmpSet);
+        if(!tmpString.isEmpty())
+            xml.writeAttribute("picURLSt", tmpString);
+
+        xml.writeCharacters(tmpSet);
         xml.writeEndElement();
     }
     const QStringList &colors = info->getColors();

cockatrice/src/carddatabase.h

@@ -100,6 +100,7 @@ private:
     QStringList colors;
     int loyalty;
     QMap<QString, QString> picURLs, picURLsHq, picURLsSt;
+    QMap<QString, int> muIds;
     bool cipt;
     int tableRow;
     QPixmap *pixmap;
@@ -139,6 +140,7 @@ public:
     QString getPicURL(const QString &set) const { return picURLs.value(set); }
     QString getPicURLHq(const QString &set) const { return picURLsHq.value(set); }
     QString getPicURLSt(const QString &set) const { return picURLsSt.value(set); }
+    int getMuId(const QString &set) const { return muIds.value(set); }
     QString getPicURL() const;
     const QMap<QString, QString> &getPicURLs() const { return picURLs; }
     QString getMainCardType() const;
@@ -149,6 +151,7 @@ public:
     void setPicURL(const QString &_set, const QString &_picURL) { picURLs.insert(_set, _picURL); }
     void setPicURLHq(const QString &_set, const QString &_picURL) { picURLsHq.insert(_set, _picURL); }
     void setPicURLSt(const QString &_set, const QString &_picURL) { picURLsSt.insert(_set, _picURL); }
+    void setMuId(const QString &_set, const int &_muId) { muIds.insert(_set, _muId); }
     void addToSet(CardSet *set);
     QPixmap *loadPixmap();
     QPixmap *getPixmap(QSize size);

oracle/src/oracleimporter.cpp

 #include "oracleimporter.h"
 #include <QtGui>
-#include <QtNetwork>
-#include <QXmlStreamReader>
-#include <QDomDocument>
 #include <QDebug>
 
 #include "qt-json/json.h"
 
 OracleImporter::OracleImporter(const QString &_dataDir, QObject *parent)
-	: CardDatabase(parent), dataDir(_dataDir), setIndex(-1)
+    : CardDatabase(parent), dataDir(_dataDir)
 {
-	buffer = new QBuffer(this);
-	http = new QHttp(this);
-	connect(http, SIGNAL(requestFinished(int, bool)), this, SLOT(httpRequestFinished(int, bool)));
-	connect(http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)), this, SLOT(readResponseHeader(const QHttpResponseHeader &)));
-	connect(http, SIGNAL(dataReadProgress(int, int)), this, SIGNAL(dataReadProgress(int, int)));
 }
 
 bool OracleImporter::readSetsFromFile(const QString &fileName)
 {
-	QFile setsFile(fileName);
-	if (!setsFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
-		QMessageBox::critical(0, tr("Error"), tr("Cannot open file '%1'.").arg(fileName));
-		return false;
-	}
+    QFile setsFile(fileName);
+    if (!setsFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
+        QMessageBox::critical(0, tr("Error"), tr("Cannot open file '%1'.").arg(fileName));
+        return false;
+    }
 
-	QXmlStreamReader xml(&setsFile);
-	return readSetsFromXml(xml);
+    return readSetsFromByteArray(setsFile.readAll());
 }
 
 bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
 {
-	QXmlStreamReader xml(data);
-	return readSetsFromXml(xml);
-}
+    QList<SetToDownload> newSetList;
+    
+    bool ok;
+    setsMap = QtJson::Json::parse(QString(data), ok).toMap();
+    if (!ok) {
+        qDebug() << "error: QtJson::Json::parse()";
+        return 0;
+    }
+    
+    QListIterator<QVariant> it(setsMap.values());
+    QVariantMap map;
 
-bool OracleImporter::readSetsFromXml(QXmlStreamReader &xml)
-{
-	QList<SetToDownload> newSetList;
-	
-	QString edition;
-	QString editionLong;
-	QString editionURL;
-	while (!xml.atEnd()) {
-		if (xml.readNext() == QXmlStreamReader::EndElement)
-			break;
-		if (xml.name() == "set") {
-			bool import = xml.attributes().value("import").toString().toInt();
-			while (!xml.atEnd()) {
-				if (xml.readNext() == QXmlStreamReader::EndElement)
-					break;
-				if (xml.name() == "name")
-					edition = xml.readElementText();
-				else if (xml.name() == "longname")
-					editionLong = xml.readElementText();
-				else if (xml.name() == "url")
-					editionURL = xml.readElementText();
-			}
-			newSetList.append(SetToDownload(edition, editionLong, editionURL, import));
-			edition = editionLong = editionURL = QString();
-		} else if (xml.name() == "picture_url")
-			pictureUrl = xml.readElementText();
-		else if (xml.name() == "picture_url_hq")
-			pictureUrlHq = xml.readElementText();
-		else if (xml.name() == "picture_url_st")
-			pictureUrlSt = xml.readElementText();
-		else if (xml.name() == "set_url")
-			setUrl = xml.readElementText();
-	}
-	if (newSetList.isEmpty())
-		return false;
-	allSets = newSetList;
-	return true;
+    QString edition;
+    QString editionLong;
+    QVariant editionCards;
+    bool import;
+
+    while (it.hasNext()) {
+        map = it.next().toMap();
+        edition = map.value("code").toString();
+        editionLong = map.value("name").toString();
+        editionCards = map.value("cards");
+
+        // core and expansion sets are marked to be imported by default
+        import = (0 == QString::compare(map.value("type").toString(), QString("core"), Qt::CaseInsensitive) ||
+            0 == QString::compare(map.value("type").toString(), QString("expansion"), Qt::CaseInsensitive));
+
+        newSetList.append(SetToDownload(edition, editionLong, editionCards, import));
+    }
+    if (newSetList.isEmpty())
+        return false;
+    allSets = newSetList;
+    return true;
 }
 
 CardInfo *OracleImporter::addCard(const QString &setName,
-								  QString cardName,
-								  bool isToken,
-								  int cardId,
-								  const QString &cardCost,
-								  const QString &cardType,
-								  const QString &cardPT,
-								  int cardLoyalty,
-								  const QStringList &cardText)
+                                  QString cardName,
+                                  bool isToken,
+                                  int cardId,
+                                  const QString &cardCost,
+                                  const QString &cardType,
+                                  const QString &cardPT,
+                                  int cardLoyalty,
+                                  const QStringList &cardText)
 {
-	QString fullCardText = cardText.join("\n");
-	bool splitCard = false;
-	if (cardName.contains('(')) {
-		cardName.remove(QRegExp(" \\(.*\\)"));
-		splitCard = true;
-	}
-	// Workaround for card name weirdness
-	if (cardName.contains("XX"))
-		cardName.remove("XX");
-	cardName = cardName.replace("Æ", "AE");
+    QString fullCardText = cardText.join("\n");
+    bool splitCard = false;
+    if (cardName.contains('(')) {
+        cardName.remove(QRegExp(" \\(.*\\)"));
+        splitCard = true;
+    }
+    // Workaround for card name weirdness
+    if (cardName.contains("XX"))
+        cardName.remove("XX");
+    cardName = cardName.replace("Æ", "AE");
         cardName = cardName.replace("’", "'");
 
-	CardInfo *card;
-	if (cardHash.contains(cardName)) {
-		card = cardHash.value(cardName);
-		if (splitCard && !card->getText().contains(fullCardText))
-			card->setText(card->getText() + "\n---\n" + fullCardText);
-	} else {
-		bool mArtifact = false;
-		if (cardType.endsWith("Artifact"))
-			for (int i = 0; i < cardText.size(); ++i)
-				if (cardText[i].contains("{T}") && cardText[i].contains("to your mana pool"))
-					mArtifact = true;
-					
-		QStringList colors;
-		QStringList allColors = QStringList() << "W" << "U" << "B" << "R" << "G";
-		for (int i = 0; i < allColors.size(); i++)
-			if (cardCost.contains(allColors[i]))
-				colors << allColors[i];
-		
-		if (cardText.contains(cardName + " is white."))
-			colors << "W";
-		if (cardText.contains(cardName + " is blue."))
-			colors << "U";
-		if (cardText.contains(cardName + " is black."))
-			colors << "B";
-		if (cardText.contains(cardName + " is red."))
-			colors << "R";
-		if (cardText.contains(cardName + " is green."))
-			colors << "G";
-		
-		bool cipt = (cardText.contains(cardName + " enters the battlefield tapped."));
-		
-		card = new CardInfo(this, cardName, isToken, cardCost, cardType, cardPT, fullCardText, colors, cardLoyalty, cipt);
-		int tableRow = 1;
-		QString mainCardType = card->getMainCardType();
-		if ((mainCardType == "Land") || mArtifact)
-			tableRow = 0;
-		else if ((mainCardType == "Sorcery") || (mainCardType == "Instant"))
-			tableRow = 3;
-		else if (mainCardType == "Creature")
-			tableRow = 2;
-		card->setTableRow(tableRow);
-		
-		cardHash.insert(cardName, card);
-	}
-	card->setPicURL(setName, getPictureUrl(pictureUrl, cardId, cardName, setName));
-	card->setPicURLHq(setName, getPictureUrl(pictureUrlHq, cardId, cardName, setName));
-	card->setPicURLSt(setName, getPictureUrl(pictureUrlSt, cardId, cardName, setName));
-	return card;
+    CardInfo *card;
+    if (cardHash.contains(cardName)) {
+        card = cardHash.value(cardName);
+        if (splitCard && !card->getText().contains(fullCardText))
+            card->setText(card->getText() + "\n---\n" + fullCardText);
+    } else {
+        bool mArtifact = false;
+        if (cardType.endsWith("Artifact"))
+            for (int i = 0; i < cardText.size(); ++i)
+                if (cardText[i].contains("{T}") && cardText[i].contains("to your mana pool"))
+                    mArtifact = true;
+                    
+        QStringList colors;
+        QStringList allColors = QStringList() << "W" << "U" << "B" << "R" << "G";
+        for (int i = 0; i < allColors.size(); i++)
+            if (cardCost.contains(allColors[i]))
+                colors << allColors[i];
+        
+        if (cardText.contains(cardName + " is white."))
+            colors << "W";
+        if (cardText.contains(cardName + " is blue."))
+            colors << "U";
+        if (cardText.contains(cardName + " is black."))
+            colors << "B";
+        if (cardText.contains(cardName + " is red."))
+            colors << "R";
+        if (cardText.contains(cardName + " is green."))
+            colors << "G";
+        
+        bool cipt = (cardText.contains(cardName + " enters the battlefield tapped."));
+        
+        card = new CardInfo(this, cardName, isToken, cardCost, cardType, cardPT, fullCardText, colors, cardLoyalty, cipt);
+        int tableRow = 1;
+        QString mainCardType = card->getMainCardType();
+        if ((mainCardType == "Land") || mArtifact)
+            tableRow = 0;
+        else if ((mainCardType == "Sorcery") || (mainCardType == "Instant"))
+            tableRow = 3;
+        else if (mainCardType == "Creature")
+            tableRow = 2;
+        card->setTableRow(tableRow);
+        
+        cardHash.insert(cardName, card);
+    }
+    card->setMuId(setName, cardId);
+
+    return card;
 }
 
-int OracleImporter::importTextSpoiler(CardSet *set, const QByteArray &data)
+int OracleImporter::importTextSpoiler(CardSet *set, const QVariant &data)
 {
     int cards = 0;
-    bool ok;
-    QVariantMap resultMap = QtJson::Json::parse(QString(data), ok).toMap();
-    if (!ok) {
-        qDebug() << "error: QtJson::Json::parse()";
-        return 0;
-    }
     
-    QListIterator<QVariant> it(resultMap.value("cards").toList());
+    QListIterator<QVariant> it(data.toList());
     QVariantMap map;
     QString cardName;
     QString cardCost;
@@ -237,111 +213,52 @@ int OracleImporter::importTextSpoiler(CardSet *set, const QByteArray &data)
     
     return cards;
 }
-
+/*
 QString OracleImporter::getPictureUrl(QString url, int cardId, QString name, const QString &setName) const
 {
-	if ((name == "Island") || (name == "Swamp") || (name == "Mountain") || (name == "Plains") || (name == "Forest"))
-		name.append("1");
-	return url.replace("!cardid!", QString::number(cardId)).replace("!set!", setName).replace("!name!", name
-		.replace("ö", "o")
-//		.remove('\'')
-		.remove(" // ")
-//		.remove(',')
-//		.remove(':')
-//		.remove('.')
-		.remove(QRegExp("\\(.*\\)"))
-		.simplified()
-//		.replace(' ', '_')
-//		.replace('-', '_')
-	);
+    if ((name == "Island") || (name == "Swamp") || (name == "Mountain") || (name == "Plains") || (name == "Forest"))
+        name.append("1");
+    return url.replace("!cardid!", QString::number(cardId)).replace("!set!", setName).replace("!name!", name
+        .replace("ö", "o")
+//      .remove('\'')
+        .remove(" // ")
+//      .remove(',')
+//      .remove(':')
+//      .remove('.')
+        .remove(QRegExp("\\(.*\\)"))
+        .simplified()
+//      .replace(' ', '_')
+//      .replace('-', '_')
+    );
 }
-
-int OracleImporter::startDownload()
+*/
+int OracleImporter::startImport()
 {
-	clear();
-	
-	setsToDownload.clear();
-	for (int i = 0; i < allSets.size(); ++i)
-		if (allSets[i].getImport())
-			setsToDownload.append(allSets[i]);
-	
-	if (setsToDownload.isEmpty())
-		return 0;
-	setIndex = 0;
-	emit setIndexChanged(0, 0, setsToDownload[0].getLongName());
-	
-	downloadNextFile();
-	return setsToDownload.size();
-}
+    clear();
 
-void OracleImporter::downloadNextFile()
-{
-	QString urlString = setsToDownload[setIndex].getUrl();
-	if (urlString.isEmpty())
-		urlString = setUrl;
-	urlString = urlString.replace("!name!", setsToDownload[setIndex].getShortName());
+    int setCards = 0, setIndex= 0;
+    QListIterator<SetToDownload> it(allSets);
+    const SetToDownload * curSet;
 
-	if (urlString.startsWith("http://")) {
-		QUrl url(urlString);
-		http->setHost(url.host(), QHttp::ConnectionModeHttp, url.port() == -1 ? 0 : url.port());
-		QString path = QUrl::toPercentEncoding(urlString.mid(url.host().size() + 7).replace(' ', '+'), "?!$&'()*+,;=:@/");
-		
-		buffer->close();
-		buffer->setData(QByteArray());
-		buffer->open(QIODevice::ReadWrite | QIODevice::Text);
-		reqId = http->get(path, buffer);
-	} else {
-		QFile file(dataDir + "/" + urlString);
-		file.open(QIODevice::ReadOnly | QIODevice::Text);
-		
-		buffer->close();
-		buffer->setData(file.readAll());
-		buffer->open(QIODevice::ReadWrite | QIODevice::Text);
-		reqId = 0;
-		httpRequestFinished(reqId, false);
-	}
-}
+    while (it.hasNext())
+    {
+        curSet = & it.next();
 
-void OracleImporter::httpRequestFinished(int requestId, bool error)
-{
-	if (error) {
-		QMessageBox::information(0, tr("HTTP"), tr("Error."));
-		return;
-	}
-	if (requestId != reqId)
-		return;
+        emit setIndexChanged(0, 0, curSet->getLongName());
+            
+        CardSet *set = new CardSet(curSet->getShortName(), curSet->getLongName());
+        if (!setHash.contains(set->getShortName()))
+            setHash.insert(set->getShortName(), set);
 
-	CardSet *set = new CardSet(setsToDownload[setIndex].getShortName(), setsToDownload[setIndex].getLongName());
-	if (!setHash.contains(set->getShortName()))
-		setHash.insert(set->getShortName(), set);
-	
-	buffer->seek(0);
-	buffer->close();
-	int cards = importTextSpoiler(set, buffer->data());
-        if (cards > 0)
-            ++setIndex;
-	
-	if (setIndex == setsToDownload.size()) {
-		emit setIndexChanged(cards, setIndex, QString());
-		setIndex = -1;
-	} else {
-		downloadNextFile();
-		emit setIndexChanged(cards, setIndex, setsToDownload[setIndex].getLongName());
-	}
-}
+        int setCards = importTextSpoiler(set, curSet->getCards());
 
-void OracleImporter::readResponseHeader(const QHttpResponseHeader &responseHeader)
-{
-	switch (responseHeader.statusCode()) {
-		case 200:
-		case 301:
-		case 302:
-		case 303:
-		case 307:
-			break;
-		default:
-			QMessageBox::information(0, tr("HTTP"), tr("Download failed: %1.").arg(responseHeader.reasonPhrase()));
-			http->abort();
-			deleteLater();
-	}
+        ++setIndex;
+            
+        emit setIndexChanged(setCards, setIndex, curSet->getLongName());
+    }
+    
+    emit setIndexChanged(setCards, setIndex, QString());
+
+    // total number of sets
+    return setIndex;
 }

oracle/src/oracleimporter.h

 #ifndef ORACLEIMPORTER_H
 #define ORACLEIMPORTER_H
 
-#include <carddatabase.h>
-#include <QHttp>
+#include <QMap>
 
-class QBuffer;
-class QXmlStreamReader;
+#include <carddatabase.h>
 
 class SetToDownload {
 private:
-	QString shortName, longName, url;
+	QString shortName, longName;
 	bool import;
+	QVariant cards;
 public:
 	const QString &getShortName() const { return shortName; }
 	const QString &getLongName() const { return longName; }
-	const QString &getUrl() const { return url; }
+	const QVariant &getCards() const { return cards; }
 	bool getImport() const { return import; }
 	void setImport(bool _import) { import = _import; }
-	SetToDownload(const QString &_shortName, const QString &_longName, const QString &_url, bool _import)
-		: shortName(_shortName), longName(_longName), url(_url), import(_import) { }
+	SetToDownload(const QString &_shortName, const QString &_longName, const QVariant &_cards, bool _import)
+		: shortName(_shortName), longName(_longName), cards(_cards), import(_import) { }
 };
 
 class OracleImporter : public CardDatabase {
 	Q_OBJECT
 private:
-	QList<SetToDownload> allSets, setsToDownload;
-	QString pictureUrl, pictureUrlHq, pictureUrlSt, setUrl;
+	QList<SetToDownload> allSets;
+	QVariantMap setsMap;
 	QString dataDir;
-	int setIndex;
-	int reqId;
-	QBuffer *buffer;
-	QHttp *http;
-	QString getPictureUrl(QString url, int cardId, QString name, const QString &setName) const;
 	
 	void downloadNextFile();
-	bool readSetsFromXml(QXmlStreamReader &xml);
 	CardInfo *addCard(const QString &setName, QString cardName, bool isToken, int cardId, const QString &cardCost, const QString &cardType, const QString &cardPT, int cardLoyalty, const QStringList &cardText);
-private slots:
-	void httpRequestFinished(int requestId, bool error);
-	void readResponseHeader(const QHttpResponseHeader &responseHeader);
 signals:
 	void setIndexChanged(int cardsImported, int setIndex, const QString &nextSetName);
 	void dataReadProgress(int bytesRead, int totalBytes);
@@ -46,8 +36,8 @@ public:
 	OracleImporter(const QString &_dataDir, QObject *parent = 0);
 	bool readSetsFromByteArray(const QByteArray &data);
 	bool readSetsFromFile(const QString &fileName);
-	int startDownload();
-	int importTextSpoiler(CardSet *set, const QByteArray &data);
+	int startImport();
+	int importTextSpoiler(CardSet *set, const QVariant &data);
 	QList<SetToDownload> &getSets() { return allSets; }
 	const QString &getDataDir() const { return dataDir; }
 };

oracle/src/window_main.cpp

@@ -19,7 +19,7 @@
 #include "window_main.h"
 #include "oracleimporter.h"
 
-const QString WindowMain::defaultSetsUrl = QString("http://www.woogerworks.com/files/sets.xml");
+const QString WindowMain::defaultSetsUrl = QString("http://mtgjson.com/json/AllSets.json");
 
 WindowMain::WindowMain(QWidget *parent)
 	: QMainWindow(parent)
@@ -45,7 +45,7 @@ WindowMain::WindowMain(QWidget *parent)
 	checkAllButtonLayout->addWidget(checkAllButton);
 	checkAllButtonLayout->addWidget(uncheckAllButton);
 	
-	startButton = new QPushButton(tr("&Start download"));
+	startButton = new QPushButton(tr("&Start import"));
 	connect(startButton, SIGNAL(clicked()), this, SLOT(actStart()));
 	
 	QVBoxLayout *settingsLayout = new QVBoxLayout;
@@ -130,7 +130,7 @@ void WindowMain::actLoadSetsFile()
 {
 	QFileDialog dialog(this, tr("Load sets file"));
 	dialog.setFileMode(QFileDialog::ExistingFile);
-	dialog.setNameFilter("Sets XML file (*.xml)");
+	dialog.setNameFilter("Sets JSON file (*.json)");
 	if (!dialog.exec())
 		return;
 
@@ -224,7 +224,7 @@ void WindowMain::actUncheckAll()
 
 void WindowMain::actStart()
 {
-	int setsCount = importer->startDownload();
+	int setsCount = importer->startImport();
 	if (!setsCount) {
 		QMessageBox::critical(this, tr("Error"), tr("No sets to download selected."));
 		return;