From d38b857e51292ae74f8c60faa5ec1a86f917a5a3 Mon Sep 17 00:00:00 2001 From: Bartosz Szatkowski Date: Tue, 24 Aug 2010 12:47:21 +0200 Subject: [PATCH] Draft of normalization routine --- trunk/src/plugins/xdxf/src/XdxfCachingDialog.cpp | 5 +- trunk/src/plugins/xdxf/src/xdxfplugin.cpp | 72 +++++++++++++++------- trunk/src/plugins/xdxf/src/xdxfplugin.h | 2 +- 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/trunk/src/plugins/xdxf/src/XdxfCachingDialog.cpp b/trunk/src/plugins/xdxf/src/XdxfCachingDialog.cpp index c3b2cfd..48e6c11 100644 --- a/trunk/src/plugins/xdxf/src/XdxfCachingDialog.cpp +++ b/trunk/src/plugins/xdxf/src/XdxfCachingDialog.cpp @@ -29,9 +29,9 @@ XdxfCachingDialog::XdxfCachingDialog(XdxfPlugin *parent) : - QDialog(0) + QDialog((QWidget*)parent) { - verticalLayout = new QVBoxLayout; + verticalLayout = new QVBoxLayout(this); setLayout(verticalLayout); setWindowTitle(tr("Caching dictionary, please wait")); @@ -61,6 +61,7 @@ XdxfCachingDialog::XdxfCachingDialog(XdxfPlugin *parent) : } void XdxfCachingDialog::updateCachingProgress(int progress, int time) { + qDebug() << "INININININININI"; cachingProgressBar->setValue(progress); if(!cachingLabel->isVisible()) diff --git a/trunk/src/plugins/xdxf/src/xdxfplugin.cpp b/trunk/src/plugins/xdxf/src/xdxfplugin.cpp index e99f6ba..518031c 100644 --- a/trunk/src/plugins/xdxf/src/xdxfplugin.cpp +++ b/trunk/src/plugins/xdxf/src/xdxfplugin.cpp @@ -31,12 +31,15 @@ XdxfPlugin::XdxfPlugin(QObject *parent) : CommonDictInterface(parent), _type(tr("xdxf")), _infoNote(tr("")) { _wordsCount = -1; _settings = new Settings(); - _dictDialog = new XdxfDictDialog(this, this); - cachingDialog = new XdxfCachingDialog(this); + _dictDialog = new XdxfDictDialog(this); + cachingDialog = new XdxfCachingDialog(); connect(cachingDialog, SIGNAL(cancelCaching()), this, SLOT(stop())); + connect(this, SIGNAL(updateCachingProgress(int,int)), + cachingDialog, SLOT(updateCachingProgress(int,int))); + _settings->setValue("type","xdxf"); stopped = false; @@ -100,7 +103,6 @@ QList XdxfPlugin::searchWordListCache(QString word, int limit) { word = word.toLower(); word = word.replace("*", "%"); word = word.replace("?", "_"); - word = removeAccents(word); QSqlQuery cur(db); if(limit !=0) @@ -111,7 +113,9 @@ QList XdxfPlugin::searchWordListCache(QString word, int limit) { if(limit !=0) cur.addBindValue(limit); cur.exec(); + bool in = false; while(cur.next()){ + in = true; bool ok=true; Translation *tran; foreach(tran,translations) { @@ -123,16 +127,42 @@ QList XdxfPlugin::searchWordListCache(QString word, int limit) { cur.value(0).toString().toLower(), _infoNote, this)); } + if(!in) { + + QSqlQuery cur(db); + if(limit !=0) + cur.prepare("select word from dict where normalized like ? limit ?"); + else + cur.prepare("select word from dict where normalized like ?"); + cur.addBindValue(word); + if(limit !=0) + cur.addBindValue(limit); + cur.exec(); + while(cur.next()){ + bool ok=true; + Translation *tran; + foreach(tran,translations) { + if(tran->key().toLower()==cur.value(0).toString().toLower()) + ok=false; + } + if(ok) /*add key word to list*/ + translations.insert(new TranslationXdxf( + cur.value(0).toString().toLower(), + _infoNote, this)); + } + } db.close(); return translations.toList(); } QList XdxfPlugin::searchWordListFile(QString word, int limit) { + QTime time; + time.start(); QSet translations; QFile dictionaryFile(path); word = word.toLower(); - word = removeAccents(word); + //word = removeAccents(word); stopped = false; QRegExp regWord(word); @@ -148,6 +178,9 @@ QList XdxfPlugin::searchWordListFile(QString word, int limit) { QXmlStreamReader reader(&dictionaryFile); /*search words list*/ QString a; + QRegExp regexp(removeAccents(a), Qt::CaseInsensitive, QRegExp::Wildcard); + regexp.setMinimal(true); + int i=0; while(!reader.atEnd() && !stopped){ reader.readNextStartElement(); @@ -156,7 +189,10 @@ QList XdxfPlugin::searchWordListFile(QString word, int limit) { reader.readNextStartElement(); if(!reader.atEnd()) a = reader.readElementText(); - if(regWord.exactMatch(removeAccents(a)) && (i XdxfPlugin::searchWordListFile(QString word, int limit) { } stopped=false; dictionaryFile.close(); + qDebug() << time.elapsed(); return translations.toList(); } @@ -378,20 +415,12 @@ void XdxfPlugin::getDictionaryInfo() { dictionaryFile.close(); } -QString XdxfPlugin::removeAccents(QString string) { - string = string.replace(QString::fromUtf8("ł"), "l", Qt::CaseInsensitive); - QString normalized = string.normalized(QString::NormalizationForm_D); - normalized = normalized; +QString XdxfPlugin::removeAccents(QString string, QChar wildcard) { + QString normalized = string.toLower(); for(int i=0; i QChar('z')))) + normalized[i] = wildcard; } return normalized; } @@ -448,7 +477,6 @@ bool XdxfPlugin::makeCache(QString dir) { .arg(name())); return 0; } - QXmlStreamReader reader(&dictionaryFile); db.setDatabaseName(cachePathN); @@ -465,7 +493,7 @@ bool XdxfPlugin::makeCache(QString dir) { cur.exec("PRAGMA synchronous = 0"); cur.exec("drop table dict"); QCoreApplication::processEvents(); - cur.exec("create table dict(word text ,translation text)"); + cur.exec("create table dict(word text, normalized text ,translation text)"); int counter = 0; cur.exec("BEGIN;"); @@ -510,13 +538,15 @@ bool XdxfPlugin::makeCache(QString dir) { temp.remove(0,1); temp=tr("") + a + tr("") + tr("") + temp+ tr(""); match=false; - cur.prepare("insert into dict values(?,?)"); + cur.prepare("insert into dict values(?,?,?)"); cur.addBindValue(a); + cur.addBindValue(removeAccents(a,'_')); cur.addBindValue(temp); cur.exec(); counter++; int prog = counter*100/_wordsCount; if(prog % 5 == 0 && lastProg != prog) { + qDebug() << prog; Q_EMIT updateCachingProgress(prog, timer.restart()); lastProg = prog; diff --git a/trunk/src/plugins/xdxf/src/xdxfplugin.h b/trunk/src/plugins/xdxf/src/xdxfplugin.h index e413e2c..e7f3ff4 100644 --- a/trunk/src/plugins/xdxf/src/xdxfplugin.h +++ b/trunk/src/plugins/xdxf/src/xdxfplugin.h @@ -120,7 +120,7 @@ Q_SIGNALS: protected: - QString removeAccents(QString); + QString removeAccents(QString, QChar wildcard= '?'); private: /*! returns true or false depending on whether the dictionary is cached -- 1.7.9.5