qt4.6的并行处理机制

google的数据后端处理大家可能听说过，所使用的并行机制是map-reduce, 也就是用map去收集整理一个数据集，之后用reduce对数据集进行处理，得出想要的结果。原理很简单，但是就是这个简单的原理构成了google搜索的强大的基石，另外google的map-reduece是分布式的，呵呵
这段时间研究qt4.6，它对于并行处理这一块做了非常不错的封装，使用很简单，不用考虑线程同步处理等问题，下面就用这个最著名的统计单词个数的例子说明，代码真实太简单了

#include <QList>
#include <QMap>
#include <QTextStream>
#include <QString>
#include <QStringList>
#include <QDir>
#include <QTime>
#include <QApplication>
#include <QDebug>
 
#include <qtconcurrentmap.h>
 
#ifndef QT_NO_CONCURRENT
 
using namespace QtConcurrent;
 
/*
    Utility function that recursivily searches for files.
*/
QStringList findFiles(const QString &startDir, QStringList filters)
{
    QStringList names;
    QDir dir(startDir);
 
    foreach (QString file, dir.entryList(filters, QDir::Files))
        names += startDir + "/" + file;
 
    foreach (QString subdir, dir.entryList(QDir::AllDirs | QDir::NoDotAndDotDot))
        names += findFiles(startDir + "/" + subdir, filters);
    return names;
}
 
typedef QMap<QString, int> WordCount;
 
/*
    Single threaded word counter function.
*/
WordCount singleThreadedWordCount(QStringList files)
{
    WordCount wordCount;
    foreach (QString file, files) {
        QFile f(file);
        f.open(QIODevice::ReadOnly);
        QTextStream textStream(&f);
        while (textStream.atEnd() == false)
            foreach(QString word, textStream.readLine().split(" "))
                wordCount[word] += 1;
 
    }
    return wordCount;
}
 
 
// countWords counts the words in a single file. This function is
// called in parallel by several threads and must be thread
// safe.
WordCount countWords(const QString &file)
{
    QFile f(file);
    f.open(QIODevice::ReadOnly);
    QTextStream textStream(&f);
    WordCount wordCount;
 
    while (textStream.atEnd() == false)
        foreach (QString word, textStream.readLine().split(" "))
            wordCount[word] += 1;
 
    return wordCount;
}
 
// reduce adds the results from map to the final
// result. This functor will only be called by one thread
// at a time.
void reduce(WordCount &result, const WordCount &w)
{
    QMapIterator<QString, int> i(w);
    while (i.hasNext()) {
        i.next();
        result[i.key()] += i.value();
    }
}
 
int main(int argc, char** argv)
{
    QApplication app(argc, argv);
    qDebug() << "finding files...";
    // 首先得到要统计目录下的所有文件
    QStringList files = findFiles("../../", QStringList() << "*.cpp" << "*.h");
    qDebug() << files.count() << "files";
 
    qDebug() << "warmup";
    {
        QTime time;
        time.start();
        WordCount total = singleThreadedWordCount(files);
    }
 
    qDebug() << "warmup done";
 
    int singleThreadTime = 0;
    {
        QTime time;
        time.start();
        // 单线程统计，好与mapreduce这种机制实现的作对比
        WordCount total = singleThreadedWordCount(files);
        singleThreadTime = time.elapsed();
        // 打印出所耗费时间
        qDebug() << "single thread" << singleThreadTime;
    }
 
    int mapReduceTime = 0;
    {
        QTime time;
        time.start();
        // mapreduce方式进行统计
        WordCount total = mappedReduced(files, countWords, reduce);
        mapReduceTime = time.elapsed();
        qDebug() << "MapReduce" << mapReduceTime;
    }
// 输出mapreduce方式比单线程处理方式要快的倍数
    qDebug() << "MapReduce speedup x" << ((double)singleThreadTime - (double)mapReduceTime) / (double)mapReduceTime + 1;
}
 
#else
 
int main()
{
	qDebug() << "Qt Concurrent is not yet supported on this platform";
}
 
#endif

 

 

通过这个测试用例，在统计文件很少的情况下，或者单核cpu的情况下，基本性能相同，但是如果统计文件一多，那么速度提升是很大的，自己的双核机器上统计300多个文件的加速比是1.3613，还是很不错的。