文件归并排序简单实现

如果要排序的数据太多了,就不能一次性加载到内存中进行排序,只能分而治之,然后再合并。 这里参考归并排序的过程简单地实现了一下,封装的还很不好。 另外在对多个文件合并时偷了懒,感觉需要用平衡二叉树,但是实现起来比较麻烦,就先用C++标准库中的multiset的排序功能了。 stl的set、map、multiset都是有序,并且是红黑树数据结构,查找、删除等性能都是log(n).

mergesort.h


#ifndef __MERGESORT_H
#define __MERGESORT_H

namespace MyNamespace
{
	typedef int (*Comparer)(const void *a, const void *b);

	template  
	int defaultCompare(T *a, T *b)
	{
		if (*a < *b) return -1;
		if (*b < *a) return 1;
		return 0;
	}

	template  
	void arrayCopy(T *a, const T *b, int n)
	{
		while (--n >= 0) a[n] = b[n];
	}

	inline int minIndex(int a, int b)
	{
		return a < b ? a : b;
	}

	template 
	int mergeSort(T a[], int n, Comparer comparer)
	{
		T *b = NULL;
		int i, j, t, start1, end1, start2, end2, len;
		int cmpResult;
		
		if (n < 2)
			return 0;

		b = new T[n];
		if (b == NULL) return -1;


		//len为每次分块的大小,即按照1,2,4,6,8,...,n增长
		len = 1;
		while (len < n)
		{
			t = 0;
			i = 0;
			j = i + len;
			while (j != n)
			{
				//start1,end1,start2,end2用于表示进行比较的两个数据块下标,
				//两个分块的下标范围为[start1,end1),[start2,end2)。
				start1 = minIndex(n, i);
				end1   = minIndex(n, i + len);
				start2 = minIndex(n, i + len);
				end2   = minIndex(n, i + len + len);

				//i,j分别表示当前进行比较的两个元素的下标
				i = start1;
				j = start2;
				while (i < end1 && j < end2)
				{
					if (comparer != NULL)
						cmpResult = comparer(a+i, a+j);
					else
						cmpResult = defaultCompare(a+i, a+j);

					b[t++] = (cmpResult <= 0) ?  a[i++] : a[j++];
				}

				if (i < end1)
				{
					while (i < end1)
						b[t++] = a[i++];
				}
				else
				{
					while (j < end2)
						b[t++] = a[j++];
				}
				//assert(i == end1 && j == end2 && t == end2);
				i = t;
			}
			len = minIndex(n, len * 2);
			arrayCopy(a, b, n);
		}

		delete []b;
		b = NULL;
		return 0;
	}

	template 
	int mergeSort(T a[], int n)
	{
		return mergeSort(a, n, NULL);
	}

};

#endif //__MERGESORT_H


filemergesort.h

#ifndef __FILEMERGESORT_H
#define __FILEMERGESORT_H

#include "mergesort.h"
#include 
#include 
#include 
#include 
#include 
#include 
using namespace std;


#define SORT_AND_WRITE_TO_FILE	\
	ret = MyNamespace::mergeSort(&v[0], v.size(), (MyNamespace::Comparer)strCompare);\
				if (ret != 0)\
				{\
					/* 内存中归并排序失败,可能是内存不足 */\
					return -3;\
				}\
				ret = ouputSortedPartFile(fileName, ++partCount, v, separator);\
				\
				if (ret != 0)\
				{\
					/* 写排序后的部分数据到文件失败 */\
					return -4;\
				}\
				v.clear();


typedef struct
{
	string elment;
	int fileIndex;
} FileElement;


int defaultStrComparer(const string *str1, const string *str2)
{
	//if (*str1 == *str2) return 0;
	//return *str1 < *str2 ? -1 : 1;


	//对逗号分割的第一个元素进行排序
	string sub1, sub2;
	sub1 = str1->substr(0, str1->find(","));
	sub2 = str2->substr(0, str2->find(","));

	if (sub1 == sub2) return 0;
	return sub1 < sub2 ? -1 : 1;
}

class ElementCompareClass
{
public:
	bool operator()(const FileElement &e1, const FileElement &e2)
	{
		return defaultStrComparer(&e1.elment, &e2.elment) < 0;
	}
};

class FileMergeSort
{
public:
	
	typedef int (*StrComparer)(const string *a, const string *b);

	FileMergeSort()
		: separator(DEFAULT_SEPARATOR), maxMemorySize(DEFAULT_MAX_MEMORY_SIZE), strCompare(defaultStrComparer)
	{
	}

	FileMergeSort(const string separatorIn, int maxMemorySizeIn = 1024*1024*10)
		: separator(separatorIn), maxMemorySize(maxMemorySizeIn), strCompare(defaultStrComparer)
	{
		adjustMaxMemorySize();
	}


	void setMaxMemorySize(int bytes)
	{
		maxMemorySize = bytes;
		adjustMaxMemorySize();
	}
	int getMaxMemorySize(int bytes)
	{
		return maxMemorySize;
	}

	void setSeparator(const string s)
	{
		separator = s;
	}
	string getSeparator()
	{
		return separator;
	}

	int readFile(ifstream &inFile, string &sBuffer, int maxSize)
	{
		char ch;
		int size = 0;
		while (size < maxSize)
		{
			ch = inFile.get();
			if (inFile.good())
			{
				size++;
				sBuffer += ch;
			}
			else
			{
				break;
			}
		}
		return size;
	}


	int sort(const char *fileName)
	{
		int ret;
		int partCount = 0;
		int partFileSize = 0;
		int readSize, maxReadSize = 1024;
		string sBuffer, element;
		vector v;

		ifstream inFile(fileName);
		if (!inFile.is_open())
		{
			//打开读文件失败
			return -1;
		}

		string::size_type index;
		while(inFile.good())
		{
			//从文件中最多读取maxReadSize个字符到sBuffer,返回实际读取个数
			readSize = readFile(inFile, sBuffer, maxReadSize);
			if (readSize <= 0)
				break;

			//根据分隔符把读取的字符串拆分成多个元素
			while (true)
			{
				index = sBuffer.find(separator);
				if (index != string::npos)
				{
					element = sBuffer.substr(0, index);
					v.push_back(element);
					sBuffer = sBuffer.substr(index + separator.length());
					partFileSize += element.size();
				}
				else
				{
					if (sBuffer.length() > maxMemorySize/2U)
					{
						//元素太长了(由separator分隔的,或许因为分隔符写错了)
						inFile.close();
						return -2;
					}
					break;
				}
			}

			/*
				如果分块容器v中的大小已经超过设置的大小了,就对这小块进行排序,并写入分块文件
				注意:这里的内存控制不那么准,可以能还差的远,跟vector的内存分分配有关;
				另外MergeSort还需要分配与v等量的内存;实际测试大概需要内存是maxMemorySize的四倍呢
			*/

			if (partFileSize >= maxMemorySize)
			{
				SORT_AND_WRITE_TO_FILE;
				partFileSize = 0;
			}
		}
		inFile.close();

		if (sBuffer.length() > 0)
		{
			v.push_back(sBuffer);
		}

		if (!v.empty())
		{
			SORT_AND_WRITE_TO_FILE;
		}

		//--------- 至此分割文件以及对分块排序完成,并且写入分块文件(filename.N.sortedpart)------------
		//--------- 下面对各个有序的分块文件进行合并 --------------------------------------------------

		
		ret = fileMerge(fileName, partCount);
		if (ret != 0)
		{
			//对分块文件进行归并出错。可能因为打开或创建文件失败、磁盘空间不足等
			return (ret - 100);
		}

		return 0;
	}

protected:	
	virtual void adjustMaxMemorySize()
	{
		if (maxMemorySize < 10240)
			maxMemorySize = 10240;
	}

	int maxMemorySize;
	int maxLineLength;
	const static int DEFAULT_MAX_MEMORY_SIZE;
	const static string DEFAULT_SEPARATOR;
	string separator;
	StrComparer strCompare;

	int fileMerge(const char *fileName, int totalPartCount)
	{
		char sPartNo[13] = {0};
		int partFileNameLength = strlen(fileName) + strlen("..sortedpart") + 13;
		char *partFileName = new char[partFileNameLength];
		
		vector inFileVector;

		int i;
		for (i = 0; i < totalPartCount; i++)
		{
			itoa(i+1, sPartNo, 10);
			memset(partFileName, 0x00, partFileNameLength);
			sprintf(partFileName, "%s.%d.sortedpart", fileName, i+1);
			
			ifstream *fin = new ifstream(partFileName);
			if (fin == NULL)
			{
				return -1;
			}
			inFileVector.push_back(fin);
			if (!inFileVector[i]->is_open())
			{
				return -2;
			}
		}


		int outFileNameLength = strlen(fileName) + strlen(".sorted") + 1;
		char *outFileName = new char[outFileNameLength];
		if (outFileName == NULL)
		{
			//内存不足
			return -3;
		}

		memset(outFileName, 0x00, outFileNameLength);
		sprintf(outFileName, "%s.sorted", fileName);
		ofstream outFile(outFileName);
		if (!outFile.is_open())
		{
			//结果文件打开失败
			return -2;
		}


		int ret;
		FileElement fileElement;

		//重要:在对多个文件合并时,为了达到log(k)的插入和查找性能,所以使用stl中的multiset(红黑树数据结构),k表示文件的分块文件的个数
		multiset dataSet;
		for (i = 0; i < totalPartCount; i++)
		{
			fileElement.fileIndex = i;
			ret = getElement(inFileVector, i, fileElement.elment);
			if (ret == -2)
			{
				//元素太长,或许分隔符不正确
				return -4;
			}
			if (ret != 0)
			{
				inFileVector[i]->close();
			}

			dataSet.insert(fileElement);
		}

		while (dataSet.size() > 0)
		{
			//把最小的数据写出去,并从这个元素对应的文件再读取一个元素放进去
			//直到所有的文件都空了
		
			fileElement = *(dataSet.begin());
			outFile.write(fileElement.elment.c_str(), fileElement.elment.length());
			outFile.write(separator.c_str(), separator.length());
			dataSet.erase(dataSet.begin());

			if (inFileVector[fileElement.fileIndex]->is_open())
			{
				ret = getElement(inFileVector, fileElement.fileIndex, fileElement.elment);
				if (ret == 0)
				{
					dataSet.insert(fileElement);
				}
				else if (ret == -2)
				{
					//元素太长,或许分隔符不正确
					return -4;
				}
				else
				{
					inFileVector[fileElement.fileIndex]->close();
					
					//TODO 此时可以删除此分块文件
				}
			}
		}

		outFile.close();
		return 0;
	}

	int getElement(vector &inFileVector, int fileIndex, string &element)
	{
		static map map_strBuffer;
		static map map_bufferSize;
		int readSize = 0;
		int index;

		if (map_bufferSize.count(fileIndex) <= 0)
		{
			map_bufferSize[fileIndex] = 0;
		}

		while (true)
		{
			index = map_strBuffer[fileIndex].find(separator);
			if (index != string::npos)
			{
				element = map_strBuffer[fileIndex].substr(0, index);
				map_strBuffer[fileIndex] = map_strBuffer[fileIndex].substr(index + separator.length());
				map_bufferSize[fileIndex] -= index +separator.length();
				return 0;
			}

			readSize = readFile(*(inFileVector[fileIndex]), map_strBuffer[fileIndex], 1024);
			if (readSize <= 0)
			{
				//读到文件结尾了
				return -1;
			}

			map_bufferSize[fileIndex] += readSize;
			if (map_bufferSize[fileIndex] > maxMemorySize/2U)
			{
				//元素太长了(由separator分隔的,或许因为分隔符写错了)
				inFileVector[fileIndex]->close();
				return -2;
			}
		}
		return 0;
	}


	int ouputSortedPartFile(const char *fileName, int partCount, vector v, const string &separator)
	{
		int partFileNameLength = strlen(fileName) + strlen("..sortedpart") + 13;
		char *partFileName = new char[partFileNameLength];

		memset(partFileName, 0x00, partFileNameLength);
		sprintf(partFileName, "%s.%d.sortedpart", fileName, partCount);

		ofstream outFile(partFileName);
		if (!outFile.is_open())
		{
			return -1;
		}

		int i, size = v.size();
		for (i = 0; i < size; i++)
		{
			outFile.write(v[i].c_str(), v[i].length());
			outFile.write(separator.c_str(), separator.length());
		}
		outFile.close();
		return 0;
	}

};

const int FileMergeSort::DEFAULT_MAX_MEMORY_SIZE = 1024*1024*10;
const string FileMergeSort::DEFAULT_SEPARATOR = "\n";

#endif //__FILEMERGESORT_H


main.c

#include 
#include 
#include 
#include "mergesort.h"
#include "filemergesort.h"

using namespace std;


int main()
{
	FileMergeSort fileMergeSort("\n", 1024*1024*10);
	int ret = fileMergeSort.sort("Quote.csv");
	std::cout << ret << std::endl;

	system("pause");
	return 0;
}


Quote.csv

1373871611,SH600000,8.56,8.57,8.56,8.55,8.71,8.41,8.49,891040704.00,1038509.00
1373871665,SH600005,2.31,2.32,2.31,2.31,2.35,2.30,2.31,45215176.00,194893.00
1373871626,SH600009,12.81,12.82,12.81,12.86,13.07,12.73,12.80,39855712.00,30992.00
1373871665,SH600010,4.08,4.09,4.08,4.05,4.13,4.01,4.05,197604608.00,485515.00
1373871607,SH600011,5.46,5.47,5.47,5.45,5.56,5.43,5.50,123869376.00,226039.00
1373871626,SH600015,9.37,9.38,9.37,9.28,9.55,9.24,9.21,290867808.00,309105.00
1373871605,SH600016,9.15,9.16,9.16,9.16,9.41,9.00,9.11,1331242880.00,1451737.00
1373871607,SH600019,4.07,4.08,4.07,4.05,4.11,4.03,4.03,73100704.00,180051.00
1373871605,SH600022,1.63,1.64,1.64,1.62,1.65,1.61,1.63,19511916.00,119705.00
1373871600,SH600027,3.14,3.15,3.15,3.12,3.20,3.12,3.13,96782808.00,307247.00
1373871605,SH600028,4.55,4.56,4.56,4.59,4.62,4.52,4.59,283333152.00,619985.00
1373871611,SH600029,2.77,2.78,2.77,2.77,2.81,2.76,2.76,52826140.00,189341.00
1373871660,SH600030,10.96,10.97,10.96,10.80,11.29,10.65,10.54,1668629248.00,1518316.00
1373871603,SH600031,7.30,7.31,7.31,7.28,7.45,7.26,7.28,211298784.00,287873.00
1373871660,SH600036,11.66,11.67,11.65,11.74,11.91,11.55,11.62,789394304.00,674696.00
1373871603,SH600037,7.18,0.00,7.18,6.60,7.18,6.55,6.53,292799488.00,416500.00

......


Quote.csv.sorted

1373871600,SH600027,3.14,3.15,3.15,3.12,3.20,3.12,3.13,96782808.00,307247.00
1373871603,SH600031,7.30,7.31,7.31,7.28,7.45,7.26,7.28,211298784.00,287873.00
1373871603,SH600037,7.18,0.00,7.18,6.60,7.18,6.55,6.53,292799488.00,416500.00
1373871605,SH600016,9.15,9.16,9.16,9.16,9.41,9.00,9.11,1331242880.00,1451737.00
1373871605,SH600022,1.63,1.64,1.64,1.62,1.65,1.61,1.63,19511916.00,119705.00
1373871605,SH600028,4.55,4.56,4.56,4.59,4.62,4.52,4.59,283333152.00,619985.00
1373871607,SH600011,5.46,5.47,5.47,5.45,5.56,5.43,5.50,123869376.00,226039.00
1373871607,SH600019,4.07,4.08,4.07,4.05,4.11,4.03,4.03,73100704.00,180051.00
1373871611,SH600000,8.56,8.57,8.56,8.55,8.71,8.41,8.49,891040704.00,1038509.00
1373871611,SH600029,2.77,2.78,2.77,2.77,2.81,2.76,2.76,52826140.00,189341.00
1373871626,SH600009,12.81,12.82,12.81,12.86,13.07,12.73,12.80,39855712.00,30992.00
1373871626,SH600015,9.37,9.38,9.37,9.28,9.55,9.24,9.21,290867808.00,309105.00
1373871660,SH600030,10.96,10.97,10.96,10.80,11.29,10.65,10.54,1668629248.00,1518316.00
1373871660,SH600036,11.66,11.67,11.65,11.74,11.91,11.55,11.62,789394304.00,674696.00
1373871665,SH600005,2.31,2.32,2.31,2.31,2.35,2.30,2.31,45215176.00,194893.00
1373871665,SH600010,4.08,4.09,4.08,4.05,4.13,4.01,4.05,197604608.00,485515.00



你可能感兴趣的:(算法)