海量不重复数据的生成

爱被打了一巴掌 2022-07-28 01:22 166阅读 0赞

![Center][]

前几天看到了一个项目需求，自己尝试写了下3亿行数据大概需要20分钟的时间，普通硬盘应该要慢些，内存映射或者内存盘应该会更快一点，懒得验证了，放出代码供学习下，算法是以前偶然看到得很巧的的算法，二次获取随机数是为了数据更随机（待验证，我觉得不需要）

#include "stdafx.h"
    #include <chrono>
    #include <iostream>
    #include <memory>
    #include <random>
    
    #include <strsafe.h>
    
    using namespace std;
    
    
    int GetRandomNum(int min, int max,int seed)
    {
    
    	//srand((unsigned)time(NULL)); //生成种子
    	srand(seed);
    	return( rand() % (max - min) + min);
    }
    
    int main()
    {
    
    	//int nSericalLenth = 10;
    
    	chrono::steady_clock::time_point t1 = chrono::steady_clock::now();
    	int64_t nArrayLength = 300000000;
    	FILE* pFile = 0;
    
    	TCHAR szBuffer[128];
    
    	memset(szBuffer, 0, sizeof(szBuffer));
    
    	try
    	{
    
    		std::random_device rd;
    		std::uniform_int_distribution<int> dist(0, 9999999);
    		//std::cout << dist(rd) << std::endl;
    
    		_tfopen_s(&pFile, _T("D:\\num.txt"), _T("wt"));
    
    		if (pFile != nullptr)
    		{
    			unique_ptr<int[]> needArray(new int[nArrayLength]);
    
    			for (int idx = 0; idx < nArrayLength; idx++)
    			{
    				needArray[idx] = idx+283823423;
    			}
    
    			int end = nArrayLength - 1;
    
    			for (int i = 0; i < nArrayLength; i++)
    			{
    				int num = GetRandomNum(0, end + 1, dist(rd));
    
    				StringCbPrintf(szBuffer, sizeof(szBuffer), _T("%010d\n"), needArray[num]);
    				fwrite(szBuffer, 1, _tcslen(szBuffer), pFile);
    				//output[i] = needArray[num];
    				needArray[num] = needArray[end];
    				end--;
    			}
    
    			fclose(pFile);
    		}
    
    	}
    	catch (...)
    	{
    		
    	}
    
    
    	chrono::steady_clock::time_point t2 = chrono::steady_clock::now();
    
    	cout << chrono::duration_cast<std::chrono::seconds>(t2 - t1).count() << endl;
    
        return 0;
    }

[Center]: /images/20220727/5a2918ed96a740a094475376c92081a5.png