67.文件映射为内存进行操作与多线程以及文件映射到内存根据索引进行内存来实现读取多线程操作

问题引出:

//有限的额内存情况怎么干活
//索引载入内存，多线程
//索引映射为内存

一.载入内存,按行读取进行多线程

定义路径以及其他信息

1 char path[256] = "kaifangX.txt";
2 char indexpath[256] = "indexkf.txt";
3 #define  N 20151574

创建索引

1 struct index
2 {
3     int *pindex;//地址
4     int length;
5 }allindex = {0};

读取文件到内存

 1 void readmem()
 2 {
 3     allindex.length = N;
 4     allindex.pindex = calloc(N, sizeof(int));
 5 
 6 
 7     FILE *pfw = fopen(indexpath, "rb");
 8     fread(allindex.pindex, sizeof(int),N,pfw);
 9     fclose(pfw);
10 }

创建索引的线程结构体,每一个线程读取length的长度的行数

 1 void readmem()
 2 {
 3     allindex.length = N;
 4     allindex.pindex = calloc(N, sizeof(int));
 5 
 6 
 7     FILE *pfw = fopen(indexpath, "rb");
 8     fread(allindex.pindex, sizeof(int),N,pfw);
 9     fclose(pfw);
10 }

多线程函数

 1 void run(void *p)
 2 {
 3     struct info *pi = p;//参数
 4     for (int  i = 0; i < pi->length-1; i++)//限定
 5     {
 6         //起始地址到结束地址之间是一个字符串,可以读取出来
 7         char *start = infoall.pfilestart + pi->pstart[i];//起始地址
 8         char *end = infoall.pfilestart + pi->pstart[i + 1];//结束地址
 9 
10 
11 
12         char *str = calloc(500, 1);//分配内存用于拷贝字符串，文件整块，没有/0,有/n
13         if (str != NULL)//分配成功
14         {
15             strncpy(str, start, pi->pstart[i + 1] - pi->pstart[i ]);//拷贝字符串
16             if (str != NULL)
17             {
18                 char *ps = strstr(str, pi->findstr);//检索
19                 if (ps != NULL)
20                 {
21                     printf("%s", str);
22                 }
23             }    
24         
25         }
26         free(str);//释放
27     }
28 }

载入到内存进行多线程检索

 1 //根据内存进行多线程索引
 2 void test1()
 3 {
 4     printf("please input");
 5     char str[100];
 6     scanf("%s", str);
 7 #define nthread 100
 8     struct info pthread[nthread] = {0};
 9     HANDLE hd[nthread] = { 0 };//线程参数，线程句柄
10     if (N%nthread==0)
11     {
12         for (int i = 0; i < nthread; i++)
13         {
14             pthread[i].id = i;
15             pthread[i].length = N / nthread;
16             pthread[i].pstart = allindex.pindex + i*(N / nthread);
17             strcpy(pthread[i].findstr,str );
18             _beginthread(run, 0, &pthread[i]);
19         }
20     }
21     else
22     {
23         for (int i = 0; i < nthread-1; i++)
24         {
25             pthread[i].id = i;
26             pthread[i].length = N / (nthread - 1);
27             pthread[i].pstart = allindex.pindex + i*(N / (nthread - 1));
28             strcpy(pthread[i].findstr, str);
29             _beginthread(run, 0, &pthread[i]);
30         }
31         int i = nthread - 1;
32         pthread[i].id = i;
33         pthread[i].length = N % (nthread - 1);
34         pthread[i].pstart = allindex.pindex + i*(N / (nthread - 1));
35         strcpy(pthread[i].findstr, str);
36         _beginthread(run, 0, &pthread[i]);
37 
38     }
39     WaitForMultipleObjects(nthread, hd, TRUE, INFINITE);//等待
40 
41 
42     system("pause");
43 
44 }

二.内存映射进行多线程检索

创建文件映射内存的首地址和尾地址

 1 struct
 2 {
 3     //文本文件首尾地址
 4     char *pfilestart;
 5     char *pfileend;
 6     //索引文件首尾地址
 7     char *pindexfilestart;
 8     char *pindexfileend;
 9 
10 }infoall = {0};

映射文本文件到内存，文件是独家享有

 1 void map1()
 2 {
 3     //打开一个文件
 4     HANDLE hfile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, 0, NULL,
 5         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
 6 
 7     if (hfile == INVALID_HANDLE_VALUE)
 8     {
 9         printf("打开文件失败");
10         system("pause");
11     }
12     //获取文件大小
13     printf("
%d", GetFileSize(hfile, NULL));
14 
15     HANDLE hmap = CreateFileMappingA(hfile,
16         NULL, PAGE_READWRITE | SEC_COMMIT,//读写
17         0,
18         GetFileSize(hfile, NULL) + 1,//大小
19         NULL);
20 
21     if (hmap == NULL)
22     {
23         printf("映射失败");
24         CloseHandle(hfile);
25         system("pause");
26     }
27 
28     //创建一个指针，存储映射以后的首地址
29     PVOID pvfile = MapViewOfFile(hmap, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
30     
31     if (pvfile == NULL)
32     {
33         printf("指针映射失败");
34         CloseHandle(hfile);
35         CloseHandle(hmap);
36         system("pause");
37 
38     }
39     puts("映射成功");
40     char *pstart = pvfile;//首地址
41     char *pend = pstart + GetFileSize(hfile, NULL);//结束地址
42     infoall.pfileend = pend;//保存地址
43     infoall.pfilestart = pstart;
44 }

映射索引文件

 1 void map2()
 2 {
 3     HANDLE hfile = CreateFileA(indexpath, GENERIC_READ | GENERIC_WRITE, 0, NULL,
 4         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);//打开一个文件
 5 
 6     if (hfile == INVALID_HANDLE_VALUE)
 7     {
 8         printf("打开文件失败");
 9         system("pause");
10     }
11     printf("
%d", GetFileSize(hfile, NULL));//获取文件大小
12     HANDLE hmap = CreateFileMappingA(hfile,
13         NULL, PAGE_READWRITE | SEC_COMMIT,//读写
14         0,
15         GetFileSize(hfile, NULL) + 1,//大小
16         NULL);
17     if (hmap == NULL)
18     {
19         printf("映射失败");
20         CloseHandle(hfile);
21         system("pause");
22     }
23     PVOID pvfile = MapViewOfFile(hmap, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
24     //创建一个指针，存储映射以后的首地址
25     if (pvfile == NULL)
26     {
27         printf("指针映射失败");
28         CloseHandle(hfile);
29         CloseHandle(hmap);
30         system("pause");
31 
32     }
33     puts("映射成功");
34     char *pstart = pvfile;//首地址
35     char *pend = pstart + GetFileSize(hfile, NULL);//结束地址
36     infoall.pindexfileend = pend;//保存地址
37     infoall.pindexfilestart = pstart;
38 }

内存映射多线程结构体

1 struct minfo
2 {
3     int*pstart;
4     int length;//0   9 10
5     int id;
6     char findstr[20];
7 };

内存映射多线程函数

 1 void mrun(void *p)
 2 {
 3     struct minfo *pi = p;
 4     for (int i = 0; i < pi->length - 1; i++)
 5     {
 6         char *start = infoall.pfilestart + pi->pstart[i];//起始地址
 7         char *end = infoall.pfilestart + pi->pstart[i + 1];//结束地址
 8         char *str = calloc(500, 1);//分配内存用于拷贝字符串
 9         if (str != NULL)//分配成功
10         {
11             strncpy(str, start, pi->pstart[i + 1] - pi->pstart[i]);//拷贝字符串
12             if (str != NULL)
13             {
14                 char *ps = strstr(str, pi->findstr);
15                 if (ps != NULL)
16                 {
17                     printf("%s", str);
18                 }
19             }
20 
21         }
22         free(str);//释放
23 
24 
25     }
26 }

创建多线程

 1 void test2()
 2 {
 3     printf("please input");
 4     char str[100];
 5     scanf("%s", str);
 6 #define nthread 100
 7     struct minfo pthread[nthread] = { 0 };
 8     HANDLE hd[nthread] = { 0 };//线程参数，线程句柄
 9     if (N%nthread == 0)
10     {
11         for (int i = 0; i < nthread; i++)
12         {
13             pthread[i].id = i;
14             pthread[i].length = N / nthread;
15             pthread[i].pstart =   ((int*)infoall.pindexfilestart) + i*(N / nthread);
16             strcpy(pthread[i].findstr, str);
17             _beginthread(mrun, 0, &pthread[i]);
18         }
19 
20 
21     }
22     else
23     {
24         for (int i = 0; i < nthread - 1; i++)
25         {
26             pthread[i].id = i;
27             pthread[i].length = N / (nthread - 1);
　　           //索引对应的位置信息
28             pthread[i].pstart = ((int*)infoall.pindexfilestart) + i*(N / (nthread - 1));
29             strcpy(pthread[i].findstr, str);
30             _beginthread(mrun, 0, &pthread[i]);
31         }
32         int i = nthread - 1;
33         pthread[i].id = i;
34         pthread[i].length = N % (nthread - 1);
35         pthread[i].pstart = ((int*)infoall.pindexfilestart) + i*(N / (nthread - 1));
36         strcpy(pthread[i].findstr, str);
37         _beginthread(mrun, 0, &pthread[i]);
38 
39     }
40     WaitForMultipleObjects(nthread, hd, TRUE, INFINITE);
41 
42 
43     system("pause");
44 
45 }

完整代码:

  1 #define _CRT_SECURE_NO_WARNINGS
  2 #include<stdio.h>
  3 #include<stdlib.h>
  4 #include<Windows.h>
  5 #include<process.h>
  6 //有限的额内存情况怎么干活
  7 //索引载入内存，多线程
  8 //索引映射为内存
  9 
 10 
 11 char path[256] = "kaifangX.txt";
 12 char indexpath[256] = "indexkf.txt";
 13 #define  N 20151574
 14 
 15 //创建文件映射内存的首地址和尾地址
 16 struct
 17 {
 18     //文本文件首尾地址
 19     char *pfilestart;
 20     char *pfileend;
 21     //索引文件首尾地址
 22     char *pindexfilestart;
 23     char *pindexfileend;
 24 
 25 }infoall = {0};
 26 
 27 
 28 //映射文本文件到内存，文件是独家享有
 29 void map1()
 30 {
 31     //打开一个文件
 32     HANDLE hfile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, 0, NULL,
 33         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
 34 
 35     if (hfile == INVALID_HANDLE_VALUE)
 36     {
 37         printf("打开文件失败");
 38         system("pause");
 39     }
 40     //获取文件大小
 41     printf("
%d", GetFileSize(hfile, NULL));
 42 
 43     HANDLE hmap = CreateFileMappingA(hfile,
 44         NULL, PAGE_READWRITE | SEC_COMMIT,//读写
 45         0,
 46         GetFileSize(hfile, NULL) + 1,//大小
 47         NULL);
 48 
 49     if (hmap == NULL)
 50     {
 51         printf("映射失败");
 52         CloseHandle(hfile);
 53         system("pause");
 54     }
 55 
 56     //创建一个指针，存储映射以后的首地址
 57     PVOID pvfile = MapViewOfFile(hmap, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
 58     
 59     if (pvfile == NULL)
 60     {
 61         printf("指针映射失败");
 62         CloseHandle(hfile);
 63         CloseHandle(hmap);
 64         system("pause");
 65 
 66     }
 67     puts("映射成功");
 68     char *pstart = pvfile;//首地址
 69     char *pend = pstart + GetFileSize(hfile, NULL);//结束地址
 70     infoall.pfileend = pend;//保存地址
 71     infoall.pfilestart = pstart;
 72 }
 73 
 74 //映射索引文件到内存
 75 void map2()
 76 {
 77     HANDLE hfile = CreateFileA(indexpath, GENERIC_READ | GENERIC_WRITE, 0, NULL,
 78         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);//打开一个文件
 79 
 80     if (hfile == INVALID_HANDLE_VALUE)
 81     {
 82         printf("打开文件失败");
 83         system("pause");
 84     }
 85     printf("
%d", GetFileSize(hfile, NULL));//获取文件大小
 86     HANDLE hmap = CreateFileMappingA(hfile,
 87         NULL, PAGE_READWRITE | SEC_COMMIT,//读写
 88         0,
 89         GetFileSize(hfile, NULL) + 1,//大小
 90         NULL);
 91     if (hmap == NULL)
 92     {
 93         printf("映射失败");
 94         CloseHandle(hfile);
 95         system("pause");
 96     }
 97     PVOID pvfile = MapViewOfFile(hmap, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
 98     //创建一个指针，存储映射以后的首地址
 99     if (pvfile == NULL)
100     {
101         printf("指针映射失败");
102         CloseHandle(hfile);
103         CloseHandle(hmap);
104         system("pause");
105 
106     }
107     puts("映射成功");
108     char *pstart = pvfile;//首地址
109     char *pend = pstart + GetFileSize(hfile, NULL);//结束地址
110     infoall.pindexfileend = pend;//保存地址
111     infoall.pindexfilestart = pstart;
112 }
113 
114 //读取索引到内存
115 struct index
116 {
117     int *pindex;//地址
118     int length;
119 }allindex = {0};
120 
121 //读取文件到内存
122 void readmem()
123 {
124     allindex.length = N;
125     allindex.pindex = calloc(N, sizeof(int));
126 
127 
128     FILE *pfw = fopen(indexpath, "rb");
129     fread(allindex.pindex, sizeof(int),N,pfw);
130     fclose(pfw);
131 }
132 
133 //创建索引的线程结构体,每一个线程读取length的长度的行数
134 struct info
135 {
136     int*pstart;//索引的收地址
137     int length;//长度
138     int id;//编号
139     char findstr[20];//查找的字符串
140 };
141 
142 //多线程函数
143 void run(void *p)
144 {
145     struct info *pi = p;//参数
146     for (int  i = 0; i < pi->length-1; i++)//限定
147     {
148         //起始地址到结束地址之间是一个字符串,可以读取出来
149         char *start = infoall.pfilestart + pi->pstart[i];//起始地址
150         char *end = infoall.pfilestart + pi->pstart[i + 1];//结束地址
151 
152 
153 
154         char *str = calloc(500, 1);//分配内存用于拷贝字符串，文件整块，没有/0,有/n
155         if (str != NULL)//分配成功
156         {
157             strncpy(str, start, pi->pstart[i + 1] - pi->pstart[i ]);//拷贝字符串
158             if (str != NULL)
159             {
160                 char *ps = strstr(str, pi->findstr);//检索
161                 if (ps != NULL)
162                 {
163                     printf("%s", str);
164                 }
165             }    
166         
167         }
168         free(str);//释放
169     }
170 }
171 
172 //根据内存进行多线程索引
173 void test1()
174 {
175     printf("please input");
176     char str[100];
177     scanf("%s", str);
178 #define nthread 100
179     struct info pthread[nthread] = {0};
180     HANDLE hd[nthread] = { 0 };//线程参数，线程句柄
181     if (N%nthread==0)
182     {
183         for (int i = 0; i < nthread; i++)
184         {
185             pthread[i].id = i;
186             pthread[i].length = N / nthread;
187             pthread[i].pstart = allindex.pindex + i*(N / nthread);
188             strcpy(pthread[i].findstr,str );
189             _beginthread(run, 0, &pthread[i]);
190         }
191     }
192     else
193     {
194         for (int i = 0; i < nthread-1; i++)
195         {
196             pthread[i].id = i;
197             pthread[i].length = N / (nthread - 1);
198             pthread[i].pstart = allindex.pindex + i*(N / (nthread - 1));
199             strcpy(pthread[i].findstr, str);
200             _beginthread(run, 0, &pthread[i]);
201         }
202         int i = nthread - 1;
203         pthread[i].id = i;
204         pthread[i].length = N % (nthread - 1);
205         pthread[i].pstart = allindex.pindex + i*(N / (nthread - 1));
206         strcpy(pthread[i].findstr, str);
207         _beginthread(run, 0, &pthread[i]);
208 
209     }
210     WaitForMultipleObjects(nthread, hd, TRUE, INFINITE);//等待
211 
212 
213     system("pause");
214 
215 }
216 
217 
218 //内存映射结构体
219 struct minfo
220 {
221     int*pstart;
222     int length;//0   9 10
223     int id;
224     char findstr[20];
225 };
226 
227 void mrun(void *p)
228 {
229     struct minfo *pi = p;
230     for (int i = 0; i < pi->length - 1; i++)
231     {
232         char *start = infoall.pfilestart + pi->pstart[i];//起始地址
233         char *end = infoall.pfilestart + pi->pstart[i + 1];//结束地址
234         char *str = calloc(500, 1);//分配内存用于拷贝字符串
235         if (str != NULL)//分配成功
236         {
237             strncpy(str, start, pi->pstart[i + 1] - pi->pstart[i]);//拷贝字符串
238             if (str != NULL)
239             {
240                 char *ps = strstr(str, pi->findstr);
241                 if (ps != NULL)
242                 {
243                     printf("%s", str);
244                 }
245             }
246 
247         }
248         free(str);//释放
249 
250 
251     }
252 }
253 
254 //根据内存映射进行多线程索引
255 void test2()
256 {
257     printf("please input");
258     char str[100];
259     scanf("%s", str);
260 #define nthread 100
261     struct minfo pthread[nthread] = { 0 };
262     HANDLE hd[nthread] = { 0 };//线程参数，线程句柄
263     if (N%nthread == 0)
264     {
265         for (int i = 0; i < nthread; i++)
266         {
267             pthread[i].id = i;
268             pthread[i].length = N / nthread;
269             pthread[i].pstart =   ((int*)infoall.pindexfilestart) + i*(N / nthread);
270             strcpy(pthread[i].findstr, str);
271             _beginthread(mrun, 0, &pthread[i]);
272         }
273 
274 
275     }
276     else
277     {
278         for (int i = 0; i < nthread - 1; i++)
279         {
280             pthread[i].id = i;
281             pthread[i].length = N / (nthread - 1);
282             pthread[i].pstart = ((int*)infoall.pindexfilestart) + i*(N / (nthread - 1));
283             strcpy(pthread[i].findstr, str);
284             _beginthread(mrun, 0, &pthread[i]);
285         }
286         int i = nthread - 1;
287         pthread[i].id = i;
288         pthread[i].length = N % (nthread - 1);
289         pthread[i].pstart = ((int*)infoall.pindexfilestart) + i*(N / (nthread - 1));
290         strcpy(pthread[i].findstr, str);
291         _beginthread(mrun, 0, &pthread[i]);
292 
293     }
294     WaitForMultipleObjects(nthread, hd, TRUE, INFINITE);
295 
296 
297     system("pause");
298 
299 }
300 
301 void main()
302 {
303     //readmem();
304     map1();//一旦被映射，文件被锁定
305     map2();
306     //test1();
307     test2();
308     getchar();
309 }

相关阅读:
Oracle中Driving_site Hint的用法
 Oracle数据库性能分析管理IO分析爱代码爱编程
 数据代码分享|Python用NLP自然语言处理LSTM神经网络Twitter推特灾难文本数据、词云可视化与SVM，KNN，多层感知器，朴素贝叶斯，随机森林，GBDT对比
 R语言单变量和多变量（多元）动态条件相关系数DCCGARCH模型分析股票收益率金融时间序列数据波动率|附代码数据
 【视频】为什么要处理缺失数据？如何R语言中进行缺失值填充？
R语言用多元ARMA,GARCH ,EWMA, ETS,随机波动率SV模型对金融时间序列数据建模|附代码数据
 美光科技股票基本分析：经济背景、行业分析财政状况（盈利、EBITDA、PP&E、D&A等）预测计算DCF
数据分享|WEKA用决策树、随机森林、支持向量机SVM、朴素贝叶斯、逻辑回归信贷违约预测报告
 python可视化探索新冠病毒与失业率是否存在线性关系
 视频|分类模型评估:精确率、召回率、ROC曲线、AUC与R语言生存分析时间依赖性ROC实现
原文地址：https://www.cnblogs.com/xiaochi/p/8443477.html

67.文件映射为内存进行操作与多线程 以及 文件映射到内存根据索引进行内存来实现读取多线程操作

67.文件映射为内存进行操作与多线程以及文件映射到内存根据索引进行内存来实现读取多线程操作