1 #include <iostream> 2 #include <fstream> 3 #include <cstring> 4 #include "Md5.h" 5 #include "Url.h" 6 #include "Document.h" 7 8 using namespace std; 9 10 int main(int argc, char* argv[]) {/* 11 * DocIndex.cpp 12 * Created on: 2011-11-9 13 * function: 14 * 将一个原始网页库进行索引， 15 * 生成网页索引文件Doc.idx 16 * 和URL索引文件Url.idx 17 */ 18 19 ifstream ifs("Tianwang.raw.1078930288"); 20 if (!ifs) { 21 cout << "不能打开原始网页库<Tianwang.raw.******>" << endl; 22 return -1; 23 } 24 25 ofstream ofsUrl("Url.idx", ios::in | ios::out | ios::trunc | ios::binary); 26 if (!ofsUrl) { 27 cout << "不能打开或者创建URL索引文件<Url.idx>" << endl; 28 cout << "error open file " << endl; 29 } 30 31 ofstream ofsDoc("Doc.idx", ios::in | ios::out | ios::trunc | ios::binary); 32 if (!ofsDoc) { 33 cout << "error open file " << endl; 34 } 35 36 ofstream ofsDocId2Url("DocId2Url.idx", 37 ios::in | ios::out | ios::trunc | ios::binary); 38 if (!ofsDocId2Url) { 39 cout << "error open file " << endl; 40 } 41 42 int cnt = 0; 43 string strLine, strPage; 44 CUrl iUrl; 45 CDocument iDocument; 46 CMD5 iMD5; 47 48 int nOffset = ifs.tellg(); //得到文件读指针距该文件头的字节数 49 cout << "tellg() is:" << nOffset << endl; 50 while (getline(ifs, strLine)) { 51 if (strLine[0] == '\0' || strLine[0] == '#' || strLine[0] == '\n') { 52 nOffset = ifs.tellg(); 53 continue; 54 } 55 56 if (!strncmp(strLine.c_str(), "version: 1.0", 12)) { 57 if (!getline(ifs, strLine)) 58 break; 59 60 if (!strncmp(strLine.c_str(), "url: ", 4)) { 61 iUrl.m_sUrl = strLine.substr(5);//保存url 62 iMD5.GenerateMD5((unsigned char*) iUrl.m_sUrl.c_str(), 63 iUrl.m_sUrl.size()); 64 iUrl.m_sChecksum = iMD5.ToString(); 65 66 } else { 67 continue; 68 } 69 70 while (getline(ifs, strLine)) {//保存文件长度信息 71 if (!strncmp(strLine.c_str(), "length: ", 8)) { 72 sscanf(strLine.substr(8).c_str(), "%d", 73 &(iDocument.m_nLength)); 74 break; 75 } 76 } 77 78 getline(ifs, strLine); 79 80 iDocument.m_nDocId = cnt; 81 iDocument.m_nPos = nOffset; 82 char *pContent = new char[iDocument.m_nLength + 1]; 83 84 memset(pContent, 0, iDocument.m_nLength + 1); 85 ifs.read(pContent, iDocument.m_nLength); 86 iMD5.GenerateMD5((unsigned char*) pContent, iDocument.m_nLength); 87 iDocument.m_sChecksum = iMD5.ToString(); 88 89 delete[] pContent; 90 91 ofsUrl << iUrl.m_sChecksum;//MD5值 92 ofsUrl << "\t" << iDocument.m_nDocId << endl;//doc的Id 93 94 ofsDoc << iDocument.m_nDocId;//文件偏移位置到MD5的映射 95 ofsDoc << "\t" << iDocument.m_nPos; 96 //ofsDoc << "\t" << iDocument.m_nLength ; 97 ofsDoc << "\t" << iDocument.m_sChecksum << endl; 98 99 ofsDocId2Url << iDocument.m_nDocId;//文件编号到url的映射 100 ofsDocId2Url << "\t" << iUrl.m_sUrl << endl; 101 102 cnt++; 103 } 104 105 nOffset = ifs.tellg(); 106 107 } 108 109 ofsDoc << cnt; 110 ofsDoc << "\t" << nOffset << endl; 111 112 return (0); 113 }