C#多线程词频统计修改

上一篇真是让大家见笑了，我今天才真正明白了“多线程”的意义。

今天拿到了真正的测试数据，大约380MB左右，用我的双线程大概能跑到16s左右。

但是惊闻有同学跑到了6s！

遂不甘心啊，果断继续修改之。

然后发现了多线程真正的奥义————建多个dictionary分别统计，最后再merge。

实际上索引是整个程序最耗时的地方，由于单词种类多达200k，所以如此庞大的一个dictionary每次的索引和更新是非常废时间的。

所以在新的多线程程序中，建了一个线程数组，每个线程负责各自的一个dictionary，

等每个线程都跑完之后，再进行Merge。

这样整个程序的时间就上去了。

380MB大约7s时间。

代码见下：

    class Program
    {
        const int ThreadNum = 7;
        static ConcurrentDictionary<string, int>[] result = new ConcurrentDictionary<string, int>[ThreadNum];
        static ConcurrentDictionary<string, int> ResultMerge;
        static int[] tablet = new int[128];
        static BlockingCollection<string> queue;
        static Thread[] WorkerTh = new Thread[ThreadNum];
        static Thread FileIOth;
        static Semaphore sem = new Semaphore(0, ThreadNum);
        static SemaphoreSlim semslim = new SemaphoreSlim(0, ThreadNum);
        static void Main(string[] args)
        {
            if (args.Length != 3)
            {
                Console.WriteLine("Command Line format: WFC rootdir N filePattern");
                Console.WriteLine("WFC: executable file name");
                Console.WriteLine("rootdir: the root directory of input files");
                Console.WriteLine("N: the count of output words");
                Console.WriteLine("filepattern: the name pattern of files to be scanned. For example, *.txt");
                return;
            }
            DateTime dt = DateTime.Now;
            string rootdir = args[0];
            int N = Convert.ToInt32(args[1]);
            string filePattern = args[2];
            if (!Directory.Exists(rootdir)) 
            {
                Console.WriteLine("The path "+ rootdir+" doesn't exist.");
                return;
            }
            string[] files = Directory.GetFiles(rootdir, filePattern, SearchOption.AllDirectories);
            if(files.Length==0)
            {
                Console.WriteLine("Can not find any "+filePattern+ " pattern's file.");
                return;
            }

            for (int i = 'a'; i <= 'z'; i++)
            {
                tablet[i] = 1;
            }
            for (int i = 'A'; i <= 'Z'; i++)
            {
                tablet[i] = 1;
            }

            //ManagementClass m = new ManagementClass(new ManagementPath( "Win32_Processor"));
            //ManagementObjectCollection moc = m.GetInstances();
            //int CPUNum = 0;

            //string NumOfCore="";
            //foreach (ManagementObject mo in moc)
            //{
            //    PropertyDataCollection properties = mo.Properties;
            //    NumOfCore += properties["NumberOfCores"].Value;
            //}
            //CPUNum = Convert.ToInt32(NumOfCore);



            queue = new BlockingCollection<string>(100);
            FileIOth = new Thread(delegate() { Read(files); });
            FileIOth.Start();

            ResultMerge = new ConcurrentDictionary<string, int>(1, 8 * N);

            for (int i = 0; i < result.Length; i++)
            {
                result[i] = new ConcurrentDictionary<string, int>(1, 20000);
            }


            int index = -1;
            for (int i = 0; i < result.Length - 1; i++)
            {
                WorkerTh[i] = new Thread(delegate()
                {
                    index++;
                    Process(index);
                    semslim.Release(1);
                });
                WorkerTh[i].Start();
            }

            WorkerTh[result.Length - 1] = new Thread(delegate()
            {
               
                Process(result.Length - 1);
                semslim.Release(1);
                while (true) 
                {
                    if (semslim.CurrentCount == ThreadNum)
                        break;
                }
                int count = 0;
                for (int i = 0; i < result.Length; i++)
                {
                    count = 0;
                    foreach (var item in result[i].OrderByDescending(k=>k.Value)) 
                    {
                        ResultMerge.AddOrUpdate(item.Key, item.Value, (k, v) => v + item.Value);
                        count++;
                        if (count > 5 * N)
                            break;
                    }
                }
                count = 0;
                var FinaloutputResult = from KVP in ResultMerge
                                        orderby KVP.Value descending
                                        select new StringBuilder(KVP.Key).Append(" ").Append(KVP.Value);
                foreach (var str in FinaloutputResult)
                {
                    Console.WriteLine(str);
                    count++;
                    if (count > N - 1) break;
                }
                DateTime ot = DateTime.Now;
                Console.WriteLine("Time: " + ((ot.Minute * 60 + ot.Second) * 1000 + ot.Millisecond - (dt.Minute * 60 + dt.Second) * 1000 - dt.Millisecond) + "ms");
                //Console.ReadKey();

            });
            WorkerTh[result.Length - 1].Start();

        }

        public static void Read(string[] files)
        {
            foreach (string file in files)
            {
                queue.TryAdd(ReadFile(file), -1);
            }
            queue.TryAdd("END", -1);
        }

        public static string ReadFile(string file)
        {
            string readLine;
            FileStream fs = new FileStream(file, FileMode.Open);
            StreamReader sr = new StreamReader(fs);
            readLine = sr.ReadToEnd();
            sr.Close();
            fs.Close();
            return readLine;
        }

        public static void Process(int index)
        {
            string readLine;
            while (true)
            {
                queue.TryTake(out readLine, -1);
                if (readLine == "END")
                {
                    queue.TryAdd("END", -1);
                    break;
                }
                Compute(readLine, index);
            }
        }

        public static void Compute(string readLine, int index)
        {
            StringBuilder sb = new StringBuilder(100);
            string strKey = "";
            int state = 0;
            for (int i = 0; i < readLine.Length; i++)
            {
                switch (state)
                {
                    case 0:
                        if ((state = (readLine[i] > 'z') ? 0 : tablet[readLine[i]]) != 0)
                        {
                            sb.Clear();
                            sb.Append(readLine[i]);
                        }
                        break;
                    default:
                        if ((state = (readLine[i] > 'z') ? 0 : tablet[readLine[i]]) == 0)
                        {
                            if (sb.Length >= 1)
                            {
                                strKey = ToLower(sb).ToString();
                                //if (!result[index].TryGetValue(strKey, out value))
                                //{
                                //    result[index].TryAdd(strKey, 1);
                                //}
                                //else
                                //{
                                //    result[index][strKey]++;
                                //}
                                result[index].AddOrUpdate(strKey, 1, (k, v) => v + 1);
                            }
                        }
                        else
                            sb.Append(readLine[i]);
                        break;
                }
            }
        }

        public static StringBuilder ToLower(StringBuilder str)
        {
            for (int i = 0; i < str.Length; i++)
            {
                if (str[i] <= 'Z')
                {
                    str[i] = (char)((int)str[i] + 32);
                }
            }
            return str;
        }

我的电脑是i7-2600, 4核8线程。本来还想根据cpu的个数来动态的调线程池大小，后来发现读取cpu参数那行代码就耗时1s。。遂果断放弃。

在多次测试后发现7,8个线程的表现均良好。

另外wencong大神用的Linq自动并行化神马神马的方法可以跑进6s。

还有惊闻guojia居然只用了1.2s。。。。。。。。。。