c#扩展出MapReduce方法

MapReduce方法主体：

public static IDictionary<TKey, TResult> MapReduce<TInput, TKey, TValue, TResult>(this IList<TInput> inputList,
            Func<MapReduceData<TInput>, KeyValueClass<TKey, TValue>> map, Func<TKey, IList<TValue>, TResult> reduce)
        {
            object locker = new object();
            ConcurrentDictionary<TKey, TResult> result = new ConcurrentDictionary<TKey, TResult>();
            //保存map出来的结果
            ConcurrentDictionary<TKey, IList<TValue>> mapDic = new ConcurrentDictionary<TKey, IList<TValue>>();
            var parallelOptions = new ParallelOptions();
            parallelOptions.MaxDegreeOfParallelism = Environment.ProcessorCount;
            //并行map
            Parallel.For(0, inputList.Count(), parallelOptions, t =>
            {
                MapReduceData<TInput> data = new MapReduceData<TInput>
                {
                    Data = inputList[t],
                    Index = t,
                    List = inputList,
                };
                var pair = map(data);
                if (pair != null && pair.Valid)
                {
                    //锁住防止并发操作list造成数据缺失
                    lock (locker)
                    {
                        //将匹配出来的结果加入结果集放入字典
                        IList<TValue> list = null;
                        if (mapDic.ContainsKey(pair.Key))
                        {
                            list = mapDic[pair.Key];
                        }
                        else
                        {
                            list = new List<TValue>();
                            mapDic[pair.Key] = list;
                        }
                        list.Add(pair.Value);
                    }
                }
            });

            //并行reduce
            Parallel.For(0, mapDic.Keys.Count, parallelOptions, t =>
            {
                KeyValuePair<TKey, IList<TValue>> pair = mapDic.ElementAt(t);
                result[pair.Key] = reduce(pair.Key, pair.Value);
            });
            return result;
        }

KeyValueClass定义：

public class KeyValueClass<K, V>
    {
        public KeyValueClass(K key, V value)
        {
            Key = key;
            Value = value;
        }

        public KeyValueClass()
        {

        }

        public K Key { get; set; }

        public V Value { get; set; }
    }

Console测试：

List<TestClass> listTestClass = new List<TestClass>();
            listTestClass.Add(new TestClass { a = "a", g = 1 });
            listTestClass.Add(new TestClass { a = "b", g = 3 });
            listTestClass.Add(new TestClass { a = "c", g = 4 });
            listTestClass.Add(new TestClass { a = "d", g = 2 });
            listTestClass.Add(new TestClass { a = "e", g = 1 });
            listTestClass.Add(new TestClass { a = "f", g = 2 });
            listTestClass.Add(new TestClass { a = "g", g = 5 });
            listTestClass.Add(new TestClass { a = "h", g = 6 });
            IDictionary<int, string> dic = listTestClass.MapReduce(t =>
            {
                if (t.g < 5)
                {
                    return new KeyValueClass<int, string>(t.g, t.a);
                }
                return null;
            }, (key, values) =>
           {
               return string.Join(",", values);
           });

TestClass定义：

public class TestClass
    {
        public string a { get; set; }
        public string b { get; set; }

        public string d { get; set; }

        //public DateTime f { get; set; }

        public int g { get; set; }

        public List<TestClass> test { get; set; }

        public Dictionary<string, string> dic { get; set; }
    }

结果：

1：a,e

2：d,f

3：b

4：c

词频性能测试