此文主要是 中国天气网和中国环境监测总站的数据抓取 打算开放全部数据抓取源代码 已在服务器上 稳定运行半个月
webapi http://api.xuzhiheng.cn/
常量
1 /// <summary> 2 /// 环保部抓取数据链接 3 /// </summary> 4 public const string HBUrl = "http://datacenter.mep.gov.cn/report/air_daily/airDairyCityHour.jsp"; 5 /// <summary> 6 /// 环境检测总站 7 /// </summary> 8 public const string HJJCZZUrl = "http://113.108.142.147:20035/emcpublish/ClientBin/Env-CnemcPublish-RiaServices-EnvCnemcPublishDomainService.svc/binary"; 9 /// <summary> 10 /// 获取所有站点AQI数据 11 /// </summary> 12 public const string AQIDataUrl = HJJCZZUrl + "/GetAQIDataPublishLives"; 13 /// <summary> 14 /// 获取所有城市AQI数据 15 /// </summary> 16 public const string CityDataUrl = HJJCZZUrl + "/GetCityAQIPublishLives"; 17 /// <summary> 18 /// 中国天气网 19 /// </summary> 20 public const string WeatherUrl = "http://www.weather.com.cn"; 21 /// <summary> 22 /// 中国天气网7天天气预报 23 /// </summary> 24 //public const string Weather7DUrl = WeatherUrl + "/weather/{0}.shtml"; 25 public const string Weather7DUrl = "http://home.cw.uitv.com.cn/Weather.ashx?service=2&cityid={0}"; 26 /// <summary> 27 /// 中国天气网实况天气预报 28 /// </summary> 29 //public const string WeatherSKUrl = "http://data.weather.com.cn/sk/{0}.html"; 30 public const string WeatherSKUrl = "http://d1.weather.com.cn/sk_2d/{0}.html"; 31 public const string WeatherSKDetailUrl = "http://flash.weather.com.cn/wmaps/xml/{0}.xml"; 32 33 34 public static readonly string[] ChinaProvinces = new string[] { "heilongjiang", "jilin", "liaoning", "hainan", "neimenggu", "xinjiang", "xizang", "qinghai", "ningxia", "gansu", "hebei", "henan", "hubei", "hunan", "shandong", "jiangsu", "anhui", "shanxi", "sanxi", "sichuan", "yunnan", "guizhou", "zhejiang", "fujian", "jiangxi", "guangdong", "guangxi", "beijing", "tianjin", "shanghai", "chongqing", "xianggang", "aomen", "taiwan", "xisha", "nanshadao", "diaoyudao" }; 35 36 /// <summary> 37 /// 风向 38 /// </summary> 39 public static readonly string[] WindDirections = new string[] { "无持续风向", "东北风", "东风", "东南风", "南风", "西南风", "西风", "西北风", "北风", "旋转风" }; 40 /// <summary> 41 /// 风力 42 /// </summary> 43 public static readonly string[] WindForces = new string[] { "微风", "3-4级", "4-5级", "5-6级", "6-7级", "7-8级", "8-9级", "9-10级", "10-11级", "11-12级" }; 44 /// <summary> 45 /// 天气 46 /// </summary> 47 public static readonly string[] Weathers = new string[] { "晴", "多云", "阴", "阵雨", "雷阵雨", "冰雹", "雨夹雪", "小雨", "中雨", "大雨", "暴雨", "大暴雨", "特大暴雨", "阵雪", "小雪", "中雪", "大雪", "暴雪", "雾", "冻雨", "沙尘暴", "小到中雨", "中到大雨", "大到暴雨", "暴雨到大暴雨", "大暴雨到特大暴雨", "小到中雪", "中到大雪", "大到暴雪", "浮尘", "扬沙", "强沙尘暴", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "霾" }; 48 /// <summary> 49 /// 质数代号 50 /// </summary> 51 public static readonly string[] Indexs = new string[] { "ag", "cl", "lk", "yh", "uv", "ls", "mf", "pj", "tr", "yd", "hc", "ac", "ct", "gj", "ys", "nl", "pk", "zs", "xc", "co", "dy", "fs", "gm", "jt", "xq", "pl", "pp" }; 52 /// <summary> 53 /// 质数名称 54 /// </summary> 55 public static readonly string[] IndexNames = new string[] { "息斯敏过敏指数", "晨练指数", "路况指数", "约会指数", "紫外线强度指数", "晾晒指数", "美发指数", "啤酒指数", "旅游指数", "运动指数", "划船指数", "空调开启指数", "穿衣指数", "逛街指数", "雨伞指数", "夜生活指数", "放风筝指数", "中暑指数", "洗车指数", "舒适度指数", "钓鱼指数", "防晒指数", "感冒指数", "交通指数", "心情指数", "空气污染扩散条件指数", "化妆指数" };
中国天气网 没什么 要说的 直接抓取 就可以了 代码如下
7天天气预报 一天发布3次
1 foreach (int areaId in weatherAreaIds) 2 { 3 if (weather7DLastGrabTime[areaId] < DateTime.Today || ((weather7DLastGrabTime[areaId] == DateTime.Today.AddHours(8) && DateTime.Now.Hour > 10) || 4 (weather7DLastGrabTime[areaId] == DateTime.Today.AddHours(11) && DateTime.Now.Hour > 17) || 5 (weather7DLastGrabTime[areaId] == DateTime.Today.AddDays(-1).AddHours(18) && DateTime.Now.Hour > 7))) 6 { 7 string res = GetRequest(string.Format(Constant.Weather7DUrl, areaId), "isexist=1"); 8 try 9 { 10 JsonForecast jsonForecast = JsonConvert.DeserializeObject<JsonForecast>(res); 11 Forecast7D forecast7D = Mapper.Map<Qtyb, Forecast7D>(jsonForecast.qtyb); 12 forecast7D.City = jsonForecast.city; 13 forecast7D.CityId = areaId; 14 forecast7D.Time = jsonForecast.ftime; 15 if (jsonForecast.ftime == weather7DLastGrabTime[areaId]) 16 { 17 Thread.Sleep(1000); 18 continue; 19 } 20 lock (aqiEntities) 21 { 22 weather7DLastGrabTime[areaId] = forecast7D.Time; 23 aqiEntities.GrabDataConfig.FirstOrDefault(p => p.Key == areaId.ToString() && p.Type == 5).Time = forecast7D.Time; 24 aqiEntities.Forecast7D.Add(forecast7D); 25 aqiEntities.SaveChanges(); 26 Log(string.Format("【天气预报】 {0} 发布时间：{1} 获取时间：{2}", forecast7D.City, forecast7D.Time, DateTime.Now)); 27 } 28 } 29 catch (Exception) 30 { 31 } 32 } 33 }
天气实况
1 string res = GetRequest(string.Format(Constant.WeatherSKDetailUrl, area), Constant.WeatherUrl, "").Replace(area, "JsonSKDetail"); 2 using (StreamReader streamReader = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(res)), Encoding.UTF8)) 3 { 4 try 5 { 6 JsonSKDetail jsonSKDetail = (JsonSKDetail)xmlSerializer.Deserialize(streamReader); 7 foreach (JsonSKDetailCity jsonSKDetailCity in jsonSKDetail.city) 8 { 9 WeatherSKDetail weatherSKDetail = Mapper.Map<JsonSKDetailCity, WeatherSKDetail>(jsonSKDetailCity); 10 weatherSKDetail.dn = jsonSKDetail.dn; 11 if (!weatherSKDetails.Exists(p => p.url == weatherSKDetail.url) && weatherSKDetailLastGrabTime[weatherSKDetail.url] != weatherSKDetail.time) 12 { 13 weatherSKDetails.Add(weatherSKDetail); 14 weatherSKDetailLastGrabTime[weatherSKDetail.url] = weatherSKDetail.time; 15 lock (aqiEntities) 16 { 17 aqiEntities.GrabDataConfig.FirstOrDefault(p => p.Type == 3 && p.Key == weatherSKDetail.url.ToString()).Time = weatherSKDetail.time; 18 aqiEntities.SaveChanges(); 19 } 20 } 21 if (!string.IsNullOrEmpty(jsonSKDetailCity.pyName)) 22 { 23 GetSKDetail(jsonSKDetailCity.pyName); 24 } 25 } 26 } 27 catch (Exception ex) 28 { 29 string s = ex.Message; 30 } 31 }
中国环境检测总站
通过抓包发现用的是wcf 返回的是二进制xml文档 需要用到 一个 https://github.com/waf/WCF-Binary-Message-Inspector 把数据解析出来 同时他也是fiddler的一个插件 可以在里面直接看出
1 private void GetCityAQI() 2 { 3 bool isWriteFile = false; 4 if (DateTime.Now >= lastGrabCityAQITime.AddHours(1)) 5 { 6 #region 获取数据 7 while (true) 8 { 9 try 10 { 11 XmlElement xmlElement = GetXmlElement(Constant.CityDataUrl); 12 XmlNode xn = xmlElement.LastChild.LastChild; 13 XmlNodeList xmlNodeList = xn.ChildNodes; 14 DateTime TimePoint = DateTime.Now; 15 if (xmlNodeList.Count > 0) 16 { 17 XmlElement xe = (XmlElement)xmlNodeList[0]; 18 TimePoint = DateTime.Parse(xe.GetElementsByTagName("b:TimePoint")[0].InnerText); 19 if (TimePoint == lastGrabCityAQITime) 20 { 21 isWriteFile = false; 22 } 23 else 24 { 25 lastGrabCityAQITime = TimePoint; 26 isWriteFile = true; 27 } 28 } 29 #region 数据操作 30 if (isWriteFile) 31 { 32 foreach (XmlElement xe in xmlNodeList) 33 { 34 string Area = xe.GetElementsByTagName("b:Area")[0].InnerText; 35 string PrimaryPollutant = xe.GetElementsByTagName("b:PrimaryPollutant")[0].InnerText; 36 if (PrimaryPollutant == "—") 37 PrimaryPollutant = null; 38 int? AQI = xe.GetElementsByTagName("b:AQI")[0].InnerText.ToNullable<int>(); 39 int? SO2 = xe.GetElementsByTagName("b:SO2")[0].InnerText.ToNullable<int>(); 40 int? PM2_5 = xe.GetElementsByTagName("b:PM2_5")[0].InnerText.ToNullable<int>(); 41 int? PM10 = xe.GetElementsByTagName("b:PM10")[0].InnerText.ToNullable<int>(); 42 int? O3 = xe.GetElementsByTagName("b:O3")[0].InnerText.ToNullable<int>(); 43 int? NO2 = xe.GetElementsByTagName("b:NO2")[0].InnerText.ToNullable<int>(); 44 float? CO = xe.GetElementsByTagName("b:CO")[0].InnerText.ToNullable<float>(); 45 int CityCode = int.Parse(xe.GetElementsByTagName("b:CityCode")[0].InnerText); 46 var model = new CityAQI() 47 { 48 CityCode = CityCode, 49 Area = Area, 50 O3 = O3, 51 CO = CO, 52 TimePoint = TimePoint, 53 AQI = AQI, 54 PrimaryPollutant = PrimaryPollutant, 55 NO2 = NO2, 56 PM10 = PM10, 57 PM2_5 = PM2_5, 58 SO2 = SO2 59 }; 60 cityAQIs.Add(model); 61 } 62 lock (aqiEntities) 63 { 64 aqiEntities.CityAQI.AddRange(cityAQIs); 65 aqiEntities.GrabDataConfig.FirstOrDefault(p => p.Key == "CityAQI").Time = lastGrabCityAQITime; 66 aqiEntities.SaveChanges(); 67 cityAQIs.Clear(); 68 } 69 lb_log.Invoke((EventHandler)delegate 70 { 71 if (lb_log.Items.Count > 200) 72 { 73 lb_log.Items.Clear(); 74 } 75 lb_log.Items.Add("中国环境检测总站 城市" + " " + TimePoint.ToString("yyyy-MM-dd HH:mm:ss") + "数据 " + DateTime.Now); 76 lb_log.SelectedIndex = lb_log.Items.Count - 1; 77 }); 78 Thread.Sleep(60 * 1000); 79 } 80 #endregion 81 break; 82 } 83 catch (Exception) 84 { 85 Thread.Sleep(60 * 1000); 86 } 87 } 88 #endregion 89 } 90 }
关注微信公众帐号获取最新更新微信号kxshzs
QQ:519872449 QQ群：77877965