TransferWebFileHelper1
1 public static class TransferWebFileHelper1 2 { 3 /// <summary> 4 /// 异步网络请求 5 /// </summary> 6 public static async Task CommunicatingWithWebServerAsync(string url,string saveFilePath, bool isGet = true, string postData = "", string contentType = "application/x-www-form-urlencoded;charset=utf-8") 7 { 8 if (string.IsNullOrWhiteSpace(url)) 9 throw new ArgumentNullException("url"); 10 if (string.IsNullOrWhiteSpace(contentType)) 11 throw new ArgumentNullException("contentType"); 12 HttpWebResponse response = null; 13 Stream stream = Stream.Null; 14 FileStream outStream = null; 15 try 16 { 17 if (url.ToUpper().Contains("HTTPS:")) //解决:“访问https请求被中止: 未能创建 SSL/TLS 安全通道” 18 { 19 ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | 20 SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12; 21 } 22 HttpWebRequest request = isGet 23 ? GenerateHttpWebRequest(new Uri(url)) 24 : GenerateHttpWebRequest(new Uri(url), postData, contentType); 25 response = await request.GetResponseAsync() as HttpWebResponse; 26 if (CategorizeResponse(response) == ResponseCategories.Success) 27 { 28 stream = response.GetResponseStream(); 29 30 outStream = new FileStream(saveFilePath, FileMode.Create); 31 int readByteCount = 0; 32 byte[] buffers = new byte[1024]; 33 while (true) 34 { 35 readByteCount = stream.Read(buffers, 0, buffers.Length); 36 if (readByteCount <= 0) 37 { 38 break; 39 } 40 outStream.Write(buffers, 0, readByteCount); 41 } 42 43 stream.Close(); 44 stream.Dispose(); 45 outStream.Close(); 46 outStream.Dispose(); 47 response.Close(); 48 response.Dispose(); 49 } 50 51 52 } 53 catch (Exception ex) 54 { 55 56 } 57 finally 58 { 59 System.GC.Collect();//解决程序提示线程用尽崩溃的问题 60 } 61 } 62 63 public static HttpWebRequest GenerateHttpWebRequest(Uri uri) 64 { 65 HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(uri); 66 httpRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";//部分网站会检查此项 67 return httpRequest; 68 } 69 70 public static HttpWebRequest GenerateHttpWebRequest(Uri uri, 71 string postData, 72 string contentType) 73 { 74 HttpWebRequest httpRequest = GenerateHttpWebRequest(uri); 75 76 byte[] bytes = Encoding.UTF8.GetBytes(postData); 77 78 httpRequest.ContentType = contentType; 79 80 httpRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)"; 81 82 httpRequest.ContentLength = postData.Length; 83 84 using (Stream requestStream = httpRequest.GetRequestStream()) 85 { 86 requestStream.Write(bytes, 0, bytes.Length); 87 } 88 return httpRequest; 89 } 90 91 public static ResponseCategories CategorizeResponse(HttpWebResponse httpResponse) 92 { 93 int statusCode = (int)httpResponse.StatusCode; 94 if ((statusCode >= 100) && (statusCode <= 199)) 95 { 96 return ResponseCategories.Informational; 97 } 98 else if ((statusCode >= 200) && (statusCode <= 299)) 99 { 100 return ResponseCategories.Success; 101 } 102 else if ((statusCode >= 300) && (statusCode <= 399)) 103 { 104 return ResponseCategories.Redirected; 105 } 106 else if ((statusCode >= 400) && (statusCode <= 499)) 107 { 108 return ResponseCategories.ClientError; 109 } 110 else if ((statusCode >= 500) && (statusCode <= 599)) 111 { 112 return ResponseCategories.ServerError; 113 } 114 return ResponseCategories.Unknown; 115 } 116 }
Enumerator
public class Enumerator { public enum HttpMethodEnum { Post, Get, Put, Delete, Patch } }
ResponseCategories
public enum ResponseCategories { Unknown, Informational, Success, Redirected, ClientError, ServerError }
Program
class Program { private static string[] randomArray = new[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" }; private static string GetRandomStr() { Random ran = new Random(); int i = ran.Next(0, 26); Thread.Sleep(1);//尽量让生成的字符每一个都不相同 return randomArray[i]; } static List<DownloadFile> ReadFileUrl(string folderName) { List<DownloadFile> list = new List<DownloadFile>(); DirectoryInfo dir; List<string> contents = new List<string>(); string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName); if (Directory.Exists(path)) { dir = new DirectoryInfo(path); FileInfo[] files = dir.GetFiles(); contents = File.ReadAllLines(files[0].FullName, Encoding.Default).ToList(); string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download"); contents.ForEach(fileName => { //TODO 完全限定文件名必须少于 260 个字符,并且目录名必须少于 248 个字符。 string tempFileName = fileName.Split('?')[0].Replace("\\", "").Replace(":", "").Replace("*", "").Replace("\"", "").Replace("<", "").Replace(">", "").Replace("|", ""); int index = tempFileName.LastIndexOf("/"); tempFileName = tempFileName.Substring(index + 1, tempFileName.Length - index - 1); int tempFileNameLength = tempFileName.Length; if ((260 - saveFolder.Length - 2) < tempFileNameLength) { int difference = tempFileNameLength - (260 - saveFolder.Length - 2);//差 tempFileName = tempFileName.Substring(difference, tempFileName.Length - difference); } tempFileName = GetRandomStr() + "_" + tempFileName; string saveFileName = Path.Combine(saveFolder, tempFileName); var model = new DownloadFile(fileName, folderName, saveFileName); list.Add(model); }); } return list; } private static void CreateFolder(string folderName) { string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download"); if (!Directory.Exists(saveFolder)) { Directory.CreateDirectory(saveFolder); } } private static void Main(string[] args) { List<string> folders = new List<string>() { "demo" }; Parallel.ForEach(folders, folder => CreateFolder(folder)); List<DownloadFile> downloadFiles = new List<DownloadFile>(); Parallel.ForEach(folders, folder => { downloadFiles.AddRange(ReadFileUrl(folder)); }); int i = 0; Task[] taskList = new Task[downloadFiles.Count]; downloadFiles.ForEach(downloadFile => { try { Task t = TransferWebFileHelper1.CommunicatingWithWebServerAsync(downloadFile.FileName,downloadFile.SaveFileName); taskList[i] = t; } catch (Exception ex) { } i++; }); Task.WaitAll(taskList); } }
测试
程序根目录下,新建一个文件夹,名demo。内新建一个txt文件,文件名随意。测试如下图片链接[随意]:
http://www.banktunnel.eu/tumblr.com/tabea-lara.2_files/tumblr_n4kxopXkJ41sq93cpo10_1280.jpg
http://www.pptok.com/wp-content/uploads/2012/08/xunguang-9.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_3163ee0990499b7.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_25d8b5e402d0391.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFNWIA9vCAAmNhS2C8eAAAPyvQApsb4ACY2d098.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOGIIc0VAAom_BUlna4AAPyvQCLDjYACicU104.png
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOqIWH2cAAXWIl1DvA4AAPyvQDbhzkABdY6485.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFPGIfiurAAKxaF37_vcAAPyvQGII2YAArGA075.jpg
http://www.hr.com.cn/attachments/2015/06/b16af76b691f47609f3397e8cdd9c1e2.jpg
http://www.hr.com.cn/attachments/2015/06/226fc8acee0d478c9588ded1d04616b1.jpg
http://www.hr.com.cn/attachments/2015/06/c36e39b67d4c410cb794f553a271377b.jpg
下载完成的图片存放在根目录下名demo_download的文件夹下
测试过支持10W左右的图片下载,不崩溃。