• 多线程下载图片


    TransferWebFileHelper1

      1 public static class TransferWebFileHelper1
      2     {
      3         /// <summary>
      4         /// 异步网络请求
      5         /// </summary>
      6         public static async Task CommunicatingWithWebServerAsync(string url,string saveFilePath, bool isGet = true, string postData = "", string contentType = "application/x-www-form-urlencoded;charset=utf-8")
      7         {
      8             if (string.IsNullOrWhiteSpace(url))
      9                 throw new ArgumentNullException("url");
     10             if (string.IsNullOrWhiteSpace(contentType))
     11                 throw new ArgumentNullException("contentType");
     12             HttpWebResponse response = null;
     13             Stream stream = Stream.Null;
     14             FileStream outStream = null;
     15             try
     16             {
     17                 if (url.ToUpper().Contains("HTTPS:")) //解决:“访问https请求被中止: 未能创建 SSL/TLS 安全通道”
     18                 {
     19                     ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls |
     20                                                            SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
     21                 }
     22                 HttpWebRequest request = isGet
     23                     ? GenerateHttpWebRequest(new Uri(url))
     24                     : GenerateHttpWebRequest(new Uri(url), postData, contentType);
     25                 response = await request.GetResponseAsync() as HttpWebResponse;
     26                 if (CategorizeResponse(response) == ResponseCategories.Success)
     27                 {
     28                     stream = response.GetResponseStream();
     29 
     30                     outStream = new FileStream(saveFilePath, FileMode.Create);
     31                     int readByteCount = 0;
     32                     byte[] buffers = new byte[1024];
     33                     while (true)
     34                     {
     35                         readByteCount = stream.Read(buffers, 0, buffers.Length);
     36                         if (readByteCount <= 0)
     37                         {
     38                             break;
     39                         }
     40                         outStream.Write(buffers, 0, readByteCount);
     41                     }
     42 
     43                     stream.Close();
     44                     stream.Dispose();
     45                     outStream.Close();
     46                     outStream.Dispose();
     47                     response.Close();
     48                     response.Dispose();
     49                 }
     50 
     51 
     52             }
     53             catch (Exception ex)
     54             {
     55 
     56             }
     57             finally
     58             {
     59                 System.GC.Collect();//解决程序提示线程用尽崩溃的问题
     60             }
     61         }
     62 
     63         public static HttpWebRequest GenerateHttpWebRequest(Uri uri)
     64         {
     65             HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(uri);
     66             httpRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";//部分网站会检查此项
     67             return httpRequest;
     68         }
     69 
     70         public static HttpWebRequest GenerateHttpWebRequest(Uri uri,
     71             string postData,
     72             string contentType)
     73         {
     74             HttpWebRequest httpRequest = GenerateHttpWebRequest(uri);
     75 
     76             byte[] bytes = Encoding.UTF8.GetBytes(postData);
     77 
     78             httpRequest.ContentType = contentType;
     79 
     80             httpRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";           
     81 
     82             httpRequest.ContentLength = postData.Length;
     83 
     84             using (Stream requestStream = httpRequest.GetRequestStream())
     85             {
     86                 requestStream.Write(bytes, 0, bytes.Length);
     87             }
     88             return httpRequest;
     89         }
     90 
     91         public static ResponseCategories CategorizeResponse(HttpWebResponse httpResponse)
     92         {
     93             int statusCode = (int)httpResponse.StatusCode;
     94             if ((statusCode >= 100) && (statusCode <= 199))
     95             {
     96                 return ResponseCategories.Informational;
     97             }
     98             else if ((statusCode >= 200) && (statusCode <= 299))
     99             {
    100                 return ResponseCategories.Success;
    101             }
    102             else if ((statusCode >= 300) && (statusCode <= 399))
    103             {
    104                 return ResponseCategories.Redirected;
    105             }
    106             else if ((statusCode >= 400) && (statusCode <= 499))
    107             {
    108                 return ResponseCategories.ClientError;
    109             }
    110             else if ((statusCode >= 500) && (statusCode <= 599))
    111             {
    112                 return ResponseCategories.ServerError;
    113             }
    114             return ResponseCategories.Unknown;
    115         }
    116     }

    Enumerator

        public class Enumerator
        {
            public enum HttpMethodEnum
            {
                Post,
                Get,
                Put,
                Delete,
                Patch
            }
        }

    ResponseCategories

        public enum ResponseCategories
        {
            Unknown,        
            Informational, 
            Success,       
            Redirected,     
            ClientError,    
            ServerError
        }

    Program

    class Program
        {
            private static string[] randomArray = new[]
            {
                "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V",
                "W", "X", "Y", "Z"
            };
            private static string GetRandomStr()
            {
                Random ran = new Random();
                int i = ran.Next(0, 26);
                Thread.Sleep(1);//尽量让生成的字符每一个都不相同
                return randomArray[i];
            }
    
            static List<DownloadFile> ReadFileUrl(string folderName)
            {
                List<DownloadFile> list = new List<DownloadFile>();
                DirectoryInfo dir;
                List<string> contents = new List<string>();
                string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName);
                if (Directory.Exists(path))
                {
                    dir = new DirectoryInfo(path);
                    FileInfo[] files = dir.GetFiles();
                    contents = File.ReadAllLines(files[0].FullName, Encoding.Default).ToList();
    
                    string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
    
                    contents.ForEach(fileName =>
                    {
                        //TODO 完全限定文件名必须少于 260 个字符,并且目录名必须少于 248 个字符。
                        string tempFileName = fileName.Split('?')[0].Replace("\\", "").Replace(":", "").Replace("*", "").Replace("\"", "").Replace("<", "").Replace(">", "").Replace("|", "");
                        int index = tempFileName.LastIndexOf("/");
                        tempFileName = tempFileName.Substring(index + 1, tempFileName.Length - index - 1);
                        int tempFileNameLength = tempFileName.Length;
                        if ((260 - saveFolder.Length - 2) < tempFileNameLength)
                        {
                            int difference = tempFileNameLength - (260 - saveFolder.Length - 2);//
                            tempFileName = tempFileName.Substring(difference, tempFileName.Length - difference);
                        }
                        tempFileName = GetRandomStr() + "_" + tempFileName;
    
    
                        string saveFileName = Path.Combine(saveFolder, tempFileName);
                        var model = new DownloadFile(fileName, folderName, saveFileName);
                        list.Add(model);
                    });
                }
                return list;
            }
    
            private static void CreateFolder(string folderName)
            {
                string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
                if (!Directory.Exists(saveFolder))
                {
                    Directory.CreateDirectory(saveFolder);
                }
            }
    
            private static void Main(string[] args)
            {
                List<string> folders = new List<string>()
                {
                    "demo"
                };
                Parallel.ForEach(folders, folder => CreateFolder(folder));
    
                List<DownloadFile> downloadFiles = new List<DownloadFile>();
    
                Parallel.ForEach(folders, folder =>
                {
                    downloadFiles.AddRange(ReadFileUrl(folder));
                });
    
                int i = 0;
                Task[] taskList = new Task[downloadFiles.Count];
                downloadFiles.ForEach(downloadFile =>
                {
                    try
                    {
                        Task t = TransferWebFileHelper1.CommunicatingWithWebServerAsync(downloadFile.FileName,downloadFile.SaveFileName);
                        taskList[i] = t;
    
                    }
                    catch (Exception ex)
                    {
    
                    }
                    i++;
                });
                Task.WaitAll(taskList);
            }
        }

    测试

    程序根目录下,新建一个文件夹,名demo。内新建一个txt文件,文件名随意。测试如下图片链接[随意]:

    http://www.banktunnel.eu/tumblr.com/tabea-lara.2_files/tumblr_n4kxopXkJ41sq93cpo10_1280.jpg
    http://www.pptok.com/wp-content/uploads/2012/08/xunguang-9.jpg
    http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_3163ee0990499b7.jpg
    http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_25d8b5e402d0391.jpg
    https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFNWIA9vCAAmNhS2C8eAAAPyvQApsb4ACY2d098.jpg
    https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOGIIc0VAAom_BUlna4AAPyvQCLDjYACicU104.png
    https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOqIWH2cAAXWIl1DvA4AAPyvQDbhzkABdY6485.jpg
    https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFPGIfiurAAKxaF37_vcAAPyvQGII2YAArGA075.jpg
    http://www.hr.com.cn/attachments/2015/06/b16af76b691f47609f3397e8cdd9c1e2.jpg
    http://www.hr.com.cn/attachments/2015/06/226fc8acee0d478c9588ded1d04616b1.jpg
    http://www.hr.com.cn/attachments/2015/06/c36e39b67d4c410cb794f553a271377b.jpg

    下载完成的图片存放在根目录下名demo_download的文件夹下

    测试过支持10W左右的图片下载,不崩溃。

  • 相关阅读:
    bstToDoublyList
    Springboot系列1_什么是Springboot
    servlet总结
    JavaFx开发桌面软件
    在Emacs中使用plantuml画UML图
    IIS7.0上传文件限制的解决方法
    开源免费的天气预报接口API以及全国所有地区代码(国家气象局提供)
    如何得到天气情况?在那个接口获取?
    Windows Update 时出现8024402C的错误
    OC中copy的使用
  • 原文地址:https://www.cnblogs.com/jeff151013/p/10594089.html
Copyright © 2020-2023  润新知