• 抓取历史天气


    工具类

    package com.example.demo.util;
    
    import org.apache.poi.hssf.usermodel.*;
    import org.apache.poi.xssf.usermodel.XSSFCell;
    import org.apache.poi.xssf.usermodel.XSSFRow;
    import org.apache.poi.xssf.usermodel.XSSFSheet;
    import org.apache.poi.xssf.usermodel.XSSFWorkbook;
    
    import java.io.FileInputStream;
    import java.io.IOException;
    import java.io.InputStream;
    import java.text.DecimalFormat;
    import java.text.SimpleDateFormat;
    import java.util.ArrayList;
    import java.util.Date;
    import java.util.List;
    
    public class ReadExcel {
    
        public static List<List<String>> readExcel(String path) throws IOException {
            if (path.endsWith(".xlsx")) {
                return readXlsx(path);
            } else if (path.endsWith(".xls")) {
                return readXlsx(path);
            } else {
                return null;
            }
        }
    
        /**
         * Read the Excel 2010
         */
        public static List<List<String>> readXlsx(String path) throws IOException {
            InputStream is = new FileInputStream(path);
            XSSFWorkbook xssfWorkbook = new XSSFWorkbook(is);
    
            List<List<String>> list = new ArrayList<List<String>>();
            // Read the Sheet
            int numSheets = xssfWorkbook.getNumberOfSheets();// 获取sheet页数
            for (int numSheet = 0; numSheet < 1; numSheet++) {
                XSSFSheet xssfSheet = xssfWorkbook.getSheetAt(numSheet);
                if (xssfSheet == null) {
                    continue;
                }
                // Read the Row
                int rowNums = xssfSheet.getLastRowNum();
                for (int rowNum = 0; rowNum <= rowNums; rowNum++) {
                    List<String> l = new ArrayList<String>();
                    // 获取对应的行数据
                    XSSFRow xssfRow = xssfSheet.getRow(rowNum);
                    if (xssfRow != null) {
                        // 获取列数
                        int columnNum = xssfRow.getLastCellNum();
                        for (int cloNum = 0; cloNum < columnNum; cloNum++) {
                            l.add(getValue(xssfRow.getCell(cloNum)));
                        }
                        list.add(l);
                    }
                }
            }
            return list;
        }
    
        /**
         * Read the Excel 2003
         */
        public void readXls(String path) throws IOException {
            InputStream is = new FileInputStream(path);
            HSSFWorkbook hssfWorkbook = new HSSFWorkbook(is);
    
            List<List<String>> list = new ArrayList<List<String>>();
            // Read the Sheet
            int numSheets = hssfWorkbook.getNumberOfSheets();// 获取sheet页数
            for (int numSheet = 0; numSheet < 1; numSheet++) {
                HSSFSheet hssfSheet = hssfWorkbook.getSheetAt(numSheet);
                if (hssfSheet == null) {
                    continue;
                }
                // Read the Row
                int rowNums = hssfSheet.getLastRowNum();
                for (int rowNum = 0; rowNum <= rowNums; rowNum++) {
                    // 获取对应的行数据
                    HSSFRow hssfRow = hssfSheet.getRow(rowNum);
                    List<String> l = new ArrayList<String>();
                    if (hssfRow != null) {
                        // 获取列数
                        int columnNum = hssfRow.getLastCellNum();
                        for (int cloNum = 0; cloNum < columnNum; cloNum++) {
                            l.add(getValue(hssfRow.getCell(cloNum)));
                        }
                        list.add(l);
                    }
                }
            }
        }
    
        /**
         * 取值2010excel
         *
         * @param cell
         * @return
         */
        private static String getValue(XSSFCell cell) {
            if (cell == null) {
                return "";
            }
    
            String value = "";
    
            switch (cell.getCellType()) {
                case HSSFCell.CELL_TYPE_STRING:
                    value = cell.getStringCellValue();
                    break;
                case HSSFCell.CELL_TYPE_NUMERIC:
                    if (HSSFDateUtil.isCellDateFormatted(cell)) {
                        Date date = cell.getDateCellValue();
                        if (date != null) {
                            value = new SimpleDateFormat("yyyy-MM-dd").format(date);
                        } else {
                            value = "";
                        }
                    } else {
                        value = new DecimalFormat("0").format(cell.getNumericCellValue());
                    }
                    break;
                case HSSFCell.CELL_TYPE_FORMULA:
                    // 导入时如果为公式生成的数据则无值
                    if (!cell.getStringCellValue().equals("")) {
                        value = cell.getStringCellValue();
                    } else {
                        value = cell.getNumericCellValue() + "";
                    }
                    break;
                case HSSFCell.CELL_TYPE_BLANK:
                    break;
                case HSSFCell.CELL_TYPE_ERROR:
                    break;
                case HSSFCell.CELL_TYPE_BOOLEAN:
                    value = (cell.getBooleanCellValue() == true ? "Y" : "N");
                    break;
                default:
                    value = "";
            }
            return value.trim();
        }
    
        /**
         * 取值2003excel
         *
         * @param cell
         * @return
         */
        private String getValue(HSSFCell cell) {
            if (cell == null) {
                return "";
            }
    
            String value = "";
    
            switch (cell.getCellType()) {
                case HSSFCell.CELL_TYPE_STRING:
                    value = cell.getStringCellValue();
                    break;
                case HSSFCell.CELL_TYPE_NUMERIC:
                    if (HSSFDateUtil.isCellDateFormatted(cell)) {
                        Date date = cell.getDateCellValue();
                        if (date != null) {
                            value = new SimpleDateFormat("yyyy-MM-dd").format(date);
                        } else {
                            value = "";
                        }
                    } else {
                        value = new DecimalFormat("0").format(cell.getNumericCellValue());
                    }
                    break;
                case HSSFCell.CELL_TYPE_FORMULA:
                    // 导入时如果为公式生成的数据则无值
                    if (!cell.getStringCellValue().equals("")) {
                        value = cell.getStringCellValue();
                    } else {
                        value = cell.getNumericCellValue() + "";
                    }
                    break;
                case HSSFCell.CELL_TYPE_BLANK:
                    break;
                case HSSFCell.CELL_TYPE_ERROR:
                    break;
                case HSSFCell.CELL_TYPE_BOOLEAN:
                    value = (cell.getBooleanCellValue() == true ? "Y" : "N");
                    break;
                default:
                    value = "";
            }
            return value.trim();
        }
    }
    View Code
    package com.example.demo.util;
    
    import com.alibaba.fastjson.JSON;
    import com.alibaba.fastjson.JSONArray;
    import com.alibaba.fastjson.JSONObject;
    
    import java.io.BufferedReader;
    import java.io.IOException;
    import java.io.InputStreamReader;
    import java.io.PrintWriter;
    import java.net.URL;
    import java.net.URLConnection;
    import java.util.List;
    import java.util.Map;
    
    public class HttpRequest {
    
        /**
         * 向指定URL发送GET方法的请求
         *
         * @param url   发送请求的URL
         * @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。
         * @return URL 所代表远程资源的响应结果
         */
        public static String sendGet(String url, String param) {
    
            String result = "";
            BufferedReader in = null;
            try {
                String urlNameString = url + "?" + param;
                URL realUrl = new URL(urlNameString);
                // 打开和URL之间的连接
                URLConnection connection = realUrl.openConnection();
                // 设置通用的请求属性
                connection.setRequestProperty("accept", "*/*");
                connection.setRequestProperty("connection", "Keep-Alive");
                connection.setRequestProperty("Charsert", "GBK"); //设置请求编码
                connection.setRequestProperty("user-agent",
                        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
                // 建立实际的连接
                connection.connect();
                // 获取所有响应头字段
                Map<String, List<String>> map = connection.getHeaderFields();
                // 遍历所有的响应头字段
                for (String key : map.keySet()) {
                    System.out.println(key + "--->" + map.get(key));
                }
                // 定义 BufferedReader输入流来读取URL的响应
                in = new BufferedReader(new InputStreamReader(
                        connection.getInputStream(), "GBK"));
                String line;
                while ((line = in.readLine()) != null) {
                    result += line;
                }
            } catch (Exception e) {
                System.out.println("get请求异常, 网站[" + url + "]");
    //            System.out.println("发送GET请求出现异常!" + e);
    //            e.printStackTrace();
            }
            // 使用finally块来关闭输入流
            finally {
                try {
                    if (in != null) {
                        in.close();
                    }
                } catch (Exception e2) {
                    e2.printStackTrace();
                }
            }
            return result;
        }
    
        /**
         * 向指定 URL 发送POST方法的请求
         *
         * @param url   发送请求的 URL
         * @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。
         * @return 所代表远程资源的响应结果
         */
        public static String sendPost(String url, String param) {
            PrintWriter out = null;
            BufferedReader in = null;
            String result = "";
            try {
                URL realUrl = new URL(url);
                // 打开和URL之间的连接
                URLConnection conn = realUrl.openConnection();
                // 设置通用的请求属性
                conn.setRequestProperty("accept", "*/*");
                conn.setRequestProperty("connection", "Keep-Alive");
                conn.setRequestProperty("Charsert", "GBK"); //设置请求编码
                conn.setRequestProperty("user-agent",
                        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
                // 发送POST请求必须设置如下两行
                conn.setDoOutput(true);
                conn.setDoInput(true);
                // 获取URLConnection对象对应的输出流
                out = new PrintWriter(conn.getOutputStream());
                // 发送请求参数
                out.print(param);
                // flush输出流的缓冲
                out.flush();
                // 定义BufferedReader输入流来读取URL的响应
                in = new BufferedReader(
                        new InputStreamReader(conn.getInputStream(), "GBK"));
                String line;
                while ((line = in.readLine()) != null) {
                    result += line;
                }
            } catch (Exception e) {
                System.out.println("发送 POST 请求出现异常!" + e);
                e.printStackTrace();
            }
            //使用finally块来关闭输出流、输入流
            finally {
                try {
                    if (out != null) {
                        out.close();
                    }
                    if (in != null) {
                        in.close();
                    }
                } catch (IOException ex) {
                    ex.printStackTrace();
                }
            }
            return result;
        }
    
    
        public static void main(String[] args) {
            //发送 GET 请求
            String s = HttpRequest.sendGet("http://tianqi.2345.com/t/wea_history/js/201701/70188_201701.js", "");
    
            s = s.substring("var weather_str=".length(), s.length() - 1);
            System.out.println(s);
    
            JSONObject object = JSON.parseObject(s);
            String city = object.get("city").toString();
            System.out.println(object.get("city"));
    
            JSONArray array = JSONArray.parseArray(object.get("tqInfo").toString());
            for (int i = 0; i < array.size(); i++) {
                String ymd = array.getJSONObject(i).get("ymd").toString();
                String bWendu = array.getJSONObject(i).get("bWendu").toString();
                String yWendu = array.getJSONObject(i).get("yWendu").toString();
                String tianqi = array.getJSONObject(i).get("tianqi").toString();
                String fengxiang = array.getJSONObject(i).get("fengxiang").toString();
                String fengli = array.getJSONObject(i).get("fengli").toString();
                String aqi = array.getJSONObject(i).get("aqi").toString();
                String aqiInfo = array.getJSONObject(i).get("aqiInfo").toString();
    
                System.out.println(fengli);
            }
    
    
    
    
          /*  //发送 POST 请求
            String sr=HttpRequest.sendPost("http://localhost:6144/Home/RequestPostString", "key=123&v=456");
            System.out.println(sr);*/
        }
    }
    View Code

    结果类 TqBean

    package com.example.demo.tianqi;
    
    public class TqBean {
        private String shen;
        private String city;
        private String url;
        private String code;
    
        private String ymd;
        private String bWendu;
        private String yWendu;
        private String tianqi;
        private String fengxiang;
        private String fengli;
        private String aqi;
        private String aqiInfo;
    
        public TqBean(String shen, String city, String ymd, String bWendu, String yWendu, String tianqi, String fengxiang, String fengli, String aqi, String aqiInfo) {
            this.shen = shen;
            this.city = city;
            this.ymd = ymd;
            this.bWendu = bWendu;
            this.yWendu = yWendu;
            this.tianqi = tianqi;
            this.fengxiang = fengxiang;
            this.fengli = fengli;
            this.aqi = aqi;
            this.aqiInfo = aqiInfo;
        }
    
        public TqBean(String shen, String city, String url, String code) {
            this.shen = shen;
            this.city = city;
            this.url = url;
            this.code = code;
        }
    
        public TqBean() {
    
        }
    
        public String getShen() {
            return shen;
        }
    
        public void setShen(String shen) {
            this.shen = shen;
        }
    
        public String getCity() {
            return city;
        }
    
        public void setCity(String city) {
            this.city = city;
        }
    
        public String getUrl() {
            return url;
        }
    
        public void setUrl(String url) {
            this.url = url;
        }
    
        public String getYmd() {
            return ymd;
        }
    
        public void setYmd(String ymd) {
            this.ymd = ymd;
        }
    
        public String getbWendu() {
            return bWendu;
        }
    
        public void setbWendu(String bWendu) {
            this.bWendu = bWendu;
        }
    
        public String getyWendu() {
            return yWendu;
        }
    
        public void setyWendu(String yWendu) {
            this.yWendu = yWendu;
        }
    
        public String getTianqi() {
            return tianqi;
        }
    
        public void setTianqi(String tianqi) {
            this.tianqi = tianqi;
        }
    
        public String getFengxiang() {
            return fengxiang;
        }
    
        public void setFengxiang(String fengxiang) {
            this.fengxiang = fengxiang;
        }
    
        public String getFengli() {
            return fengli;
        }
    
        public void setFengli(String fengli) {
            this.fengli = fengli;
        }
    
        public String getAqi() {
            return aqi;
        }
    
        public void setAqi(String aqi) {
            this.aqi = aqi;
        }
    
        public String getAqiInfo() {
            return aqiInfo;
        }
    
        public void setAqiInfo(String aqiInfo) {
            this.aqiInfo = aqiInfo;
        }
    
        public String getCode() {
            return code;
        }
    
        public void setCode(String code) {
            this.code = code;
        }
    }
    View Code

    线程类 TianQiThread

    package com.example.demo.tianqi;
    
    import com.alibaba.fastjson.JSON;
    import com.alibaba.fastjson.JSONArray;
    import com.alibaba.fastjson.JSONObject;
    import com.example.demo.util.HttpRequest;
    
    import java.util.Set;
    
    public class TianQiThread implements Runnable {
    
        private String shen;
        private String url;
        private Set<TqBean> resList;
        private Set<String> errirUrls;
    
        public TianQiThread(String shen, String url, Set<TqBean> resList, Set<String> errirUrls) {
            this.shen = shen;
            this.url = url;
            this.resList = resList;
            this.errirUrls = errirUrls;
        }
    
        @Override
        public void run() {
            String res = HttpRequest.sendGet(url, "");
            if (res.equals("")) {
                errirUrls.add(url);
            } else {
                res = res.substring("var weather_str=".length(), res.length() - 1);
                JSONObject object = JSON.parseObject(res);
                String city = object.get("city").toString();
    
                JSONArray array = JSONArray.parseArray(object.get("tqInfo").toString());
                for (int i = 0; i < array.size(); i++) {
                    String ymd = array.getJSONObject(i).get("ymd").toString();
                    String bWendu = array.getJSONObject(i).get("bWendu").toString();
                    String yWendu = array.getJSONObject(i).get("yWendu").toString();
                    String tianqi = array.getJSONObject(i).get("tianqi").toString();
                    String fengxiang = array.getJSONObject(i).get("fengxiang").toString();
                    String fengli = array.getJSONObject(i).get("fengli").toString();
                    String aqi = array.getJSONObject(i).get("aqi").toString();
                    String aqiInfo = array.getJSONObject(i).get("aqiInfo").toString();
    
                    TqBean tq = new TqBean(shen, city, ymd, bWendu, yWendu, tianqi, fengxiang, fengli, aqi, aqiInfo);
                    resList.add(tq);
                }
            }
        }
    }
    View Code

    处理类 TianQiByYear

    package com.example.demo.tianqi;
    
    import com.example.demo.util.ReadExcel;
    import org.apache.poi.xssf.usermodel.XSSFCell;
    import org.apache.poi.xssf.usermodel.XSSFRow;
    import org.apache.poi.xssf.usermodel.XSSFSheet;
    import org.apache.poi.xssf.usermodel.XSSFWorkbook;
    
    import java.io.File;
    import java.io.FileOutputStream;
    import java.util.HashSet;
    import java.util.List;
    import java.util.Set;
    import java.util.concurrent.LinkedBlockingQueue;
    import java.util.concurrent.ThreadPoolExecutor;
    import java.util.concurrent.TimeUnit;
    
    public class TianQiByYear {
    
        private static int corePoolSize = 100;
        private static int maximumPoolSize = 1000;
        private static long keepAliveTime = 200;
    
        private static Set<TqBean> resList = new HashSet<>();
        private static Set<String> errirUrls = new HashSet<>();
    
        public static void main(String[] args) {
            try {
                Set<TqBean> list = new HashSet<>();
    
                List<List<String>> citylist = ReadExcel.readExcel("d:\2345.xlsx");
    
                // 以年为周期生成excel
                for (int year = 2017; year <= 2017; year++) {
                    resList = new HashSet<>();
                    errirUrls = new HashSet<>();
    
                    for (List<String> row : citylist) {
                        String shen = row.get(0);
                        String city = row.get(1);
                        String code = row.get(2);
    
                        if (!"河南".equals(shen)) {
                            continue;
                        }
    
                        for (int month = 1; month <= 12; month++) {
                            String url = "http://tianqi.2345.com/t/wea_history/js/";
                            String m = month > 9 ? ("" + month) : "0" + month;
                            url += year + m + "/" + code + "_" + year + m + ".js";
    
                            TqBean tq = new TqBean(shen, city, url, code);
                            list.add(tq);
                        }
                    }
    
                    ThreadPoolExecutor executor = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());
    
                    for (TqBean tq : list) {
                        executor.execute(new TianQiThread(tq.getShen(), tq.getUrl(), resList, errirUrls));
                    }
    
                    executor.shutdown();
                    while (true) { //等待线程处理完毕
                        if (executor.isTerminated()) {
                            for (String s : errirUrls) {
                                System.out.println("失败url: [" + s + "]");
                            }
                            break;
                        }
                        Thread.sleep(300);
                    }
    
                    // 导出excel
                    //创建新工作簿
                    XSSFWorkbook workbook = new XSSFWorkbook();
                    //新建工作表
                    XSSFSheet sheet = workbook.createSheet("sheet");
    
                    XSSFRow row0 = sheet.createRow(0);
    
                    XSSFCell cell = row0.createCell(0);
                    XSSFCell cell1 = row0.createCell(1);
                    XSSFCell cell2 = row0.createCell(2);
                    XSSFCell cell3 = row0.createCell(3);
                    XSSFCell cell4 = row0.createCell(4);
                    XSSFCell cell5 = row0.createCell(5);
                    XSSFCell cell6 = row0.createCell(6);
                    XSSFCell cell7 = row0.createCell(7);
                    XSSFCell cell8 = row0.createCell(8);
                    XSSFCell cell9 = row0.createCell(9);
    
                    cell.setCellValue("省份");
                    cell1.setCellValue("城市");
                    cell2.setCellValue("日期");
                    cell3.setCellValue("最高气温");
                    cell4.setCellValue("最低气温");
                    cell5.setCellValue("天气");
                    cell6.setCellValue("风向");
                    cell7.setCellValue("风力");
                    cell8.setCellValue("空气质量");
                    cell9.setCellValue("质量指数");
    
                    int rownum = 1;
                    for (TqBean t : resList) {
                        //创建行,行号作为参数传递给createRow()方法,第一行从0开始计算
                        XSSFRow row = sheet.createRow(rownum++);
    
                        //创建单元格,row已经确定了行号,列号作为参数传递给createCell(),第一列从0开始计算
                        XSSFCell mcell = row.createCell(0);
                        XSSFCell mcell1 = row.createCell(1);
                        XSSFCell mcell2 = row.createCell(2);
                        XSSFCell mcell3 = row.createCell(3);
                        XSSFCell mcell4 = row.createCell(4);
                        XSSFCell mcell5 = row.createCell(5);
                        XSSFCell mcell6 = row.createCell(6);
                        XSSFCell mcell7 = row.createCell(7);
                        XSSFCell mcell8 = row.createCell(8);
                        XSSFCell mcell9 = row.createCell(9);
    
                        mcell.setCellValue(t.getShen());
                        mcell1.setCellValue(t.getCity());
                        mcell2.setCellValue(t.getYmd());
                        mcell3.setCellValue(t.getbWendu());
                        mcell4.setCellValue(t.getyWendu());
                        mcell5.setCellValue(t.getTianqi());
                        mcell6.setCellValue(t.getFengxiang());
                        mcell7.setCellValue(t.getFengli());
                        mcell8.setCellValue(t.getAqi());
                        mcell9.setCellValue(t.getAqiInfo());
                    }
    
                    //输出到磁盘中
                    FileOutputStream fos = new FileOutputStream(new File("F:\tianqi\" + year + "年天气.xlsx"));
                    workbook.write(fos);
                    fos.close();
    
                    System.out.println("===========ok========================");
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
    View Code
    不积跬步,无以至千里;不积小流,无以成江海。
  • 相关阅读:
    百战天虫中子弹的挖坑效果
    as3对象销毁是否需要全部属性置null?
    事件冒泡其实就是它给它的parent send一个事件
    25个Photoshop创作的励志展示
    15个免费的关于杂志的wordpress主题
    wordpress主题:New Horizon 博客/创意展示主题[v1.0.3][荐](类似瀑布流)
    linux nl命令与逻辑页
    PLSQL用DBMS_JOB建立作业
    PLSQL DBMS_DDL.ANALYZE_OBJECT
    PLSQL的UTL_FILE使用例子
  • 原文地址:https://www.cnblogs.com/lovedaodao/p/8006561.html
Copyright © 2020-2023  润新知