顶会热词统计

爬取CVPR2019年所有论文的题目，并提取题目中的关键字，做成按照热度显示大小的热词云。

代码：

 

# coding=utf-8import pymysqlimport requestsfrom lxml import etree

 class Spider:

    def __init__(self):

        self.url = "http://openaccess.thecvf.com/CVPR2019.py"

        self.header = {

            "user-agent": "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Mobile Safari/537.36"}

        self.db = pymysql.connect(host='localhost', port=3306, user='root', passwd='abc456', db='paperdata',

                                  charset='utf8')

        self.cursor = self.db.cursor()

        self.html_list = []

    def getHtmlList(self):

        response = requests.get(self.url, headers=self.header)

        html_body = etree.HTML(response.text)

        title = html_body.xpath("//dt[@class='ptitle']/a/@href")

        for item in title:

            self.html_list.append("http://openaccess.thecvf.com/" + item)

    def getContent(self, url):

        try:

            response = requests.get(url, headers=self.header)

            body = etree.HTML(response.text)

            title = body.xpath("//div[@id='papertitle']/text()")[0]

            abstract = body.xpath("//div[@id='abstract']/text()")[0]

            down_url = body.xpath("//div[@id='content']//a/@href")[0].replace("../../", "http://openaccess.thecvf.com/")

            sql = '''insert into data values({},"{}","{}","{}")'''.format(0, title, down_url, str(abstract))

            self.cursor.execute(sql)

            print(title + "插入成功!")

            self.db.commit()

        except Exception as e:

            print(e)

    def run(self):

        self.getHtmlList()

        for url in self.html_list:

            self.getContent(url)

 if __name__ == '__main__':

    spwder = Spider()

    spwder.run()

DataDao.java

package dao;

import java.sql.SQLException;

import java.util.List;

import org.apache.commons.dbutils.QueryRunner;

import org.apache.commons.dbutils.handlers.BeanListHandler;

import pojo.Data;

import utils.DataSourceUtils;

public class DataDao {

    public List<Data> getData() throws SQLException {

        QueryRunner queryRunner = new QueryRunner(DataSourceUtils.getDataSource());

        String sql = "select * from data ";

        List<Data> dataList = queryRunner.query(sql, new BeanListHandler<Data>(Data.class));

        return dataList;

       

       

    }

    public List<Data> getLink(String name) throws SQLException {

        QueryRunner queryRunner = new QueryRunner(DataSourceUtils.getDataSource());

        String sql = "select * from data where papername like ?";

        List<Data> dataList = queryRunner.query(sql, new BeanListHandler<Data>(Data.class),"%"+name+"%");

        return dataList;

    }

}

Data.java

package pojo;

public class Data {

    private int id;

    private String papername;

    private String paperlink;

    private String paperabstract;

    public int getId() {

        return id;

    }

    public void setId(int id) {

        this.id = id;

    }

    public String getPapername() {

        return papername;

    }

    public void setPapername(String papername) {

        this.papername = papername;

    }

    public String getPaperlink() {

        return paperlink;

    }

    public void setPaperlink(String paperlink) {

        this.paperlink = paperlink;

    }

    public String getPaperabstract() {

        return paperabstract;

    }

    public void setPaperabstract(String paperabstract) {

        this.paperabstract = paperabstract;

    }

}

World.java

package pojo;

public class Word {

    private String name;

    private int value;

    public String getName() {

        return name;

    }

    public void setName(String name) {

        this.name = name;

    }

    public int getValue() {

        return value;

    }

    public void setValue(int value) {

        this.value = value;

    }

}

DataService.java

package service;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import org.apache.commons.lang.ArrayUtils;

import dao.DataDao;

import pojo.Data;

import pojo.Word;

public class DataService {

    public List<Word> getData() throws SQLException {

        DataDao dao = new DataDao();

        List<Data>  dataList= dao.getData();

        List<Word> wordList = new ArrayList<Word>();

        String [] names = new String[100000];

        for(Data data:dataList) {

            String name = data.getPapername();

            String[] namestemp = name.split(" ");

            names = (String[]) ArrayUtils.addAll(namestemp, names);

        }

        HashMap<String, Integer> name_value = new HashMap<>();

       

        for(String name:names) {

            name_value.put(name, !name_value.containsKey(name)?1:name_value.get(name)+1);

        }

       

        for(String name:name_value.keySet()) {

            Word word = new Word();

            if(name!=null&&(name_value.get(name)>1)&&(name.length()>4)) {

                word.setName(name);

                word.setValue(name_value.get(name));

                wordList.add(word);

            }

        }

        return wordList;

    }

    public List<Data> getLink(String name) throws SQLException {

        DataDao dao = new DataDao();

        return dao.getLink(name);

    }

}

ClickFunctionServlet.java

package servlet;

import java.io.IOException;

import java.sql.SQLException;

import java.util.List;

import javax.servlet.ServletException;

import javax.servlet.annotation.WebServlet;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import pojo.Data;

import service.DataService;

/**

 * Servlet implementation class ClickFunctionServlet

 */

@WebServlet("/clickFunction")

public class ClickFunctionServlet extends HttpServlet {

    private static final long serialVersionUID = 1L;

      

    /**

     * @see HttpServlet#HttpServlet()

     */

    public ClickFunctionServlet() {

        super();

        // TODO Auto-generated constructor stub

    }

    /**

     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        // TODO Auto-generated method stub

        request.setCharacterEncoding("utf-8");

        response.setContentType("text/html;charset=UTF-8");

        String name = request.getParameter("name");

        List<Data> dataList =null;

        DataService service = new DataService();

        try {

            dataList = service.getLink(name);

        } catch (SQLException e) {

            // TODO Auto-generated catch block

            e.printStackTrace();

        }

        request.setAttribute("dataList", dataList);

        request.getRequestDispatcher("papercloud.jsp").forward(request, response);

    }

    /**

     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        // TODO Auto-generated method stub

        doGet(request, response);

    }

}

GetDataServlet.java

package servlet;

import java.io.IOException;

import java.sql.SQLException;

import java.util.List;

import javax.servlet.ServletException;

import javax.servlet.annotation.WebServlet;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import com.google.gson.Gson;

import pojo.Word;

import service.DataService;

/**

 * Servlet implementation class GetDataServlet

 */

@WebServlet("/getData")

public class GetDataServlet extends HttpServlet {

    private static final long serialVersionUID = 1L;

      

    /**

     * @see HttpServlet#HttpServlet()

     */

    public GetDataServlet() {

        super();

        // TODO Auto-generated constructor stub

    }

    /**

     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        // TODO Auto-generated method stub

        request.setCharacterEncoding("utf-8");

        response.setContentType("text/html;charset=UTF-8");

        List<Word> wordList = null;

        DataService service = new DataService();

        try {

            wordList = service.getData();

        } catch (SQLException e) {

            // TODO Auto-generated catch block

            e.printStackTrace();

        }

        Gson gson = new Gson();

        String json = gson.toJson(wordList);

        response.getWriter().write(json);

    }

    /**

     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        // TODO Auto-generated method stub

        doGet(request, response);

    }

}

DataSourceUtils.java

package utils;

import java.sql.Connection;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

import javax.sql.DataSource;

import com.mchange.v2.c3p0.ComboPooledDataSource;

public class DataSourceUtils {

    private static DataSource dataSource = new ComboPooledDataSource();

    private static ThreadLocal<Connection> tl = new ThreadLocal<Connection>();

    public static DataSource getDataSource() {

        return dataSource;

    }

   

    public static Connection getConnection() throws SQLException {

        Connection con = tl.get();

        if (con == null) {

            con = dataSource.getConnection();

            tl.set(con);

        }

        return con;

    }

   

    public static void startTransaction() throws SQLException {

        Connection con = getConnection();

        if (con != null) {

            con.setAutoCommit(false);

        }

    }

   

    public static void rollback() throws SQLException {

        Connection con = getConnection();

        if (con != null) {

            con.rollback();

        }

    }

    public static void commitAndRelease() throws SQLException {

        Connection con = getConnection();

        if (con != null) {

            con.commit();

            con.close();

            tl.remove();

        }

    }

    public static void closeConnection() throws SQLException {

        Connection con = getConnection();

        if (con != null) {

            con.close();

        }

    }

    public static void closeStatement(Statement st) throws SQLException {

        if (st != null) {

            st.close();

        }

    }

    public static void closeResultSet(ResultSet rs) throws SQLException {

        if (rs != null) {

            rs.close();

        }

    }

}

c3p0-config.xml

<?xml version="1.0" encoding="UTF-8"?>

<c3p0-config>

    <default-config>

        <property name="user">root</property>

        <property name="password">0608</property>

        <property name="driverClass">com.mysql.jdbc.Driver</property>

        <property name="jdbcUrl">jdbc:mysql://localhost:3306/paperdata?serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=UTF-8</property>

    </default-config>

</c3p0-config>

papercloud.jsp

<%@ page language="java" contentType="text/html; charset=UTF-8"

    pageEncoding="UTF-8"%>

<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>

<!DOCTYPE html>

<html>

<head>

<meta charset="UTF-8">

<title>论文云</title>

<script src='https://cdn.bootcss.com/echarts/3.7.0/echarts.simple.js'></script>

<script src="./js/echarts-wordcloud.js"></script>

<script src="./js/jquery-1.11.3.min.js"></script>

<!-- 引入Bootstrap核心样式文件 -->

<link href="css/bootstrap.css" rel="stylesheet">

<!-- 引入BootStrap核心js文件 -->

<script src="./js/bootstrap.js"></script>

<style>

html, body, #main {

    100%;

    height: 100%;

    margin: 0;

}

</style>

</head>

<body>

    <div id="main"></div>

    <div>

        <table class="table table-hover">

            <thead>

                <tr>

                    <td style="font-size: 20px;">论文链接</td>

                </tr>

            </thead>

            <tbody>

                <c:forEach items="${dataList}" var="data" varStatus="vs">

                    <tr>

                        <td><a href="${data.paperlink}">${data.papername}</a></td>

                    </tr>

                </c:forEach>

            </tbody>

        </table>

    </div>

    <script>

        var chart = echarts.init(document.getElementById('main'));

        var postURL = "/PaperData/getData";

        var mydata = new Array();

        $.ajaxSettings.async = false;

        $.post(postURL, {}, function(rs) {

            var dataList = JSON.parse(rs);

            for (var i = 0; i < dataList.length; i++) {

                var d = {};

                d['name'] = dataList[i].name;

                d['value'] = dataList[i].value;

                mydata.push(d);

            }

        });

        $.ajaxSettings.async = true;

        var option = {

            tooltip : {},

            series : [ {

                type : 'wordCloud',

                gridSize : 2,

                sizeRange : [ 20, 50 ],

                rotationRange : [ -90, 90 ],

                shape : 'pentagon',

                width : 800,

                height : 600,

                drawOutOfBound : false,

                textStyle : {

                    normal : {

                        color : function() {

                            return 'rgb('

                                    + [ Math.round(Math.random() * 160),

                                            Math.round(Math.random() * 160),

                                            Math.round(Math.random() * 160) ]

                                            .join(',') + ')';

                        }

                    },

                    emphasis : {

                        shadowBlur : 10,

                        shadowColor : '#333'

                    }

                },

                data : mydata

            } ]

        };

        chart.setOption(option);

        chart.on('click', function(params) {

            var url = "clickFunction?name=" + params.name;

            window.location.href = url;

        });

    </script>

</body>

</html>

papercloud.html

<!DOCTYPE html>

<html>

<head>

<meta charset="UTF-8">

<title>论文云</title>

<script src='https://cdn.bootcss.com/echarts/3.7.0/echarts.simple.js'></script>

<script src="./js/echarts-wordcloud.js"></script>

<script src="./js/jquery-1.11.3.min.js"></script>

<!-- 引入Bootstrap核心样式文件 -->

<link href="css/bootstrap.css" rel="stylesheet">

<!-- 引入BootStrap核心js文件 -->

<script src="./js/bootstrap.js"></script>

<style>

html, body, #main {

    100%;

    height: 100%;

    margin: 0;

}

</style>

</head>

<body>

    <div id="main"></div>

    <div>

        <table class="table table-hover">

            <thead>

                <tr>

                    <td style="font-size: 20px;">论文链接</td>

                </tr>

            </thead>

            <tbody>

                <tr>

                    <td><a>www.baidu.com</a></td>

                </tr>

            </tbody>

        </table>

    </div>

    <script>

        var chart = echarts.init(document.getElementById('main'));

        var postURL = "/PaperData/getData";

        var mydata = new Array();

        $.ajaxSettings.async = false;

        $.post(postURL, {}, function(rs) {

            var dataList = JSON.parse(rs);

            for (var i = 0; i < dataList.length; i++) {

                var d = {};

                d['name'] = dataList[i].name;

                d['value'] = dataList[i].value;

                mydata.push(d);

            }

        });

        $.ajaxSettings.async = true;

        var option = {

            tooltip : {},

            series : [ {

                type : 'wordCloud',

                gridSize : 2,

                sizeRange : [ 20, 50 ],

                rotationRange : [ -90, 90 ],

                shape : 'pentagon',

                width : 800,

                height : 600,

                drawOutOfBound : false,

                textStyle : {

                    normal : {

                        color : function() {

                            return 'rgb('

                                    + [ Math.round(Math.random() * 160),

                                            Math.round(Math.random() * 160),

                                            Math.round(Math.random() * 160) ]

                                            .join(',') + ')';

                        }

                    },

                    emphasis : {

                        shadowBlur : 10,

                        shadowColor : '#333'

                    }

                },

                data : mydata

            } ]

        };

        chart.setOption(option);

        chart.on('click', function(params) {

            var url = "clickFunction?name=" + params.name;

            window.location.href = url;

        });

    </script>

</body>

</html>