课堂练习-顶会热词统计

题目要求：

用户可给定论文列表 ◦通过论文列表，爬取论文的题目、摘要、关键词、原文链接；

可对论文列表进行增删改操作(今年、近两年、近三年)；

•对爬取的信息进行结构化处理，分析top10个热门领域或热门研究方向；可进行论文检索，当用户输入论文编号、题目、关键词等基本信息，分析返回相关的paper、source code、homepage等信息

形成如关键词图谱之类直观的查看方式；

•可对多年间、不同顶会的热词呈现热度走势对比（这里将范畴限定在计算机视觉的三大顶会CVPR、ICCV、ECCV内）。

顶会背景：

•计算机视觉领域世界三大顶级会议分别为国际计算机视觉与模式识别会议（CVPR）、国际计算机视觉大会（ICCV）和欧洲计算机视觉国际会议（ECCV）。

•以CVPR 为例，据会议官网显示，2018年大会有超过 3300 篇论文投稿，其中录取 979 篇。录取的论文有三种介绍形式：oral、spotlight、poster。

任务分解： 

1.          python爬取CVPR2019论文到数据库中；
2.          关键词拆分并排序；
3.          生成热点词汇云；利用echarts-cloud.js和echarts-min.js成词汇云
4.          点击热词云中的热词可以找到与之对应的文章题目；

Python代码：

import requests
import pymysql
from bs4 import BeautifulSoup

db = pymysql.connect('127.0.0.1',
                     port=3306,
                     user='root',
                     password='123456',
                     db='mytest',
                     charset='utf8')

cursor = db.cursor()

headers={
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
        }
url="http://openaccess.thecvf.com/CVPR2019.py"
html=requests.get(url)

soup=BeautifulSoup(html.content,'html.parser')

soup.a.contents=='pdf'

pdfs=soup.findAll(name="a",text="pdf")

lis = []
jianjie=""
for i,pdf in enumerate(pdfs):
    pdf_name=pdf["href"].split('/')[-1]
    name=pdf_name.split('.')[0].replace("_CVPR_2019_paper","")
    link="http://openaccess.thecvf.com/content_CVPR_2019/html/"+name+"_CVPR_2019_paper.html"
    url1=link
    html1 = requests.get(url1)
    soup1 = BeautifulSoup(html1.content, 'html.parser')
    weizhi = soup1.find('div', attrs={'id':'abstract'})
    if weizhi:
        jianjie =weizhi.get_text();
    print("这是第"+str(i)+"条数据")
    keyword = str(name).split('_')
    keywords = ''
    for k in range(len(keyword)):
        if (k == 0):
            keywords += keyword[k]
        else:
            keywords += ',' + keyword[k]
    info = {}
    info['title'] = name
    info['link'] =link
    info['abstract']=jianjie
    info['keywords']=keywords
    lis.append(info)

cursor = db.cursor()
for i in range(len(lis)):
    cols = ", ".join('`{}`'.format(k) for k in lis[i].keys())
    print(cols)  # '`name`, `age`'

    val_cols = ', '.join('%({})s'.format(k) for k in lis[i].keys())
    print(val_cols)  # '%(name)s, %(age)s'

    sql = "insert into lunwen(%s) values(%s)"
    res_sql = sql % (cols, val_cols)
    print(res_sql)

    cursor.execute(res_sql, lis[i])  # 将字典a传入
    db.commit()
    num=1
    print(num)
    print("ok")

dao层代码：

package dao;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.Collectors;
import  Bean.copy.*;
import jdbc.Util;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import com.sun.xml.internal.ws.policy.privateutil.PolicyUtils.Collections;

public class Dao {
    public static Map<String,Integer> getrc()
    {
        String sql="select * from lunwen";
        Map<String, Integer>map= new HashMap<String, Integer>();
        Map<String, Integer>results= new LinkedHashMap<String, Integer>();
        Connection con=null;
        Statement state=null;
        ResultSet rs=null;
        con=Util.getConn();
        try {
            state=con.createStatement();
            rs=state.executeQuery(sql);
            while(rs.next())
            {
                String keywords=rs.getString("keywords");
                String[] split = keywords.split(",");
                for(int i=0;i<split.length;i++)
                {
                    if(map.get(split[i])==null)
                    {
                        map.put(split[i],0);
                    }
                    else
                    {
                        map.replace(split[i], map.get(split[i])+1);
                    }
                }
            }
        } catch (SQLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        Util.close(rs, state, con);
        map.entrySet()                
        .stream()               
        .sorted((p1, p2) -> p2.getValue().compareTo(p1.getValue()))                
        .collect(Collectors.toList())
        .forEach(ele -> results.put(ele.getKey(), ele.getValue()));
        return results;
    }
    public List<Data> list(String keywords) { // 查询所有信息
        List<Data> list = new ArrayList<Data>(); // 创建集合
        Connection conn = Util.getConn();
        String sql = "select * from lunwen where keywords like "+"'%"+keywords+"%'"; // SQL查询语句
        try {
            PreparedStatement pst = conn.prepareStatement(sql);
            ResultSet rs = pst.executeQuery();
            Data data = null;
            while (rs.next()) {
                String title = rs.getString("title");
                String link = rs.getString("link");
                String as= rs.getString("abstract");
                data = new Data(title,link,as,keywords);
                list.add(data);
            }
            rs.close(); // 关闭
            pst.close(); // 关闭
        } catch (SQLException e1) {
            e1.printStackTrace(); // 抛出异常
        }
        return list; // 返回一个集合
    }
}

RcServlet层代码：

package servlet;

import java.io.IOException;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import dao.Dao;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;

@WebServlet("/RcServlet")
public class RcServlet extends HttpServlet {
    private static final long serialVersionUID = 1L;
    /**
     * @see HttpServlet#HttpServlet()
     */
    public RcServlet() {
        super();
        // TODO Auto-generated constructor stub
    }
    /**
     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        this.doPost(request, response);
    }
    /**
     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        request.setCharacterEncoding("utf-8");
        Map<String, Integer>sortMap=Dao.getrc();
        JSONArray json =new JSONArray();
        int k=0;
        for (Map.Entry<String, Integer> entry : sortMap.entrySet()) 
        {
            JSONObject ob=new JSONObject();
            ob.put("name", entry.getKey());
            ob.put("value", entry.getValue());
            if(!(entry.getKey().equals("for")||entry.getKey().equals("and")||entry.getKey().equals("With")||entry.getKey().equals("of")||entry.getKey().equals("in")||entry.getKey().equals("From")||entry.getKey().equals("A")||entry.getKey().equals("to")||entry.getKey().equals("a")||entry.getKey().equals("the")||entry.getKey().equals("by")))
            {
                json.add(ob);
                k++;
            }
            if(k==10)
                break;
        }
        System.out.println(json.toString());
        response.getWriter().write(json.toString());
    }
}

jsp页面代码：<%@ page language="java" contentType="text/html; charset=UTF-8"    pageEncoding="UTF-8"%><%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%><%request.setCharacterEncoding("utf-8"); 

response.setCharacterEncoding("utf-8");%>
<!DOCTYPE html>
<html>
<head>
<meta charset="ISO-8859-1">
<title>热词云</title>
</head>

<meta charset="UTF-8">
<link type="text/css" rel="stylesheet" href="css/style.css">
<script src="js/jquery-3.4.1.min.js"></script>
<script src="js/echarts.min.js"></script>
<script src="js/echarts-cloud.js"></script>
<style>              
           #main{
              width: 30%;
              height: 500px;
              border:1px solid #ddd;
              float:right;
          }
          #table{
                overflow-x: auto;
                 overflow-y: auto;
                width: 70%;
                height: 500px;
                float:left;
                margin-top:100dp;
                padding-top:100dp;
            }
        </style>
</head>

<body >
<br>
<h1>热词云</h1>
<br>
<br>
<br>
<div id="table">
  <table id='gradient-style' >
    <tr>
      <th align="center">论文连接</th>
    </tr>
    <c:forEach var="item" items="${list}">
      <tr>
        <td><a href="${item.link}">${item.title}</a></td>
      </tr>
    </c:forEach>
  </table>
</div>
<div id="main">
</div>
<script type="text/javascript">
    var dt;
          $.ajax({
                url : "RcServlet",
                async : true,
                type : "POST",
                data : {        
                },
                dataType : "json",
                success : function(data) {
                    dt = data;
                     var mydata = new Array(0);
                     for (var i = 0; i < dt.length; i++) {
                          var d = {};
                          d["name"] = dt[i].name;
                          d["value"] = dt[i].value;
                          mydata.push(d);
                      }
                     var myChart = echarts.init(document.getElementById('main'));
                     //设置点击效果
                     myChart.setOption({
                         title: {
                             text: ''
                         },
                         tooltip: {},
                         series: [{
                             type : 'wordCloud',  //类型为字符云
                                 shape:'smooth',  //平滑
                                 gridSize : 8, //网格尺寸
                                 size : ['50%','50%'],
                                 //sizeRange : [ 50, 100 ],
                                 rotationRange : [-45, 0, 45, 90], //旋转范围
                                 textStyle : {
                                     normal : {
                                         fontFamily:'微软雅黑',
                                         color: function() {
                                             return 'rgb(' + 
                                                 Math.round(Math.random() * 255) +
                                          ', ' + Math.round(Math.random() * 255) +
                                          ', ' + Math.round(Math.random() * 255) + ')'
                                                }
                                         },
                                     emphasis : {
                                         shadowBlur : 5,  //阴影距离
                                         shadowColor : '#333'  //阴影颜色
                                     }
                                 },
                                 left: 'center',
                                 top: 'center',
                                 right: null,
                                 bottom: null,
                                 '100%',
                                 height:'100%',
                                 data:mydata
                         }]
                     });
                     myChart.on('click', function (params) {
                         var url = "ClickServlet?keywords=" + params.name;
                         window.location.href = url;
                       });
                    alert("成功！");
                },
                error : function() {
                    alert("请求失败");
                },
           });
</script>
</body>
</html>

ClickServlet层代码：

package servlet;

import java.io.IOException;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import Bean.copy.*;
import dao.Dao;
/**
 * Servlet implementation class ClickServlet
 */
@WebServlet("/ClickServlet")
public class ClickServlet extends HttpServlet {
    private static final long serialVersionUID = 1L;
    /**
     * @see HttpServlet#HttpServlet()
     */
    public ClickServlet() {
        super();
        // TODO Auto-generated constructor stub
    }
    /**
     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        // TODO Auto-generated method stub
        response.getWriter().append("Served at: ").append(request.getContextPath());
        doPost(request, response);
    }
    /**
     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        request.setCharacterEncoding("UTF-8");
        String keywords=request.getParameter("keywords");
        Dao dao = new Dao();
        List<Data> list=null;
        list=dao.list(keywords);
        System.out.println(list);
        request.setAttribute("list",list); 
        request.getRequestDispatcher("RC.jsp").forward(request, response);
    }
}

运行截图：