• 各大搜索引擎蜘蛛的UserAgent


    GOOGLE 
    --------------------------------------------------------------------- 
    66.249.70.212 - - [11/Jan/2009:00:03:35 -0700] "GET www.vidun.com/user-f2fc990265c712c49d51a18a32b39f0c.html?umid=f2fc990265c712c49d51a18a32b39f0c HTTP/1.1" 200 8148 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 
    Referer: "" 
    UserAgent: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 

    66.249.70.212 - - [11/Jan/2009:03:27:23 -0700] "GET www.youxigao.com/images/pink/demo.gif HTTP/1.1" 200 2367 "-" "Googlebot-Image/1.0" 
    Referer: "" 
    UserAgent: "Googlebot-Image/1.0" 

    209.85.238.7 - - [11/Jan/2009:00:02:58 -0700] "GET www.youxigao.com/rss/c/1009 HTTP/1.1" 404 37 "-" "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 10 subscribers; feed-id=8474979256887526569)" 
    Referer: "" 
    UserAgent: "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 10 subscribers; feed-id=8474979256887526569)" 


    百度 
    --------------------------------------------------------------------- 
    60.28.22.38 - - [11/Jan/2009:01:28:09 -0700] "GET www.vidun.com/vwsoft-vwantileechs-download.html?pr=vwantileechs&vi=download HTTP/1.1" 200 27406 "http://www.vidun.com/" "Baiduspider+(+http://www.baidu.com/search/spider.htm)" 
    Referer: "" 
    UserAgent: "Baiduspider+(+http://www.baidu.com/search/spider.htm)" 


    YAHOO 
    --------------------------------------------------------------------- 
    202.160.180.81 - - [11/Jan/2009:00:02:44 -0700] "GET www.vidun.com/ HTTP/1.0" 200 14250 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" 
    Referer: "" 
    UserAgent: "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" 

    67.195.37.167 - - [11/Jan/2009:00:23:00 -0700] "GET www.vidun.com/postmsg-tech-2-120.html?type=tech&id=2&tid=120 HTTP/1.0" 200 12609 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 
    Referer: "" 
    UserAgent: "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 


    有道 
    --------------------------------------------------------------------- 
    2008-03-04 09:54:12 W3SVC226223753 222.33.192.54 GET /index.php - 80 - 61.135.219.7 Mozilla/5.0+(compatible;+YodaoBot/1.0;+http://www.yodao.com/help/webmaster/spider/;+) - 200 0 0 
    Referer: "" 
    UserAgent: "Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )" 


    61.135.249.120 - - [11/Jan/2009:09:44:46 -0700] "GET vidun.com/robots.txt HTTP/1.1" 404 - "-" "Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )" 
    Referer: "" 
    UserAgent: "Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )" 


    SOSO 
    --------------------------------------------------------------------- 
    58.61.164.207 - - [11/Jan/2009:03:13:53 -0700] "GET www.vidun.com/robots.txt HTTP/1.1" 404 - "http://www.vidun.com/robots.txt" "Sosospider+(+http://help.soso.com/webspider.htm)" 
    Referer: "" 
    UserAgent: "Sosospider+(+http://help.soso.com/webspider.htm)" 

    2008-03-04 10:48:28 W3SVC226223753 222.33.192.54 GET /index.php - 80 - 124.115.4.218 Sosoimagespider+(+http://help.soso.com/soso-image-spider.htm) http://www.tatabao.com/ 200 0 0 
    Referer: "" 
    UserAgent: "Sosoimagespider+(+http://help.soso.com/soso-image-spider.htm)" 


    Sogou 
    --------------------------------------------------------------------- 
    219.234.81.41 - - [11/Jan/2009:03:26:49 -0700] "GET www.vidun.com/ HTTP/1.0" 200 14250 "-" "Sogou Web Sprider(compatible; Mozilla 4.0; MSIE 6.0; Windows NT 5.1; SV1; Avant Browser; InfoPath.1; .NET CLR 2.0.50727; .NET CLR1.1.4322)" 
    Referer: "" 
    UserAgent: "Sogou Web Sprider(compatible; Mozilla 4.0; MSIE 6.0; Windows NT 5.1; SV1; Avant Browser; InfoPath.1; .NET CLR 2.0.50727; .NET CLR1.1.4322)" 


    220.181.61.217 - - [11/Jan/2009:13:10:57 -0700] "GET www.youxigao.com/play/3615?id=3615 HTTP/1.1" 302 5 "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 
    Referer: "" 
    UserAgent: "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 

    220.181.19.74 - - [11/Jan/2009:06:20:37 -0700] "GET www.vidun.com/vwsoft-vwantileechs-download.html?pr=vwantileechs&vi=download HTTP/1.1" 200 27406 "-" "Sogou Orion spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 
    Referer: "" 
    UserAgent: "Sogou Orion spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 


    220.181.19.78 - - [11/Jan/2009:10:55:18 -0700] "GET www.vidun.com/robots.txt HTTP/1.1" 404 - "http://pic.sogou.com/" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)" 
    Referer: "http://pic.sogou.com/" 
    UserAgent: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)" 

    219.234.81.27 - - [11/Jan/2009:23:53:41 -0700] "GET www.vidun.com/ HTTP/1.1" 200 14271 "-" "Sogou-Test-Spider/4.0 (compatible; MSIE 5.5; Windows 98)" 
    Referer: "" 
    UserAgent: "Sogou-Test-Spider/4.0 (compatible; MSIE 5.5; Windows 98)" 

    2008-03-04 17:23:59 W3SVC226223753 222.33.192.54 HEAD /index.php - 80 - 220.181.19.107 Sogou+head+spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07) - 200 0 0 

  • 相关阅读:
    如何创建支持Eclipse IDE的Maven项目
    使用Nexus搭建Maven私服
    Maven找不到java编译器的问题
    关于java -version版本问题
    win7右下角声音图标不见的解决方法
    如何用Maven创建一个普通Java项目
    如何手动把jar包添加进Maven本地仓库
    Maven是如何工作的
    Maven远程仓库
    canvas+js实现时钟效果图
  • 原文地址:https://www.cnblogs.com/feb9903/p/3430734.html
Copyright © 2020-2023  润新知