JAVA——微博热搜爬虫

小鱼儿 2022-09-08 14:50 217阅读 0赞

源代码：https://gitee.com/my-zstu/myzstu/blob/develop/myzstu-spyder/src/main/java/club/zstuca/myzstu/spyder/misc/WeiboTopSpyder.java

# 解决方案 #

package club.zstuca.myzstu.spyder.misc;
    
    import club.zstuca.myzstu.spyder.misc.entity.WeiboTopItem;
    import club.zstuca.myzstu.utils.http.HttpUtil;
    import org.apache.http.HttpResponse;
    import org.apache.http.util.EntityUtils;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    
    import java.util.ArrayList;
    import java.util.List;
    
    public class WeiboTopSpyder { 
        public String url = "https://s.weibo.com/top/summary";
        public List<WeiboTopItem> run(){ 
            try { 
                HttpResponse response = HttpUtil.doGet(url);
                String content = EntityUtils.toString(response.getEntity());
                if (content == null || content.isEmpty()) { 
                    return null;
                }
                Document document = Jsoup.parse(content);
                Elements elements = document.select("#pl_top_realtimehot").select("tbody").select("tr");
                List<WeiboTopItem> list  = new ArrayList<>();
                elements.forEach(item->{ 
                    WeiboTopItem topItem = new WeiboTopItem();
                    Element element = item.select("td").get(1).select("a").first();
                    if (element == null) { 
                        return;
                    }
                    topItem.setName(element.text());
                    topItem.setUrl("https://s.weibo.com" + element.attr("href"));
                    list.add(topItem);
                });
                return list;
            }catch (Exception e){ 
                e.printStackTrace();
            }
            return null;
        }
    }