jdk11 HttpClient 爬虫

忘是亡心i 2022-01-14 14:31 257阅读 0赞

[2019独角兽企业重金招聘Python工程师标准>>> ][2019_Python_] ![hot3.png][]

> 目的: 获得目标背单词网站中的单词, 写了一个简单的小爬虫, 使用jdk11

![eb85b63d30446ce1b3ce84577c7b17a8d38.jpg][]

![4a781b0bbebcce42efc165ef1a9970815d3.jpg][]

![b922e5b400dea2201c8faadb1befd403be8.jpg][]

> 到此, 思路明确!
> 
> 第一步,  把冰箱门...., 串词了,Sorry!!
> 
> 第一步, 调用登陆接口, 拿到sessionid!
> 
> 第二步, 带着sessionid到单词列表页, 拿到body, 转成Document, 开始"借鉴"单词!
> 
> 是不是So easy!

package com.***;
    
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    import org.junit.Test;
    
    import java.io.IOException;
    import java.net.URI;
    import java.net.http.HttpClient;
    import java.net.http.HttpRequest;
    import java.net.http.HttpResponse;
    import java.util.HashMap;
    
    /**
     * @author jqw1122@foxmail.com
     * @description 爬啊爬
     * @date 2/23/2019 17:14 
     */
    public class Crawler {
    
        @Test
        public void crawler() {
            String loginUrl = "http://www.cikuang.me/login";
            String fromBody = "username=jqw1122@foxamil.com&password=qweqwe123";
            String wordSetUrl= "http://www.cikuang.me/member/learningset?id=4573";
    
    
            HttpClient httpClient = HttpClient.newBuilder().build();
            HttpRequest request = HttpRequest.newBuilder()
                    .uri(URI.create(loginUrl))
                    .header("Content-Type","application/x-www-form-urlencoded")
                    .POST(HttpRequest.BodyPublishers.ofString(fromBody))
                    .build();
    
    
            httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
                    .thenApply(HttpResponse::headers)
                    .thenAccept(headers -> {
                //cookie有好多 我只要sid啊魂淡!
                var cookieMap = new HashMap<String, String>();
                headers.map().get("set-cookie").forEach(c -> {
                    String[] split = c.split(";");
                    for (String s : split) {
                        String[] split1 = s.split("=");
                        if (split1.length == 2)
                            cookieMap.put(split1[0], split1[1]);
                    }
                });
                //拿着sid去单词页面
                String cookie_sid = cookieMap.get("sid");
                HttpRequest request2 = HttpRequest.newBuilder()
                        .uri(URI.create(wordSetUrl))
                        .header("Content-Type","application/x-www-form-urlencoded")
                        .header("Cookie", "sid=" + cookie_sid)
                        .GET()
                        .build();
                httpClient.sendAsync(request2, HttpResponse.BodyHandlers.ofString())
                        .thenApply(HttpResponse::body)
                        .thenAccept(htmlString ->{
                    //获取到body转成Document, 方便借鉴...
                    Document htmlDocument = Jsoup.parse(htmlString);
                    
                    //获取单词table id
                    Element wordListTable = htmlDocument.getElementById("wordListTable");
                    Elements trs = wordListTable.getElementsByTag("tr");
                    trs.forEach(t -> {
                        Elements tds = t.children();
                        String en = tds.get(0).child(0).text();
                        String cn = tds.get(1).text();
                        System.out.println("单词---->>> " + en + ":" + cn);
                    });
                }).join();
            } ).join();
        }
    }

>  爬虫2: 目的:获取KMF中托福-听力-所有练习题的题目的音频

![5430b7e6bfb7cc2972907c9a7817f7de720.jpg][] ![cb3c1546a83fcbdf3963947e68669df2cf4.jpg][]

/**
     * @author jqw1122@foxmail.com
     * @description
     * @date 2/23/2019 17:14
     */
    public class Crawler {
        @Test
        public void crawlerKMF() {
            String mainUrl= "https://toefl.kmf.com";
            String mainUrl1= "https://toefl.kmf.com/listen/ets/order/";
            String localFilePath = "C:\\kmf_audio\\";
    
            HttpClient httpClient = HttpClient.newBuilder().build();
    
            List<String> detailUrlList = new ArrayList<>();
            e:for (int i = 0; i <= 5; i++) {
                for (int j = 1; j <= 4; j++) {
                    String url = mainUrl1 + i + "/0/" + j;
                    HttpRequest request = HttpRequest.newBuilder()
                            .uri(URI.create(url))
                            .header("Content-Type","application/x-www-form-urlencoded")
                            .GET()
                            .build();
                    httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()).thenApply(HttpResponse::body).thenAccept(bodyString -> {
                        Document htmlDocument = Jsoup.parse(bodyString);
                        Elements elements = htmlDocument.getElementsByAttributeValue("class", "check-links js-check-link");
                        elements.forEach(tagA -> {
                            String href = tagA.attr("href");
                            detailUrlList.add(href);
                        });
                        System.out.println("page detail number:" + elements.size());
                    }).join();
                    //test
    //            if (1==1) break e;
                }
            }
            System.out.println("page/file number:  "+detailUrlList.size());
    
            var fileList = new ArrayList<Map<String, String>>();
    
            System.out.println(LocalTime.now().toString() + " start get audio file url in detail page");
            detailUrlList.parallelStream().forEach(href -> {
                HttpRequest request = HttpRequest.newBuilder()
                        .uri(URI.create(mainUrl + href))
                        .header("Content-Type","application/x-www-form-urlencoded")
                        .GET()
                        .build();
                httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()).thenApply(HttpResponse::body).thenAccept(bodyString -> {
                    Document htmlDocument = Jsoup.parse(bodyString);
                    Elements bts = htmlDocument.getElementsByAttributeValue("class", "i-title js-top-title");
                    String fileName = bts.get(0).text();
                    Elements audios = htmlDocument.getElementsByAttributeValue("class", "question-audio-cont js-question-audio  g-player-control video-left-content js-player-record");
                    String fileUrl = audios.get(0).attr("data-url");
                    fileList.add(Map.of("fileName", fileName.toLowerCase().replace(" ", "_") + ".mp3", "fileUrl", fileUrl));
    //                System.out.println(fileName+ "--"+fileUrl);
                }).join();
            });
            System.out.println(LocalTime.now().toString() + " finish get audio file url in detail page! start downloading files to local!");
    
    
            fileList.parallelStream().forEach(t -> {
                try (InputStream ins = new URL(t.get("fileUrl")).openStream()) {
                    Path target = Paths.get(localFilePath, t.get("fileName"));
    //                Files.createDirectories(target.getParent());
                    Files.copy(ins, target, StandardCopyOption.REPLACE_EXISTING);
    
                } catch (IOException e) {
                    System.out.println("download failed! fileName:" + t.get("fileName") + " fileUrl:" + t.get("fileUrl"));
                    e.printStackTrace();
                }
            });
            System.out.println(LocalTime.now().toString() + " download completed");
    
    
        }
    
    
    
    }

![cdc4d5b2bcbc22c452196263b488c170aba.jpg][]下载成功了....

转载于:https://my.oschina.net/jiangqw/blog/3013965

[2019_Python_]: https://my.oschina.net/u/2663968/blog/3061697
[hot3.png]: /images/20220114/9984bcad527742e495fe3916f7ebf59a.png
[eb85b63d30446ce1b3ce84577c7b17a8d38.jpg]: /images/20220114/ed82db6bb1d1409a84bd7d158e166ce1.png
[4a781b0bbebcce42efc165ef1a9970815d3.jpg]: /images/20220114/3eefbab5bfd74ebf845721e7d7408964.png
[b922e5b400dea2201c8faadb1befd403be8.jpg]: /images/20220114/62f24a2cbaaf4bbba949fad4b00e55e2.png
[5430b7e6bfb7cc2972907c9a7817f7de720.jpg]: https://oscimg.oschina.net/oscnet/5430b7e6bfb7cc2972907c9a7817f7de720.jpg
[cb3c1546a83fcbdf3963947e68669df2cf4.jpg]: /images/20220114/60fdeba3d2c247c18c1dfef8b295984f.png
[cdc4d5b2bcbc22c452196263b488c170aba.jpg]: /images/20220114/0f5524889d0d46cfb4b1bca6445dbe6b.png