htmlunit爬取js异步加载后的页面

深藏阁楼爱情的钟 2022-09-27 12:29 190阅读 0赞

直接上代码：

一、 index.html  
调用后台请求获取content中的内容。

<html>
    <head>
    	<script type="text/javascript" src="./jquery.min.js"></script>
    </head>
    <body>
    <h2>Hello World!</h2>
    <div id="content"></div>
    <script type="text/javascript">
    $(document).ready(function(){
    	  $.post("/evh/test/testList",{},function(data){
    		  $("#content").text(JSON.stringify(data));
    	  }); 
    });
    </script>
    </body>
    </html>

二、TestController.java  
/test/testList接口从后台数据库获取数据。

package com.everhomes.proxy.controller;
    
    import javax.annotation.Resource;
    
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.springframework.web.bind.annotation.ExceptionHandler;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RestController;
    
    import com.everhomes.proxy.mapper.TestMapper;
    
    @RestController
    @RequestMapping("/test")
    public class TestController {
    	private static final Logger logger = LoggerFactory.getLogger(TestController.class);
    	
    	@Resource
    	private TestMapper testMapper;
    	
    	@RequestMapping("testList")
    	public Object testList(){
    		return testMapper.testList();
    	};
    	
    	@ExceptionHandler(Exception.class)
    	public Object exception(Exception e){
    		logger.error("error: ", e);
    		return "error: " + e.toString();
    	}
    }

三、Crawler.java

package com.everhomes.generate;
    
    import java.io.IOException;
    
    import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
    import com.gargoylesoftware.htmlunit.WebClient;
    import com.gargoylesoftware.htmlunit.html.HtmlPage;
    
    public class Crawler {
    	public static void main(String[] args) throws IOException, InterruptedException {
    		WebClient webClient = new WebClient(BrowserVersion.CHROME);  
    		    webClient.getOptions().setJavaScriptEnabled(true);
    		    webClient.getOptions().setCssEnabled(false);
    		    webClient.getOptions().setRedirectEnabled(true);
    		    webClient.getOptions().setThrowExceptionOnScriptError(false);
    		    webClient.getOptions().setTimeout(50000);
    		    HtmlPage rootPage = webClient.getPage("http://localhost:8080/evh/index.html");  
    		    webClient.waitForBackgroundJavaScript(10000);
    		 
    		    FileUtils.createFile(DIRECTORY+"cc.html", rootPage.asXml());
    		    webClient.close();
    	}
    }

四、pom.xml  
添加相关依赖。

<dependency>
    		<groupId>commons-lang</groupId>
    		<artifactId>commons-lang</artifactId>
    		<version>2.6</version>
    	</dependency>
    	<dependency> 
    	        <groupId>net.sourceforge.htmlunit</groupId> 
    	        <artifactId>htmlunit-core-js</artifactId> 
    	        <version>2.23</version> 
        </dependency> 
        <dependency> 
    	        <groupId>net.sourceforge.htmlunit</groupId> 
    	        <artifactId>htmlunit</artifactId> 
    	        <version>2.25</version> 
        </dependency>

--------------------

欢迎关注我的公众号“彤哥读源码”，查看更多“源码&架构&算法”系列文章, 与彤哥一起畅游源码的海洋。

![qrcode][]

[qrcode]: /images/20220708/68db38bbefad475eadeecaec54522b93.png