使用StreamLold写入 Starrocks报错：Caused by org

曾经终败给现在 2024-04-28 13:09 95阅读 0赞

### 问题描述 ###

使用StreamLoad写入Starrocks报错，报这个错误:Caused by: org.apache.http.ProtocolException: Content-Length header already present

### 代码案例 ###

#### 引入依赖 ####

<dependency>
                <groupId>org.apache.httpcomponents</groupId>
                <artifactId>httpclient</artifactId>
                <version>4.5.3</version>
            </dependency>

#### 具体demo ####

package com.song.starrocks;
    
    import org.apache.commons.codec.binary.Base64;
    import org.apache.http.HttpHeaders;
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpPut;
    import org.apache.http.entity.StringEntity;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.DefaultRedirectStrategy;
    import org.apache.http.impl.client.HttpClientBuilder;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    import java.nio.charset.StandardCharsets;
    import java.util.UUID;
    
    public class StarRocksStreamLoad {
        private final static String STARROCKS_HOST = "localhost";
        private final static String STARROCKS_DB = "starrocks_demo";
        private final static String STARROCKS_TABLE = "bitmap02";
        private final static String STARROCKS_USER = "root";
        private final static String STARROCKS_PASSWORD = "";
        private final static int STARROCKS_HTTP_PORT = 8038;
    
        public void sendData(String content) throws Exception {
            //String yyyyy = "id,name,age,age=to_bitmap(age)";
            String yyyyy = "id,process_time,bkey,bvalue,bvalue=to_bitmap(bvalue),create_time";
            final String loadUrl = String.format("http://%s:%s/api/%s/%s/_stream_load",
                    STARROCKS_HOST,
                    STARROCKS_HTTP_PORT,
                    STARROCKS_DB,
                    STARROCKS_TABLE);
    
            final HttpClientBuilder httpClientBuilder = null;
    
            int TIMEOUT = 3000;
            try (CloseableHttpClient client = httpClientBuilder.build()) {
                HttpPut put = new HttpPut(loadUrl);
                RequestConfig build = RequestConfig.custom()
                        .setAuthenticationEnabled(true)
                        .setCircularRedirectsAllowed(true)
                        .setRedirectsEnabled(true)
                        .setRelativeRedirectsAllowed(true)
                        .setExpectContinueEnabled(true)
                        .setConnectTimeout(30000).setConnectionRequestTimeout(TIMEOUT)
                        .setSocketTimeout(TIMEOUT).build();
                put.setConfig(build);
                StringEntity entity = new StringEntity(content, "UTF-8");
                put.setHeader(HttpHeaders.EXPECT, "100-continue");
                put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(STARROCKS_USER, STARROCKS_PASSWORD));
                put.setHeader("columns", yyyyy);
                put.setHeader("column_separator", "\t");
                put.setHeader("max_filter_ratio", "0.2");
                put.setHeader("label", UUID.randomUUID().toString());
                put.setEntity(entity);
    
                try (CloseableHttpResponse response = client.execute(put)) {
                    String loadResult = "";
                    if (response.getEntity() != null) {
                        loadResult = EntityUtils.toString(response.getEntity());
                    }
                    final int statusCode = response.getStatusLine().getStatusCode();
                    // statusCode 200 just indicates that starrocks be service is ok, not stream load
                    // you should see the output content to find whether stream load is success
                    if (statusCode != 200) {
                        throw new IOException(
                                String.format("Stream load failed, statusCode=%s load result=%s", statusCode, loadResult));
                    }
                    System.out.println("data {}" + loadResult);
                }
            }
        }
    
        private String basicAuthHeader(String username, String password) {
            final String tobeEncode = username + ":" + password;
            byte[] encoded = Base64.encodeBase64(tobeEncode.getBytes(StandardCharsets.UTF_8));
            return "Basic " + new String(encoded);
        }
    
    }

#### 测试类 ####

注意:因为ExtRoaringBitmap是自己封装的，所以需要换成bitmap或者RortingBitmap

@Test
        public void testBitMapWriteStarrocks() throws Exception {
            ExtRoaringBitmap extRoaringBitmap = new ExtRoaringBitmap();
            extRoaringBitmap.set(111111);
    
            LongIterator longIterator = extRoaringBitmap.longIterator();
            StringBuilder stringBuilder = new StringBuilder();
    
            while (longIterator.hasNext()) {
    
                // 生成id
                Long id = 100;
    
                // 获取ExtRoaringBitmap中的具体数据，也就是获取long类型的offset
                long bitmapValue = longIterator.next();
    
                Date date = new Date();
                String currentDate = sdf.format(date);
                String bitmapKey = "张三" + id;
    
                String oneRow = id + "\t" + currentDate + "\t" + bitmapKey + "\t" + bitmapValue + "\t" + currentDate + "\n";
                stringBuilder.append(oneRow);
            }
            stringBuilder.deleteCharAt(stringBuilder.length() - 1);
            String loadData = stringBuilder.toString();
            System.out.println("loadData :{} " + loadData);
    
            StarRocksStreamLoad starrocksStreamLoad = new StarRocksStreamLoad();
            starrocksStreamLoad.sendData(loadData);
    
        }

### 报错截图 ###

![在这里插入图片描述][53fd91217ee1410fb06b6f11b51d3a47.png]

### 分析错误原因 ###

因为代码执行是在一个父项目中调用的子项目，把写入Starrocks这块封装为一个工具类，通过其他的项目来调用， 如果个给Starrocks单独一个项目的话，是可以正常执行的 ， 一旦放入到整个项目中，通过其他的项目来进行调用，就会报错，经过排查 ，错误的原因是因为父项目中也有用到依赖httpclient这个依赖，但是版本不是4.5.3的，是其他的版本 ，虽然我子项目中有这个httpclient4.5.3的依赖，但是在项目启动的时候，父项目的依赖把子项目的依赖覆盖了，导致无法使用httpclient4.5.3的依赖，所以项目一直报Caused by: org.apache.http.ProtocolException: Content-Length header already present。

### 解决办法 ###

1.在idea中安装maven的插件:Maven Helper ,通过这个可以查到里面所有的依赖;
    2.在父项目中也添加这个httpclient4.5.3的依赖就可以正常写入数据了;

[53fd91217ee1410fb06b6f11b51d3a47.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/28/3d3b3bcc62ed4c669755e6a1752d7650.png