问题描述
使用StreamLoad写入Starrocks报错，报这个错误:Caused by: org.apache.http.ProtocolException: Content-Length header already present
代码案例
引入依赖
<!-- Starrocks使用StreamLoad发送Http请求 -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
具体demo
package com.song.starrocks;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpHeaders;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.UUID;
public class StarRocksStreamLoad {
private final static String STARROCKS_HOST = "localhost";
private final static String STARROCKS_DB = "starrocks_demo";
private final static String STARROCKS_TABLE = "bitmap02";
private final static String STARROCKS_USER = "root";
private final static String STARROCKS_PASSWORD = "";
private final static int STARROCKS_HTTP_PORT = 8038;
public void sendData(String content) throws Exception {
//String yyyyy = "id,name,age,age=to_bitmap(age)";
String yyyyy = "id,process_time,bkey,bvalue,bvalue=to_bitmap(bvalue),create_time";
final String loadUrl = String.format("http://%s:%s/api/%s/%s/_stream_load",
STARROCKS_HOST,
STARROCKS_HTTP_PORT,
STARROCKS_DB,
STARROCKS_TABLE);
final HttpClientBuilder httpClientBuilder = null;
int TIMEOUT = 3000;
try (CloseableHttpClient client = httpClientBuilder.build()) {
HttpPut put = new HttpPut(loadUrl);
RequestConfig build = RequestConfig.custom()
.setAuthenticationEnabled(true)
.setCircularRedirectsAllowed(true)
.setRedirectsEnabled(true)
.setRelativeRedirectsAllowed(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(30000).setConnectionRequestTimeout(TIMEOUT)
.setSocketTimeout(TIMEOUT).build();
put.setConfig(build);
StringEntity entity = new StringEntity(content, "UTF-8");
put.setHeader(HttpHeaders.EXPECT, "100-continue");
put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(STARROCKS_USER, STARROCKS_PASSWORD));
put.setHeader("columns", yyyyy);
put.setHeader("column_separator", "\t");
put.setHeader("max_filter_ratio", "0.2");
put.setHeader("label", UUID.randomUUID().toString());
put.setEntity(entity);
try (CloseableHttpResponse response = client.execute(put)) {
String loadResult = "";
if (response.getEntity() != null) {
loadResult = EntityUtils.toString(response.getEntity());
}
final int statusCode = response.getStatusLine().getStatusCode();
// statusCode 200 just indicates that starrocks be service is ok, not stream load
// you should see the output content to find whether stream load is success
if (statusCode != 200) {
throw new IOException(
String.format("Stream load failed, statusCode=%s load result=%s", statusCode, loadResult));
}
System.out.println("data {}" + loadResult);
}
}
}
private String basicAuthHeader(String username, String password) {
final String tobeEncode = username + ":" + password;
byte[] encoded = Base64.encodeBase64(tobeEncode.getBytes(StandardCharsets.UTF_8));
return "Basic " + new String(encoded);
}
}
测试类
注意:因为ExtRoaringBitmap是自己封装的，所以需要换成bitmap或者RortingBitmap文章来源：https://www.toymoban.com/news/detail-679773.html
@Test
public void testBitMapWriteStarrocks() throws Exception {
ExtRoaringBitmap extRoaringBitmap = new ExtRoaringBitmap();
extRoaringBitmap.set(111111);
LongIterator longIterator = extRoaringBitmap.longIterator();
StringBuilder stringBuilder = new StringBuilder();
while (longIterator.hasNext()) {
// 生成id
Long id = 100;
// 获取ExtRoaringBitmap中的具体数据，也就是获取long类型的offset
long bitmapValue = longIterator.next();
Date date = new Date();
String currentDate = sdf.format(date);
String bitmapKey = "张三" + id;
String oneRow = id + "\t" + currentDate + "\t" + bitmapKey + "\t" + bitmapValue + "\t" + currentDate + "\n";
stringBuilder.append(oneRow);
}
stringBuilder.deleteCharAt(stringBuilder.length() - 1);
String loadData = stringBuilder.toString();
System.out.println("loadData :{} " + loadData);
StarRocksStreamLoad starrocksStreamLoad = new StarRocksStreamLoad();
starrocksStreamLoad.sendData(loadData);
}
报错截图
文章来源地址https://www.toymoban.com/news/detail-679773.html
分析错误原因
因为代码执行是在一个父项目中调用的子项目，把写入Starrocks这块封装为一个工具类，通过其他的项目来调用， 如果个给Starrocks单独一个项目的话，是可以正常执行的 ， 一旦放入到整个项目中，通过其他的项目来进行调用，就会报错，经过排查 ，错误的原因是因为父项目中也有用到依赖httpclient这个依赖，但是版本不是4.5.3的，是其他的版本 ，虽然我子项目中有这个httpclient4.5.3的依赖，但是在项目启动的时候，父项目的依赖把子项目的依赖覆盖了，导致无法使用httpclient4.5.3的依赖，所以项目一直报Caused by: org.apache.http.ProtocolException: Content-Length header already present。
解决办法
1.在idea中安装maven的插件:Maven Helper ,通过这个可以查到里面所有的依赖;
2.在父项目中也添加这个httpclient4.5.3的依赖就可以正常写入数据了;
到了这里，关于使用StreamLold写入 Starrocks报错：Caused by org的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！
![[starrocks BE] 启动报错问题记录](https://imgs.yssmx.com/Uploads/2024/02/478520-1.png)
