Skip to content

Commit eb58bb7

Browse files
leo-934Ceilzcxtomsun28Aias00
authored
[feature] switch to prometheus online parser & add query datasource (apache#3215)
Signed-off-by: tomsun28 <[email protected]> Signed-off-by: leo <[email protected]> Co-authored-by: Ceilzcx <[email protected]> Co-authored-by: tomsun28 <[email protected]> Co-authored-by: aias00 <[email protected]>
1 parent 066d746 commit eb58bb7

File tree

28 files changed

+507
-1192
lines changed

28 files changed

+507
-1192
lines changed

‎hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java

Lines changed: 26 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@
1818
package org.apache.hertzbeat.collector.collect.http;
1919

2020
import static org.apache.hertzbeat.common.constants.SignConstants.RIGHT_DASH;
21+
2122
import com.google.gson.JsonArray;
2223
import com.google.gson.JsonElement;
2324
import com.google.gson.JsonObject;
2425
import com.google.gson.JsonParser;
26+
2527
import java.io.IOException;
28+
import java.io.InputStream;
2629
import java.io.InterruptedIOException;
2730
import java.io.StringReader;
2831
import java.net.ConnectException;
@@ -34,23 +37,22 @@
3437
import java.util.Map;
3538
import java.util.Objects;
3639
import java.util.Set;
37-
import java.util.concurrent.ConcurrentHashMap;
3840
import java.util.stream.Collectors;
39-
import java.util.stream.Stream;
4041
import javax.net.ssl.SSLException;
4142
import javax.xml.parsers.DocumentBuilder;
4243
import javax.xml.parsers.DocumentBuilderFactory;
4344
import javax.xml.xpath.XPath;
4445
import javax.xml.xpath.XPathConstants;
4546
import javax.xml.xpath.XPathExpressionException;
4647
import javax.xml.xpath.XPathFactory;
48+
4749
import lombok.extern.slf4j.Slf4j;
4850
import org.apache.hertzbeat.collector.collect.AbstractCollect;
4951
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
5052
import org.apache.hertzbeat.collector.collect.http.promethus.AbstractPrometheusParse;
5153
import org.apache.hertzbeat.collector.collect.http.promethus.PrometheusParseCreator;
52-
import org.apache.hertzbeat.collector.collect.http.promethus.exporter.ExporterParser;
53-
import org.apache.hertzbeat.collector.collect.http.promethus.exporter.MetricFamily;
54+
import org.apache.hertzbeat.collector.collect.prometheus.parser.MetricFamily;
55+
import org.apache.hertzbeat.collector.collect.prometheus.parser.OnlineParser;
5456
import org.apache.hertzbeat.collector.constants.CollectorConstants;
5557
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
5658
import org.apache.hertzbeat.collector.util.CollectUtil;
@@ -103,10 +105,13 @@
103105
*/
104106
@Slf4j
105107
public class HttpCollectImpl extends AbstractCollect {
106-
private static final Map<Long, ExporterParser> EXPORTER_PARSER_TABLE = new ConcurrentHashMap<>();
107-
private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
108-
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
109-
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
108+
private final Set<Integer> defaultSuccessStatusCodes = Set.of(
109+
HttpStatus.SC_OK,
110+
HttpStatus.SC_CREATED,
111+
HttpStatus.SC_ACCEPTED,
112+
HttpStatus.SC_MULTIPLE_CHOICES,
113+
HttpStatus.SC_MOVED_PERMANENTLY,
114+
HttpStatus.SC_MOVED_TEMPORARILY);
110115

111116
@Override
112117
public void preCheck(Metrics metrics) throws IllegalArgumentException {
@@ -127,7 +132,7 @@ public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
127132
if (CollectionUtils.isEmpty(httpProtocol.getSuccessCodes())) {
128133
httpProtocol.setSuccessCodes(List.of(HttpStatus.SC_OK + ""));
129134
}
130-
135+
131136
HttpContext httpContext = createHttpContext(metrics.getHttp());
132137
HttpUriRequest request = createHttpRequest(metrics.getHttp());
133138
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
@@ -139,10 +144,11 @@ public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
139144
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
140145
return;
141146
}
142-
// todo This code converts an InputStream directly to a String. For large data in Prometheus exporters,
143-
// this could create large objects, potentially impacting JVM memory space significantly.
144-
// Option 1: Parse using InputStream, but this requires significant code changes;
145-
// Option 2: Manually trigger garbage collection, similar to how it's done in Dubbo for large inputs.
147+
/*
148+
this could create large objects, potentially impacting JVM memory space significantly.
149+
Option 1: Parse using InputStream, but this requires significant code changes;
150+
Option 2: Manually trigger garbage collection, similar to how it's done in Dubbo for large inputs.
151+
*/
146152
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
147153
if (!StringUtils.hasText(resp)) {
148154
log.info("http response entity is empty, status: {}.", statusCode);
@@ -156,7 +162,7 @@ public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
156162
case DispatchConstants.PARSE_PROM_QL ->
157163
parseResponseByPromQl(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
158164
case DispatchConstants.PARSE_PROMETHEUS ->
159-
parseResponseByPrometheusExporter(resp, metrics.getAliasFields(), builder);
165+
parseResponseByPrometheusExporter(response.getEntity().getContent(), metrics.getAliasFields(), builder);
160166
case DispatchConstants.PARSE_XML_PATH ->
161167
parseResponseByXmlPath(resp, metrics, builder, responseTime);
162168
case DispatchConstants.PARSE_WEBSITE ->
@@ -594,36 +600,22 @@ private void parseResponseByPromQl(String resp, List<String> aliasFields, HttpPr
594600
prometheusParser.handle(resp, aliasFields, http, builder);
595601
}
596602

597-
private void parseResponseByPrometheusExporter(String resp, List<String> aliasFields,
598-
CollectRep.MetricsData.Builder builder) {
599-
if (!EXPORTER_PARSER_TABLE.containsKey(builder.getId())) {
600-
EXPORTER_PARSER_TABLE.put(builder.getId(), new ExporterParser());
603+
private void parseResponseByPrometheusExporter(InputStream content, List<String> aliasFields, CollectRep.MetricsData.Builder builder) throws IOException {
604+
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(content);
605+
if (metricFamilyMap == null || metricFamilyMap.isEmpty()) {
606+
return;
601607
}
602-
ExporterParser parser = EXPORTER_PARSER_TABLE.get(builder.getId());
603-
Map<String, MetricFamily> metricFamilyMap = parser.textToMetric(resp);
604608
String metrics = builder.getMetrics();
605609
if (metricFamilyMap.containsKey(metrics)) {
606610
MetricFamily metricFamily = metricFamilyMap.get(metrics);
607611
for (MetricFamily.Metric metric : metricFamily.getMetricList()) {
608-
Map<String, String> labelMap = metric.getLabelPair()
612+
Map<String, String> labelMap = metric.getLabels()
609613
.stream()
610614
.collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue));
611615
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
612616
for (String aliasField : aliasFields) {
613617
if ("value".equals(aliasField)) {
614-
if (metric.getCounter() != null) {
615-
valueRowBuilder.addColumn(String.valueOf(metric.getCounter().getValue()));
616-
} else if (metric.getGauge() != null) {
617-
valueRowBuilder.addColumn(String.valueOf(metric.getGauge().getValue()));
618-
} else if (metric.getUntyped() != null) {
619-
valueRowBuilder.addColumn(String.valueOf(metric.getUntyped().getValue()));
620-
} else if (metric.getInfo() != null) {
621-
valueRowBuilder.addColumn(String.valueOf(metric.getInfo().getValue()));
622-
} else if (metric.getSummary() != null) {
623-
valueRowBuilder.addColumn(String.valueOf(metric.getSummary().getValue()));
624-
} else if (metric.getHistogram() != null) {
625-
valueRowBuilder.addColumn(String.valueOf(metric.getHistogram().getValue()));
626-
}
618+
valueRowBuilder.addColumn(String.valueOf(metric.getValue()));
627619
} else {
628620
String columnValue = labelMap.get(aliasField);
629621
valueRowBuilder.addColumn(columnValue == null ? CommonConstants.NULL_VALUE : columnValue);

0 commit comments

Comments
 (0)