diff --git a/build.gradle b/build.gradle index 4aeb1eb93a67..bfa1f1fcaec4 100644 --- a/build.gradle +++ b/build.gradle @@ -396,6 +396,7 @@ project(':iceberg-core') { implementation libs.jackson.core implementation libs.jackson.databind implementation libs.caffeine + implementation libs.opentelemetry.httpclient implementation libs.roaringbitmap compileOnly(libs.hadoop3.client) { exclude group: 'org.apache.avro', module: 'avro' diff --git a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java index 46d9177b9571..ba0c14f3fc03 100644 --- a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java +++ b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java @@ -20,6 +20,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.instrumentation.apachehttpclient.v5_2.ApacheHttpClientTelemetry; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -114,13 +116,17 @@ private HTTPClient( ObjectMapper objectMapper, Map properties, HttpClientConnectionManager connectionManager, - AuthSession session) { + AuthSession session, + OpenTelemetry openTelemetry) { this.baseUri = baseUri; this.baseHeaders = baseHeaders; this.mapper = objectMapper; this.authSession = session; - HttpClientBuilder clientBuilder = HttpClients.custom(); + HttpClientBuilder clientBuilder = + openTelemetry != null + ? ApacheHttpClientTelemetry.builder(openTelemetry).build().newHttpClientBuilder() + : HttpClients.custom(); clientBuilder.setConnectionManager(connectionManager); @@ -491,6 +497,7 @@ public static class Builder { private HttpHost proxy; private CredentialsProvider proxyCredentialsProvider; private AuthSession authSession; + private OpenTelemetry telemetry; private Builder(Map properties) { this.properties = properties; @@ -545,6 +552,12 @@ public Builder withAuthSession(AuthSession session) { return this; } + public Builder withOpenTelemetry(OpenTelemetry openTelemetry) { + Preconditions.checkNotNull(openTelemetry, "Invalid openTelemetry for http client: null"); + this.telemetry = openTelemetry; + return this; + } + public HTTPClient build() { withHeader(CLIENT_VERSION_HEADER, IcebergBuild.fullVersion()); withHeader(CLIENT_GIT_COMMIT_SHORT_HEADER, IcebergBuild.gitCommitShortId()); @@ -588,7 +601,8 @@ public HTTPClient build() { mapper, properties, configureConnectionManager(properties), - authSession); + authSession, + telemetry); } } } diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt index ebf8a55b48ff..22a145b5e0b4 100644 --- a/flink/v1.20/flink-runtime/runtime-deps.txt +++ b/flink/v1.20/flink-runtime/runtime-deps.txt @@ -6,6 +6,14 @@ com.github.luben:zstd-jni:1.5 com.google.errorprone:error_prone_annotations:2.10 dev.failsafe:failsafe:3.3 io.airlift:aircompressor:2.0 +io.opentelemetry.instrumentation:opentelemetry-apache-httpclient-5.2:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.23 +io.opentelemetry.semconv:opentelemetry-semconv:1.37 +io.opentelemetry:opentelemetry-api-incubator:1.57 +io.opentelemetry:opentelemetry-api:1.57 +io.opentelemetry:opentelemetry-common:1.57 +io.opentelemetry:opentelemetry-context:1.57 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/flink/v2.0/flink-runtime/runtime-deps.txt b/flink/v2.0/flink-runtime/runtime-deps.txt index ebf8a55b48ff..22a145b5e0b4 100644 --- a/flink/v2.0/flink-runtime/runtime-deps.txt +++ b/flink/v2.0/flink-runtime/runtime-deps.txt @@ -6,6 +6,14 @@ com.github.luben:zstd-jni:1.5 com.google.errorprone:error_prone_annotations:2.10 dev.failsafe:failsafe:3.3 io.airlift:aircompressor:2.0 +io.opentelemetry.instrumentation:opentelemetry-apache-httpclient-5.2:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.23 +io.opentelemetry.semconv:opentelemetry-semconv:1.37 +io.opentelemetry:opentelemetry-api-incubator:1.57 +io.opentelemetry:opentelemetry-api:1.57 +io.opentelemetry:opentelemetry-common:1.57 +io.opentelemetry:opentelemetry-context:1.57 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt index ebf8a55b48ff..22a145b5e0b4 100644 --- a/flink/v2.1/flink-runtime/runtime-deps.txt +++ b/flink/v2.1/flink-runtime/runtime-deps.txt @@ -6,6 +6,14 @@ com.github.luben:zstd-jni:1.5 com.google.errorprone:error_prone_annotations:2.10 dev.failsafe:failsafe:3.3 io.airlift:aircompressor:2.0 +io.opentelemetry.instrumentation:opentelemetry-apache-httpclient-5.2:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.23 +io.opentelemetry.semconv:opentelemetry-semconv:1.37 +io.opentelemetry:opentelemetry-api-incubator:1.57 +io.opentelemetry:opentelemetry-api:1.57 +io.opentelemetry:opentelemetry-common:1.57 +io.opentelemetry:opentelemetry-context:1.57 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 032fb09dbe14..a8f217ea7c21 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -80,6 +80,7 @@ mockserver = "5.15.0" nessie = "0.107.5" netty-buffer = "4.2.13.Final" object-client-bundle = "3.3.2" +opentelemetry-httpclient = "2.23.0-alpha" orc = "1.9.8" parquet = "1.17.1" roaringbitmap = "1.6.14" @@ -170,6 +171,7 @@ microprofile-openapi-api = { module = "org.eclipse.microprofile.openapi:micropro nessie-client = { module = "org.projectnessie.nessie:nessie-client", version.ref = "nessie" } netty-buffer = { module = "io.netty:netty-buffer", version.ref = "netty-buffer" } object-client-bundle = { module = "com.emc.ecs:object-client-bundle", version.ref = "object-client-bundle" } +opentelemetry-httpclient = { module = "io.opentelemetry.instrumentation:opentelemetry-apache-httpclient-5.2", version.ref = "opentelemetry-httpclient" } orc-core = { module = "org.apache.orc:orc-core", version.ref = "orc" } parquet-avro = { module = "org.apache.parquet:parquet-avro", version.ref = "parquet" } parquet-column = { module = "org.apache.parquet:parquet-column", version.ref = "parquet" } diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt index 95ef88a87969..0e1f41027b7e 100644 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -127,7 +127,11 @@ io.netty:netty-transport:4.2 io.opencensus:opencensus-api:0.31 io.opencensus:opencensus-contrib-http-util:0.31 io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37 -io.opentelemetry.semconv:opentelemetry-semconv:1.29 +io.opentelemetry.instrumentation:opentelemetry-apache-httpclient-5.2:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.23 +io.opentelemetry.semconv:opentelemetry-semconv:1.37 +io.opentelemetry:opentelemetry-api-incubator:1.57 io.opentelemetry:opentelemetry-api:1.57 io.opentelemetry:opentelemetry-common:1.57 io.opentelemetry:opentelemetry-context:1.57 diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt index f20cb039a036..4abd82601f47 100644 --- a/spark/v3.5/spark-runtime/runtime-deps.txt +++ b/spark/v3.5/spark-runtime/runtime-deps.txt @@ -9,6 +9,14 @@ dev.failsafe:failsafe:3.3 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 +io.opentelemetry.instrumentation:opentelemetry-apache-httpclient-5.2:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.23 +io.opentelemetry.semconv:opentelemetry-semconv:1.37 +io.opentelemetry:opentelemetry-api-incubator:1.57 +io.opentelemetry:opentelemetry-api:1.57 +io.opentelemetry:opentelemetry-common:1.57 +io.opentelemetry:opentelemetry-context:1.57 org.apache.arrow:arrow-format:15.0 org.apache.arrow:arrow-memory-core:15.0 org.apache.arrow:arrow-memory-netty:15.0 diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt index f20cb039a036..4abd82601f47 100644 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -9,6 +9,14 @@ dev.failsafe:failsafe:3.3 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 +io.opentelemetry.instrumentation:opentelemetry-apache-httpclient-5.2:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.23 +io.opentelemetry.semconv:opentelemetry-semconv:1.37 +io.opentelemetry:opentelemetry-api-incubator:1.57 +io.opentelemetry:opentelemetry-api:1.57 +io.opentelemetry:opentelemetry-common:1.57 +io.opentelemetry:opentelemetry-context:1.57 org.apache.arrow:arrow-format:15.0 org.apache.arrow:arrow-memory-core:15.0 org.apache.arrow:arrow-memory-netty:15.0 diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt index f20cb039a036..4abd82601f47 100644 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -9,6 +9,14 @@ dev.failsafe:failsafe:3.3 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 +io.opentelemetry.instrumentation:opentelemetry-apache-httpclient-5.2:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.23 +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.23 +io.opentelemetry.semconv:opentelemetry-semconv:1.37 +io.opentelemetry:opentelemetry-api-incubator:1.57 +io.opentelemetry:opentelemetry-api:1.57 +io.opentelemetry:opentelemetry-common:1.57 +io.opentelemetry:opentelemetry-context:1.57 org.apache.arrow:arrow-format:15.0 org.apache.arrow:arrow-memory-core:15.0 org.apache.arrow:arrow-memory-netty:15.0