From 75dc96875a36de42161af8098bb199143b0198d5 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Thu, 9 Apr 2026 09:55:14 -0700 Subject: [PATCH 1/6] Refactor makefile scala_version --- Makefile | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index eb5b9d6b8..4f9a1ba66 100644 --- a/Makefile +++ b/Makefile @@ -70,20 +70,12 @@ doc: kafka_artifact_version=$(lastword $(subst -, ,$(1))) -# Mappings for artifacts -> scala version; any unlisted will use default 2.12 -kafka_scala_0_8_0=2.8.0 -kafka_scala_0_8_1=2.10 -kafka_scala_0_8_1_1=2.10 -kafka_scala_0_8_2_0=2.11 -kafka_scala_0_8_2_1=2.11 -kafka_scala_0_8_2_2=2.11 -kafka_scala_0_9_0_0=2.11 -kafka_scala_0_9_0_1=2.11 -kafka_scala_0_10_0_0=2.11 -kafka_scala_0_10_0_1=2.11 -kafka_scala_0_10_1_0=2.11 -kafka_scala_4_0_0=2.13 -scala_version=$(if $(SCALA_VERSION),$(SCALA_VERSION),$(if $(kafka_scala_$(subst .,_,$(1))),$(kafka_scala_$(subst .,_,$(1))),2.12)) +# Version comparison: returns "yes" if $(1) >= $(2) (using sort -V) +version_ge=$(shell printf '%s\n%s\n' '$(2)' '$(1)' | sort -V | head -n1 | grep -qx '$(2)' && echo yes) + +# Determine scala version based on kafka version thresholds +# 0.8.0 => 2.8.0, >=0.8.1 => 2.10, >=0.8.2 => 2.11, >=0.11 => 2.12, >=4.0 => 2.13 +scala_version=$(if $(SCALA_VERSION),$(SCALA_VERSION),$(if $(call version_ge,$(1),4.0),2.13,$(if $(call version_ge,$(1),0.11),2.12,$(if $(call version_ge,$(1),0.8.2),2.11,$(if $(call version_ge,$(1),0.8.1),2.10,2.8.0))))) kafka_artifact_name=kafka_$(call scala_version,$(1))-$(1).$(if $(filter 0.8.0,$(1)),tar.gz,tgz) From 5d0d64f9ec7586ab0aeed46ce01283d344254603 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Thu, 9 Apr 2026 09:55:39 -0700 Subject: [PATCH 2/6] Fixup test.integratin.fixtures.get_api_versions --- test/integration/fixtures.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/integration/fixtures.py b/test/integration/fixtures.py index d2b0131d6..9da2f3862 100644 --- a/test/integration/fixtures.py +++ b/test/integration/fixtures.py @@ -710,8 +710,8 @@ def _enrich_client_params(self, params, **defaults): def get_api_versions(): logging.basicConfig(level=logging.ERROR) - zk = ZookeeperFixture.instance() - k = KafkaFixture.instance(0, zk) + k = KafkaFixture.instance(0) + zk = k.zookeeper from kafka import KafkaClient client = KafkaClient(bootstrap_servers='localhost:{}'.format(k.port)) @@ -719,11 +719,12 @@ def get_api_versions(): from pprint import pprint - pprint(client.get_api_versions()) + print(client.get_api_versions()) client.close() k.close() - zk.close() + if zk: + zk.close() def run_brokers(): From 28fb251920e9aa11f8ec90b6a21f5578b9b04ade Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Thu, 9 Apr 2026 09:55:52 -0700 Subject: [PATCH 3/6] Add 4.1/4.2 to broker_version_data --- kafka/protocol/broker_version_data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kafka/protocol/broker_version_data.py b/kafka/protocol/broker_version_data.py index b3ca7b186..f32cf888a 100644 --- a/kafka/protocol/broker_version_data.py +++ b/kafka/protocol/broker_version_data.py @@ -107,6 +107,8 @@ def infer_broker_version_from_api_versions(api_versions): test_cases = [ # format (, ) # Make sure to update consumer_integration test check when adding newer versions. + ((4, 2), ListOffsetsRequest.API_KEY, 11), + ((4, 1), ProduceRequest.API_KEY, 13), ((4, 0), ListOffsetsRequest.API_KEY, 10), ((3, 9), FetchRequest.API_KEY, 17), ((3, 8), ProduceRequest.API_KEY, 11), @@ -223,4 +225,8 @@ def infer_broker_version_from_api_versions(api_versions): (4, 0): {0: (0, 12), 1: (4, 17), 2: (1, 10), 3: (0, 13), 8: (2, 9), 9: (1, 9), 10: (0, 6), 11: (2, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 6), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (2, 7), 20: (1, 6), 21: (0, 2), 22: (0, 5), 23: (2, 4), 24: (0, 5), 25: (0, 4), 26: (0, 5), 27: (1, 1), 28: (0, 5), 29: (1, 3), 30: (1, 3), 31: (1, 3), 32: (1, 4), 33: (0, 2), 34: (1, 2), 35: (1, 4), 36: (0, 2), 37: (0, 3), 38: (1, 3), 39: (1, 2), 40: (1, 2), 41: (1, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 55: (0, 2), 57: (0, 2), 60: (0, 2), 61: (0, 0), 64: (0, 0), 65: (0, 0), 66: (0, 1), 68: (0, 1), 69: (0, 1), 74: (0, 0), 75: (0, 0), 80: (0, 0), 81: (0, 0)}, + (4, 1): {0: (0, 13), 1: (4, 18), 2: (1, 10), 3: (0, 13), 8: (2, 9), 9: (1, 9), 10: (0, 6), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 6), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (2, 7), 20: (1, 6), 21: (0, 2), 22: (0, 5), 23: (2, 4), 24: (0, 5), 25: (0, 4), 26: (0, 5), 27: (1, 1), 28: (0, 5), 29: (1, 3), 30: (1, 3), 31: (1, 3), 32: (1, 4), 33: (0, 2), 34: (1, 2), 35: (1, 4), 36: (0, 2), 37: (0, 3), 38: (1, 3), 39: (1, 2), 40: (1, 2), 41: (1, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 1), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 55: (0, 2), 57: (0, 2), 60: (0, 2), 61: (0, 0), 64: (0, 0), 65: (0, 0), 66: (0, 2), 68: (0, 1), 69: (0, 1), 74: (0, 1), 75: (0, 0), 76: (1, 1), 77: (1, 1), 78: (1, 1), 79: (1, 1), 80: (0, 0), 81: (0, 0), 83: (0, 0), 84: (0, 0), 85: (0, 0), 86: (0, 0), 87: (0, 0), 90: (0, 0), 91: (0, 0), 92: (0, 0)}, + + (4, 2): {0: (0, 13), 1: (4, 18), 2: (1, 11), 3: (0, 13), 8: (2, 10), 9: (1, 10), 10: (0, 6), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 6), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (2, 7), 20: (1, 6), 21: (0, 2), 22: (0, 5), 23: (2, 4), 24: (0, 5), 25: (0, 4), 26: (0, 5), 27: (1, 2), 28: (0, 5), 29: (1, 3), 30: (1, 3), 31: (1, 3), 32: (1, 4), 33: (0, 2), 34: (1, 2), 35: (1, 4), 36: (0, 2), 37: (0, 3), 38: (1, 3), 39: (1, 2), 40: (1, 2), 41: (1, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 1), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 55: (0, 2), 57: (0, 2), 60: (0, 2), 61: (0, 0), 64: (0, 0), 65: (0, 0), 66: (0, 2), 68: (0, 1), 69: (0, 1), 74: (0, 1), 75: (0, 0), 76: (1, 1), 77: (1, 1), 78: (1, 2), 79: (1, 2), 80: (0, 1), 81: (0, 0), 83: (0, 0), 84: (0, 0), 85: (0, 1), 86: (0, 0), 87: (0, 1), 88: (0, 0), 89: (0, 0), 90: (0, 1), 91: (0, 0), 92: (0, 0)}, + } From b48c2139373986a879992fc06e35b906bda9fd84 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Thu, 9 Apr 2026 09:56:05 -0700 Subject: [PATCH 4/6] Bump test matrix 4.0 -> 4.2 --- .github/workflows/python-package.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 94b994c23..daffa7a72 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -33,7 +33,7 @@ jobs: - "3.0.2" - "3.5.2" - "3.9.0" - - "4.0.0" + - "4.2.0" python: - "3.14" include: @@ -41,17 +41,17 @@ jobs: # kafka: "2.6.0" # experimental: true - python: "3.8" - kafka: "4.0.0" + kafka: "4.2.0" - python: "3.9" - kafka: "4.0.0" + kafka: "4.2.0" - python: "3.10" - kafka: "4.0.0" + kafka: "4.2.0" - python: "3.11" - kafka: "4.0.0" + kafka: "4.2.0" - python: "3.12" - kafka: "4.0.0" + kafka: "4.2.0" - python: "3.13" - kafka: "4.0.0" + kafka: "4.2.0" steps: - uses: actions/checkout@v6 From 061553ea15eb629365a1bfa9d9f59c9100cb7401 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Thu, 9 Apr 2026 10:24:37 -0700 Subject: [PATCH 5/6] Try downloads.apache.org first, fallback to archive.apache.org --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4f9a1ba66..acc1f175e 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,9 @@ SHELL = bash -export KAFKA_VERSION ?= 4.0.0 -DIST_BASE_URL ?= https://archive.apache.org/dist/kafka/ +export KAFKA_VERSION ?= 4.2.0 +DIST_BASE_URL ?= https://downloads.apache.org/kafka/ +ARCHIVE_BASE_URL = https://archive.apache.org/dist/kafka/ # Required to support testing old kafka versions on newer java releases # The performance opts defaults are set in each kafka brokers bin/kafka_run_class.sh file @@ -85,8 +86,10 @@ servers/dist: mkdir -p servers/dist servers/dist/kafka_%.tgz servers/dist/kafka_%.tar.gz: + $(eval artifact_path=$(call kafka_artifact_version,$*)/$(@F)) @echo "Downloading $(@F)" - wget -nv -P servers/dist/ -N $(DIST_BASE_URL)$(call kafka_artifact_version,$*)/$(@F) + wget -nv -P servers/dist/ -N $(DIST_BASE_URL)$(artifact_path) || \ + wget -nv -P servers/dist/ -N $(ARCHIVE_BASE_URL)$(artifact_path) servers/dist/jakarta.xml.bind-api-2.3.3.jar: wget -nv -P servers/dist/ -N https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar From 30c01a298fcd4d6a2edeb5357ec0a27762bc44ca Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Thu, 9 Apr 2026 10:24:50 -0700 Subject: [PATCH 6/6] Update infer test -> 4.2 max --- test/integration/test_consumer_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_consumer_integration.py b/test/integration/test_consumer_integration.py index 9d50f7d4e..2fcff554a 100644 --- a/test/integration/test_consumer_integration.py +++ b/test/integration/test_consumer_integration.py @@ -16,7 +16,7 @@ def test_kafka_version_infer(kafka_consumer_factory): consumer = kafka_consumer_factory(api_version=None) actual = BrokerVersionData(env_kafka_version()) - expected = min((4, 0), actual.broker_version) + expected = min((4, 2), actual.broker_version) assert consumer.config['api_version'] == expected, \ "Was expecting inferred broker version to be %s but was %s" % (expected, consumer.config['api_version'])