Identify 4.1/4.2 broker_version; CI test matrix 4.0->4.2

.github/workflows/python-package.yml

@@ -33,25 +33,25 @@ jobs:
           - "3.0.2"
           - "3.5.2"
           - "3.9.0"
-          - "4.0.0"
+          - "4.2.0"
         python:
           - "3.14"
         include:
           #- python: "pypy3.9"
           #  kafka: "2.6.0"
           #  experimental: true
           - python: "3.8"
-            kafka: "4.0.0"
+            kafka: "4.2.0"
           - python: "3.9"
-            kafka: "4.0.0"
+            kafka: "4.2.0"
           - python: "3.10"
-            kafka: "4.0.0"
+            kafka: "4.2.0"
           - python: "3.11"
-            kafka: "4.0.0"
+            kafka: "4.2.0"
           - python: "3.12"
-            kafka: "4.0.0"
+            kafka: "4.2.0"
           - python: "3.13"
-            kafka: "4.0.0"
+            kafka: "4.2.0"
 
     steps:
       - uses: actions/checkout@v6

Makefile

@@ -2,8 +2,9 @@
 
 SHELL = bash
 
-export KAFKA_VERSION ?= 4.0.0
-DIST_BASE_URL ?= https://archive.apache.org/dist/kafka/
+export KAFKA_VERSION ?= 4.2.0
+DIST_BASE_URL ?= https://downloads.apache.org/kafka/
+ARCHIVE_BASE_URL = https://archive.apache.org/dist/kafka/
 
 # Required to support testing old kafka versions on newer java releases
 # The performance opts defaults are set in each kafka brokers bin/kafka_run_class.sh file
@@ -70,20 +71,12 @@ doc:
 
 kafka_artifact_version=$(lastword $(subst -, ,$(1)))
 
-# Mappings for artifacts -> scala version; any unlisted will use default 2.12
-kafka_scala_0_8_0=2.8.0
-kafka_scala_0_8_1=2.10
-kafka_scala_0_8_1_1=2.10
-kafka_scala_0_8_2_0=2.11
-kafka_scala_0_8_2_1=2.11
-kafka_scala_0_8_2_2=2.11
-kafka_scala_0_9_0_0=2.11
-kafka_scala_0_9_0_1=2.11
-kafka_scala_0_10_0_0=2.11
-kafka_scala_0_10_0_1=2.11
-kafka_scala_0_10_1_0=2.11
-kafka_scala_4_0_0=2.13
-scala_version=$(if $(SCALA_VERSION),$(SCALA_VERSION),$(if $(kafka_scala_$(subst .,_,$(1))),$(kafka_scala_$(subst .,_,$(1))),2.12))
+# Version comparison: returns "yes" if $(1) >= $(2) (using sort -V)
+version_ge=$(shell printf '%s\n%s\n' '$(2)' '$(1)' | sort -V | head -n1 | grep -qx '$(2)' && echo yes)
+
+# Determine scala version based on kafka version thresholds
+# 0.8.0 => 2.8.0, >=0.8.1 => 2.10, >=0.8.2 => 2.11, >=0.11 => 2.12, >=4.0 => 2.13
+scala_version=$(if $(SCALA_VERSION),$(SCALA_VERSION),$(if $(call version_ge,$(1),4.0),2.13,$(if $(call version_ge,$(1),0.11),2.12,$(if $(call version_ge,$(1),0.8.2),2.11,$(if $(call version_ge,$(1),0.8.1),2.10,2.8.0)))))
 
 kafka_artifact_name=kafka_$(call scala_version,$(1))-$(1).$(if $(filter 0.8.0,$(1)),tar.gz,tgz)
 
@@ -93,8 +86,10 @@ servers/dist:
 	mkdir -p servers/dist
 
 servers/dist/kafka_%.tgz servers/dist/kafka_%.tar.gz:
+	$(eval artifact_path=$(call kafka_artifact_version,$*)/$(@F))
 	@echo "Downloading $(@F)"
-	wget -nv -P servers/dist/ -N $(DIST_BASE_URL)$(call kafka_artifact_version,$*)/$(@F)
+	wget -nv -P servers/dist/ -N $(DIST_BASE_URL)$(artifact_path) || \
+	wget -nv -P servers/dist/ -N $(ARCHIVE_BASE_URL)$(artifact_path)
 
 servers/dist/jakarta.xml.bind-api-2.3.3.jar:
 	wget -nv -P servers/dist/ -N https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar

kafka/protocol/broker_version_data.py

@@ -107,6 +107,8 @@ def infer_broker_version_from_api_versions(api_versions):
     test_cases = [
         # format (<broker version>, <needed struct>)
         # Make sure to update consumer_integration test check when adding newer versions.
+        ((4, 2), ListOffsetsRequest.API_KEY, 11),
+        ((4, 1), ProduceRequest.API_KEY, 13),
         ((4, 0), ListOffsetsRequest.API_KEY, 10),
         ((3, 9), FetchRequest.API_KEY, 17),
         ((3, 8), ProduceRequest.API_KEY, 11),
@@ -223,4 +225,8 @@ def infer_broker_version_from_api_versions(api_versions):
 
     (4, 0): {0: (0, 12), 1: (4, 17), 2: (1, 10), 3: (0, 13), 8: (2, 9), 9: (1, 9), 10: (0, 6), 11: (2, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 6), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (2, 7), 20: (1, 6), 21: (0, 2), 22: (0, 5), 23: (2, 4), 24: (0, 5), 25: (0, 4), 26: (0, 5), 27: (1, 1), 28: (0, 5), 29: (1, 3), 30: (1, 3), 31: (1, 3), 32: (1, 4), 33: (0, 2), 34: (1, 2), 35: (1, 4), 36: (0, 2), 37: (0, 3), 38: (1, 3), 39: (1, 2), 40: (1, 2), 41: (1, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 55: (0, 2), 57: (0, 2), 60: (0, 2), 61: (0, 0), 64: (0, 0), 65: (0, 0), 66: (0, 1), 68: (0, 1), 69: (0, 1), 74: (0, 0), 75: (0, 0), 80: (0, 0), 81: (0, 0)},
 
+    (4, 1): {0: (0, 13), 1: (4, 18), 2: (1, 10), 3: (0, 13), 8: (2, 9), 9: (1, 9), 10: (0, 6), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 6), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (2, 7), 20: (1, 6), 21: (0, 2), 22: (0, 5), 23: (2, 4), 24: (0, 5), 25: (0, 4), 26: (0, 5), 27: (1, 1), 28: (0, 5), 29: (1, 3), 30: (1, 3), 31: (1, 3), 32: (1, 4), 33: (0, 2), 34: (1, 2), 35: (1, 4), 36: (0, 2), 37: (0, 3), 38: (1, 3), 39: (1, 2), 40: (1, 2), 41: (1, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 1), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 55: (0, 2), 57: (0, 2), 60: (0, 2), 61: (0, 0), 64: (0, 0), 65: (0, 0), 66: (0, 2), 68: (0, 1), 69: (0, 1), 74: (0, 1), 75: (0, 0), 76: (1, 1), 77: (1, 1), 78: (1, 1), 79: (1, 1), 80: (0, 0), 81: (0, 0), 83: (0, 0), 84: (0, 0), 85: (0, 0), 86: (0, 0), 87: (0, 0), 90: (0, 0), 91: (0, 0), 92: (0, 0)},
+
+    (4, 2): {0: (0, 13), 1: (4, 18), 2: (1, 11), 3: (0, 13), 8: (2, 10), 9: (1, 10), 10: (0, 6), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 6), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (2, 7), 20: (1, 6), 21: (0, 2), 22: (0, 5), 23: (2, 4), 24: (0, 5), 25: (0, 4), 26: (0, 5), 27: (1, 2), 28: (0, 5), 29: (1, 3), 30: (1, 3), 31: (1, 3), 32: (1, 4), 33: (0, 2), 34: (1, 2), 35: (1, 4), 36: (0, 2), 37: (0, 3), 38: (1, 3), 39: (1, 2), 40: (1, 2), 41: (1, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 1), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 55: (0, 2), 57: (0, 2), 60: (0, 2), 61: (0, 0), 64: (0, 0), 65: (0, 0), 66: (0, 2), 68: (0, 1), 69: (0, 1), 74: (0, 1), 75: (0, 0), 76: (1, 1), 77: (1, 1), 78: (1, 2), 79: (1, 2), 80: (0, 1), 81: (0, 0), 83: (0, 0), 84: (0, 0), 85: (0, 1), 86: (0, 0), 87: (0, 1), 88: (0, 0), 89: (0, 0), 90: (0, 1), 91: (0, 0), 92: (0, 0)},
+
 }

test/integration/fixtures.py

@@ -710,20 +710,21 @@ def _enrich_client_params(self, params, **defaults):
 
 def get_api_versions():
     logging.basicConfig(level=logging.ERROR)
-    zk = ZookeeperFixture.instance()
-    k = KafkaFixture.instance(0, zk)
+    k = KafkaFixture.instance(0)
+    zk = k.zookeeper
 
     from kafka import KafkaClient
     client = KafkaClient(bootstrap_servers='localhost:{}'.format(k.port))
     client.check_version()
 
     from pprint import pprint
 
-    pprint(client.get_api_versions())
+    print(client.get_api_versions())
 
     client.close()
     k.close()
-    zk.close()
+    if zk:
+        zk.close()
 
 
 def run_brokers():

test/integration/test_consumer_integration.py

@@ -16,7 +16,7 @@
 def test_kafka_version_infer(kafka_consumer_factory):
     consumer = kafka_consumer_factory(api_version=None)
     actual = BrokerVersionData(env_kafka_version())
-    expected = min((4, 0), actual.broker_version)
+    expected = min((4, 2), actual.broker_version)
     assert consumer.config['api_version'] == expected, \
         "Was expecting inferred broker version to be %s but was %s" % (expected, consumer.config['api_version'])