feat(dind): multi-queue support and Docker mode fixes

charts/sourcegraph-executor/dind/README.md

@@ -63,11 +63,13 @@ In addition to the documented values, the `executor` and `private-docker-registr
 | executor.image.defaultTag | string | `"6.0.0@sha256:0be94a7c91f8273db10fdf46718c6596340ab2acc570e7b85353806e67a27508"` |  |
 | executor.image.name | string | `"executor"` |  |
 | executor.replicaCount | int | `1` |  |
+| executor.resources | object | `{}` | Resource requests and limits for the executor container. Each queue can override this with its own resources field. |
 | privateDockerRegistry.enabled | bool | `true` | Whether to deploy the private registry. Only one registry is needed when deploying multiple executors. More information: https://docs.sourcegraph.com/admin/executors/deploy_executors#using-private-registries |
 | privateDockerRegistry.image.registry | string | `"index.docker.io"` |  |
-| privateDockerRegistry.image.repository | string | `"docker/regisry"` |  |
-| privateDockerRegistry.image.tag | int | `2` |  |
+| privateDockerRegistry.image.repository | string | `"registry"` |  |
+| privateDockerRegistry.image.tag | int | `3` |  |
 | privateDockerRegistry.storageSize | string | `"10Gi"` |  |
+| queues | list | `[]` | Optional list of queues to deploy as standalone Deployments. When set, the single executor Deployment is not rendered. Each entry supports: name (required), replicaCount, resources, env (merged with executor.env, queue overrides). |
 | sourcegraph.affinity | object | `{}` | Affinity, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) |
 | sourcegraph.image.defaultTag | string | `"{{ .Chart.AppVersion }}"` | Global docker image tag |
 | sourcegraph.image.pullPolicy | string | `"IfNotPresent"` | Global docker image pull policy |

charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.executor.enabled -}}
+{{- if or .Values.queues .Values.executor.enabled -}}
 apiVersion: v1
 data:
   daemon.json: |

charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml

@@ -1,7 +1,179 @@
-{{- if .Values.executor.enabled -}}
+{{- if .Values.queues }}
+{{- range .Values.queues }}
+{{- $queue := . }}
+{{- $mergedEnv := mergeOverwrite (deepCopy $.Values.executor.env) ($queue.env | default dict) }}
+{{- $replicaCount := ($queue.replicaCount | default $.Values.executor.replicaCount) }}
+{{- $resources := ($queue.resources | default $.Values.executor.resources) }}
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  name: executor-{{ $queue.name }}
+  annotations:
+    description: Runs sourcegraph executors
+    kubectl.kubernetes.io/default-container: executor
+  labels:
+    {{- include "sourcegraph.labels" $ | nindent 4 }}
+    {{- if $.Values.executor.labels }}
+      {{- toYaml $.Values.executor.labels | nindent 4 }}
+    {{- end }}
+    app: executor-{{ $queue.name }}
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: executor
+spec:
+  selector:
+    matchLabels:
+      {{- include "sourcegraph.selectorLabels" $ | nindent 6 }}
+      app: executor-{{ $queue.name }}
+  minReadySeconds: 10
+  replicas: {{ $replicaCount }}
+  revisionHistoryLimit: 10
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: executor
+      {{- if $.Values.sourcegraph.podAnnotations }}
+      {{- toYaml $.Values.sourcegraph.podAnnotations | nindent 8 }}
+      {{- end }}
+      {{- if $.Values.executor.podAnnotations }}
+      {{- toYaml $.Values.executor.podAnnotations | nindent 8 }}
+      {{- end }}
+      labels:
+      {{- include "sourcegraph.selectorLabels" $ | nindent 8 }}
+      {{- if $.Values.sourcegraph.podLabels }}
+      {{- toYaml $.Values.sourcegraph.podLabels | nindent 8 }}
+      {{- end }}
+      {{- if $.Values.executor.podLabels }}
+      {{- toYaml $.Values.executor.podLabels | nindent 8 }}
+      {{- end }}
+        app: executor-{{ $queue.name }}
+        deploy: sourcegraph
+        sourcegraph-resource-requires: no-cluster-admin
+        app.kubernetes.io/component: executor
+    spec:
+      containers:
+        - name: executor
+          image: {{ include "sourcegraph.image" (list $ "executor") }}
+          imagePullPolicy: {{ $.Values.sourcegraph.image.pullPolicy }}
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http-debug
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http-debug
+              scheme: HTTP
+            periodSeconds: 5
+            timeoutSeconds: 5
+          ports:
+            - name: http-debug
+              containerPort: 8080
+          terminationMessagePolicy: FallbackToLogsOnError
+          env:
+            {{- range $name, $item := $mergedEnv }}
+            - name: {{ $name }}
+              {{- $item | toYaml | nindent 14 }}
+            {{- end }}
+            - name: EXECUTOR_USE_FIRECRACKER
+              value: "false"
+            - name: EXECUTOR_HEALTH_SERVER_ADDR
+              value: ":8080"
+            - name: EXECUTOR_JOB_NUM_CPUS
+              value: "0"
+            - name: EXECUTOR_JOB_MEMORY
+              value: "0"
+            - name: DOCKER_HOST
+              value: tcp://localhost:2375
+            - name: TMPDIR
+              value: /scratch
+            - name: EXECUTOR_USE_KUBERNETES
+              value: "false"
+          volumeMounts:
+            - mountPath: /scratch
+              name: executor-scratch
+          {{- with $resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: dind
+          image: "{{ $.Values.dind.image.registry}}/{{ $.Values.dind.image.repository}}:{{ $.Values.dind.image.tag}}"
+          imagePullPolicy: {{ $.Values.sourcegraph.image.pullPolicy }}
+          securityContext:
+            privileged: true
+          command:
+            - 'dockerd'
+            - '--tls=false'
+            - '--mtu=1200'
+            - '--registry-mirror=http://private-docker-registry:5000'
+            - '--host=tcp://0.0.0.0:2375'
+          livenessProbe:
+            tcpSocket:
+              port: 2375
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 5
+          readinessProbe:
+            tcpSocket:
+              port: 2375
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 5
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+          ports:
+            - containerPort: 2375
+              protocol: TCP
+          volumeMounts:
+            - mountPath: /scratch
+              name: executor-scratch
+            - mountPath: /etc/docker/daemon.json
+              subPath: daemon.json
+              name: docker-config
+      enableServiceLinks: false
+      {{- with $.Values.sourcegraph.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $.Values.sourcegraph.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with include "sourcegraph.priorityClassName" (list $ "executor") | trim }}{{ . | nindent 6 }}{{- end }}
+      {{- with $.Values.sourcegraph.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $.Values.sourcegraph.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: executor-scratch
+          emptyDir: {}
+        - name: docker-config
+          configMap:
+            defaultMode: 420
+            name: docker-config
+{{- end }}
+{{- else if .Values.executor.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "executor.name" . }}
   annotations:
     description: Runs sourcegraph executors
     kubectl.kubernetes.io/default-container: executor
@@ -57,14 +229,14 @@ spec:
             timeoutSeconds: 5
           readinessProbe:
             httpGet:
-              path: /ready
+              path: /healthz
               port: http-debug
               scheme: HTTP
             periodSeconds: 5
             timeoutSeconds: 5
           ports:
             - name: http-debug
-              containerPort: 6060
+              containerPort: 8080
           terminationMessagePolicy: FallbackToLogsOnError
           env:
             {{- range $name, $item := .Values.executor.env }}
@@ -73,6 +245,8 @@ spec:
             {{- end }}
             - name: EXECUTOR_USE_FIRECRACKER
               value: "false"
+            - name: EXECUTOR_HEALTH_SERVER_ADDR
+              value: ":8080"
             - name: EXECUTOR_JOB_NUM_CPUS
               value: "0"
             - name: EXECUTOR_JOB_MEMORY
@@ -81,9 +255,15 @@ spec:
               value: tcp://localhost:2375
             - name: TMPDIR
               value: /scratch
+            - name: EXECUTOR_USE_KUBERNETES
+              value: "false"
           volumeMounts:
             - mountPath: /scratch
               name: executor-scratch
+          {{- with .Values.executor.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
         - name: dind
           image: "{{ .Values.dind.image.registry}}/{{ .Values.dind.image.repository}}:{{ .Values.dind.image.tag}}"
           imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }}
@@ -93,7 +273,7 @@ spec:
             - 'dockerd'
             - '--tls=false'
             - '--mtu=1200'
-            - '--registry-mirror=http://executor:5000'
+            - '--registry-mirror=http://private-docker-registry:5000'
             - '--host=tcp://0.0.0.0:2375'
           livenessProbe:
             tcpSocket:
@@ -122,6 +302,7 @@ spec:
             - mountPath: /etc/docker/daemon.json
               subPath: daemon.json
               name: docker-config
+      enableServiceLinks: false
       {{- with .Values.sourcegraph.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Deployment.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.privateDockerRegistry.enabled -}}
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: private-docker-registry
   labels:
@@ -11,13 +11,14 @@ metadata:
     deploy: sourcegraph
     app.kubernetes.io/component: private-docker-registry
 spec:
-  replicas: {{ .Values.privateDockerRegistry.replicaCount }}
+  replicas: 1
+  serviceName: private-docker-registry
   selector:
     matchLabels:
       {{- include "sourcegraph.selectorLabels" . | nindent 6 }}
       app: private-docker-registry
-  strategy:
-    type: Recreate
+  updateStrategy:
+    type: RollingUpdate
   template:
     metadata:
       annotations:
@@ -83,8 +84,17 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      volumes:
-        - name: cache
-          persistentVolumeClaim:
-            claimName: private-docker-registry
+  volumeClaimTemplates:
+    - metadata:
+        name: cache
+        labels:
+          deploy: sourcegraph
+          app.kubernetes.io/component: private-docker-registry
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: {{ .Values.privateDockerRegistry.storageSize }}
+        storageClassName: {{ .Values.storageClass.name }}
 {{- end }}