1
# Setup Service to provide access to Zookeeper for clients
2
apiVersion: v1
3
kind: Service
4
metadata:
5
  # DNS would be like zookeeper.zoons
6
  name: zookeeper
7
  labels:
8
    app: zookeeper
9
spec:
10
  ports:
11
    - port: 2181
12
      name: client
13
    - port: 7000
14
      name: prometheus
15
  selector:
16
    app: zookeeper
17
    what: node
18
---
19
# Setup Headless Service for StatefulSet
20
apiVersion: v1
21
kind: Service
22
metadata:
23
  # DNS would be like zookeeper-0.zookeepers.etc
24
  name: zookeepers
25
  labels:
26
    app: zookeeper
27
spec:
28
  ports:
29
    - port: 2888
30
      name: server
31
    - port: 3888
32
      name: leader-election
33
  clusterIP: None
34
  selector:
35
    app: zookeeper
36
    what: node
37
---
38
# Setup max number of unavailable pods in StatefulSet
39
apiVersion: policy/v1
40
kind: PodDisruptionBudget
41
metadata:
42
  name: zookeeper-pod-disruption-budget
43
spec:
44
  selector:
45
    matchLabels:
46
      app: zookeeper
47
  maxUnavailable: 1
48
---
49
# Setup Zookeeper StatefulSet
50
# Possible params:
51
# 1. replicas
52
# 2. memory
53
# 3. cpu
54
# 4. storage
55
# 5. storageClassName
56
# 6. user to run app
57
apiVersion: apps/v1
58
kind: StatefulSet
59
metadata:
60
  # nodes would be named as zookeeper-0, zookeeper-1, zookeeper-2
61
  name: zookeeper
62
  labels:
63
    app: zookeeper
64
spec:
65
  selector:
66
    matchLabels:
67
      app: zookeeper
68
  serviceName: zookeepers
69
  replicas: 3
70
  updateStrategy:
71
    type: RollingUpdate
72
  podManagementPolicy: OrderedReady
73
  template:
74
    metadata:
75
      labels:
76
        app: zookeeper
77
        what: node
78
      annotations:
79
        prometheus.io/port: '7000'
80
        prometheus.io/scrape: 'true'
81
    spec:
82
      affinity: {}
83
      containers:
84
        - name: kubernetes-zookeeper
85
          imagePullPolicy: IfNotPresent
86
          image: "docker.io/zookeeper:3.8.4"
87
          resources:
88
            requests:
89
              memory: "512M"
90
              cpu: "1"
91
            limits:
92
              memory: "4Gi"
93
              cpu: "2"
94
          ports:
95
            - containerPort: 2181
96
              name: client
97
            - containerPort: 2888
98
              name: server
99
            - containerPort: 3888
100
              name: leader-election
101
            - containerPort: 7000
102
              name: prometheus
103
          env:
104
            - name: SERVERS
105
              value: "3"
106

107
# See those links for proper startup settings:
108
# https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper
109
# https://clickhouse.yandex/docs/en/operations/tips/#zookeeper
110
# https://github.com/ClickHouse/ClickHouse/issues/11781
111
          command:
112
            - bash
113
            - -x
114
            - -c
115
            - |
116
              HOST=`hostname -s` &&
117
              DOMAIN=`hostname -d` &&
118
              CLIENT_PORT=2181 &&
119
              SERVER_PORT=2888 &&
120
              ELECTION_PORT=3888 &&
121
              PROMETHEUS_PORT=7000 &&
122
              ZOO_DATA_DIR=/var/lib/zookeeper/data &&
123
              ZOO_DATA_LOG_DIR=/var/lib/zookeeper/datalog &&
124
              {
125
                echo "clientPort=${CLIENT_PORT}"
126
                echo 'tickTime=2000'
127
                echo 'initLimit=300'
128
                echo 'syncLimit=10'
129
                echo 'maxClientCnxns=2000'
130
                echo 'maxTimeToWaitForEpoch=2000'
131
                echo 'maxSessionTimeout=60000000'
132
                echo "dataDir=${ZOO_DATA_DIR}"
133
                echo "dataLogDir=${ZOO_DATA_LOG_DIR}"
134
                echo 'autopurge.snapRetainCount=10'
135
                echo 'autopurge.purgeInterval=1'
136
                echo 'preAllocSize=131072'
137
                echo 'snapCount=3000000'
138
                echo 'leaderServes=yes'
139
                echo 'standaloneEnabled=false'
140
                echo '4lw.commands.whitelist=*'
141
                echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider'
142
                echo "metricsProvider.httpPort=${PROMETHEUS_PORT}"
143
                echo "skipACL=true"
144
                echo "fastleader.maxNotificationInterval=10000"
145
              } > /conf/zoo.cfg &&
146
              {
147
                echo "zookeeper.root.logger=CONSOLE"
148
                echo "zookeeper.console.threshold=INFO"
149
                echo "log4j.rootLogger=\${zookeeper.root.logger}"
150
                echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender"
151
                echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}"
152
                echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout"
153
                echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"
154
              } > /conf/log4j.properties &&
155
              echo 'JVMFLAGS="-Xms128M -Xmx4G -XX:ActiveProcessorCount=8 -XX:+AlwaysPreTouch -Djute.maxbuffer=8388608 -XX:MaxGCPauseMillis=50"' > /conf/java.env &&
156
              if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
157
                  NAME=${BASH_REMATCH[1]} &&
158
                  ORD=${BASH_REMATCH[2]};
159
              else
160
                  echo "Failed to parse name and ordinal of Pod" &&
161
                  exit 1;
162
              fi &&
163
              mkdir -pv ${ZOO_DATA_DIR} &&
164
              mkdir -pv ${ZOO_DATA_LOG_DIR} &&
165
              whoami &&
166
              chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" &&
167
              export MY_ID=$((ORD+1)) &&
168
              echo $MY_ID > $ZOO_DATA_DIR/myid &&
169
              for (( i=1; i<=$SERVERS; i++ )); do
170
                  echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
171
              done &&
172
              if [[ $SERVERS -eq 1 ]]; then
173
                  echo "group.1=1" >> /conf/zoo.cfg;
174
              else
175
                  echo "group.1=1:2:3" >> /conf/zoo.cfg;
176
              fi &&
177
              for (( i=1; i<=$SERVERS; i++ )); do
178
                  WEIGHT=1
179
                  if [[ $i == 1 ]]; then
180
                    WEIGHT=10
181
                  fi
182
                  echo "weight.$i=$WEIGHT" >> /conf/zoo.cfg;
183
              done &&
184
              zkServer.sh start-foreground
185
          readinessProbe:
186
            exec:
187
              command:
188
                - bash
189
                - -c
190
                - '
191
                  IFS=;
192
                  MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;);
193
                  while [[ "$MNTR" == "This ZooKeeper instance is not currently serving requests" ]];
194
                  do
195
                    echo "wait mntr works";
196
                    sleep 1;
197
                    MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;);
198
                  done;
199
                  STATE=$(echo -e $MNTR | grep zk_server_state | cut -d " " -f 2);
200
                  if [[ "$STATE" =~ "leader" ]]; then
201
                    echo "check leader state";
202
                    SYNCED_FOLLOWERS=$(echo -e $MNTR | grep zk_synced_followers | awk -F"[[:space:]]+" "{print \$2}" | cut -d "." -f 1);
203
                    if [[ "$SYNCED_FOLLOWERS" != "0" ]]; then
204
                      ./bin/zkCli.sh ls /;
205
                      exit $?;
206
                    else
207
                      exit 0;
208
                    fi;
209
                  elif [[ "$STATE" =~ "follower" ]]; then
210
                    echo "check follower state";
211
                    PEER_STATE=$(echo -e $MNTR | grep zk_peer_state);
212
                    if [[ "$PEER_STATE" =~ "following - broadcast" ]]; then
213
                      ./bin/zkCli.sh ls /;
214
                      exit $?;
215
                    else
216
                      exit 1;
217
                    fi;
218
                  else
219
                    exit 1;
220
                  fi
221
                   '
222
            initialDelaySeconds: 15
223
            periodSeconds: 10
224
            timeoutSeconds: 60
225
          livenessProbe:
226
            exec:
227
              command:
228
                - bash
229
                - -xc
230
                - 'date && OK=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "ruok" >&3 ; IFS=; tee <&3; exec 3<&- ;); if [[ "$OK" == "imok" ]]; then exit 0; else exit 1; fi'
231
            initialDelaySeconds: 10
232
            periodSeconds: 30
233
            timeoutSeconds: 5
234
          volumeMounts:
235
            - name: datadir-volume
236
              mountPath: /var/lib/zookeeper
237
      # Run as a non-privileged user
238
      securityContext:
239
        runAsUser: 1000
240
        fsGroup: 1000
241
      volumes:
242
        - name: datadir-volume
243
          emptyDir:
244
            medium: "" #accepted values:  empty str (means node's default medium) or Memory
245
            sizeLimit: 1Gi

1
apiVersion: "clickhouse.altinity.com/v1"
2
kind: "ClickHouseInstallation"
3

4
metadata:
5
  name: "repl-05"
6

7
spec:
8
  defaults:
9
    templates:
10
      dataVolumeClaimTemplate: default
11
      podTemplate: clickhouse-20.7
12

13
  configuration:
14
    zookeeper:
15
      nodes:
16
      - host: zookeeper.zoo3ns
17
    clusters:
18
      - name: replicated
19
        layout:
20
          shardsCount: 1
21
          replicasCount: 2
22

23
  templates:
24
    volumeClaimTemplates:
25
      - name: default
26
        spec:
27
          accessModes:
28
            - ReadWriteOnce
29
          resources:
30
            requests:
31
              storage: 500Mi
32
    podTemplates:
33
      - name: clickhouse-20.7
34
        spec:
35
          containers:
36
            - name: clickhouse-pod
37
              image: clickhouse/clickhouse-server:24.8