From 1edeab4c87d5e820bb0469b79d48aeb6e2b48338 Mon Sep 17 00:00:00 2001 From: Slach Date: Fri, 8 Apr 2022 22:05:27 +0500 Subject: [PATCH 1/3] update clickhouse-keeper manifests, update clickhouse image to clickhouse/clickhouse-server:22.3, update backup clickhouse-backup:1.3.1, zookeeper:3.8.0 (acm not affected), tests passed add test_clickhouse_keeper to XFail https://github.com/ClickHouse/ClickHouse/issues/35464, https://github.com/ClickHouse/ClickHouse/issues/3546 --- .../delete-zookeeper-operator.sh | 3 + .../install-zookeeper-operator.sh | 9 + .../zookeeper-operator-1-node.yaml | 21 + .../zookeeper-operator-3-node.yaml | 21 + ...-node-1GB-for-tests-only-scaleout-pvc.yaml | 612 ++++++++++++++++++ .../zookeeper-1-node-1GB-for-tests-only.yaml | 1 + ...nodes-1GB-for-tests-only-scaleout-pvc.yaml | 612 ++++++++++++++++++ .../zookeeper-3-nodes-1GB-for-tests-only.yaml | 1 + tests/e2e/kubectl.py | 22 +- ...test-cluster-for-zookeeper-operator-1.yaml | 21 + ...test-cluster-for-zookeeper-operator-2.yaml | 21 + tests/e2e/test_keeper.py | 57 +- tests/e2e/util.py | 34 +- 13 files changed, 1405 insertions(+), 30 deletions(-) create mode 100644 deploy/zookeeper-operator/delete-zookeeper-operator.sh create mode 100644 deploy/zookeeper-operator/install-zookeeper-operator.sh create mode 100644 deploy/zookeeper-operator/zookeeper-operator-1-node.yaml create mode 100644 deploy/zookeeper-operator/zookeeper-operator-3-node.yaml create mode 100644 deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml create mode 100644 deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml create mode 100644 tests/e2e/manifests/chi/test-cluster-for-zookeeper-operator-1.yaml create mode 100644 tests/e2e/manifests/chi/test-cluster-for-zookeeper-operator-2.yaml diff --git a/deploy/zookeeper-operator/delete-zookeeper-operator.sh b/deploy/zookeeper-operator/delete-zookeeper-operator.sh new file mode 100644 index 000000000..6c84fa2cc --- /dev/null +++ b/deploy/zookeeper-operator/delete-zookeeper-operator.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +ZOOKEEPER_OPERATOR_NAMESPACE=${ZOOKEEPER_OPERATOR_NAMESPACE:-zookeeper-operator} +kubectl delete ns "$ZOOKEEPER_OPERATOR_NAMESPACE" diff --git a/deploy/zookeeper-operator/install-zookeeper-operator.sh b/deploy/zookeeper-operator/install-zookeeper-operator.sh new file mode 100644 index 000000000..64ff94f41 --- /dev/null +++ b/deploy/zookeeper-operator/install-zookeeper-operator.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +ZOOKEEPER_OPERATOR_VERSION=${ZOOKEEPER_OPERATOR_VERSION:-v0.2.13} +ZOOKEEPER_OPERATOR_NAMESPACE=${ZOOKEEPER_OPERATOR_NAMESPACE:-zookeeper-operator} +kubectl create ns "${ZOOKEEPER_OPERATOR_NAMESPACE}" || true +kubectl apply -f "https://github.com/pravega/zookeeper-operator/raw/${ZOOKEEPER_OPERATOR_VERSION}/deploy/crds/zookeeper.pravega.io_zookeeperclusters_crd.yaml" +kubectl apply -n "${ZOOKEEPER_OPERATOR_NAMESPACE}" -f <( + curl -sL "https://github.com/pravega/zookeeper-operator/raw/${ZOOKEEPER_OPERATOR_VERSION}/deploy/all_ns/rbac.yaml" | sed -e "s/namespace: default/namespace: ${ZOOKEEPER_OPERATOR_NAMESPACE}/g" | sed -e "s/v1beta1/v1/g" +) +kubectl apply -n "${ZOOKEEPER_OPERATOR_NAMESPACE}" -f "https://github.com/pravega/zookeeper-operator/raw/${ZOOKEEPER_OPERATOR_VERSION}/deploy/all_ns/operator.yaml" diff --git a/deploy/zookeeper-operator/zookeeper-operator-1-node.yaml b/deploy/zookeeper-operator/zookeeper-operator-1-node.yaml new file mode 100644 index 000000000..fd100dec1 --- /dev/null +++ b/deploy/zookeeper-operator/zookeeper-operator-1-node.yaml @@ -0,0 +1,21 @@ +apiVersion: zookeeper.pravega.io/v1beta1 +kind: ZookeeperCluster +metadata: + name: zookeeper + labels: + app: zookeeper-operator +spec: + replicas: 1 + image: + repository: pravega/zookeeper + tag: 0.2.13 + labels: + app: zookeeper-operator + storageType: persistence + persistence: + reclaimPolicy: Delete + spec: + storageClassName: "standard" + resources: + requests: + storage: 1Gi diff --git a/deploy/zookeeper-operator/zookeeper-operator-3-node.yaml b/deploy/zookeeper-operator/zookeeper-operator-3-node.yaml new file mode 100644 index 000000000..04c259204 --- /dev/null +++ b/deploy/zookeeper-operator/zookeeper-operator-3-node.yaml @@ -0,0 +1,21 @@ +apiVersion: zookeeper.pravega.io/v1beta1 +kind: ZookeeperCluster +metadata: + name: zookeeper + labels: + app: zookeeper-operator +spec: + replicas: 3 + image: + repository: pravega/zookeeper + tag: 0.2.13 + labels: + app: zookeeper-operator + storageType: persistence + persistence: + reclaimPolicy: Delete + spec: + storageClassName: "standard" + resources: + requests: + storage: 1Gi diff --git a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml new file mode 100644 index 000000000..5d3db1afa --- /dev/null +++ b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml @@ -0,0 +1,612 @@ +# Following scripts based from https://github.com/pravega/zookeeper-operator/ +--- +# service for REST administration and reconfiguration +apiVersion: v1 +kind: Service +metadata: + labels: + app: zookeeper + name: zookeeper-admin-server +spec: + type: ClusterIP + ports: + - name: tcp-admin-server + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: zookeeper + +--- +# service for clickhouse zookeeper client connections +apiVersion: v1 +kind: Service +metadata: + labels: + app: zookeeper + name: zookeeper +spec: + type: ClusterIP + ports: + - name: tcp-client + port: 2181 + protocol: TCP + targetPort: 2181 + selector: + app: zookeeper + what: node +--- +# headless service for Zookeeper Quorum Election and service name +apiVersion: v1 +kind: Service +metadata: + labels: + app: zookeeper + name: zookeeper-headless +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-client + port: 2181 + protocol: TCP + targetPort: 2181 + - name: tcp-quorum + port: 2888 + protocol: TCP + targetPort: 2888 + - name: tcp-leader-election + port: 3888 + protocol: TCP + targetPort: 3888 + - name: tcp-metrics + port: 7000 + protocol: TCP + targetPort: 7000 + - name: tcp-admin-server + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: zookeeper +--- +# ConfigMap with common startup scripts and base config +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: zookeeper + name: zookeeper-scripts +data: + env.sh: | + #!/usr/bin/env bash + DOMAIN=`hostname -d` + QUORUM_PORT=2888 + LEADER_PORT=3888 + CLIENT_HOST=zookeeper + CLIENT_PORT=2181 + ADMIN_SERVER_HOST=zookeeper-admin-server + ADMIN_SERVER_PORT=8080 + CLUSTER_NAME=zookeeper + log4j-quiet.properties: | + log4j.rootLogger=CONSOLE + log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender + log4j.appender.CONSOLE.Threshold=ERROR + log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout + log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + log4j.properties: | + zookeeper.root.logger=CONSOLE + zookeeper.console.threshold=WARN + log4j.rootLogger=${zookeeper.root.logger} + log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender + log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} + log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout + log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + zoo.cfg: | + 4lw.commands.whitelist=* + dataDir=/data + standaloneEnabled=false + reconfigEnabled=true + skipACL=yes + metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider + metricsProvider.httpPort=7000 + metricsProvider.exportJvmInfo=true + tickTime=2000 + initLimit=300 + syncLimit=10 + maxClientCnxns=2000 + maxTimeToWaitForEpoch=2000 + globalOutstandingLimit=1000 + preAllocSize=65536 + snapCount=10000 + commitLogCount=500 + snapSizeLimitInKb=4194304 + maxCnxns=0 + maxClientCnxns=60 + minSessionTimeout=4000 + maxSessionTimeout=40000 + autopurge.snapRetainCount=3 + autopurge.purgeInterval=1 + quorumListenOnAllIPs=false + admin.serverPort=8080 + dynamicConfigFile=/data/zoo.cfg.dynamic + zookeeperFunctions.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + set -ex + function zkConfig() { + echo "$HOST.$DOMAIN:$QUORUM_PORT:$LEADER_PORT:$ROLE;$CLIENT_PORT" + } + function zkConnectionString() { + # If the client service address is not yet available, then return localhost + set +e + getent hosts "${CLIENT_HOST}" 2>/dev/null 1>/dev/null + if [[ $? -ne 0 ]]; then + set -e + echo "localhost:${CLIENT_PORT}" + else + set -e + echo "${CLIENT_HOST}:${CLIENT_PORT}" + fi + } + zookeeperStart.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + + set -ex + # TODO think how to add nslookup to docker image + apt-get update && apt-get install --no-install-recommends -y dnsutils curl procps + source /cluster-size/cluster-size.sh + source /conf/env.sh + source /conf/zookeeperFunctions.sh + + HOST=`hostname -s` + DATA_DIR=/data + MYID_FILE=$DATA_DIR/myid + LOG4J_CONF=/conf/log4j-quiet.properties + DYNCONFIG=$DATA_DIR/zoo.cfg.dynamic + STATIC_CONFIG=/data/conf/zoo.cfg + + # Extract resource name and this members ordinal value from pod hostname + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + ORD=${BASH_REMATCH[2]} + else + echo Failed to parse name and ordinal of Pod + exit 1 + fi + + MYID=$((ORD+1)) + + # Values for first startup + WRITE_CONFIGURATION=true + REGISTER_NODE=true + ONDISK_MYID_CONFIG=false + ONDISK_DYN_CONFIG=false + + # Check validity of on-disk configuration + if [ -f $MYID_FILE ]; then + EXISTING_ID="`cat $DATA_DIR/myid`" + if [[ "$EXISTING_ID" == "$MYID" && -f $STATIC_CONFIG ]]; then + # If Id is correct and configuration is present under `/data/conf` + ONDISK_MYID_CONFIG=true + fi + fi + + if [ -f $DYNCONFIG ]; then + ONDISK_DYN_CONFIG=true + fi + + set +e + # Check if envoy is up and running + if [[ -n "$ENVOY_SIDECAR_STATUS" ]]; then + COUNT=0 + MAXCOUNT=${1:-30} + HEALTHYSTATUSCODE="200" + while true; do + COUNT=$(expr $COUNT + 1) + SC=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:15000/ready) + echo "waiting for envoy proxy to come up"; + sleep 1; + if (( "$SC" == "$HEALTHYSTATUSCODE" || "$MAXCOUNT" == "$COUNT" )); then + break + fi + done + fi + set -e + + # Determine if there is an ensemble available to join by checking the service domain + set +e + getent hosts $DOMAIN # This only performs a dns lookup + if [[ $? -eq 0 ]]; then + ACTIVE_ENSEMBLE=true + elif nslookup $DOMAIN | grep -q "server can't find $DOMAIN"; then + echo "there is no active ensemble" + ACTIVE_ENSEMBLE=false + else + # If an nslookup of the headless service domain fails, then there is no + # active ensemble yet, but in certain cases nslookup of headless service + # takes a while to come up even if there is active ensemble + ACTIVE_ENSEMBLE=false + declare -i count=20 + while [[ $count -ge 0 ]] + do + sleep 2 + ((count=count-1)) + getent hosts $DOMAIN + if [[ $? -eq 0 ]]; then + ACTIVE_ENSEMBLE=true + break + fi + done + fi + + if [[ "$ONDISK_MYID_CONFIG" == true && "$ONDISK_DYN_CONFIG" == true ]]; then + # If Configuration is present, we assume, there is no need to write configuration. + WRITE_CONFIGURATION=false + else + WRITE_CONFIGURATION=true + fi + + if [[ "$ACTIVE_ENSEMBLE" == false ]]; then + # This is the first node being added to the cluster or headless service not yet available + REGISTER_NODE=false + else + # An ensemble exists, check to see if this node is already a member. + if [[ "$ONDISK_MYID_CONFIG" == false || "$ONDISK_DYN_CONFIG" == false ]]; then + REGISTER_NODE=true + else + REGISTER_NODE=false + fi + fi + + if [[ "$WRITE_CONFIGURATION" == true ]]; then + echo "Writing myid: $MYID to: $MYID_FILE." + echo $MYID > $MYID_FILE + if [[ $MYID -eq 1 ]]; then + ROLE=participant + echo Initial initialization of ordinal 0 pod, creating new config. + ZKCONFIG=$(zkConfig) + echo Writing bootstrap configuration with the following config: + echo $ZKCONFIG + echo $MYID > $MYID_FILE + echo "server.${MYID}=${ZKCONFIG}" > $DYNCONFIG + fi + fi + + if [[ "$REGISTER_NODE" == true ]]; then + ROLE=observer + ZKURL=$(zkConnectionString) + ZKCONFIG=$(zkConfig) + + set -e + echo Registering node and writing local configuration to disk. + zkCli.sh -server ${ZKURL} reconfig -add "\nserver.$MYID=$ZKCONFIG" | grep -E '^server\.[0-9]+=' > $DYNCONFIG + set +e + fi + + ZOOCFGDIR=/data/conf + export ZOOCFGDIR + echo Copying /conf contents to writable directory, to support Zookeeper dynamic reconfiguration + if [[ ! -d "$ZOOCFGDIR" ]]; then + mkdir $ZOOCFGDIR + cp -f /conf/zoo.cfg $ZOOCFGDIR + else + echo Copying the /conf/zoo.cfg contents except the dynamic config file during restart + echo -e "$( head -n -1 /conf/zoo.cfg )""\n""$( tail -n 1 "$STATIC_CONFIG" )" > $STATIC_CONFIG + fi + cp -f /conf/log4j.properties $ZOOCFGDIR + cp -f /conf/log4j-quiet.properties $ZOOCFGDIR + cp -f /conf/env.sh $ZOOCFGDIR + + if [ -f $DYNCONFIG ]; then + # Node registered, start server + echo Starting zookeeper service + zkServer.sh --config $ZOOCFGDIR start-foreground + else + echo "Node failed to register!" + exit 1 + fi + + zookeeperTeardown.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + + set -ex + + source /cluster-size/cluster-size.sh + source /conf/env.sh + source /conf/zookeeperFunctions.sh + + DATA_DIR=/data + MYID_FILE=$DATA_DIR/myid + LOG4J_CONF=/conf/log4j-quiet.properties + + # Wait for client connections to drain. Kubernetes will wait until the confiugred + # "terminationGracePeriodSeconds" before focibly killing the container + CONN_COUNT=`echo cons | nc localhost 2181 | grep -v "^$" |grep -v "/127.0.0.1:" | wc -l` + for (( i = 0; i < 6; i++ )); do + if [[ "$CONN_COUNT" -gt 0 ]]; then + echo "$CONN_COUNT non-local connections still connected." + sleep 5 + else + echo "$CONN_COUNT non-local connections" + break + fi + done + + # Check to see if zookeeper service for this node is a participant + set +e + ZKURL=$(zkConnectionString) + set -e + MYID=`cat $MYID_FILE` + + echo "CLUSTER_SIZE=$CLUSTER_SIZE, MyId=$MYID" + if [[ -n "$CLUSTER_SIZE" && "$CLUSTER_SIZE" -lt "$MYID" ]]; then + # If ClusterSize < MyId, this server is being permanantly removed. + zkCli.sh -server ${ZKURL} reconfig -remove $MYID | grep -E '^server\.\d+=' > $DYNCONFIG + echo $? + fi + + # Kill the primary process ourselves to circumvent the terminationGracePeriodSeconds + ps -ef | grep zoo.cfg | grep -v grep | awk '{print $2}' | xargs kill + + + zookeeperLive.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + set -ex + source /conf/env.sh + OK=$(echo ruok | nc 127.0.0.1 $CLIENT_PORT) + # Check to see if zookeeper service answers + if [[ "$OK" == "imok" ]]; then + exit 0 + else + exit 1 + fi + + zookeeperReady.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + set -ex + source /cluster-size/cluster-size.sh + source /conf/env.sh + source /conf/zookeeperFunctions.sh + + HOST=`hostname -s` + DATA_DIR=/data + MYID_FILE=$DATA_DIR/myid + LOG4J_CONF=/conf/log4j-quiet.properties + STATIC_CONFIG=/data/conf/zoo.cfg + + OK=$(echo ruok | nc 127.0.0.1 $CLIENT_PORT) + + # Check to see if zookeeper service answers + if [[ "$OK" == "imok" ]]; then + set +e + getent hosts $DOMAIN + if [[ $? -ne 0 ]]; then + set -e + echo "There is no active ensemble, skipping readiness probe..." + exit 0 + else + set -e + # An ensemble exists, check to see if this node is already a member. + # Check to see if zookeeper service for this node is a participant + set +e + # Extract resource name and this members' ordinal value from pod hostname + HOST=`hostname -s` + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + ORD=${BASH_REMATCH[2]} + else + echo Failed to parse name and ordinal of Pod + exit 1 + fi + MYID=$((ORD+1)) + ONDISK_CONFIG=false + if [ -f $MYID_FILE ]; then + EXISTING_ID="`cat $DATA_DIR/myid`" + if [[ "$EXISTING_ID" == "$MYID" && -f $STATIC_CONFIG ]]; then + #If Id is correct and configuration is present under `/data/conf` + ONDISK_CONFIG=true + DYN_CFG_FILE_LINE=`cat $STATIC_CONFIG|grep "dynamicConfigFile\="` + DYN_CFG_FILE=${DYN_CFG_FILE_LINE##dynamicConfigFile=} + SERVER_FOUND=`cat $DYN_CFG_FILE | grep "server.${MYID}=" | wc -l` + if [[ "$SERVER_FOUND" == "0" ]]; then + echo "Server not found in ensemble. Exiting ..." + exit 1 + fi + SERVER=`cat $DYN_CFG_FILE | grep "server.${MYID}="` + if [[ "$SERVER" == *"participant"* ]]; then + ROLE=participant + elif [[ "$SERVER" == *"observer"* ]]; then + ROLE=observer + fi + fi + fi + + if [[ "$ROLE" == "participant" ]]; then + echo "Zookeeper service is available and an active participant" + exit 0 + elif [[ "$ROLE" == "observer" ]]; then + echo "Zookeeper service is ready to be upgraded from observer to participant." + ROLE=participant + ZKURL=$(zkConnectionString) + ZKCONFIG=$(zkConfig) + zkCli.sh -server ${ZKURL} reconfig -remove $MYID | grep -E '^server\.[0-9]+=' > $DYNCONFIG + sleep 1 + zkCli.sh -server ${ZKURL} reconfig -add "\nserver.$MYID=$ZKCONFIG" + exit 0 + else + echo "Something has gone wrong. Unable to determine zookeeper role." + exit 1 + fi + fi + + else + echo "Zookeeper service is not available for requests" + exit 1 + fi + +--- +# ConfigMap for current cluster size +apiVersion: v1 +kind: ConfigMap +metadata: + name: zookeeper-cluster-size + labels: + app: zookeeper +data: + cluster-size.sh: | + #!/usr/bin/env bash + CLUSTER_SIZE=1 + +--- +# Main StatefulSet +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: zookeeper + name: zookeeper + +spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + podManagementPolicy: OrderedReady + replicas: 1 + selector: + matchLabels: + app: zookeeper + serviceName: zookeeper-headless + template: + metadata: + generateName: zookeeper + labels: + app: zookeeper + what: node + annotations: + prometheus.io/port: '7000' + prometheus.io/scrape: 'true' + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - zookeeper + topologyKey: kubernetes.io/hostname + weight: 20 + containers: + - name: zookeeper + command: + - /conf/zookeeperStart.sh + env: + - name: ENVOY_SIDECAR_STATUS + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['sidecar.istio.io/status'] + image: zookeeper:3.8.0 + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: + - /conf/zookeeperTeardown.sh + livenessProbe: + exec: + command: + - /conf/zookeeperLive.sh + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /conf/zookeeperReady.sh + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 + ports: + - containerPort: 2181 + name: client + protocol: TCP + - containerPort: 2888 + name: quorum + protocol: TCP + - containerPort: 3888 + name: leader-election + protocol: TCP + - containerPort: 7000 + name: metrics + protocol: TCP + - containerPort: 8080 + name: admin-server + protocol: TCP + volumeMounts: + - mountPath: /data + name: data + - mountPath: /conf + name: conf + - mountPath: /cluster-size + name: cluster-size + restartPolicy: Always + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + name: zookeeper-scripts + defaultMode: 0755 + name: conf + - configMap: + name: zookeeper-cluster-size + defaultMode: 0755 + name: cluster-size + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + labels: + app: zookeeper + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem diff --git a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml index 322b700da..47fb75f01 100644 --- a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml +++ b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml @@ -123,6 +123,7 @@ spec: echo 'initLimit=300' echo 'syncLimit=10' echo 'maxClientCnxns=2000' + echo 'maxTimeToWaitForEpoch=2000' echo 'maxSessionTimeout=60000000' echo "dataDir=${ZOO_DATA_DIR}" echo "dataLogDir=${ZOO_DATA_LOG_DIR}" diff --git a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml new file mode 100644 index 000000000..d97148fe8 --- /dev/null +++ b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml @@ -0,0 +1,612 @@ +# Following scripts based from https://github.com/pravega/zookeeper-operator/ +--- +# service for REST administration and reconfiguration +apiVersion: v1 +kind: Service +metadata: + labels: + app: zookeeper + name: zookeeper-admin-server +spec: + type: ClusterIP + ports: + - name: tcp-admin-server + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: zookeeper + +--- +# service for clickhouse zookeeper client connections +apiVersion: v1 +kind: Service +metadata: + labels: + app: zookeeper + name: zookeeper +spec: + type: ClusterIP + ports: + - name: tcp-client + port: 2181 + protocol: TCP + targetPort: 2181 + selector: + app: zookeeper + what: node +--- +# headless service for Zookeeper Quorum Election and service name +apiVersion: v1 +kind: Service +metadata: + labels: + app: zookeeper + name: zookeeper-headless +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-client + port: 2181 + protocol: TCP + targetPort: 2181 + - name: tcp-quorum + port: 2888 + protocol: TCP + targetPort: 2888 + - name: tcp-leader-election + port: 3888 + protocol: TCP + targetPort: 3888 + - name: tcp-metrics + port: 7000 + protocol: TCP + targetPort: 7000 + - name: tcp-admin-server + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: zookeeper +--- +# ConfigMap with common startup scripts and base config +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: zookeeper + name: zookeeper-scripts +data: + env.sh: | + #!/usr/bin/env bash + DOMAIN=`hostname -d` + QUORUM_PORT=2888 + LEADER_PORT=3888 + CLIENT_HOST=zookeeper + CLIENT_PORT=2181 + ADMIN_SERVER_HOST=zookeeper-admin-server + ADMIN_SERVER_PORT=8080 + CLUSTER_NAME=zookeeper + log4j-quiet.properties: | + log4j.rootLogger=CONSOLE + log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender + log4j.appender.CONSOLE.Threshold=ERROR + log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout + log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + log4j.properties: | + zookeeper.root.logger=CONSOLE + zookeeper.console.threshold=WARN + log4j.rootLogger=${zookeeper.root.logger} + log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender + log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} + log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout + log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + zoo.cfg: | + 4lw.commands.whitelist=* + dataDir=/data + standaloneEnabled=false + reconfigEnabled=true + skipACL=yes + metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider + metricsProvider.httpPort=7000 + metricsProvider.exportJvmInfo=true + tickTime=2000 + initLimit=300 + syncLimit=10 + maxClientCnxns=2000 + maxTimeToWaitForEpoch=2000 + globalOutstandingLimit=1000 + preAllocSize=65536 + snapCount=10000 + commitLogCount=500 + snapSizeLimitInKb=4194304 + maxCnxns=0 + maxClientCnxns=60 + minSessionTimeout=4000 + maxSessionTimeout=40000 + autopurge.snapRetainCount=3 + autopurge.purgeInterval=1 + quorumListenOnAllIPs=false + admin.serverPort=8080 + dynamicConfigFile=/data/zoo.cfg.dynamic + zookeeperFunctions.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + set -ex + function zkConfig() { + echo "$HOST.$DOMAIN:$QUORUM_PORT:$LEADER_PORT:$ROLE;$CLIENT_PORT" + } + function zkConnectionString() { + # If the client service address is not yet available, then return localhost + set +e + getent hosts "${CLIENT_HOST}" 2>/dev/null 1>/dev/null + if [[ $? -ne 0 ]]; then + set -e + echo "localhost:${CLIENT_PORT}" + else + set -e + echo "${CLIENT_HOST}:${CLIENT_PORT}" + fi + } + zookeeperStart.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + + set -ex + # TODO think how to add nslookup to docker image + apt-get update && apt-get install --no-install-recommends -y dnsutils curl procps + source /cluster-size/cluster-size.sh + source /conf/env.sh + source /conf/zookeeperFunctions.sh + + HOST=`hostname -s` + DATA_DIR=/data + MYID_FILE=$DATA_DIR/myid + LOG4J_CONF=/conf/log4j-quiet.properties + DYNCONFIG=$DATA_DIR/zoo.cfg.dynamic + STATIC_CONFIG=/data/conf/zoo.cfg + + # Extract resource name and this members ordinal value from pod hostname + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + ORD=${BASH_REMATCH[2]} + else + echo Failed to parse name and ordinal of Pod + exit 1 + fi + + MYID=$((ORD+1)) + + # Values for first startup + WRITE_CONFIGURATION=true + REGISTER_NODE=true + ONDISK_MYID_CONFIG=false + ONDISK_DYN_CONFIG=false + + # Check validity of on-disk configuration + if [ -f $MYID_FILE ]; then + EXISTING_ID="`cat $DATA_DIR/myid`" + if [[ "$EXISTING_ID" == "$MYID" && -f $STATIC_CONFIG ]]; then + # If Id is correct and configuration is present under `/data/conf` + ONDISK_MYID_CONFIG=true + fi + fi + + if [ -f $DYNCONFIG ]; then + ONDISK_DYN_CONFIG=true + fi + + set +e + # Check if envoy is up and running + if [[ -n "$ENVOY_SIDECAR_STATUS" ]]; then + COUNT=0 + MAXCOUNT=${1:-30} + HEALTHYSTATUSCODE="200" + while true; do + COUNT=$(expr $COUNT + 1) + SC=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:15000/ready) + echo "waiting for envoy proxy to come up"; + sleep 1; + if (( "$SC" == "$HEALTHYSTATUSCODE" || "$MAXCOUNT" == "$COUNT" )); then + break + fi + done + fi + set -e + + # Determine if there is an ensemble available to join by checking the service domain + set +e + getent hosts $DOMAIN # This only performs a dns lookup + if [[ $? -eq 0 ]]; then + ACTIVE_ENSEMBLE=true + elif nslookup $DOMAIN | grep -q "server can't find $DOMAIN"; then + echo "there is no active ensemble" + ACTIVE_ENSEMBLE=false + else + # If an nslookup of the headless service domain fails, then there is no + # active ensemble yet, but in certain cases nslookup of headless service + # takes a while to come up even if there is active ensemble + ACTIVE_ENSEMBLE=false + declare -i count=20 + while [[ $count -ge 0 ]] + do + sleep 2 + ((count=count-1)) + getent hosts $DOMAIN + if [[ $? -eq 0 ]]; then + ACTIVE_ENSEMBLE=true + break + fi + done + fi + + if [[ "$ONDISK_MYID_CONFIG" == true && "$ONDISK_DYN_CONFIG" == true ]]; then + # If Configuration is present, we assume, there is no need to write configuration. + WRITE_CONFIGURATION=false + else + WRITE_CONFIGURATION=true + fi + + if [[ "$ACTIVE_ENSEMBLE" == false ]]; then + # This is the first node being added to the cluster or headless service not yet available + REGISTER_NODE=false + else + # An ensemble exists, check to see if this node is already a member. + if [[ "$ONDISK_MYID_CONFIG" == false || "$ONDISK_DYN_CONFIG" == false ]]; then + REGISTER_NODE=true + else + REGISTER_NODE=false + fi + fi + + if [[ "$WRITE_CONFIGURATION" == true ]]; then + echo "Writing myid: $MYID to: $MYID_FILE." + echo $MYID > $MYID_FILE + if [[ $MYID -eq 1 ]]; then + ROLE=participant + echo Initial initialization of ordinal 0 pod, creating new config. + ZKCONFIG=$(zkConfig) + echo Writing bootstrap configuration with the following config: + echo $ZKCONFIG + echo $MYID > $MYID_FILE + echo "server.${MYID}=${ZKCONFIG}" > $DYNCONFIG + fi + fi + + if [[ "$REGISTER_NODE" == true ]]; then + ROLE=observer + ZKURL=$(zkConnectionString) + ZKCONFIG=$(zkConfig) + + set -e + echo Registering node and writing local configuration to disk. + zkCli.sh -server ${ZKURL} reconfig -add "\nserver.$MYID=$ZKCONFIG" | grep -E '^server\.[0-9]+=' > $DYNCONFIG + set +e + fi + + ZOOCFGDIR=/data/conf + export ZOOCFGDIR + echo Copying /conf contents to writable directory, to support Zookeeper dynamic reconfiguration + if [[ ! -d "$ZOOCFGDIR" ]]; then + mkdir $ZOOCFGDIR + cp -f /conf/zoo.cfg $ZOOCFGDIR + else + echo Copying the /conf/zoo.cfg contents except the dynamic config file during restart + echo -e "$( head -n -1 /conf/zoo.cfg )""\n""$( tail -n 1 "$STATIC_CONFIG" )" > $STATIC_CONFIG + fi + cp -f /conf/log4j.properties $ZOOCFGDIR + cp -f /conf/log4j-quiet.properties $ZOOCFGDIR + cp -f /conf/env.sh $ZOOCFGDIR + + if [ -f $DYNCONFIG ]; then + # Node registered, start server + echo Starting zookeeper service + zkServer.sh --config $ZOOCFGDIR start-foreground + else + echo "Node failed to register!" + exit 1 + fi + + zookeeperTeardown.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + + set -ex + + source /cluster-size/cluster-size.sh + source /conf/env.sh + source /conf/zookeeperFunctions.sh + + DATA_DIR=/data + MYID_FILE=$DATA_DIR/myid + LOG4J_CONF=/conf/log4j-quiet.properties + + # Wait for client connections to drain. Kubernetes will wait until the confiugred + # "terminationGracePeriodSeconds" before focibly killing the container + CONN_COUNT=`echo cons | nc localhost 2181 | grep -v "^$" |grep -v "/127.0.0.1:" | wc -l` + for (( i = 0; i < 6; i++ )); do + if [[ "$CONN_COUNT" -gt 0 ]]; then + echo "$CONN_COUNT non-local connections still connected." + sleep 5 + else + echo "$CONN_COUNT non-local connections" + break + fi + done + + # Check to see if zookeeper service for this node is a participant + set +e + ZKURL=$(zkConnectionString) + set -e + MYID=`cat $MYID_FILE` + + echo "CLUSTER_SIZE=$CLUSTER_SIZE, MyId=$MYID" + if [[ -n "$CLUSTER_SIZE" && "$CLUSTER_SIZE" -lt "$MYID" ]]; then + # If ClusterSize < MyId, this server is being permanantly removed. + zkCli.sh -server ${ZKURL} reconfig -remove $MYID | grep -E '^server\.\d+=' > $DYNCONFIG + echo $? + fi + + # Kill the primary process ourselves to circumvent the terminationGracePeriodSeconds + ps -ef | grep zoo.cfg | grep -v grep | awk '{print $2}' | xargs kill + + + zookeeperLive.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + set -ex + source /conf/env.sh + OK=$(echo ruok | nc 127.0.0.1 $CLIENT_PORT) + # Check to see if zookeeper service answers + if [[ "$OK" == "imok" ]]; then + exit 0 + else + exit 1 + fi + + zookeeperReady.sh: | + #!/usr/bin/env bash + # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # http://www.apache.org/licenses/LICENSE-2.0 + set -ex + source /cluster-size/cluster-size.sh + source /conf/env.sh + source /conf/zookeeperFunctions.sh + + HOST=`hostname -s` + DATA_DIR=/data + MYID_FILE=$DATA_DIR/myid + LOG4J_CONF=/conf/log4j-quiet.properties + STATIC_CONFIG=/data/conf/zoo.cfg + + OK=$(echo ruok | nc 127.0.0.1 $CLIENT_PORT) + + # Check to see if zookeeper service answers + if [[ "$OK" == "imok" ]]; then + set +e + getent hosts $DOMAIN + if [[ $? -ne 0 ]]; then + set -e + echo "There is no active ensemble, skipping readiness probe..." + exit 0 + else + set -e + # An ensemble exists, check to see if this node is already a member. + # Check to see if zookeeper service for this node is a participant + set +e + # Extract resource name and this members' ordinal value from pod hostname + HOST=`hostname -s` + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + ORD=${BASH_REMATCH[2]} + else + echo Failed to parse name and ordinal of Pod + exit 1 + fi + MYID=$((ORD+1)) + ONDISK_CONFIG=false + if [ -f $MYID_FILE ]; then + EXISTING_ID="`cat $DATA_DIR/myid`" + if [[ "$EXISTING_ID" == "$MYID" && -f $STATIC_CONFIG ]]; then + #If Id is correct and configuration is present under `/data/conf` + ONDISK_CONFIG=true + DYN_CFG_FILE_LINE=`cat $STATIC_CONFIG|grep "dynamicConfigFile\="` + DYN_CFG_FILE=${DYN_CFG_FILE_LINE##dynamicConfigFile=} + SERVER_FOUND=`cat $DYN_CFG_FILE | grep "server.${MYID}=" | wc -l` + if [[ "$SERVER_FOUND" == "0" ]]; then + echo "Server not found in ensemble. Exiting ..." + exit 1 + fi + SERVER=`cat $DYN_CFG_FILE | grep "server.${MYID}="` + if [[ "$SERVER" == *"participant"* ]]; then + ROLE=participant + elif [[ "$SERVER" == *"observer"* ]]; then + ROLE=observer + fi + fi + fi + + if [[ "$ROLE" == "participant" ]]; then + echo "Zookeeper service is available and an active participant" + exit 0 + elif [[ "$ROLE" == "observer" ]]; then + echo "Zookeeper service is ready to be upgraded from observer to participant." + ROLE=participant + ZKURL=$(zkConnectionString) + ZKCONFIG=$(zkConfig) + zkCli.sh -server ${ZKURL} reconfig -remove $MYID | grep -E '^server\.[0-9]+=' > $DYNCONFIG + sleep 1 + zkCli.sh -server ${ZKURL} reconfig -add "\nserver.$MYID=$ZKCONFIG" + exit 0 + else + echo "Something has gone wrong. Unable to determine zookeeper role." + exit 1 + fi + fi + + else + echo "Zookeeper service is not available for requests" + exit 1 + fi + +--- +# ConfigMap for current cluster size +apiVersion: v1 +kind: ConfigMap +metadata: + name: zookeeper-cluster-size + labels: + app: zookeeper +data: + cluster-size.sh: | + #!/usr/bin/env bash + CLUSTER_SIZE=3 + +--- +# Main StatefulSet +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: zookeeper + name: zookeeper + +spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + podManagementPolicy: OrderedReady + replicas: 3 + selector: + matchLabels: + app: zookeeper + serviceName: zookeeper-headless + template: + metadata: + generateName: zookeeper + labels: + app: zookeeper + what: node + annotations: + prometheus.io/port: '7000' + prometheus.io/scrape: 'true' + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - zookeeper + topologyKey: kubernetes.io/hostname + weight: 20 + containers: + - name: zookeeper + command: + - /conf/zookeeperStart.sh + env: + - name: ENVOY_SIDECAR_STATUS + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['sidecar.istio.io/status'] + image: zookeeper:3.8.0 + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: + - /conf/zookeeperTeardown.sh + livenessProbe: + exec: + command: + - /conf/zookeeperLive.sh + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /conf/zookeeperReady.sh + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 + ports: + - containerPort: 2181 + name: client + protocol: TCP + - containerPort: 2888 + name: quorum + protocol: TCP + - containerPort: 3888 + name: leader-election + protocol: TCP + - containerPort: 7000 + name: metrics + protocol: TCP + - containerPort: 8080 + name: admin-server + protocol: TCP + volumeMounts: + - mountPath: /data + name: data + - mountPath: /conf + name: conf + - mountPath: /cluster-size + name: cluster-size + restartPolicy: Always + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + name: zookeeper-scripts + defaultMode: 0755 + name: conf + - configMap: + name: zookeeper-cluster-size + defaultMode: 0755 + name: cluster-size + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + labels: + app: zookeeper + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem diff --git a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml index 11dcfbeee..d9b91b5c2 100644 --- a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml +++ b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml @@ -123,6 +123,7 @@ spec: echo 'initLimit=300' echo 'syncLimit=10' echo 'maxClientCnxns=2000' + echo 'maxTimeToWaitForEpoch=2000' echo 'maxSessionTimeout=60000000' echo "dataDir=${ZOO_DATA_DIR}" echo "dataLogDir=${ZOO_DATA_LOG_DIR}" diff --git a/tests/e2e/kubectl.py b/tests/e2e/kubectl.py index 43f5a31fb..5aaebb109 100644 --- a/tests/e2e/kubectl.py +++ b/tests/e2e/kubectl.py @@ -74,16 +74,18 @@ def delete_all_chi(ns=namespace): delete_chi(chi["metadata"]["name"], ns) -def delete_all_keeper(keeper_type="zookeeper", ns=namespace): - for resource_type in ("sts", "pvc"): - try: - zks = get(resource_type, "", label=f"-l app={keeper_type}", ns=ns, ok_to_fail=True) - except Exception as e: - zks = {} - if "items" in zks: - for item in zks["items"]: - name = item['metadata']['name'] - launch(f"delete {resource_type} -n {ns} {name}") +def delete_all_keeper(ns=namespace): + for keeper_type in ('zookeeper-operator', 'zookeeper', 'clickhouse-keeper'): + expected_resource_types = ("zookeepercluster", ) if keeper_type == "zookeeper-operator" else ("sts", "pvc", "cm", "svc") + for resource_type in expected_resource_types: + try: + item_list = get(resource_type, "", label=f"-l app={keeper_type}", ns=ns, ok_to_fail=True) + except Exception as e: + item_list = {} + if "items" in item_list: + for item in item_list["items"]: + name = item['metadata']['name'] + launch(f"delete {resource_type} -n {ns} {name}", ok_to_fail=True) def create_and_check(manifest, check, ns=namespace, timeout=900): diff --git a/tests/e2e/manifests/chi/test-cluster-for-zookeeper-operator-1.yaml b/tests/e2e/manifests/chi/test-cluster-for-zookeeper-operator-1.yaml new file mode 100644 index 000000000..1e1f8d465 --- /dev/null +++ b/tests/e2e/manifests/chi/test-cluster-for-zookeeper-operator-1.yaml @@ -0,0 +1,21 @@ +apiVersion: "clickhouse.altinity.com/v1" + +kind: "ClickHouseInstallation" + +metadata: + name: test-cluster-for-zk + +spec: + useTemplates: + - name: clickhouse-latest-version + - name: persistent-volume + configuration: + zookeeper: + nodes: + - host: zookeeper-client + port: 2181 + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 1 diff --git a/tests/e2e/manifests/chi/test-cluster-for-zookeeper-operator-2.yaml b/tests/e2e/manifests/chi/test-cluster-for-zookeeper-operator-2.yaml new file mode 100644 index 000000000..a23f6251e --- /dev/null +++ b/tests/e2e/manifests/chi/test-cluster-for-zookeeper-operator-2.yaml @@ -0,0 +1,21 @@ +apiVersion: "clickhouse.altinity.com/v1" + +kind: "ClickHouseInstallation" + +metadata: + name: test-cluster-for-zk + +spec: + useTemplates: + - name: clickhouse-latest-version + - name: persistent-volume + configuration: + zookeeper: + nodes: + - host: zookeeper-client + port: 2181 + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 2 diff --git a/tests/e2e/test_keeper.py b/tests/e2e/test_keeper.py index 27de3b723..031ef7785 100644 --- a/tests/e2e/test_keeper.py +++ b/tests/e2e/test_keeper.py @@ -8,10 +8,12 @@ from testflows.core import * -def wait_keeper_ready(svc_name='zookeeper', keeper_type='zookeeper', pod_count=3, retries=10): +def wait_keeper_ready(keeper_type='zookeeper', pod_count=3, retries=10): + svc_name = 'zookeeper-client' if keeper_type == "zookeeper-operator" else 'zookeeper' expected_containers = "2/2" if keeper_type == "clickhouse-keeper" else "1/1" + expected_pod_prefix = "clickhouse-keeper" if keeper_type == "clickhouse-keeper" else "zookeeper" for i in range(retries): - ready_pods = kubectl.launch(f"get pods | grep {keeper_type} | grep Running | grep '{expected_containers}' | wc -l") + ready_pods = kubectl.launch(f"get pods | grep {expected_pod_prefix} | grep Running | grep '{expected_containers}' | wc -l") ready_endpoints = "0" if ready_pods == str(pod_count): ready_endpoints = kubectl.launch(f"get endpoints {svc_name} -o json | jq '.subsets[].addresses[].ip' | wc -l") @@ -79,9 +81,14 @@ def check_zk_root_znode(chi, pod_count, retry_count=5): out = "" expected_out = "" for i in range(retry_count): - if keeper_type == "zookeeper": + if keeper_type == "zookeeper-operator": + expected_out = "[clickhouse, zookeeper, zookeeper-operator]" + keeper_cmd = './bin/zkCli.sh ls /' + pod_prefix = "zookeeper" + elif keeper_type == "zookeeper": expected_out = "[clickhouse, zookeeper]" keeper_cmd = './bin/zkCli.sh ls /' + pod_prefix = "zookeeper" else: expected_out = "clickhouse" keeper_cmd = "if [[ ! $(command -v zookeepercli) ]]; then " @@ -89,8 +96,9 @@ def check_zk_root_znode(chi, pod_count, retry_count=5): keeper_cmd += "dpkg -i /tmp/zookeepercli.deb; " keeper_cmd += "fi; " keeper_cmd += "zookeepercli -servers 127.0.0.1:2181 -c ls /" + pod_prefix = "clickhouse-keeper" - out = kubectl.launch(f"exec {keeper_type}-{pod_num} -- bash -ce '{keeper_cmd}'", ns=settings.test_namespace, ok_to_fail=True) + out = kubectl.launch(f"exec {pod_prefix}-{pod_num} -- bash -ce '{keeper_cmd}'", ns=settings.test_namespace, ok_to_fail=True) if expected_out in out: break else: @@ -99,8 +107,12 @@ def check_zk_root_znode(chi, pod_count, retry_count=5): assert expected_out in out, f"Unexpected {keeper_type} `ls /` output" out = clickhouse.query(chi["metadata"]["name"], "SELECT count() FROM system.zookeeper WHERE path='/'") - expected_out = "1" if keeper_type == "clickhouse-keeper" else "2" - assert expected_out == out.strip(" \t\r\n"), f"Unexpected `SELECT count() FROM system.zookeeper WHERE path='/'` output {out}" + expected_out = { + "zookeeper": "2", + "zookeeper-operator": "3", + "clickhouse-keeper": "1", + } + assert expected_out[keeper_type] == out.strip(" \t\r\n"), f"Unexpected `SELECT count() FROM system.zookeeper WHERE path='/'` output {out}" def rescale_zk_and_clickhouse(ch_node_count, keeper_node_count, first_install=False): keeper_manifest = keeper_manifest_1_node if keeper_node_count == 1 else keeper_manifest_3_node @@ -117,9 +129,9 @@ def rescale_zk_and_clickhouse(ch_node_count, keeper_node_count, first_install=Fa ) return chi - with When("Clean exists ClickHouse and Keeper"): - kubectl.delete_all_keeper(self.context.keeper_type, settings.test_namespace) + with When("Clean exists ClickHouse Keeper and ZooKeeper"): kubectl.delete_all_chi(settings.test_namespace) + kubectl.delete_all_keeper(settings.test_namespace) with When("Install CH 1 node ZK 1 node"): chi = rescale_zk_and_clickhouse(ch_node_count=1, keeper_node_count=1, first_install=True) @@ -157,11 +169,11 @@ def rescale_zk_and_clickhouse(ch_node_count, keeper_node_count, first_install=Fa check_zk_root_znode(chi, pod_count=3) with Then('check data in tables'): - for table, exptected_rows in {"test_repl1": str(1000 + 2000 * total_iterations), "test_repl2": str(2000 * total_iterations), "test_repl3": str(1000 * total_iterations)}.items(): + for table_name, exptected_rows in {"test_repl1": str(1000 + 2000 * total_iterations), "test_repl2": str(2000 * total_iterations), "test_repl3": str(1000 * total_iterations)}.items(): actual_rows = clickhouse.query( - chi['metadata']['name'], f'SELECT count() FROM default.{table}', pod="chi-test-cluster-for-zk-default-0-1-0" + chi['metadata']['name'], f'SELECT count() FROM default.{table_name}', pod="chi-test-cluster-for-zk-default-0-1-0" ) - assert actual_rows == exptected_rows, f"Invalid rows counter after inserts {table} expected={exptected_rows} actual={actual_rows}" + assert actual_rows == exptected_rows, f"Invalid rows counter after inserts {table_name} expected={exptected_rows} actual={actual_rows}" with Then('drop all created tables'): for i in range(3): @@ -190,11 +202,34 @@ def test_clickhouse_keeper_rescale(self): ) +@TestScenario +@Name("test_zookeeper_operator_rescale. Check Zookeeper OPERATOR scale-up / scale-down cases") +def test_zookeeper_operator_rescale(self): + test_keeper_outline( + keeper_type="zookeeper-operator", + pod_for_insert_data="chi-test-cluster-for-zk-default-0-1-0", + keeper_manifest_1_node='zookeeper-operator-1-node.yaml', + keeper_manifest_3_node='zookeeper-operator-3-node.yaml', + ) + +@TestScenario +@Name("test_zookeeper_pvc_scaleout_rescale. Check ZK+PVC scale-up / scale-down cases") +def test_zookeeper_pvc_scaleout_rescale(self): + test_keeper_outline( + keeper_type="zookeeper", + pod_for_insert_data="chi-test-cluster-for-zk-default-0-1-0", + keeper_manifest_1_node='zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml', + keeper_manifest_3_node='zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml', + ) + + @TestModule @Name("e2e.test_keeper") def test(self): all_tests = [ + test_zookeeper_operator_rescale, test_clickhouse_keeper_rescale, + test_zookeeper_pvc_scaleout_rescale, test_zookeeper_rescale, ] diff --git a/tests/e2e/util.py b/tests/e2e/util.py index 925aa732c..f779b3bfb 100644 --- a/tests/e2e/util.py +++ b/tests/e2e/util.py @@ -58,24 +58,34 @@ def require_keeper(keeper_manifest='', keeper_type='zookeeper', force_install=Fa if keeper_type == "clickhouse-keeper": keeper_manifest = 'clickhouse-keeper-1-node-256M-for-test-only.yaml' if keeper_manifest == '' else keeper_manifest keeper_manifest = f"../../deploy/clickhouse-keeper/{keeper_manifest}" + if keeper_type == "zookeeper-operator": + keeper_manifest = 'zookeeper-operator-1-node.yaml' if keeper_manifest == '' else keeper_manifest + keeper_manifest = f"../../deploy/zookeeper-operator/{keeper_manifest}" + multi_doc = yaml_manifest.get_multidoc_manifest_data(get_full_path(keeper_manifest, lookup_in_host=True)) keeper_nodes = 1 docs_count = 0 for doc in multi_doc: docs_count += 1 - if doc["kind"] == "StatefulSet": + if doc["kind"] in ("StatefulSet", "ZookeeperCluster"): keeper_nodes = doc["spec"]["replicas"] expected_docs = { - "zookeeper": 4, - "clickhouse-keeper": 6 + "zookeeper": 6 if 'scaleout-pvc' in keeper_manifest else 4, + "clickhouse-keeper": 6, + "zookeeper-operator": 1, + } + expected_pod_prefix = { + "zookeeper": "zookeeper", + "zookeeper-operator": "zookeeper", + "clickhouse-keeper": "clickhouse-keeper", } assert docs_count == expected_docs[keeper_type], f"invalid {keeper_type} manifest, expected {expected_docs[keeper_type]}, actual {docs_count} documents in {keeper_manifest} file" with Given(f"Install {keeper_type} {keeper_nodes} nodes"): kubectl.apply(get_full_path(keeper_manifest, lookup_in_host=False)) - for docs_count in range(keeper_nodes): - kubectl.wait_object("pod", f"{keeper_type}-{docs_count}") - for docs_count in range(keeper_nodes): - kubectl.wait_pod_status(f"{keeper_type}-{docs_count}", "Running") + for pod_num in range(keeper_nodes): + kubectl.wait_object("pod", f"{expected_pod_prefix[keeper_type]}-{pod_num}") + for pod_num in range(keeper_nodes): + kubectl.wait_pod_status(f"{expected_pod_prefix[keeper_type]}-{pod_num}", "Running") def wait_clickhouse_cluster_ready(chi): @@ -87,7 +97,7 @@ def wait_clickhouse_cluster_ready(chi): for pod in chi['status']['pods']: cluster_response = clickhouse.query( chi["metadata"]["name"], - "SYSTEM RELOAD CONFIG; SELECT host_name FROM system.clusters WHERE cluster='all-sharded'", + "SELECT host_name FROM system.clusters WHERE cluster='all-sharded'", pod=pod ) for host in chi['status']['fqdns']: @@ -102,7 +112,13 @@ def install_clickhouse_and_keeper(chi_file, chi_template_file, chi_name, keeper_type='zookeeper', keeper_manifest='', force_keeper_install=False, clean_ns=True, keeper_install_first=True, make_object_count=True): if keeper_manifest == '': - keeper_manifest = 'zookeeper-1-node-1GB-for-tests-only.yaml' if keeper_type == 'zookeeper' else 'clickhouse-keeper-1-node-256M-for-test-only.yaml' + if keeper_type == 'zookeeper': + keeper_manifest = 'zookeeper-1-node-1GB-for-tests-only.yaml' + if keeper_type == 'clickhouse-keeper': + keeper_manifest = 'clickhouse-keeper-1-node-256M-for-test-only.yaml' + if keeper_type == 'zookeeper-operator': + keeper_manifest = 'zookeeper-operator-1-node.yaml' + with Given("install zookeeper/clickhouse-keeper + clickhouse"): if clean_ns: kubectl.delete_ns(settings.test_namespace, ok_to_fail=True, timeout=600) From 7775e4445e53ea7165d25aa06e59f07378d56457 Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 10 Apr 2022 13:57:19 +0500 Subject: [PATCH 2/3] test_keeper.py pass --- ...-node-1GB-for-tests-only-scaleout-pvc.yaml | 42 +++++++++++++------ ...nodes-1GB-for-tests-only-scaleout-pvc.yaml | 42 +++++++++++++------ 2 files changed, 60 insertions(+), 24 deletions(-) diff --git a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml index 5d3db1afa..9ab587acb 100644 --- a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml +++ b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml @@ -80,14 +80,15 @@ metadata: data: env.sh: | #!/usr/bin/env bash - DOMAIN=`hostname -d` - QUORUM_PORT=2888 - LEADER_PORT=3888 - CLIENT_HOST=zookeeper - CLIENT_PORT=2181 - ADMIN_SERVER_HOST=zookeeper-admin-server - ADMIN_SERVER_PORT=8080 - CLUSTER_NAME=zookeeper + export DOMAIN=`hostname -d` + export QUORUM_PORT=2888 + export LEADER_PORT=3888 + export CLIENT_HOST=zookeeper + export CLIENT_PORT=2181 + export ADMIN_SERVER_HOST=zookeeper-admin-server + export ADMIN_SERVER_PORT=8080 + export CLUSTER_NAME=zookeeper + export ZOO_LOG4J_PROP="WARN, CONSOLE" log4j-quiet.properties: | log4j.rootLogger=CONSOLE log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender @@ -96,12 +97,28 @@ data: log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n log4j.properties: | zookeeper.root.logger=CONSOLE - zookeeper.console.threshold=WARN + zookeeper.console.threshold=INFO log4j.rootLogger=${zookeeper.root.logger} log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + logback.xml: | + + + + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + + ${zookeeper.console.threshold} + + + + + + + zoo.cfg: | 4lw.commands.whitelist=* dataDir=/data @@ -305,6 +322,7 @@ data: fi cp -f /conf/log4j.properties $ZOOCFGDIR cp -f /conf/log4j-quiet.properties $ZOOCFGDIR + cp -f /conf/logback.xml $ZOOCFGDIR cp -f /conf/env.sh $ZOOCFGDIR if [ -f $DYNCONFIG ]; then @@ -352,11 +370,11 @@ data: ZKURL=$(zkConnectionString) set -e MYID=`cat $MYID_FILE` - + CLUSTER_SIZE=$(zkCli.sh -server ${ZKURL} config | grep -c -E '^server\.[0-9]+=') echo "CLUSTER_SIZE=$CLUSTER_SIZE, MyId=$MYID" - if [[ -n "$CLUSTER_SIZE" && "$CLUSTER_SIZE" -lt "$MYID" ]]; then + if [[ "$MYID" -gt "1" ]]; then # If ClusterSize < MyId, this server is being permanantly removed. - zkCli.sh -server ${ZKURL} reconfig -remove $MYID | grep -E '^server\.\d+=' > $DYNCONFIG + zkCli.sh -server ${ZKURL} reconfig -remove $MYID | grep -E '^server\.[0-9]+=' > $DYNCONFIG echo $? fi diff --git a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml index d97148fe8..0bca404fb 100644 --- a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml +++ b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml @@ -80,14 +80,15 @@ metadata: data: env.sh: | #!/usr/bin/env bash - DOMAIN=`hostname -d` - QUORUM_PORT=2888 - LEADER_PORT=3888 - CLIENT_HOST=zookeeper - CLIENT_PORT=2181 - ADMIN_SERVER_HOST=zookeeper-admin-server - ADMIN_SERVER_PORT=8080 - CLUSTER_NAME=zookeeper + export DOMAIN=`hostname -d` + export QUORUM_PORT=2888 + export LEADER_PORT=3888 + export CLIENT_HOST=zookeeper + export CLIENT_PORT=2181 + export ADMIN_SERVER_HOST=zookeeper-admin-server + export ADMIN_SERVER_PORT=8080 + export CLUSTER_NAME=zookeeper + export ZOO_LOG4J_PROP="WARN, CONSOLE" log4j-quiet.properties: | log4j.rootLogger=CONSOLE log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender @@ -96,12 +97,28 @@ data: log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n log4j.properties: | zookeeper.root.logger=CONSOLE - zookeeper.console.threshold=WARN + zookeeper.console.threshold=INFO log4j.rootLogger=${zookeeper.root.logger} log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + logback.xml: | + + + + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + + ${zookeeper.console.threshold} + + + + + + + zoo.cfg: | 4lw.commands.whitelist=* dataDir=/data @@ -305,6 +322,7 @@ data: fi cp -f /conf/log4j.properties $ZOOCFGDIR cp -f /conf/log4j-quiet.properties $ZOOCFGDIR + cp -f /conf/logback.xml $ZOOCFGDIR cp -f /conf/env.sh $ZOOCFGDIR if [ -f $DYNCONFIG ]; then @@ -352,11 +370,11 @@ data: ZKURL=$(zkConnectionString) set -e MYID=`cat $MYID_FILE` - + CLUSTER_SIZE=$(zkCli.sh -server ${ZKURL} config | grep -c -E '^server\.[0-9]+=') echo "CLUSTER_SIZE=$CLUSTER_SIZE, MyId=$MYID" - if [[ -n "$CLUSTER_SIZE" && "$CLUSTER_SIZE" -lt "$MYID" ]]; then + if [[ "$MYID" -gt "1" ]]; then # If ClusterSize < MyId, this server is being permanantly removed. - zkCli.sh -server ${ZKURL} reconfig -remove $MYID | grep -E '^server\.\d+=' > $DYNCONFIG + zkCli.sh -server ${ZKURL} reconfig -remove $MYID | grep -E '^server\.[0-9]+=' > $DYNCONFIG echo $? fi From 57d5ee93e8fdc499b50fa5c2468ee541d527a7a5 Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 10 Apr 2022 16:10:42 +0500 Subject: [PATCH 3/3] some final polishing, unfortunately --feature-gates=StatefulSetAutoDeletePVC=true required --- ...keeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml | 11 +++++------ ...eeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml index 9ab587acb..8c7f9fc5c 100644 --- a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml +++ b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml @@ -139,7 +139,6 @@ data: commitLogCount=500 snapSizeLimitInKb=4194304 maxCnxns=0 - maxClientCnxns=60 minSessionTimeout=4000 maxSessionTimeout=40000 autopurge.snapRetainCount=3 @@ -153,7 +152,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex function zkConfig() { echo "$HOST.$DOMAIN:$QUORUM_PORT:$LEADER_PORT:$ROLE;$CLIENT_PORT" @@ -176,7 +175,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex # TODO think how to add nslookup to docker image @@ -340,7 +339,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex @@ -388,7 +387,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex source /conf/env.sh OK=$(echo ruok | nc 127.0.0.1 $CLIENT_PORT) @@ -405,7 +404,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex source /cluster-size/cluster-size.sh source /conf/env.sh diff --git a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml index 0bca404fb..00343745f 100644 --- a/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml +++ b/deploy/zookeeper/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml @@ -139,7 +139,6 @@ data: commitLogCount=500 snapSizeLimitInKb=4194304 maxCnxns=0 - maxClientCnxns=60 minSessionTimeout=4000 maxSessionTimeout=40000 autopurge.snapRetainCount=3 @@ -153,7 +152,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex function zkConfig() { echo "$HOST.$DOMAIN:$QUORUM_PORT:$LEADER_PORT:$ROLE;$CLIENT_PORT" @@ -176,7 +175,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex # TODO think how to add nslookup to docker image @@ -340,7 +339,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex @@ -388,7 +387,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex source /conf/env.sh OK=$(echo ruok | nc 127.0.0.1 $CLIENT_PORT) @@ -405,7 +404,7 @@ data: # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 + # https://www.apache.org/licenses/LICENSE-2.0 set -ex source /cluster-size/cluster-size.sh source /conf/env.sh