搜索文章 Ctrl K

6-2-1-k8s实际应用-zookeeper集群部署(一)

熊 1 阅读 容器

zookeeper是一个开源的分布式协调服务 , 由知名互联网公司Yahoo创建 , 它是Chubby的开源实现；换句话讲 , zookeeper是一个典型的分布式数据一致性解决方案 , 分布式应用程序可以基于它实现数据的发布/订阅、负载均衡、名称服务、分布式协调/通知、集群管理、Master选举、分布式锁和分布式队列;

kubernetnes-zookeeper-1

1. 构建zookeeper镜像

  1. 下载java环境基础镜像，将对应镜像修改本地harbor地址上传至harbor

    root@k8s-master01:~# nerdctl pull elevy/slim_java:8
WARN[0000] skipping verifying HTTPS certs for "docker.io" 
docker.io/elevy/slim_java:8:                                                      resolved       |++++++++++++++++++++++++++++++++++++++| 
manifest-sha256:044e42fb89cda51e83701349a9b79e8117300f4841511ed853f73caf7fc98a51: done           |++++++++++++++++++++++++++++++++++++++| 
config-sha256:0776147f4957ebff0e3314117258e09933273d45b83f07f1a4e1863201eac2b0:   done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:fd529fe251b34db45de24e46ae4d8f57c5b8bbcfb1b8d8c6fb7fa9fcdca8905e:    done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:88286f41530e93dffd4b964e1db22ce4939fffa4a4c665dab8591fbab03d4926:    done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:7141511c4dad1bb64345a3cd38e009b1bcd876bba3e92be040ab2602e9de7d1e:    done           |++++++++++++++++++++++++++++++++++++++| 
elapsed: 15.4s                                                                    total:  31.0 M (2.0 MiB/s)                                       
root@k8s-master01:~# nerdctl tag  elevy/slim_java:8 harbor.cbear.top/baseimages/slim_java:8
root@k8s-master01:~# nerdctl push harbor.cbear.top/baseimages/slim_java:8
INFO[0000] pushing as a reduced-platform image (application/vnd.docker.distribution.manifest.v2+json, sha256:044e42fb89cda51e83701349a9b79e8117300f4841511ed853f73caf7fc98a51) 
WARN[0000] skipping verifying HTTPS certs for "harbor.cbear.top" 
manifest-sha256:044e42fb89cda51e83701349a9b79e8117300f4841511ed853f73caf7fc98a51: done           |++++++++++++++++++++++++++++++++++++++| 
config-sha256:0776147f4957ebff0e3314117258e09933273d45b83f07f1a4e1863201eac2b0:   done           |++++++++++++++++++++++++++++++++++++++| 
elapsed: 1.2 s                                                                    total:  18.0 K (15.0 KiB/s)                                      
​

  2. 准备相关配置文件

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# cat bin/zkReady.sh 
#!/bin/bash
​
/zookeeper/bin/zkServer.sh status | egrep 'Mode: (standalone|leading|following|observing)'
root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# cat entrypoint.sh 
#!/bin/bash
​
##将$MYID的值写入MYID文件,如果变量为空就默认为1,$MYID为pod中的系统级别环境变量,该变量由部署zk的yaml清单中的环境变量指定
echo ${MYID:-1} > /zookeeper/data/myid
​
#如果$SERVERS不为空则向下执行,SERVERS为pod中的系统级别环境变量,该变量同样也是在zk部署yaml清单中指定的环境变量的
#IFS为bash内置变量用于分割字符并将结果形成一个数组
#${!servers[@]}表示获取servers中每个元素的索引值,此索引值会用做当前ZK的ID
##打印结果并输出重定向到文件/zookeeper/conf/zoo.cfg，其中%i和%s的值来分别自于后面变量"$((1 + $i))" "${servers[$i]}"
if [ -n "$SERVERS" ]; then
    IFS=\, read -a servers <<<"$SERVERS"
    for i in "${!servers[@]}"; do 
        printf "\nserver.%i=%s:2888:3888" "$((1 + $i))" "${servers[$i]}" >> /zookeeper/conf/zoo.cfg
    done
fi
​
#$@变量用于引用给脚本传递的所有参数,传递的所有参数会被作为一个数组列表,exec为终止当前进程、保留当前进程id、新建一个进程执行新的任务,即CMD [ "zkServer.sh", "start-foreground" ]
cd /zookeeper
exec "$@"
root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# cat conf/
log4j.properties  zoo.cfg           
root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# cat conf/zoo.cfg 
tickTime=2000
initLimit=20
syncLimit=20
dataDir=/zookeeper/data
dataLogDir=/zookeeper/wal
#snapCount=100000
autopurge.purgeInterval=1
clientPort=2181
quorumListenOnAllIPs=false
root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# cat conf/log4j.properties 
# Define some default values that can be overridden by system properties
zookeeper.root.logger=INFO, CONSOLE, ROLLINGFILE
zookeeper.console.threshold=INFO
zookeeper.log.dir=/zookeeper/log
zookeeper.log.file=zookeeper.log
zookeeper.log.threshold=INFO
zookeeper.tracelog.dir=/zookeeper/log
zookeeper.tracelog.file=zookeeper_trace.log
​
#
# ZooKeeper Logging Configuration
#
​
# Format is "<default threshold> (, <appender>)+
​
# DEFAULT: console appender only
log4j.rootLogger=${zookeeper.root.logger}
​
# Example with rolling log file
#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
​
# Example with rolling log file and tracing
#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE
​
#
# Log INFO level and above messages to the console
#
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold}
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
​
#
# Add ROLLINGFILE to rootLogger to get log file output
#    Log DEBUG level and above messages to a log file
log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold}
log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file}
​
# Max log file size of 10MB
log4j.appender.ROLLINGFILE.MaxFileSize=10MB
# uncomment the next line to limit number of backup files
log4j.appender.ROLLINGFILE.MaxBackupIndex=5
​
log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
​
​
#
# Add TRACEFILE to rootLogger to get log file output
#    Log DEBUG level and above messages to a log file
log4j.appender.TRACEFILE=org.apache.log4j.FileAppender
log4j.appender.TRACEFILE.Threshold=TRACE
log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file}
​
log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout
### Notice we are including log4j's NDC here (%x)
log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n
​

  3. 为对应文件添加执行权限

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# chmod +x entrypoint.sh bin/zkReady.sh

  4. 配置对应Dockerfile文件

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# cat Dockerfile
#FROM harbor-linux38.local.com/linux38/slim_java:8 
FROM harbor.cbear.top/pub-images/jdk-17-base:v2
​
ENV ZK_VERSION 3.9.2
​
RUN yum install -y \
      ca-certificates   \
      gnupg             \
      tar               \
      wget &&           \
    #
    # Install dependencies
    yum install -y  \
      bash &&           \
    #
    # Download Zookeeper
    wget -nv -O /tmp/zk.tgz "https://dlcdn.apache.org/zookeeper/current/apache-zookeeper-3.9.2-bin.tar.gz" && \
    wget -nv -O /tmp/zk.tgz.asc "https://dlcdn.apache.org/zookeeper/current/apache-zookeeper-3.9.2-bin.tar.gz.asc" && \
    wget -nv -O /tmp/KEYS https://archive.apache.org/dist/zookeeper/KEYS && \
    #
    # Verify the signature
    export GNUPGHOME="$(mktemp -d)" && \
    gpg -q --batch --import /tmp/KEYS && \
    gpg -q --batch --no-auto-key-retrieve --verify /tmp/zk.tgz.asc /tmp/zk.tgz && \
    #
    # Set up directories
    #
    mkdir -p /zookeeper/data /zookeeper/wal /zookeeper/log && \
    #
    # Install
    tar -x -C /zookeeper --strip-components=1 --no-same-owner -f /tmp/zk.tgz && \
    #
    # Slim down
    cd /zookeeper && \
    #cp dist-maven/zookeeper-${ZK_VERSION}.jar . && \
    rm -rf \
      *.txt \
      *.xml \
      *.md \
      bin/README.txt \
      bin/*.cmd \
      conf/* \
      zookeeper-contrib \
      dist-maven \
      docs \
      lib/*.txt \
      lib/cobertura \
      lib/jdiff \
      recipes \
      src \
      zookeeper-*.asc \
      zookeeper-*.md5 \
      zookeeper-*.sha1 && \
    #
    # Clean up
    #apk del .build-deps && \
    rm -rf /tmp/* "$GNUPGHOME"
​
## 拷贝配置文件和脚本
COPY conf /zookeeper/conf/
COPY bin/zkReady.sh /zookeeper/bin/
COPY entrypoint.sh /
​
ENV PATH=/zookeeper/bin:${PATH} \
    ZOO_LOG_DIR=/zookeeper/log \
    ZOO_LOG4J_PROP="INFO, CONSOLE, ROLLINGFILE" \
    JMXPORT=9010 \
    ZOOKEEPER_HOME=/zookeeper
​
#启动zookeeper，entrypoint 脚本和cmd联合使用，entrypoint会把cmd当作参数传递给entrypoint脚本执行，即entrypoint脚本通常用来做一些环境初始化；比如这里就是用来在线生成zk集群配置
ENTRYPOINT [ "/entrypoint.sh" ]
​
CMD [ "zkServer.sh", "start-foreground" ]
​
EXPOSE 2181 2888 3888 9010
​

  5. 创建构建脚本

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# cat build-command.sh 
#!/bin/bash
TAG=$1
nerdctl build -t harbor.cbear.top/cbear/zookeeper:${TAG} .
echo "镜像构建完成 , 即将上传到harbor"
sleep 1
nerdctl push harbor.cbear.top/cbear/zookeeper
:${TAG}
echo "镜像上传到harbor完成"

  6. 运行构建脚本

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# bash build-command.sh v3.9.2
​

  7. 验证上传的zookeeper容器镜像是否正常

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/zookeeper# nerdctl run -it --rm -p 2181:2181 harbor.cbear.top/cbear/zookeeper:v3.9.2
    image-20240603195106076

2. 在K8s环境中运行zookeeper集群

  1. 在nfs上准备pv目录

    root@harbor:~# mkdir -pv /data/k8sdata/cbear/zookeeper-datadir-{1,2,3}
mkdir: created directory '/data/k8sdata/cbear/zookeeper-datadir-1'
mkdir: created directory '/data/k8sdata/cbear/zookeeper-datadir-2'
mkdir: created directory '/data/k8sdata/cbear/zookeeper-datadir-3'
​
root@harbor:~# cat /etc/exports 
# /etc/exports: the access control list for filesystems which may be exported
#       to NFS clients.  See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes       hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4        gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes  gss/krb5i(rw,sync,no_subtree_check)
#
/data/k8sdata/cbear/nginx/static *(rw,no_root_squash,no_subtree_check)
/data/k8sdata/cbear/nginx/images *(rw,no_root_squash,no_subtree_check)
/data/html *(rw,no_root_squash,no_subtree_check)
/data/v1 *(rw,no_root_squash,no_subtree_check)
/data/volumes *(rw,no_root_squash,no_subtree_check)
/data/k8sdata/cbear/zookeeper-datadir-1 *(rw,no_root_squash,no_subtree_check)
/data/k8sdata/cbear/zookeeper-datadir-2 *(rw,no_root_squash,no_subtree_check)
/data/k8sdata/cbear/zookeeper-datadir-3 *(rw,no_root_squash,no_subtree_check)
​
​
root@harbor:~# exportfs  -rv
exporting *:/data/k8sdata/cbear/zookeeper-datadir-3
exporting *:/data/k8sdata/cbear/zookeeper-datadir-2
exporting *:/data/k8sdata/cbear/zookeeper-datadir-1
exporting *:/data/volumes
exporting *:/data/v1
exporting *:/data/html
exporting *:/data/k8sdata/cbear/nginx/images
exporting *:/data/k8sdata/cbear/nginx/static
​

  2. 在k8s上创建pv

    root@k8s-master01:/opt/k8s-data/yaml/cbear/zookeeper# vi zookeeper-pv.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-1
spec:
  capacity:
    storage: 20Gi
  accessModes:
    - ReadWriteOnce 
  nfs:
    server: 192.168.71.34
    path: /data/k8sdata/cbear/zookeeper-datadir-1 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-2
spec:
  capacity:
    storage: 20Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/k8sdata/cbear/zookeeper-datadir-2 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-3
spec:
  capacity:
    storage: 20Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34 
    path: /data/k8sdata/cbear/zookeeper-datadir-3

    启用并验证配置

    root@k8s-master01:/opt/k8s-data/yaml/cbear/zookeeper# kubectl apply -f zookeeper-pv.yaml 
persistentvolume/zookeeper-datadir-pv-1 created
persistentvolume/zookeeper-datadir-pv-2 created
persistentvolume/zookeeper-datadir-pv-3 created

root@k8s-master01:/opt/k8s-data/yaml/cbear/zookeeper# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                STORAGECLASS          VOLUMEATTRIBUTESCLASS   REASON   AGE
pvc-26e5524a-c11a-4416-89b7-66ea8576fa5e   1Gi        RWO            Retain           Bound       default/www-web-1    managed-nfs-storage   <unset>                          16h
pvc-4c91cd11-fca0-42ab-a986-8ef75e396d75   1Gi        RWO            Retain           Bound       default/www-web-0    managed-nfs-storage   <unset>                          16h
pvc-935cb882-eff4-4438-b3ca-65e12ad2bb40   100Mi      RWX            Retain           Bound       default/test-claim   managed-nfs-storage   <unset>                          17h
zookeeper-datadir-pv-1                     20Gi       RWO            Retain           Available                                              <unset>                          35s
zookeeper-datadir-pv-2                     20Gi       RWO            Retain           Available                                              <unset>                          35s
zookeeper-datadir-pv-3                     20Gi       RWO            Retain           Available                                              <unset>                          35s

  3. 在k8s上创建pvc

    root@k8s-master01:/opt/k8s-data/yaml/cbear/zookeeper# cat zookeeper-pvc.yaml 
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-1
  namespace: cbear
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-1
  resources:
    requests:
      storage: 10Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-2
  namespace: cbear
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-2
  resources:
    requests:
      storage: 10Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-3
  namespace: cbear
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-3
  resources:
    requests:
      storage: 10Gi

    启用并验证配置

    root@k8s-master01:/opt/k8s-data/yaml/cbear/zookeeper# kubectl apply -f zookeeper-pvc.yaml 
persistentvolumeclaim/zookeeper-datadir-pvc-1 created
persistentvolumeclaim/zookeeper-datadir-pvc-2 created
persistentvolumeclaim/zookeeper-datadir-pvc-3 created

root@k8s-master01:/opt/k8s-data/yaml/cbear/zookeeper# kubectl get pvc -n cbear
NAME                      STATUS   VOLUME                   CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
zookeeper-datadir-pvc-1   Bound    zookeeper-datadir-pv-1   20Gi       RWO                           <unset>                 12s
zookeeper-datadir-pvc-2   Bound    zookeeper-datadir-pv-2   20Gi       RWO                           <unset>                 12s
zookeeper-datadir-pvc-3   Bound    zookeeper-datadir-pv-3   20Gi       RWO                           <unset>                 12s

  4. 部署zookeeper

    root@k8s-master01:/opt/k8s-data/yaml/cbear/zookeeper# cat zookeeper.yaml
apiVersion: v1
kind: Service
metadata:
  name: zookeeper
  namespace: cbear
spec:
  ports:
    - name: client
      port: 2181
  selector:
    app: zookeeper
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper1
  namespace: cbear
spec:
  clusterIP: None       
  ports:
    - name: client
      port: 2181
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "1"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper2
  namespace: cbear
spec:
  clusterIP: None       
  ports:
    - name: client
      port: 2181
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "2"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper3
  namespace: cbear
spec:
  clusterIP: None       
  ports:
    - name: client
      port: 2181
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "3"
---
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  name: zookeeper1
  namespace: cbear
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "1"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.cbear.top/cbear/zookeeper:v2 
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "1"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-1 
      volumes:
        - name: zookeeper-datadir-pvc-1 
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-1
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: zookeeper2
  namespace: cbear
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "2"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.cbear.top/cbear/zookeeper:v2 
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "2"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-2 
      volumes:
        - name: zookeeper-datadir-pvc-2
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-2
---
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  name: zookeeper3
  namespace: cbear
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "3"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.cbear.top/cbear/zookeeper:v2 
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "3"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-3
      volumes:
        - name: zookeeper-datadir-pvc-3
          persistentVolumeClaim:
           claimName: zookeeper-datadir-pvc-3

3. 验证zookeeper集群状态

  1. 进入任意一个pod内部, 查看集群角色和配置文件

    image-20240616163035118

参考链接

k8s实战案例之部署Zookeeper集群 - Linux-1874 - 博客园 (cnblogs.com)