搜索文章 Ctrl K

6-3-k8s实际应用-redis单机和redis cluster

熊 0 阅读 容器

1. 在k8s上部署redis单机

1.1 redis简介

redis是一款基于BSD协议 , 开源的非关系型数据库( nosql数据库 ) , 作者是意大利开发者Salvatore Sanfilippo在2009年发布 , 使用C语言编写。redis是基于内存存储 , 而且是目前比较流行的键值数据库( key-value database ) , 它提供将内存通过网络远程共享的一种服务 , 提供类似功能的还有memcache , 但相比 memcache , redis 还提供了易扩展、高性能、具备数据持久性等功能。主要的应用场景有session共享 , 常用于web集群中的tomcat或PHP中多web服务器的session共享。消息队列 , ELK的日志缓存 , 部分业务的订阅发布系统。计数器 , 常用于访问排行榜 , 商品浏览数等和次数相关的数值统计场景。缓存 , 常用于数据查询、电商网站商品信息、新闻内容等；相对memcache , redis支持数据的持久化 , 可以将内存的数据保存在磁盘中 , 重启redis服务或者服务器之后可以从备份文件中恢复数据到内存继续使用。

1.2 PV/PVC 及 Redis 单机

kubernetes-applyation-redis-1

由于redis的数据(主要是redis快照)都存放在存储系统中 , 即便redis pod挂掉 , 对应数据都不会丢。因为在k8s上部署redis单机 , redis pod挂了 , k8s会将对应pod重建 , 重建时会把对应pvc挂载至pod中 , 加载快照 , 从而使得redis的数据不被pod的挂掉而丢数据。

1.3 构建redis镜像

对应配置文件

root@k8s-master01:/opt/k8s-data/dockerfile/cbear/redis# ll
total 3432
drwxr-xr-x 3 root root    4096 Jun 18 07:37 ./
drwxr-xr-x 4 root root    4096 Jun 17 12:15 ../
-rw-r--r-- 1 root root     133 Jun 18 07:37 build-command.sh
-rw-r--r-- 1 root root     705 Jun 18 07:35 Dockerfile
drwxrwxr-x 8 root root    4096 May 19 06:12 redis-7.2.5/
-rw-r--r-- 1 root root 3427473 Jun 18 06:47 redis-7.2.5.tar.gz
-rw-r--r-- 1 root root   58783 Jun 18 07:34 redis.conf
-rwxr-xr-x 1 root root      85 Jun 18 07:34 run_redis.sh*

  • 准备对应构建镜像文件

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/redis# cat Dockerfile 
#Redis Image
# 导入自定义centos基础镜像
FROM harbor.cbear.top/baseimages/centos:7.9.2009 
# 添加redis源码包至/usr/local/src
ADD redis-7.2.5.tar.gz /usr/local/src
# 编译安装redis
RUN ln -sv /usr/local/src/redis-7.2.5 /usr/local/redis && cd /usr/local/redis && make && cp src/redis-cli /usr/sbin/ && cp src/redis-server  /usr/sbin/ && mkdir -pv /data/redis-data 
# 添加redis配置文件
ADD redis.conf /usr/local/redis/redis.conf 
# 暴露redis服务端口
EXPOSE 6379

#ADD run_redis.sh /usr/local/redis/run_redis.sh
#CMD ["/usr/local/redis/run_redis.sh"]
# 添加启动脚本
ADD run_redis.sh /usr/local/redis/entrypoint.sh
# 启动redis
ENTRYPOINT ["/usr/local/redis/entrypoint.sh"]

  • 构建镜像脚本

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/redis# cat build-command.sh 
#!/bin/bash
TAG=$1
nerdctl build -t harbor.cbear.top/cbear/redis:${TAG} .
sleep 3
nerdctl push  harbor.cbear.top/cbear/redis:${TAG}

  • 准备运行脚本

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/redis# cat run_redis.sh 
#!/bin/bash

/usr/sbin/redis-server /usr/local/redis/redis.conf

tail -f  /etc/hosts

  • redis.conf配置文件

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/redis# grep -v '^#\|^$' redis.conf 
bind 0.0.0.0
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile ""
databases 16
always-show-logo yes
save 900 1
save 5 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error no
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /data/redis-data
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
requirepass 123456
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble no
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes

  • 直接运行构建脚本

    image-20240618154809892

  • 远程连接, 测试是否可以正常连接

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/redis# nerdctl run -it --rm -p 6379:6379  harbor.cbear.top/cbear/redis:v1
# <nerdctl>
127.0.0.1	localhost localhost.localdomain
::1		localhost localhost.localdomain
10.4.0.79       70d94eaf1f7c redis-70d94
10.4.0.45       aaa402cd6389 zookeeper-aaa40
10.4.0.46       b483fc412552 zookeeper-b483f
10.4.0.49       b9b4b6208b70 zookeeper-b9b4b
10.4.0.48       ddc9eee32742 zookeeper-ddc9e
# </nerdctl>


root@k8s-master01:/opt/k8s-data/dockerfile/cbear/redis# telnet 127.0.0.1 6379
Trying 127.0.0.1...
Connected to 127.0.0.1.
Escape character is '^]'.
auth 123456
+OK
set k1 v1
+OK
get k1 
$2
v1

    能够将redis镜像运行为容器 , 并且能够通过远程主机连接至redis进行数据读写 , 说明我们构建的redis镜像没有问题。

1.4 创建pv和pvc

1.4.1 在nfs服务器上准备redis数据存储目录

  • 准备共享目录

    root@harbor:/data# mkdir redis-datadir-1
root@harbor:/data# ll /data/redis-datadir-1/ -d
drwxr-xr-x 2 root root 4096 Jun 18 07:55 /data/redis-datadir-1//

  • 配置共享目录的nfs文件

    root@harbor:/data# cat /etc/exports 
# /etc/exports: the access control list for filesystems which may be exported
#		to NFS clients.  See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes       hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4        gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes  gss/krb5i(rw,sync,no_subtree_check)
#
/data/k8sdata/cbear/nginx/static *(rw,no_root_squash,no_subtree_check)
/data/k8sdata/cbear/nginx/images *(rw,no_root_squash,no_subtree_check)
/data/html *(rw,no_root_squash,no_subtree_check)
/data/v1 *(rw,no_root_squash,no_subtree_check)
/data/v4 *(rw,no_root_squash,no_subtree_check)
/data/v5 *(rw,no_root_squash,no_subtree_check)
/data/v6 *(rw,no_root_squash,no_subtree_check)
/data/volumes *(rw,no_root_squash,no_subtree_check)
/data/k8sdata/cbear/zookeeper-datadir-1 *(rw,no_root_squash,no_subtree_check)
/data/k8sdata/cbear/zookeeper-datadir-2 *(rw,no_root_squash,no_subtree_check)
/data/k8sdata/cbear/zookeeper-datadir-3 *(rw,no_root_squash,no_subtree_check)
/data/statefulset-1 *(rw,no_root_squash,no_subtree_check)
/data/statefulset-2 *(rw,no_root_squash,no_subtree_check)
/data/statefulset-3 *(rw,no_root_squash,no_subtree_check)
/data/statefulset-4 *(rw,no_root_squash,no_subtree_check)
/data/statefulset-5 *(rw,no_root_squash,no_subtree_check)
/data/statefulset-6 *(rw,no_root_squash,no_subtree_check)
/data/redis-datadir-1/ *(rw,no_root_squash,no_subtree_check)

  • 重新加载对应配置文件

    root@harbor:/data# exportfs  -av
exporting *:/data/redis-datadir-1
exporting *:/data/statefulset-6
exporting *:/data/statefulset-5
exporting *:/data/statefulset-4
exporting *:/data/statefulset-3
exporting *:/data/statefulset-2
exporting *:/data/statefulset-1
exporting *:/data/k8sdata/cbear/zookeeper-datadir-3
exporting *:/data/k8sdata/cbear/zookeeper-datadir-2
exporting *:/data/k8sdata/cbear/zookeeper-datadir-1
exporting *:/data/volumes
exporting *:/data/v6
exporting *:/data/v5
exporting *:/data/v4
exporting *:/data/v1
exporting *:/data/html
exporting *:/data/k8sdata/cbear/nginx/images
exporting *:/data/k8sdata/cbear/nginx/static

1.4.2 准备redis存储目录

  • 创建pv

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis# cat redis-persistentvolume.yaml 
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-datadir-pv-1
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/redis-datadir-1 
    server: 192.168.71.34

  • 创建pvc

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis# cat redis-persistentvolumeclaim.yaml 
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: redis-datadir-pvc-1 
  namespace: cbear
spec:
  volumeName: redis-datadir-pv-1 
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi

  • 启用对应存储配置

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis# kubectl apply -f ./
persistentvolume/redis-datadir-pv-1 created
persistentvolumeclaim/redis-datadir-pvc-1 created
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis# kubectl get pvc -n cbear
NAME                  STATUS   VOLUME               CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
redis-datadir-pvc-1   Bound    redis-datadir-pv-1   10Gi       RWO                           <unset>                 15s
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis# kubectl get pv -n cbear
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS     CLAIM                       STORAGECLASS          VOLUMEATTRIBUTESCLASS   REASON   AGE

redis-datadir-pv-1                         10Gi       RWO            Retain           Bound      cbear/redis-datadir-pvc-1

1.5 部署redis服务

  • 准备对应yaml文件

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis# cat redis.yaml 
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: devops-redis 
  name: deploy-devops-redis
  namespace: cbear
spec:
  replicas: 1 
  selector:
    matchLabels:
      app: devops-redis
  template:
    metadata:
      labels:
        app: devops-redis
    spec:
      containers:
        - name: redis-container
          image: harbor.cbear.top/cbear/redis:v1
          imagePullPolicy: Always
          volumeMounts:
          - mountPath: "/data/redis-data/"
            name: redis-datadir
      volumes:
        - name: redis-datadir
          persistentVolumeClaim:
            claimName: redis-datadir-pvc-1 

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: devops-redis
  name: srv-devops-redis
  namespace: cbear
spec:
  type: NodePort
  ports:
  - name: http
    port: 6379 
    targetPort: 6379
    nodePort: 36379 
  selector:
    app: devops-redis
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 10800

    上述报错说我们的服务端口超出范围 , 这是因为我们在初始化k8s集群时指定的服务端口范围 。

  • 修改对应Nodeport端口范围

    kubernetes-applyation-redis-8

    编辑/etc/systemd/system/kube-apiserver.service,将其--service-node-port-range选项指定的值修改即可 ; 其他两个master节点也需要修改哦

  • 重载kube-apiserver.service,重启kube-apiserver

    root@k8s-master01:~# systemctl daemon-reload                 
root@k8s-master01:~# systemctl restart kube-apiserver.service
root@k8s-master01:~#

1.6 验证redis数据读写

  • 用客户端连接kubernetes 的36379端口

    image-20240618163836016

  • 在客户端中写入新的数据

    image-20240618164017115

  • 验证快照文件是否存储在存储上

    root@harbor:/data# ll redis-datadir-1/dump.rdb 
-rw-r--r-- 1 root root 125 Jun 18 08:41 redis-datadir-1/dump.rdb

    可以看到刚才我们向redis写入数据 , 对应redis在规定时间内发现key的变化就做了快照 , 因为redis数据目录时通过pv/pvc挂载的nfs , 所以我们在nfs对应目录里时可以正常看到这个快照文件的。

  • 删除redis pod 等待k8s重建redis pod

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis# kubectl delete pods -n cbear deploy-devops-redis-6db6dc99d5-2vtn7
pod "deploy-devops-redis-6db6dc99d5-2vtn7" deleted
^Croot@k8s-master01:/opt/k8s-data/yaml/cbear/redis# kubectl get pods -n cbear
NAME                                   READY   STATUS    RESTARTS   AGE
deploy-devops-redis-6db6dc99d5-krg58   1/1     Running   0          32s

    用客户端连接, 可以看到k8s重建后的redis pod 还保留着原有pod的数据；这说明k8s重建时挂载了前一个pod的pvc;

2. 在k8s上部署redis集群

2.1 PV/PVC及Redis Cluster-StatefulSet

kubernetes-applyation-redis-13

redis cluster相比redis单机要稍微复杂一点 , 我们也是通过pv/pvc将redis cluster数据存放在存储系统中 , 不同于redis单机 , redis cluster对存入的数据会做crc16计算 , 然后和16384做取模计算 , 得出一个数字 , 这个数字就是存入redis cluster的一个槽位；即redis cluster将16384个槽位 , 平均分配给集群所有master节点 , 每个master节点存放整个集群数据的一部分。这样一来就存在一个问题 , 如果master宕机 , 那么对应槽位的数据也就不可用 , 为了防止master单点故障 , 我们还需要对master做高可用 , 即专门用一个slave节点对master做备份 , master宕机的情况下 , 对应slave会接管master继续向集群提供服务 , 从而实现redis cluster master的高可用；如上图所示 , 我们使用3主3从的redis cluster , redis0,1,2为master , 那么3 , 4 , 5就对应为0 , 1 , 2的slave , 负责备份各自对应的master的数据；这六个pod都是通过k8s集群的pv/pvc将数据存放在存储系统中 。

2.2 创建PV

2.2.1 在nfs上准备redis cluster 数据目录

  • 准备目录

    root@harbor:/data# mkdir -pv /data/redis{0,1,2,3,4,5}
mkdir: created directory '/data/redis0'
mkdir: created directory '/data/redis1'
mkdir: created directory '/data/redis2'
mkdir: created directory '/data/redis3'
mkdir: created directory '/data/redis4'
mkdir: created directory '/data/redis5'

  • 准备nfs配置文件

    root@harbor:/data# cat /etc/exports 
# /etc/exports: the access control list for filesystems which may be exported
#		to NFS clients.  See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes       hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4        gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes  gss/krb5i(rw,sync,no_subtree_check)
#
/data/redis-datadir-1/ *(rw,no_root_squash,no_subtree_check)
/data/redis0/ *(rw,no_root_squash,no_subtree_check)
/data/redis1/ *(rw,no_root_squash,no_subtree_check)
/data/redis2/ *(rw,no_root_squash,no_subtree_check)
/data/redis3/ *(rw,no_root_squash,no_subtree_check)
/data/redis4/ *(rw,no_root_squash,no_subtree_check)
/data/redis5/ *(rw,no_root_squash,no_subtree_check)

  • 重载配置文件

    root@harbor:/data# exportfs -rv
exporting *:/data/redis5
exporting *:/data/redis4
exporting *:/data/redis3
exporting *:/data/redis2
exporting *:/data/redis1
exporting *:/data/redis0
exporting *:/data/redis-datadir-1

2.2.2 创建pv

  • 准备yaml文件

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# cat redis-cluster-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv0
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/redis0 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv1
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/redis1 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv2
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/redis2 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv3
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/redis3 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv4
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/redis4 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv5
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/redis5

  • 应用配置文件

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl apply -f ./
persistentvolume/redis-cluster-pv0 created
persistentvolume/redis-cluster-pv1 created
persistentvolume/redis-cluster-pv2 created
persistentvolume/redis-cluster-pv3 created
persistentvolume/redis-cluster-pv4 created
persistentvolume/redis-cluster-pv5 created
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl get pv 
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                       STORAGECLASS          VOLUMEATTRIBUTESCLASS   REASON   AGE

redis-cluster-pv0                          5Gi        RWO            Retain           Available                                                     <unset>                          4s
redis-cluster-pv1                          5Gi        RWO            Retain           Available                                                     <unset>                          4s
redis-cluster-pv2                          5Gi        RWO            Retain           Available                                                     <unset>                          4s
redis-cluster-pv3                          5Gi        RWO            Retain           Available                                                     <unset>                          4s
redis-cluster-pv4                          5Gi        RWO            Retain           Available                                                     <unset>                          4s
redis-cluster-pv5                          5Gi        RWO            Retain           Available                                                     <unset>                          4s

2.3 部署redis cluster

  • 准备redis.conf文件创建configmap目录

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# cat redis.conf 
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379

  • 创建并验证configmap

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl create cm redis-conf --from-file=./redis.conf -n cbear
configmap/redis-conf created
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl get cm -n cbear
NAME               DATA   AGE
kube-root-ca.crt   1      11d
redis-conf         1      15s
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl describe cm/redis-conf -n cbear
Name:         redis-conf
Namespace:    cbear
Labels:       <none>
Annotations:  <none>

Data
====
redis.conf:
----
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379


BinaryData
====

Events:  <none>

  • 准备redis cluster的yaml文件

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# cat redis-cluster.yaml 
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: cbear
  labels:
    app: redis
spec:
  selector:
    app: redis
    appCluster: redis-cluster
  ports:
  - name: redis
    port: 6379
  clusterIP: None
  
---
apiVersion: v1
kind: Service
metadata:
  name: redis-access
  namespace: cbear
  labels:
    app: redis
spec:
  type: NodePort
  selector:
    app: redis
    appCluster: redis-cluster
  ports:
  - name: redis-access
    protocol: TCP
    port: 6379
    targetPort: 6379
    nodePort: 36379

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: cbear
spec:
  serviceName: redis
  replicas: 6
  selector:
    matchLabels:
      app: redis
      appCluster: redis-cluster
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: harbor.cbear.top/cbear/redis:v1
        env:
          - name: MY_POD_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP 
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
          - "--cluster-announce-ip"
          - "$(MY_POD_IP)"
        resources:
          requests:
            cpu: "500m"
            memory: "500Mi"
        ports:
        - containerPort: 6379
          name: redis
          protocol: TCP
        - containerPort: 16379
          name: cluster
          protocol: TCP
        volumeMounts:
        - name: conf
          mountPath: /etc/redis
        - name: data
          mountPath: /var/lib/redis
      volumes:
      - name: conf
        configMap:
          name: redis-conf
          items:
          - key: redis.conf
            path: redis.conf
  volumeClaimTemplates:
  - metadata:
      name: data
      namespace: magedu
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 5Gi

    上述配置清单 , 主要用sts控制器创建了6个pod副本 , 每个副本都使用configmap中的配置文件作为redis配置文件 , 使用pvc模板指定pod在k8s上自动关联pv , 并在magedu名称空间创建pvc,即只要k8s上有空余的pv , 对应pod就会在magedu这个名称空间按pvc模板信息创建pvc。当然我们可以使用存储类自动创建pvc , 也可以提前创建好pvc , 一般情况下使用sts控制器 , 我们可以使用pvc模板的方式来指定pod自动创建pvc(前提是k8s有足够的pv可用)。

    这里需要额外说明一个redis-server参数。--cluster-announce-ip 参数在 Redis 配置中用于指定集群节点的对外 IP 地址。这个参数在容器化环境(如Docker)中特别有用，因为这些环境中的 NAT(网络地址转换)可能导致 Redis 节点无法通过默认的内部 IP 地址互相发现和通信。通过使用 --cluster-announce-ip，可以手动设置每个节点的公共 IP 地址，确保集群中的其他节点能够正确地识别和连接到该节点。

  • 应用配置清单部署redis cluster

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl apply -f ./
persistentvolume/redis-cluster-pv0 unchanged
persistentvolume/redis-cluster-pv1 unchanged
persistentvolume/redis-cluster-pv2 unchanged
persistentvolume/redis-cluster-pv3 unchanged
persistentvolume/redis-cluster-pv4 unchanged
persistentvolume/redis-cluster-pv5 unchanged
service/redis unchanged
service/redis-access created
statefulset.apps/redis configured


root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl get pvc -n cbear
NAME           STATUS   VOLUME              CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
data-redis-0   Bound    redis-cluster-pv2   5Gi        RWO                           <unset>                 10m
data-redis-1   Bound    redis-cluster-pv3   5Gi        RWO                           <unset>                 10m
data-redis-2   Bound    redis-cluster-pv0   5Gi        RWO                           <unset>                 9m34s
data-redis-3   Bound    redis-cluster-pv5   5Gi        RWO                           <unset>                 9m1s
data-redis-4   Bound    redis-cluster-pv1   5Gi        RWO                           <unset>                 8m57s
data-redis-5   Bound    redis-cluster-pv4   5Gi        RWO                           <unset>                 8m54s
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl get pods -n cbear
NAME      READY   STATUS    RESTARTS   AGE
redis-0   1/1     Running   0          10m
redis-1   1/1     Running   0          10m
redis-2   1/1     Running   0          9m46s
redis-3   1/1     Running   0          9m13s
redis-4   1/1     Running   0          9m9s
redis-5   1/1     Running   0          9m6s

    使用sts控制器创建pod , pod名称是sts控制器的名称-id , 使用pvc模板创建pvc的名称为pvc模板名称-pod名称,即pvc模板名-sts控制器名-id;

2.4 初始化redis cluster

redis有自带的集群初始化命令, 这里直接使用以下命令即可

root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it redis-0 -n cbear -- redis-cli --cluster create --cluster-replicas 1 redis-0.redis.cbear.svc.cbear.local:6379 redis-1.redis.cbear.svc.cbear.local:6379 redis-2.redis.cbear.svc.cbear.local:6379 redis-3.redis.cbear.svc.cbear.local:6379 redis-4.redis.cbear.svc.cbear.local:6379 redis-5.redis.cbear.svc.cbear.local:6379
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica redis-4.redis.cbear.svc.cbear.local:6379 to redis-0.redis.cbear.svc.cbear.local:6379
Adding replica redis-5.redis.cbear.svc.cbear.local:6379 to redis-1.redis.cbear.svc.cbear.local:6379
Adding replica redis-3.redis.cbear.svc.cbear.local:6379 to redis-2.redis.cbear.svc.cbear.local:6379
M: ed99d74fb668725eca3914e2abc026641dda5ccf redis-0.redis.cbear.svc.cbear.local:6379
   slots:[0-5460] (5461 slots) master
M: 925b5147d93d3ced93004ada33cdd6006e0670a0 redis-1.redis.cbear.svc.cbear.local:6379
   slots:[5461-10922] (5462 slots) master
M: 7dc3e18bc955bf46760e89271bc646993a0b93b8 redis-2.redis.cbear.svc.cbear.local:6379
   slots:[10923-16383] (5461 slots) master
S: 05952a34ab368f31e65d6d96283d8d1d8971fdc3 redis-3.redis.cbear.svc.cbear.local:6379
   replicates 7dc3e18bc955bf46760e89271bc646993a0b93b8
S: 2b4b5f4cc8134fbdd6fb48faac0ddd96398005f7 redis-4.redis.cbear.svc.cbear.local:6379
   replicates ed99d74fb668725eca3914e2abc026641dda5ccf
S: 6a42ae8ba108a0f8073d95651302c6406d6fddc2 redis-5.redis.cbear.svc.cbear.local:6379
   replicates 925b5147d93d3ced93004ada33cdd6006e0670a0
Can I set the above configuration? (type 'yes' to accept): yes            #这里需要额外输入yes命令
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join

>>> Performing Cluster Check (using node redis-0.redis.cbear.svc.cbear.local:6379)
M: ed99d74fb668725eca3914e2abc026641dda5ccf redis-0.redis.cbear.svc.cbear.local:6379
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
M: 925b5147d93d3ced93004ada33cdd6006e0670a0 172.20.13.205:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
S: 6a42ae8ba108a0f8073d95651302c6406d6fddc2 172.20.25.90:6379
   slots: (0 slots) slave
   replicates 925b5147d93d3ced93004ada33cdd6006e0670a0
M: 7dc3e18bc955bf46760e89271bc646993a0b93b8 172.20.25.89:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: 2b4b5f4cc8134fbdd6fb48faac0ddd96398005f7 172.20.37.116:6379
   slots: (0 slots) slave
   replicates ed99d74fb668725eca3914e2abc026641dda5ccf
S: 05952a34ab368f31e65d6d96283d8d1d8971fdc3 172.20.13.207:6379
   slots: (0 slots) slave
   replicates 7dc3e18bc955bf46760e89271bc646993a0b93b8
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

2.5 验证redis cluster状态

  • 进入redis cluster任意pod查看集群信息

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it redis-0 -n cbear -- bash
#这里cluster_state状态，显示为ok
[root@redis-0 /]# redis-cli  cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:2782
cluster_stats_messages_pong_sent:2688
cluster_stats_messages_sent:5470
cluster_stats_messages_ping_received:2683
cluster_stats_messages_pong_received:2782
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:5470
total_cluster_links_buffer_limit_exceeded:0

  • 查看集群节点状态

    [root@redis-0 /]# redis-cli  cluster nodes
ed99d74fb668725eca3914e2abc026641dda5ccf 172.20.37.115:6379@16379 myself,master - 0 1718733994000 1 connected 0-5460
925b5147d93d3ced93004ada33cdd6006e0670a0 172.20.13.205:6379@16379 master - 0 1718733994000 2 connected 5461-10922
6a42ae8ba108a0f8073d95651302c6406d6fddc2 172.20.25.90:6379@16379 slave 925b5147d93d3ced93004ada33cdd6006e0670a0 0 1718733993527 2 connected
7dc3e18bc955bf46760e89271bc646993a0b93b8 172.20.25.89:6379@16379 master - 0 1718733994532 3 connected 10923-16383
2b4b5f4cc8134fbdd6fb48faac0ddd96398005f7 172.20.37.116:6379@16379 slave ed99d74fb668725eca3914e2abc026641dda5ccf 0 1718733995034 1 connected
05952a34ab368f31e65d6d96283d8d1d8971fdc3 172.20.13.207:6379@16379 slave 7dc3e18bc955bf46760e89271bc646993a0b93b8 0 1718733994934 3 connected

    集群节点信息中记录了master节点id和slave id , 其中slave后面会对应master的id , 表示该slave备份对应master数据；

  • 查看当前节点信息

    [root@redis-0 /]# redis-cli  info         
# Server
redis_version:7.2.5
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:35436023d831d5b2
redis_mode:cluster
os:Linux 5.15.0-107-generic x86_64
arch_bits:64
monotonic_clock:POSIX clock_gettime
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:4.8.5
process_id:1
process_supervised:no
run_id:9805791cf400efc382f071261a7ca3ebc5f3f1ca
tcp_port:6379
server_time_usec:1718734087143111
uptime_in_seconds:1715
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:7458055
executable:/redis-server
config_file:/etc/redis/redis.conf
io_threads_active:0
listener0:name=tcp,bind=*,bind=-::*,port=6379

# Clients
connected_clients:1
cluster_connections:10
maxclients:10000
client_recent_max_input_buffer:20480
client_recent_max_output_buffer:20504
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
total_blocking_keys:0
total_blocking_keys_on_nokey:0

# Memory
used_memory:1890688
used_memory_human:1.80M
used_memory_rss:7168000
used_memory_rss_human:6.84M
used_memory_peak:1890688
used_memory_peak_human:1.80M
used_memory_peak_perc:100.08%
used_memory_overhead:1614124
used_memory_startup:1582704
used_memory_dataset:276564
used_memory_dataset_perc:89.80%
allocator_allocated:1903168
allocator_active:2203648
allocator_resident:5795840
total_system_memory:16605708288
total_system_memory_human:15.47G
used_memory_lua:31744
used_memory_vm_eval:31744
used_memory_lua_human:31.00K
used_memory_scripts_eval:0
number_of_cached_scripts:0
number_of_functions:0
number_of_libraries:0
used_memory_vm_functions:32768
used_memory_vm_total:64512
used_memory_vm_total_human:63.00K
used_memory_functions:184
used_memory_scripts:184
used_memory_scripts_human:184B
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.16
allocator_frag_bytes:300480
allocator_rss_ratio:2.63
allocator_rss_bytes:3592192
rss_overhead_ratio:1.24
rss_overhead_bytes:1372160
mem_fragmentation_ratio:3.87
mem_fragmentation_bytes:5317344
mem_not_counted_for_evict:8
mem_replication_backlog:20508
mem_total_replication_buffers:20504
mem_clients_slaves:0
mem_clients_normal:0
mem_cluster_links:10720
mem_aof_buffer:8
mem_allocator:jemalloc-5.3.0
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

# Persistence
loading:0
async_loading:0
current_cow_peak:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1718732372
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:0
rdb_current_bgsave_time_sec:-1
rdb_saves:0
rdb_last_cow_size:368640
rdb_last_load_keys_expired:0
rdb_last_load_keys_loaded:0
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_rewrites:0
aof_rewrites_consecutive_failures:0
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0
aof_current_size:0
aof_base_size:0
aof_pending_rewrite:0
aof_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:0

# Stats
total_connections_received:6
total_commands_processed:1642
instantaneous_ops_per_sec:1
total_net_input_bytes:142629
total_net_output_bytes:23322
total_net_repl_input_bytes:0
total_net_repl_output_bytes:2526
instantaneous_input_kbps:0.07
instantaneous_output_kbps:0.00
instantaneous_input_repl_kbps:0.00
instantaneous_output_repl_kbps:0.00
rejected_connections:0
sync_full:1
sync_partial_ok:0
sync_partial_err:1
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:24
evicted_keys:0
evicted_clients:0
total_eviction_exceeded_time:0
current_eviction_exceeded_time:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
pubsubshard_channels:0
latest_fork_usec:600
total_forks:1
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
total_active_defrag_time:0
current_active_defrag_time:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:0
dump_payload_sanitizations:0
total_reads_processed:1649
total_writes_processed:179
io_threaded_reads_processed:0
io_threaded_writes_processed:0
reply_buffer_shrinks:2
reply_buffer_expands:0
eventloop_cycles:30602
eventloop_duration_sum:5528755
eventloop_duration_cmd_sum:6247
instantaneous_eventloop_cycles_per_sec:17
instantaneous_eventloop_duration_usec:184
acl_access_denied_auth:0
acl_access_denied_cmd:0
acl_access_denied_key:0
acl_access_denied_channel:0

# Replication
role:master
connected_slaves:1
slave0:ip=192.168.71.36,port=6379,state=online,offset=2282,lag=1
master_failover_state:no-failover
master_replid:5e708257352a010d6a9656a412e297511f6a3cde
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:2282
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:2282

# CPU
used_cpu_sys:2.939753
used_cpu_user:3.088222
used_cpu_sys_children:0.002285
used_cpu_user_children:0.000000
used_cpu_sys_main_thread:2.938229
used_cpu_user_main_thread:3.084656

# Modules

# Errorstats

# Cluster
cluster_enabled:1

# Keyspace

验证redis cluster读写数据是否正常？

  • 手动连接redis cluster 进行数据读写

    [root@redis-0 /]# redis-cli 
127.0.0.1:6379> KEYS *
(empty array)
127.0.0.1:6379> set key1 values1
(error) MOVED 9189 172.20.13.205:6379
127.0.0.1:6379> set key2 values2
OK
127.0.0.1:6379> set key3 values3
OK
127.0.0.1:6379> set key4 values4
(error) MOVED 13120 172.20.25.89:6379

    手动连接redis 集群master节点进行数据读写 , 存在一个问题就是当我们写入的key经过crc16计算对16384取模后 , 对应槽位可能不在当前节点 , redis它会告诉我们该key该在哪里去写；从上面的命令可以看到 , 现在redis cluster 是可用正常读写数据的

  • 使用python脚本连接redis cluster 进行数据读写

    root@k8s-master01:~/k8s-data/yaml/magedu/redis-cluster# cat redis-client-test.py
#!/usr/bin/env python
#coding:utf-8
#Author:Zhang ShiJie
#python 2.7/3.8
#pip install redis-py-cluster

import sys,time
from rediscluster import RedisCluster
def init_redis():
    startup_nodes = [
        {'host': '192.168.0.34', 'port': 36379},
        {'host': '192.168.0.35', 'port': 36379},
        {'host': '192.168.0.36', 'port': 36379},
        {'host': '192.168.0.34', 'port': 36379},
        {'host': '192.168.0.35', 'port': 36379},
        {'host': '192.168.0.36', 'port': 36379},
    ]
    try:
        conn = RedisCluster(startup_nodes=startup_nodes,
                            # 有密码要加上密码哦
                            decode_responses=True, password='')
        print('连接成功！！！！！1', conn)
        #conn.set("key-cluster","value-cluster")
        for i in range(100):
            conn.set("key%s" % i, "value%s" % i)
            time.sleep(0.1)
            data = conn.get("key%s" % i)
            print(data)

        #return conn

    except Exception as e:
        print("connect error ", str(e))
        sys.exit(1)

init_redis()
root@k8s-master01:~/k8s-data/yaml/magedu/redis-cluster#

    运行脚本 , 向redis cluster 写入数据

    root@k8s-master01:~/k8s-data/yaml/magedu/redis-cluster# python redis-client-test.py
Traceback (most recent call last):
  File "/root/k8s-data/yaml/magedu/redis-cluster/redis-client-test.py", line 8, in <module>
    from rediscluster import RedisCluster
ModuleNotFoundError: No module named 'rediscluster'
root@k8s-master01:~/k8s-data/yaml/magedu/redis-cluster#

    这里提示没有找到rediscluster模块 , 解决办法就是通过pip安装redis-py-cluster模块即可；

    安装redis-py-cluster模块

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# apt install python3-pip
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# pip install redis-py-cluster
Collecting redis-py-cluster
  Downloading redis_py_cluster-2.1.3-py2.py3-none-any.whl (42 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 42.6/42.6 KB 470.6 kB/s eta 0:00:00
Collecting redis<4.0.0,>=3.0.0
  Downloading redis-3.5.3-py2.py3-none-any.whl (72 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 72.1/72.1 KB 524.6 kB/s eta 0:00:00
Installing collected packages: redis, redis-py-cluster
Successfully installed redis-3.5.3 redis-py-cluster-2.1.3
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
    kubernetes-applyation-redis-23

    运行脚本连接redis cluster进行数据读写

    kubernetes-applyation-redis-24

  • 连接redis pod , 验证数据是否正常写入？

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it  redis-0 -n cbear -- bash
[root@redis-0 /]# redis-cli 
127.0.0.1:6379> KEYS *
 1) "key68"
 2) "key24"
 3) "key37"
 4) "key99"
 5) "key91"
 6) "key82"
 7) "key64"
 8) "key60"
 9) "key59"
10) "key33"
11) "key25"
12) "key36"
13) "key7"
14) "key28"
15) "key20"
16) "key46"
17) "key2"
18) "key51"
19) "key15"
20) "key86"
21) "key77"
22) "key3"
23) "key6"
24) "key55"
25) "key19"
26) "key42"
27) "key95"
28) "key21"
29) "key29"
30) "key11"
31) "key73"
32) "key32"

root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it  redis-2 -n cbear -- bash
[root@redis-2 /]# redis-cli 
127.0.0.1:6379> KEYS *
 1) "key38"
 2) "key70"
 3) "key41"
 4) "key84"
 5) "key81"
 6) "key75"
 7) "key63"
 8) "key23"
 9) "key85"
10) "key92"
11) "key8"
12) "key80"
13) "key62"
14) "key49"
15) "key27"
16) "key88"
17) "key96"
18) "key97"
19) "key34"
20) "key79"
21) "key16"
22) "key93"
23) "key0"
24) "key56"
25) "key12"
26) "key66"
27) "key78"
28) "key52"
29) "key71"
30) "key4"
31) "key89"
32) "key45"
33) "key57"
34) "key67"
35) "key74"
36) "key30"

root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it  redis-3 -n cbear -- bash
[root@redis-3 /]# redis-cli 
127.0.0.1:6379> KEYS *
 1) "key12"
 2) "key89"
 3) "key30"
 4) "key70"
 5) "key79"
 6) "key97"
 7) "key88"
 8) "key66"
 9) "key80"
10) "key85"
11) "key23"
12) "key34"
13) "key62"
14) "key75"
15) "key93"
16) "key4"
17) "key41"
18) "key63"
19) "key38"
20) "key16"
21) "key78"
22) "key96"
23) "key56"
24) "key71"
25) "key84"
26) "key92"
27) "key8"
28) "key81"
29) "key49"
30) "key0"
31) "key74"
32) "key57"
33) "key45"
34) "key27"
35) "key52"
36) "key67"

    从上面的命令可用看到三个reids cluster master pod各自都存放了一部分key,并非全部 。说明刚才我们用python脚本把数据正常写入了redis cluster。

    slave节点是不可以读取数据；

上述验证说明了redis cluster 只有master可以读写数据 , slave只是对master数据做备份 , 不可以在slave上读写数据；

2.6 验证验证redis cluster高可用

2.6.1 测试集群重建后的高可用状态

  • 修改镜像tag， 并上传对应镜像

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# nerdctl tag harbor.cbear.top/cbear/redis:v1 harbor.cbear.top/cbear/redis:v2
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# nerdctl push harbor.cbear.top/cbear/redis:v2
INFO[0000] pushing as a reduced-platform image (application/vnd.docker.distribution.manifest.v2+json, sha256:29aa765dfaa67f9aa322e785f46bef1523e456c57576a80eba7e5f66b7179393) 
WARN[0000] skipping verifying HTTPS certs for "harbor.cbear.top" 
manifest-sha256:29aa765dfaa67f9aa322e785f46bef1523e456c57576a80eba7e5f66b7179393: done           |++++++++++++++++++++++++++++++++++++++| 
config-sha256:245f310475b7852ef3fdfd5e16839972ca921ad26aa223cbfbf1beaa1ecbf0f4:   done           |++++++++++++++++++++++++++++++++++++++| 
elapsed: 0.4 s                                                                    total:  5.5 Ki (13.8 KiB/s)

  • 修改对应yaml文件中的image , 设置为刚指定的镜像文件

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# cat redis-cluster.yaml 
.....
      containers:
      - name: redis
        image: harbor.cbear.top/cbear/redis:v2
        imagePullPolicy: Always
        env:
....

    修改镜像为本地harbor镜像和拉取策略是方便我们测试redis cluster的高可用。

  • 重新应用对应redis cluster部署清单

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl apply -f ./
persistentvolume/redis-cluster-pv0 unchanged
persistentvolume/redis-cluster-pv1 unchanged
persistentvolume/redis-cluster-pv2 unchanged
persistentvolume/redis-cluster-pv3 unchanged
persistentvolume/redis-cluster-pv4 unchanged
persistentvolume/redis-cluster-pv5 unchanged
service/redis unchanged
service/redis-access unchanged
statefulset.apps/redis configured

    这里相当于给redis cluster更新 , 他们之间的集群关系还存在 , 因为集群关系配置都保存在远端存储之上。

  • 验证pod是否都正常running?

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl get pods -n cbear
NAME      READY   STATUS    RESTARTS   AGE
redis-0   1/1     Running   0          26s
redis-1   1/1     Running   0          29s
redis-2   1/1     Running   0          32s
redis-3   1/1     Running   0          40s
redis-4   1/1     Running   0          43s
redis-5   1/1     Running   0          46s

  • 验证集群状态和集群关系

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it redis-0 -n cbear -- bash
[root@redis-0 /]# redis-cli 
127.0.0.1:6379> CLUSTER INFO
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:253
cluster_stats_messages_pong_sent:253
cluster_stats_messages_sent:506
cluster_stats_messages_ping_received:253
cluster_stats_messages_pong_received:253
cluster_stats_messages_received:506
total_cluster_links_buffer_limit_exceeded:0
127.0.0.1:6379> CLUSTER NODES 
05952a34ab368f31e65d6d96283d8d1d8971fdc3 172.20.13.206:6379@16379 slave 7dc3e18bc955bf46760e89271bc646993a0b93b8 0 1718737042000 3 connected
2b4b5f4cc8134fbdd6fb48faac0ddd96398005f7 172.20.37.117:6379@16379 slave ed99d74fb668725eca3914e2abc026641dda5ccf 0 1718737043844 1 connected
6a42ae8ba108a0f8073d95651302c6406d6fddc2 172.20.25.91:6379@16379 slave 925b5147d93d3ced93004ada33cdd6006e0670a0 0 1718737042840 2 connected
ed99d74fb668725eca3914e2abc026641dda5ccf 172.20.37.118:6379@16379 myself,master - 0 1718737041000 1 connected 0-5460
925b5147d93d3ced93004ada33cdd6006e0670a0 172.20.13.208:6379@16379 master - 0 1718737043000 2 connected 5461-10922
7dc3e18bc955bf46760e89271bc646993a0b93b8 172.20.25.92:6379@16379 master - 0 1718737043543 3 connected 10923-16383

    从上面集群状态可以看到, 已经重新形成了对应集群

2.6.2 停掉本地harbor , 删除redis master pod , 看看对应slave是否会提升为master？

  • 停止harbor服务

    root@harbor:/usr/local/harbor# docker-compose down 
Stopping nginx             ... done
Stopping harbor-jobservice ... done
Stopping harbor-core       ... done
Stopping registry          ... done
Stopping harbor-portal     ... done
Stopping harbor-db         ... done
Stopping registryctl       ... done
Stopping redis             ... done
Stopping harbor-log        ... done
Removing nginx             ... done
Removing harbor-jobservice ... done
Removing harbor-core       ... done
Removing registry          ... done
Removing harbor-portal     ... done
Removing harbor-db         ... done
Removing registryctl       ... done
Removing redis             ... done
Removing harbor-log        ... done
Removing network harbor_harbor

  • 删除redis-0,看看redis-4是否会提升为master？

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it redis-4 -n cbear -- bash
[root@redis-4 /]# redis-cli 
127.0.0.1:6379> info 
# Server
redis_version:7.2.5
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:35436023d831d5b2
redis_mode:cluster
os:Linux 5.15.0-107-generic x86_64
arch_bits:64
monotonic_clock:POSIX clock_gettime
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:4.8.5
process_id:1
process_supervised:no
run_id:3641d567f7ff48bd956b9cbdd7ee0c41df6bd926
tcp_port:6379
server_time_usec:1718737926972710
uptime_in_seconds:77
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:7461894
executable:/redis-server
config_file:/etc/redis/redis.conf
io_threads_active:0
listener0:name=tcp,bind=*,bind=-::*,port=6379

# Clients
connected_clients:1
cluster_connections:10
maxclients:10000
client_recent_max_input_buffer:20480
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
total_blocking_keys:0
total_blocking_keys_on_nokey:0

# Memory
used_memory:1706040
used_memory_human:1.63M
used_memory_rss:7385088
used_memory_rss_human:7.04M
used_memory_peak:1924096
used_memory_peak_human:1.83M
used_memory_peak_perc:88.67%
used_memory_overhead:1630996
used_memory_startup:1596632
used_memory_dataset:75044
used_memory_dataset_perc:68.59%
allocator_allocated:2034304
allocator_active:2367488
allocator_resident:6131712
total_system_memory:16605708288
total_system_memory_human:15.47G
used_memory_lua:31744
used_memory_vm_eval:31744
used_memory_lua_human:31.00K
used_memory_scripts_eval:0
number_of_cached_scripts:0
number_of_functions:0
number_of_libraries:0
used_memory_vm_functions:32768
used_memory_vm_total:64512
used_memory_vm_total_human:63.00K
used_memory_functions:184
used_memory_scripts:184
used_memory_scripts_human:184B
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.16
allocator_frag_bytes:333184
allocator_rss_ratio:2.59
allocator_rss_bytes:3764224
rss_overhead_ratio:1.20
rss_overhead_bytes:1253376
mem_fragmentation_ratio:4.39
mem_fragmentation_bytes:5701936
mem_not_counted_for_evict:8
mem_replication_backlog:20508
mem_total_replication_buffers:20504
mem_clients_slaves:0
mem_clients_normal:1928
mem_cluster_links:9648
mem_aof_buffer:8
mem_allocator:jemalloc-5.3.0
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

# Persistence
loading:0
async_loading:0
current_cow_peak:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1718737849
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_saves:0
rdb_last_cow_size:0
rdb_last_load_keys_expired:0
rdb_last_load_keys_loaded:32
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:0
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_rewrites:1
aof_rewrites_consecutive_failures:0
aof_last_write_status:ok
aof_last_cow_size:491520
module_fork_in_progress:0
module_fork_last_cow_size:0
aof_current_size:565
aof_base_size:565
aof_pending_rewrite:0
aof_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:0

# Stats
total_connections_received:1
total_commands_processed:3
instantaneous_ops_per_sec:0
total_net_input_bytes:805
total_net_output_bytes:206496
total_net_repl_input_bytes:764
total_net_repl_output_bytes:0
instantaneous_input_kbps:0.00
instantaneous_output_kbps:0.00
instantaneous_input_repl_kbps:0.00
instantaneous_output_repl_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
evicted_clients:0
total_eviction_exceeded_time:0
current_eviction_exceeded_time:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
pubsubshard_channels:0
latest_fork_usec:649
total_forks:1
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
total_active_defrag_time:0
current_active_defrag_time:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:0
dump_payload_sanitizations:0
total_reads_processed:5
total_writes_processed:26
io_threaded_reads_processed:0
io_threaded_writes_processed:0
reply_buffer_shrinks:2
reply_buffer_expands:0
eventloop_cycles:1320
eventloop_duration_sum:559292
eventloop_duration_cmd_sum:3935
instantaneous_eventloop_cycles_per_sec:17
instantaneous_eventloop_duration_usec:221
acl_access_denied_auth:0
acl_access_denied_cmd:0
acl_access_denied_key:0
acl_access_denied_channel:0

# Replication
role:master
connected_slaves:0
master_failover_state:no-failover
master_replid:28570b4dd3321d5e6a6c30caf4c6afafe19dca3c
master_replid2:d42f400ba6a010323f0ef5acae792bf5877beac7
master_repl_offset:1316
second_repl_offset:1317
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1289
repl_backlog_histlen:28

# CPU
used_cpu_sys:0.177991
used_cpu_user:0.170509
used_cpu_sys_children:0.003415
used_cpu_user_children:0.000000
used_cpu_sys_main_thread:0.175905
used_cpu_user_main_thread:0.167971

# Modules

# Errorstats

# Cluster
cluster_enabled:1

# Keyspace
db0:keys=32,expires=0,avg_ttl=0
127.0.0.1:6379> CLUSTER NODES
ed99d74fb668725eca3914e2abc026641dda5ccf 172.20.37.118:6379@16379 master,fail - 1718737877056 1718737875000 1 connected
925b5147d93d3ced93004ada33cdd6006e0670a0 172.20.13.208:6379@16379 master - 0 1718737947435 2 connected 5461-10922
2b4b5f4cc8134fbdd6fb48faac0ddd96398005f7 172.20.37.119:6379@16379 myself,master - 0 1718737946000 7 connected 0-5460
7dc3e18bc955bf46760e89271bc646993a0b93b8 172.20.25.92:6379@16379 master - 0 1718737945524 3 connected 10923-16383
6a42ae8ba108a0f8073d95651302c6406d6fddc2 172.20.25.91:6379@16379 slave 925b5147d93d3ced93004ada33cdd6006e0670a0 0 1718737946430 2 connected
05952a34ab368f31e65d6d96283d8d1d8971fdc3 172.20.13.206:6379@16379 slave 7dc3e18bc955bf46760e89271bc646993a0b93b8 0 1718737945424 3 connected

    可用看到我们把redis-0删除 ( 相当于master宕机) 以后 , 对应slave提升为master了。

2.6.3 恢复harbor服务 , 看看对应redis-0恢复会议后是否还是redis-4的slave呢？

  • 恢复harbor服务

    root@harbor:/usr/local/harbor# docker-compose up -d 
Creating network "harbor_harbor" with the default driver
Creating harbor-log ... done
Creating redis         ... done
Creating registryctl   ... done
Creating harbor-db     ... done
Creating harbor-portal ... done
Creating registry      ... done
Creating harbor-core   ... done
Creating nginx             ... done
Creating harbor-jobservice ... done

  • 验证redis-0 pod是否恢复？

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl get pods -n cbear -o wide
NAME      READY   STATUS             RESTARTS   AGE    IP              NODE                   NOMINATED NODE   READINESS GATES
redis-0   0/1     ImagePullBackOff   0          2m1s   172.20.37.120   k8s-node02.cbear.top   <none>           <none>
redis-1   1/1     Running            0          18m    172.20.13.208   k8s-node01.cbear.top   <none>           <none>
redis-2   1/1     Running            0          18m    172.20.25.92    k8s-node03.cbear.top   <none>           <none>
redis-3   1/1     Running            0          18m    172.20.13.206   k8s-node01.cbear.top   <none>           <none>
redis-4   1/1     Running            0          3m8s   172.20.37.119   k8s-node02.cbear.top   <none>           <none>
redis-5   1/1     Running            0          18m    172.20.25.91    k8s-node03.cbear.top   <none>           <none>

root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl delete pods -n cbear redis-0 
pod "redis-0" deleted
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl get pods -n cbear -o wide
NAME      READY   STATUS    RESTARTS   AGE    IP              NODE                   NOMINATED NODE   READINESS GATES
redis-0   1/1     Running   0          10s    172.20.37.121   k8s-node02.cbear.top   <none>           <none>
redis-1   1/1     Running   0          23m    172.20.13.208   k8s-node01.cbear.top   <none>           <none>
redis-2   1/1     Running   0          23m    172.20.25.92    k8s-node03.cbear.top   <none>           <none>
redis-3   1/1     Running   0          23m    172.20.13.206   k8s-node01.cbear.top   <none>           <none>
redis-4   1/1     Running   0          8m7s   172.20.37.119   k8s-node02.cbear.top   <none>           <none>
redis-5   1/1     Running   0          23m    172.20.25.91    k8s-node03.cbear.top   <none>           <none>

    再次删除redis-0以后 , 对应pod正常被重建 , 并处于running状态；

  • 验证redis-4的主从关系、

    root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it redis-4 -n cbear -- bash
[root@redis-4 /]# redis-cli 
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:1
slave0:ip=192.168.71.36,port=6379,state=online,offset=1610,lag=1
master_failover_state:no-failover
master_replid:28570b4dd3321d5e6a6c30caf4c6afafe19dca3c
master_replid2:d42f400ba6a010323f0ef5acae792bf5877beac7
master_repl_offset:1610
second_repl_offset:1317
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1289
repl_backlog_histlen:322
127.0.0.1:6379> exit
[root@redis-4 /]# exit
exit
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it redis-0 -n cbear -- redis-cli info replications
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl get pods -n cbear -o wide
NAME      READY   STATUS    RESTARTS   AGE     IP              NODE                   NOMINATED NODE   READINESS GATES
redis-0   1/1     Running   0          3m19s   172.20.37.121   k8s-node02.cbear.top   <none>           <none>
redis-1   1/1     Running   0          26m     172.20.13.208   k8s-node01.cbear.top   <none>           <none>
redis-2   1/1     Running   0          26m     172.20.25.92    k8s-node03.cbear.top   <none>           <none>
redis-3   1/1     Running   0          26m     172.20.13.206   k8s-node01.cbear.top   <none>           <none>
redis-4   1/1     Running   0          11m     172.20.37.119   k8s-node02.cbear.top   <none>           <none>
redis-5   1/1     Running   0          26m     172.20.25.91    k8s-node03.cbear.top   <none>           <none>
root@k8s-master01:/opt/k8s-data/yaml/cbear/redis-cluster# kubectl exec -it redis-0 -n cbear -- bash 
[root@redis-0 /]# redis-cli 
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:172.20.37.119
master_port:6379
master_link_status:up
master_last_io_seconds_ago:8
master_sync_in_progress:0
slave_read_repl_offset:1708
slave_repl_offset:1708
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:28570b4dd3321d5e6a6c30caf4c6afafe19dca3c
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:1708
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1415
repl_backlog_histlen:294
127.0.0.1:6379> CLUSTER NODES
2b4b5f4cc8134fbdd6fb48faac0ddd96398005f7 172.20.37.119:6379@16379 master - 0 1718738534325 7 connected 0-5460
6a42ae8ba108a0f8073d95651302c6406d6fddc2 172.20.25.91:6379@16379 slave 925b5147d93d3ced93004ada33cdd6006e0670a0 0 1718738535331 2 connected
925b5147d93d3ced93004ada33cdd6006e0670a0 172.20.13.208:6379@16379 master - 0 1718738534000 2 connected 5461-10922
05952a34ab368f31e65d6d96283d8d1d8971fdc3 172.20.13.206:6379@16379 slave 7dc3e18bc955bf46760e89271bc646993a0b93b8 0 1718738535532 3 connected
7dc3e18bc955bf46760e89271bc646993a0b93b8 172.20.25.92:6379@16379 master - 0 1718738534828 3 connected 10923-16383
ed99d74fb668725eca3914e2abc026641dda5ccf 172.20.37.121:6379@16379 myself,slave 2b4b5f4cc8134fbdd6fb48faac0ddd96398005f7 0 1718738534000 7 connected

    可以看到redis-0恢复以后 , 对应自动加入集群成为redis-4的slave。

3. 其他yaml示例

  1. 如果集群节大于6台，可以放开116-127行的注释。

  2. redis集群密码修改

    63-64，163行 "PASSWORD" ,可根据自己情况进行修改。

  3. 存储类修改

　　修改192行 storageClassName: YOU-K8S-SC 为自己的存储类.

　　195行存储大小根据自己情况确定

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: init-cluster
data:
  init_cluster.sh: |
    #!/bin/bash
    #ns=$NAMESPACE
    RedisPwd=$1
    cluster_domain=`awk '/search/{print $2}'  /etc/resolv.conf`

    function cluster_init(){
        sleep 3
        n0=`/usr/bin/dig  +short redis-cluster-0.redis-service.${cluster_domain}`
        n1=$(/usr/bin/dig +short redis-cluster-1.redis-service.${cluster_domain})
        n2=$(/usr/bin/dig +short redis-cluster-2.redis-service.${cluster_domain})
        n3=$(/usr/bin/dig +short redis-cluster-3.redis-service.${cluster_domain})
        n4=$(/usr/bin/dig +short redis-cluster-4.redis-service.${cluster_domain})
        n5=`ip a|grep inet|tail -1|awk '{print $2}'|awk -F "/" '{print $1}'`
        #n5=`/usr/bin/dig  +short redis-cluster-5.redis-service.${ns}.svc.cluster.local`
        #n5=$(/usr/bin/dig +short redis-cluster-5.redis-service.${ns}.svc.cluster.local)
        echo "0:$n0 1:$n1 2:$n2 3:$n3 4:$n4 5:$n5"
        echo yes | /usr/local/bin/redis-cli -a $RedisPwd  --cluster create ${n0}:6379 ${n1}:6379 ${n2}:6379 ${n3}:6379 ${n4}:6379 ${n5}:6379 --cluster-replicas 1 && echo "redis-cluster init finshed" || echo "xxxx.... "
    }

    function set_password(){
        sleep 5
        echo "start save cluster password"
        /usr/local/bin/redis-cli -h redis-cluster -p 6379 config set masterauth $RedisPwd && \
        /usr/local/bin/redis-cli -h redis-cluster config set requirepass $RedisPwd
        echo "end save cluster password"
        sleep 1

       # for i in $(seq 0 5)
       # do
       #     redis_host=redis-cluster-${i}.redis-service.${ns}.svc.${cluster_domain}
       #     /usr/local/bin/redis-cli -h ${redis_host} -a $RedisPwd config set masterauth  $RedisPwd
       #     /usr/local/bin/redis-cli -h ${redis_host} config set requirepass $RedisPwd
       #     /usr/local/bin/redis-cli -h ${redis_host} -a $RedisPwd  config rewrite
       # done
    }
    cluster_init && set_password
    echo "ok......"

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cluster-config
data:
  redis.conf: |
    bind 0.0.0.0
    appendonly yes
    cluster-enabled yes
    cluster-config-file nodes.conf
    cluster-node-timeout 5000
    dir /data/
    port 6379
    pidfile /var/run/redis.pid
    logfile "redis.log"
    protected-mode no
    masterauth "PASSWORD"
    requirepass "PASSWORD"

---
apiVersion: v1
kind: Service
metadata:
  name: redis-service
  labels:
    app: redis-cluster
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis-cluster
    appCluster: redis-cluster
---
apiVersion: v1
kind: Service
metadata:
  name: redis-cluster
  labels:
    app: redis-cluster
spec:
  ports:
  - name: redis-port
    protocol: "TCP"
    port: 6379
    targetPort: 6379
  selector:
    app: redis-cluster
    appCluster: redis-cluster

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-cluster
spec:
  serviceName: "redis-service"
  replicas: 6
  selector:
    matchLabels:
      app: redis-cluster
  template:
    metadata:
      labels:
        app: redis-cluster
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
#      affinity:
#        podAntiAffinity:
#          preferredDuringSchedulingIgnoredDuringExecution:
#          - weight: 100
#            podAffinityTerm:
#              labelSelector:
#                matchExpressions:
#                - key: app
#                  operator: In
#                  values:
#                  - redis-cluster
#              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: linuxpy/redis:cluster-v3
        imagePullPolicy: Always
        env:
          - name: NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
          - name: POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: MY_POD_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
        command:
          - "redis-server"
        args:
          - /usr/local/etc/redis.conf
          - --cluster-announce-ip
          - "$(MY_POD_IP)"
        resources:
          requests:
            cpu: "100m"
            memory: "100Mi"
        lifecycle:
          postStart:
            exec:
              command:
              - /bin/bash
              - -c
              - >
                if [ `hostname` = "redis-cluster-5" ]; then
                bash /opt/init_cluster.sh "PASSWORD" 1>/tmp/init.log 2>&1;
                fi;
        ports:
          - name: redis
            containerPort: 6379
            protocol: "TCP"
          - name: cluster
            containerPort: 16379
            protocol: "TCP"
        volumeMounts:
          - name: "data"
            mountPath: "/data/"
          - name: redis-cluster-config
            mountPath: /usr/local/etc/
          - name: init-cluster
            mountPath: /opt/
      volumes:
        - name: redis-cluster-config
          configMap:
            name: redis-cluster-config
        - name: init-cluster
          configMap:
            name: init-cluster

  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: YOU-K8S-SC
      resources:
        requests:
          storage: 1Gi

参考链接

k8s实战案例之部署redis单机和redis cluster - Linux-1874 - 博客园 (cnblogs.com)