搜索文章 Ctrl K

6-5-k8s实际应用-jenkins部署

熊 0 阅读 容器
kubernetes-applyation-jenkins-1

基于java命令，运行java war包或jar包，本次以jenkins.war 包部署⽅式为例，且要求jenkins的数据保存⾄外部存储(NFS或者PVC)，其他java应⽤看实际需求是否需要将数据保存至外部存储。

1. 镜像准备

构建镜像准备的文件

root@k8s-master01:/opt/k8s-data/dockerfile/cbear/jenkins# ll
total 91236
drwxr-xr-x 2 root root     4096 Jun 20 07:50 ./
drwxr-xr-x 5 root root     4096 Jun 20 06:59 ../
-rwxr-xr-x 1 root root      501 Jun 20 07:50 build-command.sh*
-rwxr-xr-x 1 root root      192 Jun 20 07:44 Dockerfile*
-rwxr-xr-x 1 root root 93404074 Jun 12 14:59 jenkins.war*
-rwxr-xr-x 1 root root      144 Jun 20 07:46 run_jenkins.sh*
​

  • 构建jenkins镜像Dockerfile

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/jenkins# cat Dockerfile 
#Jenkins Version 2.452.2
FROM harbor.cbear.top/pub-images/jdk-17-base:v2
​
ADD jenkins.war /apps/jenkins/jenkins.war
ADD run_jenkins.sh /usr/bin/
RUN yum install freetype fontconfig -y
EXPOSE 8080 
​
CMD ["/usr/bin/run_jenkins.sh"]
​

    上述Dockerfile主要引用了一个jdk-17-base的镜像，该镜像有java环境，然后再这基础之上将jenkins.war包和运行脚本加进去，然后暴露8080端口，最后给出运行jenkins的cmd命令。

  • jenkins运行脚本

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/jenkins# cat run_jenkins.sh 
#!/bin/bash
cd /apps/jenkins && java -server -Xms1024m -Xmx1024m -Xss512k -jar jenkins.war --webroot=/apps/jenkins/jenkins-data --httpPort=8080
​

  • 准备构建脚本

    #!/bin/bash
echo "即将开始就像构建,请稍等!" && echo 3 && sleep 1 && echo 2 && sleep 1 && echo 1
nerdctl build -t  harbor.cbear.top/cbear/jenkins:v2.452.2 .
if [ $? -eq 0 ];then
  echo "即将开始镜像上传,请稍等!" && echo 3 && sleep 1 && echo 2 && sleep 1 && echo 1
  nerdctl push harbor.cbear.top/cbear/jenkins:v2.452.2
  if [ $? -eq 0 ];then
    echo "镜像上传成功!"
  else
    echo "镜像上传失败"
  fi
else
  echo "镜像构建失败,请检查构建输出信息!"
fi

  • 运行构建脚本

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/jenkins# bash build-command.sh

  • 测试jenkins镜像是否可以正常运行。看到如下界面判断jenkins镜像构建成功

    root@k8s-master01:/opt/k8s-data/dockerfile/cbear/jenkins# nerdctl run -it --rm -p 8080:8080 harbor.cbear.top/cbear/jenkins:v2.452.2
    image-20240620161653498

2. 准备k8s上运行jenkins

2.1 准备pv/pvc

2.1.1 准备pv

  • 在nfs服务器上准备jenkins数据目录

    root@harbor:~# mkdir -p /data/cbear/{jenkins-data,jenkins-root-data}

  • 配置nfs配置文件

    root@harbor:~# cat /etc/exports
# /etc/exports: the access control list for filesystems which may be exported
#       to NFS clients.  See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes       hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4        gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes  gss/krb5i(rw,sync,no_subtree_check)
#
/data/cbear/jenkins-data  *(rw,no_root_squash,no_subtree_check)
/data/cbear/jenkins-root-data  *(rw,no_root_squash,no_subtree_check)

  • 重新加载nfs配置文件

    root@harbor:~# exportfs  -rv
exporting *:/data/cbear/jenkins-root-data
exporting *:/data/cbear/jenkins-data

  • 在k8s上创建pv

    ---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: jenkins-datadir-pv
  namespace: cbear
spec:
  capacity:
    storage: 100Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/cbear/jenkins-data 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: jenkins-root-datadir-pv
  namespace: cbear
spec:
  capacity:
    storage: 100Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.71.34
    path: /data/cbear/jenkins-root-data

  • 创建并验证的pv

    root@k8s-master01:/opt/k8s-data/yaml/cbear/jenkins# kubectl apply -f jenkins-pv.yaml 
persistentvolume/jenkins-datadir-pv created
persistentvolume/jenkins-root-datadir-pv created

root@k8s-master01:/opt/k8s-data/yaml/cbear/jenkins# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                       STORAGECLASS          VOLUMEATTRIBUTESCLASS   REASON   AGE
jenkins-datadir-pv                         100Gi      RWO            Retain           Available                                                     <unset>                          12s
jenkins-root-datadir-pv                    100Gi      RWO            Retain           Available                                                     <unset>                          12s

2.1.2 准备pvc

  • 准备pvc的资源配置文件

    ---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-datadir-pvc
  namespace: cbear
spec:
  volumeName: jenkins-datadir-pv
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 80Gi

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-root-data-pvc
  namespace: cbear
spec:
  volumeName: jenkins-root-datadir-pv 
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 80Gi

  • 启用对应pvc文件

    root@k8s-master01:/opt/k8s-data/yaml/cbear/jenkins# kubectl apply -f jenkins-pvc.yaml 
persistentvolumeclaim/jenkins-datadir-pvc created
persistentvolumeclaim/jenkins-root-data-pvc created

  • 验证对应pvc

    root@k8s-master01:/opt/k8s-data/yaml/cbear/jenkins# kubectl get pvc -n cbear
NAME                    STATUS   VOLUME                    CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
jenkins-datadir-pvc     Bound    jenkins-datadir-pv        100Gi      RWO                           <unset>                 21s
jenkins-root-data-pvc   Bound    jenkins-root-datadir-pv   100Gi      RWO                           <unset>                 21s

2.2 准备jenkins的yaml文件

  • 准备对应资源配置文件

    kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: cbear-jenkins
  name: cbear-jenkins-deployment
  namespace: cbear
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cbear-jenkins
  template:
    metadata:
      labels:
        app: cbear-jenkins
    spec:
      containers:
      - name: cbear-jenkins-container
        image: harbor.cbear.top/cbear/jenkins:v2.452.2
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          protocol: TCP
          name: http
        volumeMounts:
        - mountPath: "/apps/jenkins/jenkins-data/"
          name: jenkins-datadir-cbear
        - mountPath: "/root/.jenkins"
          name: jenkins-root-datadir
      volumes:
        - name: jenkins-datadir-cbear
          persistentVolumeClaim:
            claimName: jenkins-datadir-pvc
        - name: jenkins-root-datadir
          persistentVolumeClaim:
            claimName: jenkins-root-data-pvc

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: cbear-jenkins
  name: cbear-jenkins-service
  namespace: cbear
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8080
    nodePort: 38080
  selector:
    app: cbear-jenkins

  • 启用对应配置清单文件

    root@k8s-master01:/opt/k8s-data/yaml/cbear/jenkins# kubectl apply -f ./
persistentvolume/jenkins-datadir-pv unchanged
persistentvolume/jenkins-root-datadir-pv unchanged
persistentvolumeclaim/jenkins-datadir-pvc unchanged
persistentvolumeclaim/jenkins-root-data-pvc unchanged
deployment.apps/cbear-jenkins-deployment created
service/cbear-jenkins-service created

3. 验证

  • 验证Jenkins Pod是否正常运行

    root@k8s-master01:/opt/k8s-data/yaml/cbear/jenkins# kubectl get pods -n cbear
NAME                                        READY   STATUS    RESTARTS   AGE
cbear-jenkins-deployment-78d5f47755-9w4wz   1/1     Running   0          5m21s
root@k8s-master01:/opt/k8s-data/yaml/cbear/jenkins# kubectl get svc -n cbear
NAME                    TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
cbear-jenkins-service   NodePort    10.68.62.59     <none>        80:38080/TCP     5m27s

  • 验证web访问jenkins是否可正常访问？

    image-20240620164056163

    查看对应jenkins密码文件, 这里密码文件已经保存在nfs远程目录的/data/cbear/jenkins-root-data目录中

    root@harbor:~# ls /data/cbear/jenkins-root-data/
config.xml                     jenkins.telemetry.Correlator.xml  nodeMonitors.xml  secret.key                secrets  userContent
hudson.model.UpdateCenter.xml  jobs                              plugins           secret.key.not-so-secret  updates  users
root@harbor:~# cat /data/cbear/jenkins-root-data/secrets/initialAdminPassword 
0543975f0e4b40258388ada98e4fa043

    将对应密码拷贝到对应页面中之后 使jenkins页面进行初始化

4. 配置高可用

  • 配置haproxy

    root@ha2:~# cat /etc/haproxy/haproxy.cfg 
global
	log /dev/log	local0
	log /dev/log	local1 notice
	chroot /var/lib/haproxy
	stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
	stats timeout 30s
	user haproxy
	group haproxy
	daemon

	# Default SSL material locations
	ca-base /etc/ssl/certs
	crt-base /etc/ssl/private

	# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
        ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
        ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
        ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets

defaults
	log	global
	mode	http
	option	httplog
	option	dontlognull
        timeout connect 5000
        timeout client  50000
        timeout server  50000
	errorfile 400 /etc/haproxy/errors/400.http
	errorfile 403 /etc/haproxy/errors/403.http
	errorfile 408 /etc/haproxy/errors/408.http
	errorfile 500 /etc/haproxy/errors/500.http
	errorfile 502 /etc/haproxy/errors/502.http
	errorfile 503 /etc/haproxy/errors/503.http
	errorfile 504 /etc/haproxy/errors/504.http
listen stats 
       mode http 
       bind 0.0.0.0:9999 
       stats enable 
       log global 
       stats uri     /haproxy-status 
       stats auth    haadmin:123456 
 
listen k8s-6443 
       bind 192.168.71.150:6443 
       mode tcp 
       balance roundrobin 
       server 192.168.71.29 192.168.71.29:6443 check inter 2s fall 3 rise 5 
       server 192.168.71.30 192.168.71.30:6443 check inter 2s fall 3 rise 5 
       server 192.168.71.31 192.168.71.31:6443 check inter 2s fall 3 rise 5 
listen nginx-svc-80
       bind 192.168.71.150:80
       mode tcp
       server k8s-node01 192.168.71.37:30090 check inter 2000 fall 3 rise 5
       server k8s-node02 192.168.71.35:30090 check inter 2000 fall 3 rise 5
       server k8s-node03 192.168.71.36:30090 check inter 2000 fall 3 rise 5

listen k8s-jenkins-8080
       bind 192.168.71.150:8080
       mode tcp
       server k8s-node01 192.168.71.37:38080 check inter 2000 fall 3 rise 5
       server k8s-node02 192.168.71.35:38080 check inter 2000 fall 3 rise 5
       server k8s-node03 192.168.71.36:38080 check inter 2000 fall 3 rise 5

  • 重启对应haproxy和keepalived

之后直接访问192.168.71.150:8080即可