jenkins 所有的资源全部在 kubernetes 中进行,并配置 kubernetes 云
针对 jenkins slave 做动态资源伸缩,构建时申请 pod, 操作完毕后自动删除 pod
基于 Kubernetes Jenkins slave 动态资源伸缩 pod
环境说明
请自行配置 kubernetes 集群, kubernetes 版本为 1.9.1
1
2
3
| 172.16.1.64 - kubernetes master
172.16.1.65 - kubernetes master or node
172.16.1.66 - kubernetes node
|
先决条件
后端存储, 用于存放 jenkins home 目录,以及 slave 构建依赖包目录
1
2
3
4
| # 配置jenkins 后端存储, 如 nfs, gfs, cephfs 等, 请自行配置,或者参考 之前的文章。
# 基于单独的 nfs 最为后端 文档在这里 https://jicki.cn/2017/01/01/kubernetes-nfs/
|
创建 jenkins master
1
2
3
4
5
6
7
8
9
10
| # 首先创建一个 namespace
vi jenkins-namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: jenkins
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
| # 导入文件
kubectl apply -f jenkins-namespace.yaml
namespace "jenkins" created
# 查看
[root@kubernetes-64 jenkins]# kubectl get namespace
NAME STATUS AGE
default Active 26d
ingress-nginx Active 15d
jenkins Active 23s
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
| # 创建 jenkins 的 rbac , jenkins 与 api-service 通讯必须要 rabc 认证
vi jenkins-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: jenkins
name: jenkins-admin
namespace: jenkins
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: jenkins-admin
namespace: jenkins
labels:
k8s-app: jenkins
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: jenkins-admin
namespace: jenkins
|
1
2
3
4
5
6
7
8
9
10
11
12
13
| # 导入文件
[root@kubernetes-64 jenkins]# kubectl apply -f jenkins-rbac.yaml
serviceaccount "jenkins-admin" created
clusterrolebinding "jenkins-admin" created
# 查看
[root@kubernetes-64 jenkins]# kubectl get serviceaccount -n jenkins
NAME SECRETS AGE
default 1 3h
jenkins-admin 1 1m
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
| vi jenkins-deployment.yaml
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: jenkins
namespace: jenkins
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 2
maxUnavailable: 0
template:
metadata:
labels:
app: jenkins
spec:
securityContext:
fsGroup: 1000
serviceAccount: "jenkins-admin"
containers:
- name: jenkins
image: jenkins/jenkins
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: web
protocol: TCP
- containerPort: 50000
name: agent
protocol: TCP
volumeMounts:
- name: jenkinshome
mountPath: /var/jenkins_home
env:
- name: JAVA_OPTS
value: "-Xms512m -Xmx1024m -XX:PermSize=512m -XX:MaxPermSize=1024m -Duser.timezone=Asia/Shanghai"
- name: TRY_UPGRADE_IF_NO_MARKER
value: "true"
volumes:
- name: jenkinshome
nfs:
server: 172.16.1.66
path: "/opt/data/jenkins"
---
kind: Service
apiVersion: v1
metadata:
labels:
app: jenkins
name: jenkins
namespace: jenkins
spec:
ports:
- port: 8080
targetPort: 8080
name: web
- port: 50000
targetPort: 50000
name: agent
selector:
app: jenkins
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: jenkins
namespace: jenkins
spec:
rules:
- host: jenkins.jicki.cn
http:
paths:
- path: /
backend:
serviceName: jenkins
servicePort: 8080
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
| # 导入 文件
[root@kubernetes-64 jenkins]# kubectl apply -f jenkins-deployment.yaml
deployment "jenkins" created
service "jenkins" created
ingress "jenkins" created
# 查看 服务
[root@kubernetes-64 jenkins]# kubectl get pods -n jenkins
NAME READY STATUS RESTARTS AGE
jenkins-c5f45c84-8vlcp 1/1 Running 0 6m
[root@kubernetes-64 jenkins]#
[root@kubernetes-64 jenkins]# kubectl get svc -n jenkins
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
jenkins ClusterIP 10.254.47.35 <none> 8080/TCP,50000/TCP 6m
[root@kubernetes-64 jenkins]#
[root@kubernetes-64 jenkins]# kubectl get ingress -n jenkins
NAME HOSTS ADDRESS PORTS AGE
jenkins jenkins.jicki.cn 80 6m
# 查看 jenkins 初始密码
[root@kubernetes-64 jenkins]# kubectl logs pods/jenkins-c5f45c84-8vlcp -n jenkins
*************************************************************
*************************************************************
*************************************************************
Jenkins initial setup is required. An admin user has been created and a password generated.
Please use the following password to proceed to installation:
b2458cfb46724268a4db025c54f172ab
This may also be found at: /var/jenkins_home/secrets/initialAdminPassword
*************************************************************
*************************************************************
*************************************************************
|
配置 jenkins
1
2
3
4
| # 登录 web ui
http://jenkins.jicki.cn
|

1
2
3
4
5
6
7
8
| # Master 需要安装的插件
1. Pipeline
2. kubernetes plugin
3. Build WIth Parameters (构建 输入参数的插件)
4. Git Parameter Plug-In ( git 分支 构建选择)
|

配置 kubernetes 云
1
| # 点击【系统管理】-【系统设置】-【新增一个云】-【Kubernetes】
|


基于 pipeline 模式
1
2
3
4
5
6
7
8
9
10
11
| # 创建一个基于 pipeline 的配置 cloud 字段配置为 kubernetes 云的 name .
podTemplate(label: 'jenkins-pod', cloud: 'Jenkins-Cloud') {
node('jenkins-pod') {
stage('Run shell') {
sh 'echo hello world'
}
}
}
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
| # 点击立即构建, 控制台输出如下:
Started by user 小炒肉
Running in Durability level: MAX_SURVIVABILITY
[Pipeline] podTemplate
[Pipeline] {
[Pipeline] node
Still waiting to schedule task
jenkins-slave-bkcfk-vpzz0 is offline
Running on jenkins-slave-bkcfk-vpzz0 in /home/jenkins/workspace/test-k8s
[Pipeline] {
[Pipeline] stage
[Pipeline] { (Run shell)
[Pipeline] sh
[test-k8s] Running shell script
+ echo hello world
hello world
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
Finished: SUCCESS
|
基于 自由风格模式
1
2
3
4
5
6
|
# 点击【系统管理】-【系统设置】-【新增一个云】-【Kubernetes】- 【 Kubernetes Pod Template 】
- 【 Container Template 】
具体参数 如下:
|


制作 slave 镜像
官方默认的 jenkins-slave 里面不存在任何的工具,依赖,项目环境等,我们需要制作一个属于自己项目的 jenkins-slave 镜像。
官方 jenkins-slave dockerfile 地址 https://github.com/jenkinsci/docker-jnlp-slave
官方 slave https://github.com/jenkinsci/docker-slave
1
2
3
4
5
6
7
8
| # 个人的 oracle-jdk-8 镜像 java -version = 1.8.0_161
docker pull jicki/oracle-jdk:8
# 个人的 gradle 镜像
docker pull jicki/gradle:4.5
|
1
2
3
4
5
| # 基于 jicki/gradle:4.5 的一个 jenkins slave
docker pull jicki/slave:3.16
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
|
# 获取官方的git,主要是 jenkins-slave 脚本文件
git clone https://github.com/jenkinsci/docker-jnlp-slave
# 重新编辑 dockerfile 官方默认使用 openjdk-1.8
vi dockerfile
FROM jicki/slave:3.16
COPY jenkins-slave /usr/local/bin/jenkins-slave
USER root
RUN chmod +x /usr/local/bin/jenkins-slave
USER jenkins
ENTRYPOINT ["jenkins-slave"]
|
1
2
| docker build -t="jicki/jenkins-jnlp" .
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
| # 测试自己的镜像
podTemplate(label: 'jicki-jenkins', cloud: 'Jenkins-Cloud', containers: [
containerTemplate(
name: 'jnlp',
image: 'jicki/jenkins-jnlp',
alwaysPullImage: true,
args: '${computer.jnlpmac} ${computer.name}'),
],
volumes: [
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
],)
{
node('jicki-jenkins') {
stage('Task-1') {
stage('show java version') {
sh 'java -version'
}
}
}
node('jicki-jenkins') {
stage('Task-2') {
stage('show gradle version') {
sh 'gradle -version'
}
}
}
}
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
Started by user 小炒肉
Running in Durability level: MAX_SURVIVABILITY
[Pipeline] podTemplate
[Pipeline] {
[Pipeline] node
Still waiting to schedule task
Waiting for next available executor on jenkins-slave-9v0g5-73tng
Running on jenkins-slave-9v0g5-73tng in /home/jenkins/workspace/test-2
[Pipeline] {
[Pipeline] stage
[Pipeline] { (Task-1)
[Pipeline] stage
[Pipeline] { (show java version)
[Pipeline] sh
[test-2] Running shell script
+ java -version
java version "1.8.0_161"
Java(TM) SE Runtime Environment (build 1.8.0_161-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // node
[Pipeline] node
Still waiting to schedule task
jenkins-slave-9v0g5-nxdf8 is offline
Running on jenkins-slave-9v0g5-nxdf8 in /home/jenkins/workspace/test-2
[Pipeline] {
[Pipeline] stage
[Pipeline] { (Task-2)
[Pipeline] stage
[Pipeline] { (show gradle version)
[Pipeline] sh
[test-2] Running shell script
+ gradle -version
------------------------------------------------------------
Gradle 4.5
------------------------------------------------------------
Build time: 2018-01-24 17:04:52 UTC
Revision: 77d0ec90636f43669dc794ca17ef80dd65457bec
Groovy: 2.4.12
Ant: Apache Ant(TM) version 1.9.9 compiled on February 2 2017
JVM: 1.8.0_161 (Oracle Corporation 25.161-b12)
OS: Linux 4.4.111-1.el7.elrepo.x86_64 amd64
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
Finished: SUCCESS
|