쿠버네티스를 이용해 테스팅 환경 구현해보기

kops create cluster \
--cloud aws \
--name {clustername} \
--state s3://{s3-store} \
--topology private \
--zones ap-northeast-2a,ap-northeast-2c \
--master-zones ap-northeast-2a \
--networking calico \
--bastion \
--ssh-public-key {ssh-public-key} \
--node-count=2 \
--node-size=t2.medium \
--master-size=t2.medium \
--master-count=1

def frontWebImage = "이미지 레포지토리"
def frontWebImageTag

node {
    stage('Build And Push Front') {
        if (frontWebImageTag == null) {
            podTemplate(label: 'build-front-web'
            containers: [
                containerTemplate(name: 'front-web-builder', image: '{docker-image-repo}:{docker-image-tag}', ttyEnabled: true, command: 'cat', resourceLimitMemory: '768Mi'),
                containerTemplate(name: 'docker', image: 'docker:17.11', ttyEnabled: true, command: 'cat', resourceLimitMemory: '64Mi'),
            ],
            volumes: [
                hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock')
            ]) {
                node('build-frontWeb') {
                    stage('Build Front Image') {
                        sshagent (credentials: ['eng-fresh']) {
                            // 별도로 container 문을 사용하지 않으면 Jenkins 슬레이브 컨테이너에서 명령어가 실행된다.
                            git branch: "${params.branch}", credentialsId: '{credential-id}', url: "${params.repo}"
                            gitCommit = sh(returnStdout: true, script: 'git rev-parse HEAD').trim().take(8)
                            frontWebImageTag = "${BUILD_NUMBER}-${gitCommit}"
                            ARTIFACT_ID = sh(returnStdout: true, script: "echo fresh-frontWeb2-b${BUILD_NUMBER}").trim()
                            PACKAGE = sh(returnStdout: true, script: "echo target/${ARTIFACT_ID}.tar.gz").trim()

                            container('front-web-builder') { // 레거시 웹 컨테이너에서 쉘을 실행한다.
                                sh "build/build.sh package ${ARTIFACT_ID}" //build.sh. 는 composer 와 nom 을 사용한다.
                            }

                            container('docker') { // 도커 컨테이너에서 쉘을 실행한다.
                                sh "docker build -t ${frontWebImage}:${frontWebImageTag} --build-arg PACKAGE=${PACKAGE} --build-arg ARTIFACT_ID=${ARTIFACT_ID} ."
                            }
                        }
                    }
                   stage("Push Front Image") {
                        container('docker') {
                            docker.withRegistry('{eco-uri}', '{docker-credential}') {
                                docker.image("${frontWebImage}:${frontWebImageTag}").push()
                            }
                        }
                    }
                }
            }
        }
    }
    .
    .
    .
}

node {
    .
    . //도커 이미지 빌드 & 푸쉬
    .
    stage ("Helm Deploy") {
        podTemplate(label: 'helm'
        containers: [
            containerTemplate(name: 'helm', image: 'lachlanevenson/k8s-helm:latest', ttyEnabled: true, command: 'cat')
        ]) {
            node('helm') {
                stage("Deploy Alpha") {
                    git branch: 'master', credentialsId: '{Git Credential}', url: '{Git Repo}'

                    container('helm') {
                        sh "helm init" //helm init 명령어는 클러스터에 Tiller 를 배포하면서 로컬환경에서 사용하는 차트 저장소를 만든다.
                        sh "helm package helm/alpha/router" # 패키징 된 차트는 로컬 차트 저장소에 저장된다.
                        sh "helm package helm/alpha/front-web"
                        sh "helm package helm/alpha/msa1"
                        sh "helm package helm/alpha/api-gateway"
                        sh "helm dependency update helm/alpha/alpha" #의존하는 모든 패키지를 가져와 알파 패키지에 추가한다. 위에서 패키징한 차트는 로컬 저장소에 저장되어 있어 가져올 수 있다.
                        sh "helm install --debug --name ${params.release} --namespace default --set front-web.frontWebTag=${frontWebImageTag} --set msa1.msa1ImageTag=${msa1ImageTag} --set api-gateway.apiGatewayImageTag=${apiGatewayImageTag} helm/alpha/alpha"
                        // 도커 이미지의 태그 값은 중복되지 않도록 git commit ID 와 Jenkins 빌드 넘버의 조합이다. 이 값은 helm install 명령에 파라미터로써 제공되어 젠킨스 파이프라인에서 빌드한 이미지를 팟에서 사용하는것이 가능하게 해준다.
                        // ${params.release} 깂은 빌드할 때 설정하는 값으로 alpha-cluster01~05 의 값중에 하나를 사용한다.
                    }
                }
            }
        }
    }
}

$ helm ls
NAME      	        REVISION	UPDATED                 	STATUS  	CHART         	NAMESPACE
alpha-cluster01  	1       	Sat DEC  9 14:36:53 2017	DEPLOYED	alpha-1.0.0   	default
jenkins   	        1       	Thu Dec 14 15:29:29 2017	DEPLOYED	jenkins-0.10.1	jenkins

$ kubectl get po
NAME                                           READY     STATUS    RESTARTS   AGE
alpha-cluster01-api-gateway-4117715186-f23mv   1/1       Running   0          1m
alpha-cluster01-front-web-1457034248-qvnsr     1/1       Running   0          1m
alpha-cluster01-msa1-3928334195-11092          1/1       Running   0          1m
alpha-cluster01-router-1217976428-8xp7g        1/1       Running   0          1m

//--name alpha-cluster01 값으로 설치된 차트의 서비스들은 alpha-cluster01 접두사를 가지게 되어 다른 테스팅 환경의 서비스와 간편하게 구분되고 사용될 수 있다.
$ kubectl get svc
NAME                          CLUSTER-IP       EXTERNAL-IP   PORT(S)          AGE
alpha-cluster01-api-gateway   100.68.48.102    <none>        80/TCP           1m
alpha-cluster01-api-proxy     100.64.121.224   <none>        80/TCP,443/TCP   1m
alpha-cluster01-front-web     100.66.0.18      <none>        80/TCP,443/TCP   1m
alpha-cluster01-msa1       100.68.79.194       <none>        80/TCP           1m
alpha-cluster01-router        100.66.130.177   <none>        80/TCP,443/TCP   1m

$ kubectl top node
NAME                                            CPU(cores)   CPU%      MEMORY(bytes)   MEMORY%
ip-10-xx-xx-xx.ap-northeast-2.compute.internal   97m          4%        2642Mi          68%
ip-10-xx-xx-xx.ap-northeast-2.compute.internal   68m          3%        3678Mi          46%
ip-10-xx-xx-xx.ap-northeast-2.compute.internal   54m          2%        3339Mi          42%

$ kubectl top po
NAME                                           CPU(cores)   MEMORY(bytes)
alpha-cluster01-router-1217976428-8xp7g        0m           10Mi
alpha-cluster01-front-web-3938765134-hvxwr     0m           124Mi
alpha-cluster01-api-gateway-4117715186-f23mv   0m           630Mi
alpha-cluster01-msa1-3928334195-11092          0m           726Mi

$ kubectl top po -n jenkins //젠킨스와 젠킨스 슬레이브 팟은 jenkins 네임스페이스를 가진다.
NAME                              CPU(cores)   MEMORY(bytes)
jenkins-jenkins-322528892-n58cn   0m           1143Mi
jenkins-slave-3928334195-12824    0m           938Mi        // 배포중일때 kubectl top 명령어를 사용해 보았다.

podTemplate(label: 'build-msa1',
    containers: [
		//젠킨스 슬레이브 컨테이너는 선언하지 않아도 자동으로 포함되지만 리소스 제한을 설정하기 위해서는 직접 선언해줘야 한다.
        containerTemplate(name: 'jnlp', image: 'jenkins/jnlp-slave:3.10-1-alpine', args: '${computer.jnlpmac} ${computer.name}', resourceLimitMemory: '384Mi',
            envVars: [
                envVar(key: 'JVM_HEAP_MIN', value: '-Xmx192m'),
                envVar(key: 'JVM_HEAP_MAX', value: '-Xmx192m')
            ]
        ),
        containerTemplate(name: 'java', image: 'java:8-jdk', ttyEnabled: true, command: 'cat', resourceLimitMemory: '768Mi',
            envVars: [
                envVar(key: 'JVM_HEAP_MIN', value: '-Xmx256m'),
                envVar(key: 'JVM_HEAP_MAX', value: '-Xmx256m')
            ]
        ),
        containerTemplate(name: 'docker', image: 'docker:17.11', ttyEnabled: true, command: 'cat', resourceLimitMemory: '64Mi'),
    ]
);

$ kubectl get node --show-labels
NAME                                            STATUS    AGE       VERSION   LABELS
ip-10-xx-xx-xx.ap-northeast-2.compute.internal   Ready     11d       v1.7.11   alpha-pod-affinity=deploy ....
ip-10-yy-yy-yy.ap-northeast-2.compute.internal   Ready     11d       v1.7.11   alpha-pod-affinity=service ....

podTemplate(label: 'build-msa1', nodeSelector: 'alpha-pod-affinity=deploy', // 노드셀렉터 설정을 추가하였다.
    containers: [
        containerTemplate(name: 'jnlp', image: 'jenkins/jnlp-slave:3.10-1-alpine', args: '${computer.jnlpmac} ${computer.name}', resourceLimitMemory: '384Mi',
            envVars: [
                envVar(key: 'JVM_HEAP_MIN', value: '-Xmx192m'),
                envVar(key: 'JVM_HEAP_MAX', value: '-Xmx192m')
            ]
        ),
        containerTemplate(name: 'java', image: 'java:8-jdk', ttyEnabled: true, command: 'cat', resourceLimitMemory: '768Mi',
            envVars: [
                envVar(key: 'JVM_HEAP_MIN', value: '-Xmx256m'),
                envVar(key: 'JVM_HEAP_MAX', value: '-Xmx256m')
            ]
        ),
        containerTemplate(name: 'docker', image: 'docker:17.11', ttyEnabled: true, command: 'cat', resourceLimitMemory: '64Mi'),
    ]
    volumes: [
        hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock')
    ]
);

$ kubectl get po -o wide
NAME                                           READY     STATUS    RESTARTS   AGE       IP                NODE
alpha-cluster01-api-gateway-4117715186-f23mv   1/1       Running   0          1h        100.113.255.189   ip-10-xx-xx-xx.ap-northeast-2.compute.internal
alpha-cluster01-front-web-3938765134-hvxwr     1/1       Running   0          1h        100.113.255.149   ip-10-xx-xx-xx.ap-northeast-2.compute.internal
alpha-cluster01-msa1-3928334195-11092          1/1       Running   0          1h        100.113.255.148   ip-10-xx-xx-xx.ap-northeast-2.compute.internal
alpha-cluster01-router-1217976428-8xp7g        1/1       Running   0          1h        100.113.255.187   ip-10-xx-xx-xx.ap-northeast-2.compute.internal

$ kubectl get po -o wide -n jenkins
NAME                              READY     STATUS    RESTARTS   AGE       IP              NODE
jenkins-jenkins-322528892-n58cn   1/1       Running   0          7d        100.111.50.79   ip-10-yy-yy-yy.ap-northeast-2.compute.internal
jenkins-slave-4028334195-12824    4/4       Running   0          2m        100.111.50.82   ip-10-yy-yy-yy.ap-northeast-2.compute.internal

podTemplate(label: 'build-msa1', nodeSelector: 'alpha-pod-affinity=deploy',
    containers: [
        containerTemplate(name: 'jnlp', image: 'jenkins/jnlp-slave:3.10-1-alpine', args: '${computer.jnlpmac} ${computer.name}', resourceLimitMemory: '384Mi',
            envVars: [
                envVar(key: 'JVM_HEAP_MIN', value: '-Xmx192m'),
                envVar(key: 'JVM_HEAP_MAX', value: '-Xmx192m')
            ]
        ),
        containerTemplate(name: 'java', image: 'java:8-jdk', ttyEnabled: true, command: 'cat', resourceLimitMemory: '768Mi',
            envVars: [
                envVar(key: 'JVM_HEAP_MIN', value: '-Xmx256m'),
                envVar(key: 'JVM_HEAP_MAX', value: '-Xmx256m')
            ]
        ),
        containerTemplate(name: 'docker', image: 'docker:17.11', ttyEnabled: true, command: 'cat', resourceLimitMemory: '64Mi'),
    ],
    volumes: [
        hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock'),
        persistentVolumeClaim(mountPath: '/home/jenkins/.gradle', claimName: 'msa1', readOnly: false) //Gradle 캐시를 저장할 EBS 를 마운트한다.
    ]
);

def msa1Image = "{image Repo}"
def msa1ImageTag

node {
    stage('Partial Deploy Msa1') {
        if (msa1ImageTag == null) {
            podTemplate(label: 'build-msa1', nodeSelector: 'alpha-pod-affinity=deploy',
                containers: [
                    containerTemplate(name: 'jnlp', image: 'jenkins/jnlp-slave:3.10-1-alpine', args: '${computer.jnlpmac} ${computer.name}', resourceLimitMemory: '384Mi',
                        envVars: [
                            envVar(key: 'JVM_HEAP_MIN', value: '-Xmx192m'),
                            envVar(key: 'JVM_HEAP_MAX', value: '-Xmx192m')
                        ]
                    ),
                    containerTemplate(name: 'java', image: 'java:8-jdk', ttyEnabled: true, command: 'cat', resourceLimitMemory: '768Mi',
                        envVars: [
                            envVar(key: 'GRADLE_OPTS', value: '-Xmx384m'),
                            envVar(key: 'JVM_HEAP_MIN', value: '-Xmx256m'),
                            envVar(key: 'JVM_HEAP_MAX', value: '-Xmx256m')
                        ]
                    ),
                    containerTemplate(name: 'docker', image: 'docker:17.11', ttyEnabled: true, command: 'cat', resourceLimitMemory: '64Mi'),
                    containerTemplate(name: 'kubectl', image: 'lachlanevenson/k8s-kubectl:latest', ttyEnabled: true, command: 'cat', resourceLimitMemory: '64Mi')
                ],
                volumes: [
                    hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock'),
                    persistentVolumeClaim(mountPath: '/home/jenkins/.gradle', claimName: 'msa1', readOnly: false)
                ]
            ) {
                node('build-msa1') {
                    stage("Build Msa1") {
                        git branch: "${params.branch}", credentialsId: '{git credential}', url: "${params.repo}"
                        gitCommit = sh(returnStdout: true, script: 'git rev-parse HEAD').trim().take(8)
                        msa1ImageTag = "partial-${BUILD_NUMBER}-${gitCommit}"

                        container('java') {
                            sh './gradlew --gradle-user-home=/home/jenkins/.gradle --no-daemon --stacktrace -x test -x findBugsMain -x findBugsTest clean build'
                        }

                        container('docker') {
                            sh "docker build -t ${msa1Image}:${msa1ImageTag} ."
                        }
                    }
                    stage ('Push Msa1') {
                        container('docker') {
                            docker.withRegistry('{docker-repo}', '{docker-credential}') {
                                docker.image("${msa1Image}:${msa1ImageTag}").push()
                            }
                        }
                    }
                    stage('Update Deployment Image') {
                        container('kubectl') {
                            sh "kubectl set image deployment/${params.release}-msa1 msa1=${msa1Image}:${msa1ImageTag} -n default"
                        }
                    }
                }
            }
        }
    }
}