From 6b9fa5b279ef612d41a5797a1dedb95fd936cf7e Mon Sep 17 00:00:00 2001 From: Yuanle Song Date: Wed, 2 May 2018 17:28:08 +0800 Subject: [PATCH] v1.2.0 support test, build and deploy in gocd - added make ci-build and make deploy - added app.yaml for k8s based deploy - after-install.sh no longer run make install - utils/build-deb grep add -q option - Dockerfile.template apt use -qq option --- operational | 30 ++++++- project.clj | 2 +- src/leiningen/new/python.clj | 1 + src/leiningen/new/python/Dockerfile.template | 4 +- src/leiningen/new/python/Makefile | 15 +++- src/leiningen/new/python/app.yaml | 85 +++++++++++++++++++ .../new/python/deb-scripts/after-install.sh | 3 +- src/leiningen/new/python/utils/build-deb | 4 +- 8 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 src/leiningen/new/python/app.yaml diff --git a/operational b/operational index 2fd4e4f..3db2d47 100644 --- a/operational +++ b/operational @@ -1,7 +1,29 @@ * COMMENT -*- mode: org -*- #+Date: 2014-05-09 -Time-stamp: <2016-09-08> +Time-stamp: <2018-05-02> #+STARTUP: content +* notes :entry: +** 2018-05-02 support test, build and deploy in gocd +- stage 1: testAndBuild + make ci-build + + required env var: + DOCKER_USER + DOCKER_PASSWORD + docker login info for private registry. + +- stage 2: deploy + make deploy + + required env var: + KUBECTL_API + KUBECTL_TOKEN + kubectl auth info for running kubectl apply. + +- implemented in v1.2.0 + + in gocd, use the python-app pipeline template to automate stages and jobs. + * current :entry: ** ** 2016-09-08 support updating site file, deb scripts etc after project is generated. @@ -26,9 +48,6 @@ unit tests against defined python in tox.ini. ** 2016-08-07 in python project template, add a project wide pylintrc file. see example ~/.pylintrc -** 2016-07-27 package name should not have -, auto convert it to _. -- package name is part of module name, - is not valid there. - ** 2016-06-30 update it for use with default debian config. (non-yy env) - update nginx config file - add support for systemd @@ -39,6 +58,9 @@ see example ~/.pylintrc ** 2014-05-09 add sphinx support * later :entry: * done :entry: +** 2016-07-27 package name should not have -, auto convert it to _. +- package name is part of module name, - is not valid there. + ** 2016-09-08 when bootstrap, create /var/log/{{name}} dir using sudo and chown to current user. this makes it easier when developing. diff --git a/project.clj b/project.clj index 824218f..19cb630 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject python/lein-template "1.1.0" +(defproject python/lein-template "1.2.0" :description "lein template for a python project" :repositories [["snapshots" {:url "http://devserv.game.yy.com/nexus/content/repositories/snapshots" diff --git a/src/leiningen/new/python.clj b/src/leiningen/new/python.clj index 4ac8f8a..f0824db 100644 --- a/src/leiningen/new/python.clj +++ b/src/leiningen/new/python.clj @@ -46,6 +46,7 @@ "requirements-dev.txt" "setup.py" "Dockerfile.template" + "app.yaml" "README.rst" "utils/versionutils.py" "utils/build-deb" diff --git a/src/leiningen/new/python/Dockerfile.template b/src/leiningen/new/python/Dockerfile.template index dda64ee..4393d9f 100644 --- a/src/leiningen/new/python/Dockerfile.template +++ b/src/leiningen/new/python/Dockerfile.template @@ -3,8 +3,8 @@ FROM sylecn/debian:stretch-slim-cn #LABEL description="TODO add image description and uncomment this" RUN set -x \ - && apt-get update \ - && apt-get install -y --no-install-recommends --no-install-suggests \ + && apt-get update -qq \ + && apt-get install -qq -y --no-install-recommends --no-install-suggests \ DEPENDS \ && rm -rf /var/lib/apt/lists/* diff --git a/src/leiningen/new/python/Makefile b/src/leiningen/new/python/Makefile index e39989b..3bbaf76 100644 --- a/src/leiningen/new/python/Makefile +++ b/src/leiningen/new/python/Makefile @@ -15,7 +15,9 @@ DEV_REQUIREMENTS := -r requirements-dev.txt VERSION := $(shell grep '__version__' {{python-pkg-name}}/__init__.py |cut -d'"' -f 2) # docker hub user name or private registry URL with username. -DOCKER_IMAGE_PREFIX := sylecn +DOCKER_IMAGE_PREFIX := de02-reg.emacsos.com/sylecn + +BUILD_DOCKER_IMAGE := de02-reg.emacsos.com/sylecn/python-build-image:1.0.0 default: check-coding-style build: @@ -77,6 +79,17 @@ test: check-coding-style just-test: $(PYTEST) $(PYTHON_MODULES) +# run test, build deb in python build image container, then create docker +# image and push image on host. +ci-build: + docker login -u $(DOCKER_USER) -p $(DOCKER_PASSWORD) $(BUILD_DOCKER_IMAGE) + docker run --rm --name {{ name }}-test -v "$(CURDIR)":/app $(BUILD_DOCKER_IMAGE) make test deb -C /app + docker build -t $(DOCKER_IMAGE_PREFIX)/{{ name }}:$(VERSION) . + docker push $(DOCKER_IMAGE_PREFIX)/{{ name }}:$(VERSION) +deploy: + sed "s/IMAGE_TAG/$(VERSION)/" app.yaml > $(VERSION).yaml + kubectl --certificate-authority=/etc/kubernetes/pki/ca.crt -s $(KUBECTL_API) --token=$(KUBECTL_TOKEN) apply -f $(VERSION).yaml + install-dev-no-wheel: $(PIP) install -q -i https://pypi.tuna.tsinghua.edu.cn/simple $(DEV_REQUIREMENTS) test-prod: install-dev-no-wheel test diff --git a/src/leiningen/new/python/app.yaml b/src/leiningen/new/python/app.yaml new file mode 100644 index 0000000..f9ca139 --- /dev/null +++ b/src/leiningen/new/python/app.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ name }} +spec: + selector: + matchLabels: + app: {{ name }} + replicas: 1 + minReadySeconds: 10 + revisionHistoryLimit: 2 + template: + metadata: + labels: + app: {{ name }} + spec: + imagePullSecrets: + - name: pull-from-de02-reg + containers: + - name: {{ name }} + image: de02-reg.emacsos.com/sylecn/{{ name }}:IMAGE_TAG + env: + - name: ENV + value: prod + - name: HOST + value: "0.0.0.0" + - name: PORT + value: "8082" + ports: + - containerPort: 8082 + # livenessProbe: + # httpGet: + # path: / + # port: 8082 + # httpHeaders: + # - name: Host + # value: foo.emacsos.com + # initialDelaySeconds: 60 + # periodSeconds: 30 + # readinessProbe: + # httpGet: + # path: / + # port: 8082 + # httpHeaders: + # - name: Host + # value: foo.emacsos.com + # initialDelaySeconds: 30 + # periodSeconds: 5 + # resources: + # requests: + # cpu: 50m + # memory: 50Mi + # limits: + # cpu: 50m + # memory: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ name }} +spec: + ports: + - port: 8082 + targetPort: 8082 + selector: + app: {{ name }} +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ name }} + annotations: + kubernetes.io/ingress.class: nginx +spec: + tls: + - secretName: emacsos-com-cert + hosts: + - foo.emacsos.com + rules: + - host: foo.emacsos.com + http: + paths: + - backend: + serviceName: {{ name }} + servicePort: 8082 diff --git a/src/leiningen/new/python/deb-scripts/after-install.sh b/src/leiningen/new/python/deb-scripts/after-install.sh index 3671ba6..25b1c15 100644 --- a/src/leiningen/new/python/deb-scripts/after-install.sh +++ b/src/leiningen/new/python/deb-scripts/after-install.sh @@ -12,7 +12,8 @@ if [ "$RUN_AS_USER" != root ]; then fi cd "$PREFIX" -make bootstrap install +make bootstrap +# make install # install crontab, upstart job, nginx site file, logrotate config etc diff --git a/src/leiningen/new/python/utils/build-deb b/src/leiningen/new/python/utils/build-deb index 5748095..c9bfe21 100644 --- a/src/leiningen/new/python/utils/build-deb +++ b/src/leiningen/new/python/utils/build-deb @@ -37,8 +37,8 @@ fpm -t deb -s dir -n "$DEB_PKG_NAME" -v "$VERSION" --prefix "$DEST_DIR" -f \ # create Dockerfile and .dockerignore file DEB_PKG="{{name}}_${VERSION}_amd64.deb" -DEPENDS=`dpkg-deb -f "$DEB_PKG" Depends` -if ! echo "$DEPENDS" | grep python; then +DEPENDS=`dpkg-deb -f "$DEB_PKG" Depends | sed "s/, / /g"` +if ! echo "$DEPENDS" | grep -q python; then DEPENDS="$DEPENDS python3" fi sed "s/DEB_PKG/$DEB_PKG/; s/DEPENDS/$DEPENDS/" Dockerfile.template > Dockerfile -- GitLab