Commit 360e6496 authored by Yuanle Song's avatar Yuanle Song
Browse files

v1.11.0 "make ci-build" now use cached venv 

and wheelhouse to speed up build.

bootstrap and bootstrap-dev now don't check for library if requirements
file have not changed.

"make wheel" will be no-op if requirements file have not changed.

dropped usage of CI_BUILD=1 env variable, it is not used.
parent b1dacd16

project.clj

+1 −1
Original line number Diff line number Diff line 
(defproject python/lein-template "1.10.0"

(defproject python/lein-template "1.11.0"

  :description "lein template for a python project"

  :url "https://gitlab.emacsos.com/sylecn/python-project-template"

  :license {:name "Apache License 2.0"

src/leiningen/new/python/Makefile

+46 −21
Original line number Diff line number Diff line 
PYTHON_MODULES := {{python-pkg-name}}

PYTHONPATH := .

VENV := .venv

WHEELHOUSE := wheelhouse

PYTEST := env PYTHONPATH=$(PYTHONPATH) PYTEST=1 $(VENV)/bin/py.test

PYLINT := env PYTHONPATH=$(PYTHONPATH) $(VENV)/bin/pylint --disable=I0011 --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}"

PEP8 := env PYTHONPATH=$(PYTHONPATH) $(VENV)/bin/pycodestyle --repeat --ignore=E202,E501,E402,W503,W504,E122,E129
@@ -13,8 +14,8 @@ PYPI_MIRROR ?= https://mirrors.aliyun.com/pypi/simple 
DEFAULT_PYTHON ?= $(shell ./utils/choose_default_python.sh)

VIRTUALENV := $(wildcard ./utils/virtualenv-*/virtualenv.py)



REQUIREMENTS := -r requirements.txt

REQUIREMENTS_DEV := -r requirements-dev.txt

REQUIREMENTS := requirements.txt

REQUIREMENTS_DEV := requirements-dev.txt



VERSION := $(shell grep '__version__' {{python-pkg-name}}/__init__.py |cut -d'"' -f 2)

# docker hub user name or private registry URL with username. do not end with /.
@@ -23,6 +24,13 @@ KUBECTL_CA_PATH ?= /etc/kubernetes/pki/ca.crt 


BUILD_DOCKER_IMAGE := $(DOCKER_IMAGE_PREFIX)/python-build-image:1.2.1-stretch



CI_BUILD_GLOBAL_CACHE := $(HOME)/.cache/ci_build/$(GO_PIPELINE_NAME)

wheelhouse_hash_dir := wheelhouse-$(shell sha1sum $(REQUIREMENTS) | cut -d' ' -f1)

venv_hash_dir := venv-$(shell cat $(REQUIREMENTS) $(REQUIREMENTS_DEV) | sha1sum | cut -d' ' -f1)

pip_cache_dir := $(CI_BUILD_GLOBAL_CACHE)/cache

host_venvs_dir := $(CI_BUILD_GLOBAL_CACHE)/venvs

host_wheelhouses_dir := $(CI_BUILD_GLOBAL_CACHE)/wheelhouses



default: check-coding-style

dist: bootstrap

	rm -rf dist/*
@@ -49,24 +57,25 @@ uwsgi: 
shell:

	$(PYTHON) -i



venv:

	test -d $(VENV) || $(DEFAULT_PYTHON) $(VIRTUALENV) --no-download -q $(VENV)

requirements:

	@if [ -d wheelhouse ]; then \

		$(PIP) install -q --isolated --no-index --find-links=wheelhouse $(REQUIREMENTS); \

$(VENV)/runtime-venv-ready: $(REQUIREMENTS)

	test -d "$(VENV)/bin" || $(DEFAULT_PYTHON) $(VIRTUALENV) --no-download -q "$(VENV)"

	@if [ -d "$(WHEELHOUSE)" ]; then \

		$(PIP) install -q --isolated --no-index --find-links="$(WHEELHOUSE)" -r $(REQUIREMENTS); \

	else \

		$(PIP) install -q -i $(PYPI_MIRROR) $(REQUIREMENTS); \

		$(PIP) install -q -i $(PYPI_MIRROR) -r $(REQUIREMENTS); \

	fi

requirements-dev:

	touch "$(VENV)/runtime-venv-ready"

$(VENV)/dev-venv-ready: $(VENV)/runtime-venv-ready $(REQUIREMENTS_DEV)

	./utils/ensure_pkg_installed $(DEV_PKGS)

	$(PIP) install -q -i $(PYPI_MIRROR) $(REQUIREMENTS_DEV)

	$(PIP) install -q -i $(PYPI_MIRROR) -r $(REQUIREMENTS_DEV)

	touch "$(VENV)/dev-venv-ready"

logdir:

	@if [ ! -d /var/log/{{name}} ]; then \

		sudo mkdir /var/log/{{name}}; \

		sudo chown -R ${USER} /var/log/{{name}}; \

	fi

bootstrap: venv requirements

bootstrap-dev: venv requirements requirements-dev

bootstrap: $(VENV)/runtime-venv-ready

bootstrap-dev: $(VENV)/dev-venv-ready

install: bootstrap

	$(PIP) install -q --no-index .

uninstall: bootstrap
@@ -78,11 +87,13 @@ deb: 
	utils/build-deb

pipfreeze:

	$(PIP) freeze

wheel: bootstrap-dev

	@echo "building wheelhouse..."

$(WHEELHOUSE)/ready: $(REQUIREMENTS)

	test -d "$(VENV)/bin" || $(DEFAULT_PYTHON) $(VIRTUALENV) --no-download -q "$(VENV)"

	@echo "building wheelhouse in $(WHEELHOUSE)..."

	$(PIP) install wheel

	$(PIP) wheel -w wheelhouse $(REQUIREMENTS)



	$(PIP) wheel -w wheelhouse -r $(REQUIREMENTS)

	touch "$(WHEELHOUSE)/ready"

wheel: $(WHEELHOUSE)/ready

check: just-test

sanity-check: bootstrap

	$(PYTHON) {{python-pkg-name}}/sanity_check.py
@@ -101,21 +112,35 @@ just-test: 
# image and push image on host.

ci-build:

	echo '$(value DOCKER_PASSWORD)' | docker login -u '$(DOCKER_USER)' --password-stdin '$(BUILD_DOCKER_IMAGE)'

	mkdir -p .cache

	docker run --rm --name {{ name }}-test -v "$(CURDIR)":/app -v "$(CURDIR)/.cache":/.cache -u $(shell id -u):$(shell id -g) -e CI_BUILD=1 '$(BUILD_DOCKER_IMAGE)' make test deb -C /app

	mkdir -p $(pip_cache_dir) \

		$(host_venvs_dir)/$(venv_hash_dir) \

		$(host_wheelhouses_dir)/$(wheelhouse_hash_dir)

	docker run --rm --name {{ name }}-test \

		-v "$(CURDIR)":/app \

		-v "$(pip_cache_dir)":/.cache \

		-v "$(host_venvs_dir)/$(venv_hash_dir)":/app/$(VENV) \

		-v "$(host_wheelhouses_dir)/$(wheelhouse_hash_dir)":/app/$(WHEELHOUSE) \

		-u $(shell id -u):$(shell id -g) \

		$(BUILD_DOCKER_IMAGE) \

		make test deb -C /app

	docker build -t $(DOCKER_IMAGE_PREFIX)/{{ name }}:$(VERSION) .

	docker push $(DOCKER_IMAGE_PREFIX)/{{ name }}:$(VERSION)

	@echo "removing obsolete .venv-<hash> and wheelhouse-<hash> dirs..."

	find $(CI_BUILD_GLOBAL_CACHE) -maxdepth 2 -type d \

		\( \( -name "venv-*" -a ! -name "$(venv_hash_dir)" \) -o \

		   \( -name "wheelhouse-*" -a ! -name "$(wheelhouse_hash_dir)" \) \) \

		-mtime +180 -execdir rm -rf {} \;

deploy:

	sed "s/IMAGE_TAG/$(VERSION)/" app.yaml > $(VERSION).yaml

	kubectl --certificate-authority='$(KUBECTL_CA_PATH)' -s '$(KUBECTL_API)' --token='$(KUBECTL_TOKEN)' apply -f $(VERSION).yaml



test-prod: requirements-dev test

test-prod: bootstrap-dev test



clean:

	find . -name "__pycache__" -type d -exec rm -rf {} \; || true

	find . -name "*.pyc" -type f -exec rm -rf {} \; || true

full-clean:

	rm -rf $(VENV) build/ dist/ wheelhouse/ {{python-pkg-name}}.egg-info/ distribute-*.tar.gz

	rm -rf "$(VENV)" build/ dist/ "$(WHEELHOUSE)" {{python-pkg-name}}.egg-info/ distribute-*.tar.gz .venv-* wheelhouse-*

	find . -name "__pycache__" -type d -exec rm -rf {} \; || true

	find . -name "*.pyc" -type f -exec rm -rf {} \; || true

todo:
@@ -130,4 +155,4 @@ install-git-hooks-force: 
loc:

	find . -regex '.*\.pyw?$$' -exec wc -l {} \+ | tail -n 1

	which cloc >/dev/null && cloc {{python-pkg-name}}/

.PHONY: default build debug run uwsgi shell venv bootstrap bootstrap-dev install uninstall deb pipfreeze wheel check check-coding-style pylint-full test just-test test-prod clean full-clean todo TAGS update-git-hooks install-git-hooks install-git-hooks-force loc
 
.PHONY: default build debug run uwsgi shell bootstrap bootstrap-dev install uninstall deb pipfreeze wheel check check-coding-style pylint-full test just-test test-prod clean full-clean todo TAGS update-git-hooks install-git-hooks install-git-hooks-force loc

src/leiningen/new/python/utils/build-deb

+1 −3
Original line number Diff line number Diff line
@@ -40,9 +40,7 @@ DEB_PKG_NAME="{{name}}" 
DEST_DIR="/opt/$DEB_PKG_NAME"

PYTHON_MODULES=`ls */__init__.py|cut -d'/' -f1`



if [ ! -d wheelhouse ]; then

make wheel

fi



build_dest_dir