e0166ecba0
Each time a new pipeline is triggered, some jobs may fail due to temporary issue with a Gitlab runner (network, power supply, docker or maintainance). Most of the problems are "runner system failure" [1] and require to retart each failed jobs manually by maintainers to complete the pipeline with only real failures if any. The "retry" keyword allows to configure how many times a job is retried if it fails. "retry:when" allows to retry a failed job only on specific failure types like "runner_system_failure". While at it, retry a job if it failed due to a timeout failure (this timeout means that the job was pending for more than 24h) [2]. Such timeout failures occur on pipelines testing each Buildroot's defconfig since there is not enough gitlab runner available to build all of them within 24h. Retry only jobs that are more likely to wait for a runner (generate-gitlab-ci-yml, runtime_test_base, defconfig_base and test_pkg). [1] https://gitlab.com/buildroot.org/buildroot/-/jobs/4936949397 (runner system failure) [2] https://gitlab.com/buildroot.org/buildroot/-/jobs/4936949530 (timeout failure or the job got stuck) https://docs.gitlab.com/ee/ci/yaml/#retrywhen Signed-off-by: Romain Naour <romain.naour@gmail.com> Cc: Arnout Vandecappelle <arnout@mind.be> Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
146 lines
3.9 KiB
YAML
146 lines
3.9 KiB
YAML
stages:
|
|
- download
|
|
- test
|
|
|
|
before_script:
|
|
- git config --global --add safe.directory ${CI_PROJECT_DIR}
|
|
|
|
.check-check-package_base:
|
|
stage: test
|
|
script:
|
|
- python3 -m pytest -v utils/checkpackagelib/
|
|
|
|
.check-check-symbol_base:
|
|
stage: test
|
|
script:
|
|
- python3 -m pytest -v utils/checksymbolslib/
|
|
|
|
.check-DEVELOPERS_base:
|
|
stage: test
|
|
script:
|
|
- utils/get-developers -v
|
|
|
|
.check-package_base:
|
|
stage: test
|
|
script:
|
|
- make check-package
|
|
|
|
.check-symbol_base:
|
|
stage: test
|
|
script:
|
|
- utils/check-symbols
|
|
|
|
.defconfig_check:
|
|
stage: test
|
|
script:
|
|
- DEFCONFIG_NAME=$(echo ${CI_JOB_NAME} | sed -e 's,_check$,,g')
|
|
- echo "Configure Buildroot for ${DEFCONFIG_NAME}"
|
|
- make ${DEFCONFIG_NAME}
|
|
- support/scripts/check-dotconfig.py .config configs/${DEFCONFIG_NAME}
|
|
artifacts:
|
|
when: on_failure
|
|
expire_in: 2 weeks
|
|
paths:
|
|
- .config
|
|
|
|
.run_make: &run_make
|
|
|
|
|
make O=${OUTPUT_DIR} > >(tee build.log |grep '>>>') 2>&1 || {
|
|
echo 'Failed build last output'
|
|
tail -200 build.log
|
|
exit 1
|
|
}
|
|
|
|
.defconfig_base:
|
|
stage: test
|
|
script:
|
|
- DEFCONFIG_NAME=${CI_JOB_NAME}
|
|
- OUTPUT_DIR=output
|
|
- echo "Configure Buildroot for ${DEFCONFIG_NAME}"
|
|
- make ${DEFCONFIG_NAME}
|
|
- ./support/scripts/check-dotconfig.py .config ./configs/${DEFCONFIG_NAME}
|
|
- echo 'Build buildroot'
|
|
- *run_make
|
|
- |
|
|
./support/scripts/boot-qemu-image.py "${DEFCONFIG_NAME}" > >(tee runtime-test.log) 2>&1 || {
|
|
echo 'Failed runtime test last output'
|
|
tail -200 runtime-test.log
|
|
exit 1
|
|
}
|
|
retry:
|
|
max: 2
|
|
when:
|
|
- runner_system_failure
|
|
- stuck_or_timeout_failure
|
|
artifacts:
|
|
when: always
|
|
expire_in: 2 weeks
|
|
paths:
|
|
- .config
|
|
- build.log
|
|
- output/images/
|
|
- output/build/build-time.log
|
|
- output/build/packages-file-list.txt
|
|
- output/build/*/.config
|
|
- runtime-test.log
|
|
|
|
.runtime_test_download:
|
|
stage: download
|
|
# Keep test-dl directory so the downloaded files can be an artifact of
|
|
# the job passed to all jobs of next stages.
|
|
script: ./support/testing/run-tests -d test-dl/ --prepare-only
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- test-dl/
|
|
|
|
.runtime_test_base:
|
|
stage: test
|
|
# Keep build directories so the rootfs can be an artifact of the job. The
|
|
# runner will clean up those files for us.
|
|
# Multiply every emulator timeout by 10 to avoid sporadic failures in
|
|
# elastic runners.
|
|
script:
|
|
- TEST_CASE_NAME=${CI_JOB_NAME}
|
|
- echo "Starting runtime test ${TEST_CASE_NAME}"
|
|
- ./support/testing/run-tests -o test-output/ -d test-dl/ -k --timeout-multiplier 10 ${TEST_CASE_NAME}
|
|
retry:
|
|
max: 2
|
|
when:
|
|
- runner_system_failure
|
|
- stuck_or_timeout_failure
|
|
artifacts:
|
|
when: always
|
|
expire_in: 2 weeks
|
|
paths:
|
|
- test-output/*.log
|
|
- test-output/*/.config
|
|
- test-output/*/images/*
|
|
|
|
.test_pkg:
|
|
stage: test
|
|
script:
|
|
- OUTPUT_DIR=${CI_JOB_NAME}
|
|
- echo "Configure Buildroot for ${OUTPUT_DIR}"
|
|
- make O=${OUTPUT_DIR} syncconfig
|
|
- make O=${OUTPUT_DIR} savedefconfig
|
|
- echo 'Build buildroot'
|
|
- *run_make
|
|
needs:
|
|
- pipeline: $PARENT_PIPELINE_ID
|
|
job: generate-gitlab-ci-yml
|
|
retry:
|
|
max: 2
|
|
when:
|
|
- runner_system_failure
|
|
- stuck_or_timeout_failure
|
|
artifacts:
|
|
when: always
|
|
expire_in: 2 weeks
|
|
paths:
|
|
- build.log
|
|
- br-test-pkg/*/.config
|
|
- br-test-pkg/*/defconfig
|
|
- br-test-pkg/*/build/build-time.log
|
|
- br-test-pkg/*/build/packages-file-list*.txt
|