CI: Use local mirrors for submodules
The CI uses the mirror-submodule-update.sh for non-permanent reassignment
of 3rdparty sub-modules to local mirrors (only for 'non-master' branches).
List of the mirrors in mirror-list.txt
You can use the script locally to speed up data retrieval on network problems
check_submodule_sync job uses 'GIT_STRATEGY: clone' to check the availability of public sources
This commit is contained in:
Anton Maklakov
parent
f52519d7a2
commit
52445ee541
|
|
@ -37,7 +37,6 @@ before_script:
|
|||
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
|
||||
|
||||
# Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not
|
||||
# and adjust "our" submodule URLs if IS_PRIVATE
|
||||
#
|
||||
# (the same regular expressions are used to set these are used in 'only:' sections below
|
||||
- source tools/ci/configure_ci_environment.sh
|
||||
|
|
@ -374,6 +373,8 @@ check_submodule_sync:
|
|||
- /^release\/v/
|
||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||
dependencies: []
|
||||
variables:
|
||||
GIT_STRATEGY: clone
|
||||
before_script:
|
||||
- echo "Not setting up GitLab key, not fetching submodules"
|
||||
script:
|
||||
|
|
|
|||
|
|
@ -1,16 +1,15 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Short script that is sourced in to the CI environment
|
||||
# This file is sourced in to the CI environment
|
||||
# in .gitlab-ci.yml
|
||||
#
|
||||
# Sets IS_PUBLIC and IS_PRIVATE based on branch type
|
||||
|
||||
# Sets the error behaviour options for shell throughout the CI environment
|
||||
#
|
||||
# Tweaks .gitmodules file for private builds
|
||||
set -o errexit # Exit if command failed.
|
||||
|
||||
[ -z $CI_COMMIT_REF_NAME ] && echo "This internal script should only be run by a Gitlab CI runner." && exit 1
|
||||
|
||||
REF=$CI_COMMIT_REF_NAME
|
||||
|
||||
# Sets IS_PUBLIC and IS_PRIVATE based on branch type
|
||||
#
|
||||
# Public branches are:
|
||||
# release branches - start with release/
|
||||
# release tags - look like vXX.YY or vXX.YY.ZZ with an optional dash followed by anything on the end
|
||||
|
|
@ -18,18 +17,10 @@ REF=$CI_COMMIT_REF_NAME
|
|||
#
|
||||
# These POSIX REs are equivalent to the REs in some "only:" sections of the gitlab-ci.yml file
|
||||
#
|
||||
REF=$CI_COMMIT_REF_NAME
|
||||
if [[ $REF = "master" || $REF =~ ^release/v || $REF =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?(-|$) ]]; then
|
||||
export IS_PUBLIC=1
|
||||
else
|
||||
export IS_PRIVATE=1
|
||||
fi
|
||||
|
||||
unset REF
|
||||
|
||||
set -e
|
||||
|
||||
if [[ $IS_PRIVATE ]]; then
|
||||
# Redirect git submodules from public github to our private gitlab server
|
||||
sed -i "s%https://github.com/espressif/esp32-wifi-lib%${GITLAB_SSH_SERVER}/idf/esp32-wifi-lib%" .gitmodules
|
||||
sed -i "s%https://github.com/espressif/esp32-bt-lib%${GITLAB_SSH_SERVER}/idf/esp32-bt-lib%" .gitmodules
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -16,6 +16,18 @@ die() {
|
|||
}
|
||||
|
||||
[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner."
|
||||
[[ ( -z ${IS_PRIVATE} ) && ( -z ${IS_PUBLIC} ) ]] && die "IS_PRIVATE or IS_PUBLIC should be defined in the CI environment."
|
||||
|
||||
SCRIPT_DIR=$(dirname -- "${0}")
|
||||
update_submodules() {
|
||||
if [ "${IS_PRIVATE}" ]; then
|
||||
${SCRIPT_DIR}/mirror-submodule-update.sh
|
||||
else
|
||||
git submodule foreach "git submodule deinit --force ."
|
||||
git submodule deinit --force .
|
||||
git submodule update --init --recursive
|
||||
fi
|
||||
}
|
||||
|
||||
DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX)
|
||||
del_files() {
|
||||
|
|
@ -32,7 +44,7 @@ RETRIES=10
|
|||
# For the first time, we try the fastest way.
|
||||
for try in `seq $RETRIES`; do
|
||||
echo "Trying to add submodules to existing repo..."
|
||||
git submodule update --init --recursive &&
|
||||
update_submodules &&
|
||||
echo "Fetch strategy submodules succeeded" &&
|
||||
exit 0
|
||||
done
|
||||
|
|
@ -44,7 +56,7 @@ for try in `seq $RETRIES`; do
|
|||
del_files
|
||||
git clone ${CI_REPOSITORY_URL} . &&
|
||||
git checkout ${CI_COMMIT_SHA} &&
|
||||
git submodule update --init --recursive &&
|
||||
update_submodules &&
|
||||
echo "Clone strategy succeeded" &&
|
||||
del_files_confirm &&
|
||||
exit 0
|
||||
|
|
@ -52,6 +64,4 @@ for try in `seq $RETRIES`; do
|
|||
echo "Clean clone failed..."
|
||||
done
|
||||
|
||||
echo "Failed to clone repo & submodules together"
|
||||
|
||||
exit 1
|
||||
die "Failed to clone repo & submodules together"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
components/esp32/lib @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git
|
||||
components/bt/lib @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git
|
||||
components/aws_iot/aws-iot-device-sdk-embedded-C @GENERAL_MIRROR_SERVER@/idf/aws-iot-device-sdk-embedded-C.git
|
||||
components/coap/libcoap @GENERAL_MIRROR_SERVER@/idf/libcoap.git
|
||||
components/esptool_py/esptool @GENERAL_MIRROR_SERVER@/idf/esptool.git
|
||||
components/libsodium/libsodium @GENERAL_MIRROR_SERVER@/idf/libsodium.git
|
||||
components/micro-ecc/micro-ecc @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git
|
||||
components/nghttp/nghttp2 @GENERAL_MIRROR_SERVER@/idf/nghttp2.git
|
||||
third-party/mruby @GENERAL_MIRROR_SERVER@/idf/mruby.git
|
||||
third-party/neverbleed @GENERAL_MIRROR_SERVER@/idf/neverbleed.git
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Redirects git submodules to the specified local mirrors and updates these recursively.
|
||||
#
|
||||
# To revert the changed URLs use 'git submodule deinit .'
|
||||
#
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Common bash
|
||||
|
||||
if [[ ! -z ${DEBUG} ]]
|
||||
then
|
||||
set -x # Activate the expand mode if DEBUG is anything but empty.
|
||||
fi
|
||||
|
||||
set -o errexit # Exit if command failed.
|
||||
set -o pipefail # Exit if pipe failed.
|
||||
set -o nounset # Exit if variable not set.
|
||||
|
||||
die() {
|
||||
echo "${1:-"Unknown Error"}" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
|
||||
|
||||
REPO_DIR=${1:-"${PWD}"}
|
||||
REPO_DIR=$(readlink -f -- "${REPO_DIR}")
|
||||
|
||||
SCRIPT_DIR=$(dirname -- "${0}")
|
||||
SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
|
||||
|
||||
SCRIPT_SH=$(readlink -f -- "${0}")
|
||||
|
||||
MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
|
||||
|
||||
[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
|
||||
[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
|
||||
[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
|
||||
|
||||
pushd ${REPO_DIR} >/dev/null
|
||||
|
||||
# 0
|
||||
[ -f ".gitmodules" ] || exit 0
|
||||
|
||||
# 1
|
||||
git submodule init
|
||||
|
||||
|
||||
# 2
|
||||
# Replacing each submodule URL of the current repository
|
||||
# according to the one found in the MIRRORLIST
|
||||
|
||||
# SED parses the strings like:
|
||||
#
|
||||
#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
|
||||
#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
|
||||
#
|
||||
for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
|
||||
do
|
||||
SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
|
||||
[ ${SUBMIRROR} ] || continue
|
||||
SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
|
||||
echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR"
|
||||
|
||||
git config submodule.${SUBPATH}.url ${SUBMIRROR}
|
||||
done
|
||||
|
||||
# 3
|
||||
# Getting submodules of the current repository from the local mirrors
|
||||
git submodule update
|
||||
|
||||
# 4
|
||||
# Replacing URLs for each sub-submodule.
|
||||
# The script runs recursively
|
||||
git submodule foreach "${SCRIPT_SH}" # No '--recursive'
|
||||
|
||||
popd >/dev/null
|
||||
Loading…
Reference in New Issue