Files
mesa/.gitlab-ci/setup-test-env.sh
Vignesh Raman 7959250d1e s3_upload: improve url validation and error message
Ensure s3_upload correctly validates the S3 folder url by requiring it
to end with /. This prevents wrong uploads to invalid paths, such as
file urls. Also improve the error message.

Signed-off-by: Vignesh Raman <vignesh.raman@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34255>
2025-04-04 13:32:45 +00:00

312 lines
8.2 KiB
Bash

#!/usr/bin/env bash
# shellcheck disable=SC2048
# shellcheck disable=SC2086 # we want word splitting
# shellcheck disable=SC2155 # mktemp usually not failing
shopt -s expand_aliases
function _x_store_state {
if [[ "$-" == *"x"* ]]; then
previous_state_x=1
else
previous_state_x=0
fi
}
_x_store_state
alias x_store_state='{ _x_store_state; } >/dev/null 2>/dev/null'
function _x_off {
x_store_state
set +x
}
alias x_off='{ _x_off; } >/dev/null 2>/dev/null'
function _x_restore {
[ $previous_state_x -eq 0 ] || set -x
}
alias x_restore='{ _x_restore; } >/dev/null 2>/dev/null'
function _error_msg() (
x_off
RED="\e[0;31m"
ENDCOLOR="\e[0m"
echo -e "${RED}$*${ENDCOLOR}"
)
export JOB_START_S=$(date -u +"%s" -d "${CI_JOB_STARTED_AT:?}")
function get_current_minsec {
DATE_S=$(date -u +"%s")
CURR_TIME=$((DATE_S-JOB_START_S))
printf "%02d:%02d" $((CURR_TIME/60)) $((CURR_TIME%60))
}
function _build_section_start {
local section_params=$1
shift
local section_name=$1
CURRENT_SECTION=$section_name
shift
CYAN="\e[0;36m"
ENDCOLOR="\e[0m"
CURR_MINSEC=$(get_current_minsec)
echo -e "\n\e[0Ksection_start:$(date +%s):$section_name$section_params\r\e[0K${CYAN}[${CURR_MINSEC}] $*${ENDCOLOR}\n"
x_restore
}
alias build_section_start="x_off; _build_section_start"
function _section_start {
build_section_start "[collapsed=true]" $*
x_restore
}
alias section_start="x_off; _section_start"
function _uncollapsed_section_start {
build_section_start "" $*
x_restore
}
alias uncollapsed_section_start="x_off; _uncollapsed_section_start"
function _build_section_end {
echo -e "\e[0Ksection_end:$(date +%s):$1\r\e[0K"
CURRENT_SECTION=""
x_restore
}
alias build_section_end="x_off; _build_section_end"
function _section_end {
build_section_end $*
x_restore
}
alias section_end="x_off; _section_end"
function _section_switch {
if [ -n "$CURRENT_SECTION" ]
then
build_section_end $CURRENT_SECTION
x_off
fi
build_section_start "[collapsed=true]" $*
x_restore
}
alias section_switch="x_off; _section_switch"
function _uncollapsed_section_switch {
if [ -n "$CURRENT_SECTION" ]
then
build_section_end $CURRENT_SECTION
x_off
fi
build_section_start "" $*
x_restore
}
alias uncollapsed_section_switch="x_off; _uncollapsed_section_switch"
export -f _x_store_state
export -f _x_off
export -f _x_restore
export -f get_current_minsec
export -f _build_section_start
export -f _section_start
export -f _build_section_end
export -f _section_end
export -f _section_switch
export -f _uncollapsed_section_switch
export -f _error_msg
# Freedesktop requirement (needed for Wayland)
[ -n "${XDG_RUNTIME_DIR:-}" ] || export XDG_RUNTIME_DIR="$(mktemp -p "$PWD" -d xdg-runtime-XXXXXX)"
if [ -z "${RESULTS_DIR:-}" ]; then
export RESULTS_DIR="${PWD%/}/results"
if [ -e "${RESULTS_DIR}" ]; then
rm -rf "${RESULTS_DIR}"
fi
mkdir -p "${RESULTS_DIR}"
fi
function error {
# we force the following to be not in a section
if [ -n "${CURRENT_SECTION:-}" ]; then
section_end $CURRENT_SECTION
x_off
fi
CURR_MINSEC=$(get_current_minsec)
echo -e "\n"
_error_msg "[$CURR_MINSEC] ERROR: $*"
x_restore
}
function trap_err {
x_off
error ${CURRENT_SECTION:-'unknown-section'}: ret code: $*
}
# ------ Structured tagging
export _CI_TAG_CHECK_DIR="/mesa-ci-build-tag"
_ci_tag_from_name_to_var() {
# Transforms MY_COMPONENT_TAG to my-component
echo "${1%_TAG}" | tr '[:upper:]' '[:lower:]' | tr '_' '-'
}
_ci_tag_check() (
x_off
_declared_name="${1}"
declare -n _declared="${_declared_name}"
_calculated="${2}"
local component_lower=$(_ci_tag_from_name_to_var "${_declared_name}")
if [ -z "${_declared:-}" ]; then
# Close the section
error "Fatal error"
_error_msg "Structured tag is not set: ${_declared_name}"
_error_msg ""
echo "If you are adding a new component, please run:"
echo "bin/ci/update_tag.py --include ${component_lower}"
echo "This will automatically update the YAML file for you."
echo "Or manually edit .gitlab-ci/conditional-build-image-tags.yml to add the new"
echo "component."
error ""
exit 2
fi
if [ "${_declared}" != "${_calculated}" ]; then
# Close the section
error "Fatal error"
_error_msg "Mismatch in declared and calculated tags:"
_error_msg " ${_declared_name} from the YAML is \"${_declared}\""
_error_msg " ... but I think it should be \"${_calculated}\""
_error_msg ""
echo "Usually this happens when you change what you want to be built without also"
echo "changing the YAML declaration. For example, you've bumped SKQP to version"
echo "1.2.3, but you still have 'SKQP_VER: 1.2.2' in"
echo ".gitlab-ci/conditional-build-image-tags.yml."
echo ""
echo "If you meant to change the component I'm talking about, please change the"
echo "tag and resubmit. You can also run:"
echo "bin/ci/update_tag.py --include ${component_lower}"
echo "to update the tag automatically."
echo ""
echo "If you didn't mean to change the component, please ping @mesa/ci-helpers and we"
echo "can help you figure out what's gone wrong."
echo ""
echo "But for now, I've got to fail this build. Sorry."
exit 2
fi
x_restore
)
_ci_tag_check_build() {
x_off
if [ -n "${NEW_TAG_DRY_RUN:-}" ]; then
echo "${2}"
exit 0
fi
_ci_tag_check "${1}" "${2}"
if [ -n "${CI_NOT_BUILDING_ANYTHING:-}" ]; then
exit 0
fi
x_restore
}
get_tag_file() {
x_off
# If no tag name is provided, return the directory
echo "${_CI_TAG_CHECK_DIR}/${1:-}"
x_restore
}
_ci_tag_write() (
set +x
local tag_name="${1}"
local tag_value="${2}"
mkdir -p "${_CI_TAG_CHECK_DIR}"
echo -n "${tag_value}" > "$(get_tag_file "${tag_name}")"
)
_ci_calculate_tag() {
x_off
# the args are files that can affect the build output
mapfile -t extra_files < <(printf '%s\n' "$@")
(
for extra_file in "${extra_files[@]}"; do
if [ ! -f "${extra_file}" ]; then
error "File '${extra_file}' does not exist"
exit 1
fi
cat "${extra_file}"
done
) | md5sum | cut -d' ' -f1
x_restore
}
ci_tag_build_time_check() {
# Get the caller script and hash its contents plus the extra files
x_off
local tag_name="${1}"
local build_script_file="build-$(_ci_tag_from_name_to_var "${tag_name}").sh"
local build_script=".gitlab-ci/container/${build_script_file}"
shift
# now $@ has the extra files
local calculated_tag=$(_ci_calculate_tag "${build_script}" "$@")
_ci_tag_check_build "${tag_name}" "${calculated_tag}"
_ci_tag_write "${tag_name}" "${calculated_tag}"
x_restore
}
ci_tag_test_time_check() {
x_off
local tag_file=$(get_tag_file "${1}")
if [ ! -f "${tag_file}" ]; then
_error_msg "Structured tag file ${tag_file} does not exist"
_error_msg "Please run the ci_tag_build_time_check first and rebuild the image/rootfs"
exit 2
fi
_ci_tag_check "${1}" "$(cat "${tag_file}")"
x_restore
}
# Export all functions
export -f _ci_calculate_tag
export -f _ci_tag_check
export -f _ci_tag_check_build
export -f _ci_tag_from_name_to_var
export -f _ci_tag_write
export -f ci_tag_build_time_check
export -f ci_tag_test_time_check
export -f get_tag_file
# Structured tagging ------
s3_upload() {
x_off
local file=$1 s3_folder_url=$2
if [ ! -f "$file" ] || [[ "$s3_folder_url" != https://*/ ]]
then
echo "Error: s3_upload incorrect usage."
echo "Usage: s3_upload <file> <s3_folder_url>"
echo " - <file> must exist."
echo " - <s3_folder_url> must start with 'https://' and end with '/'."
exit 1
fi
curl --fail --retry-all-errors --retry 4 --retry-delay 60 \
--header "Authorization: Bearer $(cat "${S3_JWT_FILE}")" \
-X PUT --form file=@"$file" \
"$s3_folder_url"
x_restore
}
export -f s3_upload
export -f error
export -f trap_err
set -E
trap 'trap_err $?' ERR