From d1caf07271971d0d3936ec87999c65fd9e62c3ae Mon Sep 17 00:00:00 2001 From: wangchong1995924 <15229716099@163.com> Date: Mon, 29 Jul 2024 17:26:03 +0800 Subject: [PATCH] adapt eulermaker --- build.py | 80 ++++------ script/common_function.sh | 42 +----- script/config/repo_conf/obs-repo.conf | 42 ------ script/config/repo_conf/repofile.conf | 11 ++ script/internal.sh | 40 +++-- script/make_version.sh | 14 +- script/setup_env.sh | 113 ++++----------- script/step/get_epol_rpms.sh | 193 +++++++++++-------------- script/step/make_debug_everything.sh | 8 +- script/step/make_desktop_iso.sh | 5 +- script/step/make_docker_image.sh | 5 +- script/step/make_edge_iso.sh | 6 +- script/step/make_hmi.sh | 30 +++- script/step/make_iso.sh | 19 ++- script/step/make_iso_everysrc.sh | 9 +- script/step/make_iso_everything.sh | 9 +- script/step/make_microvm_image.sh | 2 +- script/step/make_netinst_iso.sh | 6 +- script/step/make_raspi_image.sh | 4 +- script/step/make_riscv64_qemu_image.sh | 4 +- script/step/release_dir.sh | 29 ++-- script/tools/chroot.sh | 53 ++++++- 22 files changed, 306 insertions(+), 418 deletions(-) delete mode 100644 script/config/repo_conf/obs-repo.conf create mode 100644 script/config/repo_conf/repofile.conf diff --git a/build.py b/build.py index 50a0972..3f24245 100644 --- a/build.py +++ b/build.py @@ -60,37 +60,24 @@ class Build(object): self.env["ISCI"] = "0" return True - def set_obs_project(self, obs_standard_prj, obs_epol_prj, obs_extras_prj, obs_repo_ip, check_dep): + def set_obs_project(self, standard_prj, standard_prj_repo, epol_prj_repo, check_dep): """ - obs_standard_prj: - obs_epo_prj: - obs_extras_prj: + standard_prj: + standard_prj_repo: + epo_prj_repo: """ - cmd = "sed -i 's/checkdep=.*/checkdep=%s/g' script/setup_env.sh" % check_dep + cmd = "sed -i 's/STANDARD_PROJECT=.*/STANDARD_PROJECT=%s/g' script/setup_env.sh" % standard_prj rmsg = os.popen(cmd).read() print(rmsg) - if obs_repo_ip: - cmd = "sed -i 's/OBS_SERVER_IP=.*/OBS_SERVER_IP=%s/g' script/setup_env.sh" % obs_repo_ip - rmsg = os.popen(cmd).read() - print(rmsg) - cmd = "sed -i 's/OBS_STANDARD_PROJECT=.*/OBS_STANDARD_PROJECT=%s/g' script/setup_env.sh" % obs_standard_prj + cmd = "perl -pi -e 's#STANDARD_PROJECT_REPO=.*#STANDARD_PROJECT_REPO=%s#g' script/setup_env.sh" % standard_prj_repo rmsg = os.popen(cmd).read() print(rmsg) - cmd = "sed -i 's/OBS_EPOL_PROJECT=.*/OBS_EPOL_PROJECT=%s/g' script/setup_env.sh" % obs_epol_prj + cmd = "perl -pi -e 's#EPOL_PROJECT_REPO=.*#EPOL_PROJECT_REPO=%s#g' script/setup_env.sh" % epol_prj_repo rmsg = os.popen(cmd).read() print(rmsg) - cmd = "sed -i 's/OBS_EXTRAS_PROJECT=.*/OBS_EXTRAS_PROJECT=%s/g' script/setup_env.sh" % obs_extras_prj + cmd = "sed -i 's/checkdep=.*/checkdep=%s/g' script/setup_env.sh" % check_dep rmsg = os.popen(cmd).read() print(rmsg) - cmd = "cat script/setup_env.sh | grep OBS_EPOL_MULTI_VERSION_LIST" - rmsg = os.popen(cmd).read().split("=")[1].replace("\n", "") - if not rmsg: - cmd = "osc list | grep ^%s:Multi" % obs_epol_prj - rmsg = os.popen(cmd).read().replace("\n", " ").strip() - cmd = "sed -i 's/OBS_EPOL_MULTI_VERSION_LIST=.*/OBS_EPOL_MULTI_VERSION_LIST=\"{0}\"/g' \ - script/setup_env.sh".format(rmsg) - rmsg = os.popen(cmd).read() - print(rmsg) def clean(self): """ @@ -119,7 +106,11 @@ class Build(object): if os.system(cmd) != 0: logger.error("build fail") return -1 - code = trace_execute("bash -x {0} {1}".format( + cmd = "bash -x reset_release_server_ip.sh" + if os.system(cmd) != 0: + logger.error("build fail") + return -1 + code = trace_execute("bash {0} {1}".format( self.local_build_shell_path, step), env=self.env, logger=logger) if code != 0 and code != "0": logger.error("build fail") @@ -145,27 +136,20 @@ def usage(): print("* Commands: *") print("* all default *") print("* clean *") - print("* make_tar *") - print("* make_hmi *") + print("* set_release_dir *") + print("* get_epol_rpms *") print("* make_iso *") print("* make_edge_iso *") print("* make_desktop_iso *") - print("* make_livecd *") - print("* build_and_wait *") - print("* make_iso_debug *") print("* make_netinst_iso *") - print("* make_compile_env *") - print("* make_euleros_certdb *") - print("* update_release_info *") - print("* make_upgrade_patch *") - print("* make_docker_image *") + print("* make_iso_everysrc *") + print("* make_iso_everything *") + print("* make_debug_everything *") + print("* make_hmi *") print("* make_raspi_image *") - print("* make_riscv64_qemu_image *") - print("* make_container_tools *") - print("* make_tools_othertools *") - print("* make_tools_lib_storage *") - print("* make_tools_debug_tools *") - print("* get_epol_rpms *") + print("* make_docker_image *") + print("* make_microvm_image *") + print("* make_riscv64_qemu_image *") print("* *") print("***********************************") return -1 @@ -176,22 +160,20 @@ if __name__ == "__main__": par = argparse.ArgumentParser() par.add_argument("-i", "--step_info", help="what you want to do", required=True) - par.add_argument("-s", "--obs_standard_prj", - help="obs standard project", required=True) - par.add_argument("-ep", "--obs_epol_prj", - help="obs epol project", required=True) - par.add_argument("-ex", "--obs_extras_prj", - help="obs extras project", required=True) - par.add_argument("-ip", "--obs_repo_ip", - help="obs repo ip", required=False) + par.add_argument("-sp", "--standard_prj", + help="standard project", required=True) + par.add_argument("-spr", "--standard_prj_repo", + help="standard project repo", required=True) + par.add_argument("-epr", "--epol_prj_repo", + help="epol project repo", required=True) par.add_argument("-c", "--check_dep", default="false", help="check rpm dependence", required=False) args = par.parse_args() build = Build() - build.set_obs_project(args.obs_standard_prj, - args.obs_epol_prj, args.obs_extras_prj, - args.obs_repo_ip, args.check_dep) + build.set_obs_project(args.standard_prj, + args.standard_prj_repo, args.epol_prj_repo, + args.check_dep) one_step = args.step_info if one_step == "clean": ret = build.clean() diff --git a/script/common_function.sh b/script/common_function.sh index fc2dcc2..1e84865 100644 --- a/script/common_function.sh +++ b/script/common_function.sh @@ -85,53 +85,23 @@ function run_srcipt_all() done } -#source "${BUILD_SCRIPT_DIR}"/setup_env.sh -source "${BUILD_SCRIPT_DIR}"/make_tools/init.sh -source "${BUILD_SCRIPT_DIR}"/make_tools/common.sh -#source "${BUILD_SCRIPT_DIR}"/make_tools/container_tools.sh -source "${BUILD_SCRIPT_DIR}"/make_tools/debug_tools.sh -source "${BUILD_SCRIPT_DIR}"/make_tools/lib_storage_tools.sh ## x86 -source "${BUILD_SCRIPT_DIR}"/make_tools/docker_tools.sh -#source "${BUILD_SCRIPT_DIR}"/make_tools/other_tool.sh + source "${BUILD_SCRIPT_DIR}"/step/get_version.sh source "${BUILD_SCRIPT_DIR}"/step/release_dir.sh -#source "${BUILD_SCRIPT_DIR}"/step/createrepo.sh +source "${BUILD_SCRIPT_DIR}"/step/get_epol_rpms.sh +source "${BUILD_SCRIPT_DIR}"/step/make_hmi.sh source "${BUILD_SCRIPT_DIR}"/step/make_docker_image.sh -if [ "$STEP" == "make_raspi_image" ];then +if [[ ${STEP} == "make_raspi_image" ]];then source "${BUILD_SCRIPT_DIR}"/step/make_raspi_image.sh -elif [ "$STEP" == "make_riscv64_qemu_image" ];then +elif [[ ${STEP} == "make_riscv64_qemu_image" ]];then source "${BUILD_SCRIPT_DIR}"/step/make_riscv64_qemu_image.sh -else +elif [[ ${STEP} == "make_microvm_image" ]];then source "${BUILD_SCRIPT_DIR}"/step/make_microvm_image.sh fi -source "${BUILD_SCRIPT_DIR}"/step/make_gcov.sh -source "${BUILD_SCRIPT_DIR}"/step/merge_release_html.sh -#source "${BUILD_SCRIPT_DIR}"/step/build_and_wait.sh source "${BUILD_SCRIPT_DIR}"/step/make_iso.sh -source "${BUILD_SCRIPT_DIR}"/step/get_epol_rpms.sh -source "${BUILD_SCRIPT_DIR}"/step/make_iso_debug.sh source "${BUILD_SCRIPT_DIR}"/step/make_netinst_iso.sh source "${BUILD_SCRIPT_DIR}"/step/make_edge_iso.sh source "${BUILD_SCRIPT_DIR}"/step/make_desktop_iso.sh source "${BUILD_SCRIPT_DIR}"/step/make_iso_everything.sh source "${BUILD_SCRIPT_DIR}"/step/make_debug_everything.sh source "${BUILD_SCRIPT_DIR}"/step/make_iso_everysrc.sh -source "${BUILD_SCRIPT_DIR}"/step/make_tar.sh -#source "${BUILD_SCRIPT_DIR}"/step/make_compile_env.sh -#source "${BUILD_SCRIPT_DIR}"/step/make_compile_env_cross.sh -#source "${BUILD_SCRIPT_DIR}"/step/make_livecd.sh -source "${BUILD_SCRIPT_DIR}"/step/make_images_slim.sh -#source "${BUILD_SCRIPT_DIR}"/step/make_euleros_certdb.sh -#source "${BUILD_SCRIPT_DIR}"/step/make_compile_tools.sh -#source "${BUILD_SCRIPT_DIR}"/step/make_upgrade_patch.sh -source "${BUILD_SCRIPT_DIR}"/step/make_hmi.sh -#source "${BUILD_SCRIPT_DIR}"/step/push_lts_dir.sh -### common -source "${BUILD_SCRIPT_DIR}"/step/merge_release_html.sh -#source "${BUILD_SCRIPT_DIR}"/step/make_upload_cmc_image.sh -#source "${BUILD_SCRIPT_DIR}"/step/upload_to_cmc.sh -source "${BUILD_SCRIPT_DIR}"/step/update_release_info.sh -#source "${BUILD_SCRIPT_DIR}"/step/make_cmc_file.sh -### arm -source "${BUILD_SCRIPT_DIR}"/step/make_vm_qcow2.sh -source "${BUILD_SCRIPT_DIR}"/step/mkdliso_hyperstackos.sh diff --git a/script/config/repo_conf/obs-repo.conf b/script/config/repo_conf/obs-repo.conf deleted file mode 100644 index 27805b3..0000000 --- a/script/config/repo_conf/obs-repo.conf +++ /dev/null @@ -1,42 +0,0 @@ -[main] -cachedir=/var/cache/yum/xxx -keepcache=0 -debuglevel=2 -logfile=/var/log/yum.log -exactarch=1 -obsoletes=1 -gpgcheck=1 -plugins=1 -installonly_limit=3 -reposdir=/xxx - -# This is the default, if you make this bigger yum won't see if the metadata -# is newer on the remote and so you'll "gain" the bandwidth of not having to -# download the new metadata and "pay" for it by yum not having correct -# information. -# It is esp. important, to have correct metadata, for distributions like -# Fedora which don't keep old packages around. If you don't like this checking -# interupting your command line usage, it's much better to have something -# manually check the metadata once an hour (yum-updatesd will do this). -# metadata_expire=90m - -# PUT YOUR REPOS HERE OR IN separate files named file.repo -# in /etc/yum.repos.d -[obs-standard] -name=obs-standard -baseurl=http://172.16.1.95:82/openEuler:/20.03:/LTS:/SP1/standard_aarch64/ -enabled=1 -gpgcheck=0 - -[obs-Extras] -name=obs-Extras -baseurl=http://172.16.1.95/openEuler:/20.03:/LTS:/SP1:/Extras/standard_aarch64/ -enabled=1 -gpgcheck=0 - -[obs-Epol] -name=obs-Epol -baseurl=http://172.16.1.95/openEuler:/20.03:/LTS:/SP1:/Epol/standard_aarch64/ -enabled=1 -gpgcheck=0 - diff --git a/script/config/repo_conf/repofile.conf b/script/config/repo_conf/repofile.conf new file mode 100644 index 0000000..6870ba1 --- /dev/null +++ b/script/config/repo_conf/repofile.conf @@ -0,0 +1,11 @@ +[main] +cachedir=/var/cache/yum/xxx +keepcache=0 +debuglevel=2 +logfile=/var/log/yum.log +exactarch=1 +obsoletes=1 +gpgcheck=1 +plugins=1 +installonly_limit=3 +reposdir=/xxx diff --git a/script/internal.sh b/script/internal.sh index 16d72b8..c79854c 100644 --- a/script/internal.sh +++ b/script/internal.sh @@ -1,34 +1,30 @@ #!/bin/bash set -e -if echo "${BUILD_SCRIPT_DIR}" | grep 'gcov-'; then - export CI_PROJECT="openeuler_${ARCH}_gcov" -else - export CI_PROJECT="openeuler_${ARCH}" +if [[ ${ARCH} == "aarch64" ]];then + arch_name="ARM64" +elif [[ ${ARCH} == "x86_64" ]];then + arch_name="X86" +elif [[ ${ARCH} == "loongarch64" ]];then + arch_name="LOONGARCH64" +elif [[ ${ARCH} == "riscv64" ]];then + arch_name="RISCV64" +elif [[ ${ARCH} == "ppc64le" ]];then + arch_name="PPC64LE" fi -if [ "x${arm2x86}" = "xx86_64" ]; then - export CI_PROJECT=$(echo "$CI_PROJECT" |sed 's/x86_64/aarch64/') +if echo "${BUILD_SCRIPT_DIR}" | grep 'gcov-'; then + export CI_PROJECT="openeuler_${arch_name}_gcov" +else + export CI_PROJECT="openeuler_${arch_name}" fi -export CI_PROJECT=$(echo "$CI_PROJECT" |sed 's/x86_64/X86/') -export CI_PROJECT=$(echo "$CI_PROJECT" |sed 's/aarch64/ARM64/') export WORK_DIR="${OUTPUT_PATH}/work_dir/${CI_PROJECT}/" -#configure for release to remote -set +ue -export RELEASE_ROOT_PATH=${RELEASE_ROOT_PATH} -if [ "x${jenkins_build}" != "x" ]; then - export USER="" - export PASSWD="" - export HTTP_DIR=${RELEASE_ROOT_PATH} - export PRE_VERSION=${RELEASE_VERSION_DIR} -else - export IP="127.0.0.1" - export HTTP_DIR="${OUTPUT_PATH}/release" - export PRE_VERSION="openeuler/${MYVERSION}" -fi -set -ue +export USER="" +export PASSWD="" +export HTTP_DIR=${RELEASE_ROOT_PATH} +export PRE_VERSION=${RELEASE_VERSION_DIR} export VERSION="${CI_PROJECT}" export CMC_BASEDIR="/usr1" diff --git a/script/make_version.sh b/script/make_version.sh index ca0c1c2..9813ad9 100644 --- a/script/make_version.sh +++ b/script/make_version.sh @@ -16,7 +16,6 @@ if [ ! -d "${OUTPUT_PATH}" ]; then mkdir -p "${OUTPUT_PATH}" fi - if [ "${ISCI}" = "1" ]; then STARTTIME=$(perl -e 'print time()') exec 1> >(exec -a 'build logging' perl -e '$|=1;select(F);$|=1;while(){my $p=sprintf("[%5ds] ", time()-'"${STARTTIME}"');print STDOUT $p.$_;}') 2>&1 @@ -27,18 +26,9 @@ source "${BUILD_SCRIPT_DIR}"/internal.sh || exit 1 source "${BUILD_SCRIPT_DIR}"/tools/util.sh || exit 1 source "${BUILD_SCRIPT_DIR}"/tools/common.sh || exit 1 source "${BUILD_SCRIPT_DIR}"/tools/chroot.sh || exit 1 -#source "${BUILD_SCRIPT_DIR}"/tools/obs.sh || exit 1 - -#sh "${BUILD_SCRIPT_DIR}"/tools/safe_sshcmd.sh -c "ip add | grep 172 | awk -F '/' '{print \$1}'| awk '{print \$2}'" -m ${RELEASE_SERVER_IP} -u "root" -p "xxxx" -t 120 -n 3 -#if echo "${CI_PROJECT}" | grep '_gcov'; then -# modify_for_gcov "${BUILD_SCRIPT_DIR}" -# source "${BUILD_SCRIPT_DIR}"/config.sh || exit 1 -#fi source "${BUILD_SCRIPT_DIR}"/common_function.sh || exit 1 export EXCLUDE_REGISTER_SCRIPT=('set_release_dir') -sed -i 's/container=.*>/container=\"'${CONTAINER_NAME}'\">/g' "${BUILD_SCRIPT_DIR}"/config/docker_image/config.xml - ###################### # 使用说明 # Globals: @@ -50,11 +40,9 @@ function usage() echo "$(basename $0) [script_name|all]" } -export ALL_SCRIPT=('build_and_wait' 'update_release_info' 'make_tar' 'make_hmi' 'make_iso' 'make_iso_debug' 'make_iso_singleoss' 'make_livecd' 'make_compile_env' 'make_docker_image' 'make_raspi_image' 'make_riscv64_qemu_image' 'make_microvm_image' 'make_euleros_certdb' 'make_tools_lib_storage' 'make_container_tools' 'make_tools_debug_tools' 'make_upgrade_patch' 'make_iso_everything' 'make_iso_everysrc' 'make_debug_everything' 'push_lts_dir' 'make_netinst_iso' 'get_epol_rpms' 'make_edge_iso' 'make_desktop_iso') +export ALL_SCRIPT=('make_hmi' 'make_iso' 'make_docker_image' 'make_raspi_image' 'make_riscv64_qemu_image' 'make_microvm_image' 'make_iso_everything' 'make_iso_everysrc' 'make_debug_everything' 'make_netinst_iso' 'get_epol_rpms' 'make_edge_iso' 'make_desktop_iso' 'make_riscv64_image') [[ "${DEBUG}" -eq 1 ]] && set -x -#check_env - arg1="$1" shift arg2="$@" diff --git a/script/setup_env.sh b/script/setup_env.sh index 0df6abf..2dbdb6e 100644 --- a/script/setup_env.sh +++ b/script/setup_env.sh @@ -1,18 +1,25 @@ #======= release version info ====== -export OBS_STANDARD_PROJECT="openEuler:Mainline" -export OBS_EPOL_PROJECT="openEuler:Epol" -export OBS_EXTRAS_PROJECT="openEuler:Extras" -export OBS_EPOL_MULTI_VERSION_LIST= -#=================================== -export OPENEULER_CHROOT_PATH="/usr1/openeuler" +export STANDARD_PROJECT= +export STANDARD_PROJECT_REPO= +export EPOL_PROJECT_REPO= +export THIRD_REPO= + +export checkdep=true +export ARCH=$(uname -m) export OS_NAME="openEuler" -tmp="$(echo ${OBS_STANDARD_PROJECT#*:})" +export PRODUCTS=${OS_NAME} +tmp="$(echo ${STANDARD_PROJECT#*:})" export OS_VERSION="$(echo ${tmp//:/-})" +export SHA256SUM=".sha256sum" +export OPENEULER_CHROOT_PATH="/usr1/openeuler" -export OBS_UI_IP="172.16.1.81" -export OBS_SERVER_IP="172.16.1.95" export RELEASE_SERVER_IP="172.16.1.236" export RELEASE_SERVER_PORT="30322" +if [[ ${ARCH} == "loongarch64" ]] || [[ ${ARCH} == "ppc64le" ]];then + export RELEASE_SERVER_IP="121.36.84.172" + export RELEASE_SERVER_PORT="" +fi +export RELEASE_HTTP_URL="http://121.36.84.172/dailybuild" if [ -n "${RELEASE_SERVER_PORT}" ];then export SSHPORT="-p ${RELEASE_SERVER_PORT}" export SCPPORT="-P ${RELEASE_SERVER_PORT}" @@ -20,55 +27,20 @@ else export SSHPORT="" export SCPPORT="" fi -export ARCH=$(uname -m) -export RELEASE_HTTP_URL="http://121.36.84.172/dailybuild" - -SUB_STANDARD_REPO_URL="$(echo ${OBS_STANDARD_PROJECT//:/:\/})" -SUB_EPOL_REPO_URL="$(echo ${OBS_EPOL_PROJECT//:/:\/})" -SUB_EXTRAS_REPO_URL="$(echo ${OBS_EXTRAS_PROJECT//:/:\/})" -if [[ "${OBS_STANDARD_PROJECT}" =~ "RISC-V" ]] ;then - export OBS_STANDARD_REPO_URL="http://${OBS_SERVER_IP}:82/${SUB_STANDARD_REPO_URL}/BaseOS" -else - export OBS_STANDARD_REPO_URL="http://${OBS_SERVER_IP}:82/${SUB_STANDARD_REPO_URL}/standard_${ARCH}" -fi -export OBS_EPOL_REPO_URL="http://${OBS_SERVER_IP}:82/${SUB_EPOL_REPO_URL}/standard_${ARCH}" -export OBS_EXTRAS_REPO_URL="http://${OBS_SERVER_IP}:82/${SUB_EXTRAS_REPO_URL}/standard_${ARCH}" -export OBS_BRINGINRELY_URL= -export OBS_STANDARD_THIRD_REPO_URL= - -obs_master_project=(openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Erlang openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Perl openEuler:Python openEuler:Qt openEuler:Ruby) -if [[ ${OBS_STANDARD_PROJECT} == "openEuler:Mainline" ]];then - for p in ${obs_master_project[@]} - do - tmp="$(echo ${p//:/:\/})" - tmp_url="http://${OBS_SERVER_IP}:82/${tmp}/standard_${ARCH}" - TMP_STANDARD_REPO_URL="${tmp_url} ${TMP_STANDARD_REPO_URL}" - done - export OBS_STANDARD_REPO_URL="${TMP_STANDARD_REPO_URL}${OBS_STANDARD_REPO_URL}" -fi - export RELEASE_ROOT_PATH="/repo/openeuler/dailybuild" -export RELEASE_VERSION_DIR="${OS_NAME}-${OS_VERSION}" +export RELEASE_VERSION_DIR="EBS-${OS_NAME}-${OS_VERSION}" export RELEASE_ARCH_DIR="openEuler_${ARCH}" -export CONTAINER_NAME=$(echo ${RELEASE_VERSION_DIR} | tr A-Z a-z) -export VM_IMAGE_NAME="${RELEASE_VERSION_DIR}-${ARCH}.qcow2" -export RASPI_IMAGE_NAME="${RELEASE_VERSION_DIR}-raspi-${ARCH}.img" -export RISCV64_QEMU_IMAGE_NAME="${RELEASE_VERSION_DIR}-qemu-${ARCH}.qcow2" -export MICROVM_IMAGE_NAME="${RELEASE_VERSION_DIR}-stratovirt-${ARCH}.img" +export CONTAINER_NAME=$(echo ${OS_NAME}-${OS_VERSION} | tr A-Z a-z) +export VM_IMAGE_NAME="${OS_NAME}-${OS_VERSION}-${ARCH}.qcow2" +export RASPI_IMAGE_NAME="${OS_NAME}-${OS_VERSION}-raspi-${ARCH}.img" +export RISCV64_QEMU_IMAGE_NAME="${OS_NAME}-${OS_VERSION}-qemu-${ARCH}.qcow2" +export MICROVM_IMAGE_NAME="${OS_NAME}-${OS_VERSION}-stratovirt-${ARCH}.img" export MICROVM_KERNEL_NAME="vmlinux.bin" export STDANDARD_VM_KERNEL_NAME="std-vmlinux" -export PRODUCTS=${OS_NAME} - -export SHA256SUM=".sha256sum" - -export jenkins_build="1" - -export checkdep=true - -OBS_REPO_CONF=$(find -iname "obs-repo.conf") -cat>${OBS_REPO_CONF}<<-EOF +REPO_CONF=$(find -iname "repofile.conf") +cat>${REPO_CONF}<<-EOF [main] cachedir=/var/cache/yum/xxx keepcache=0 @@ -81,41 +53,18 @@ plugins=1 installonly_limit=3 reposdir=/xxx -[obs-Extras] -name=obs-Extras -baseurl=${OBS_EXTRAS_REPO_URL}/ +[standard] +name=standard +baseurl=${STANDARD_PROJECT_REPO} enabled=1 gpgcheck=0 -[obs-Epol] -name=obs-Epol -baseurl=${OBS_EPOL_REPO_URL}/ +[epol] +name=epol +baseurl=${EPOL_PROJECT_REPO} enabled=1 gpgcheck=0 EOF -if [[ ${OBS_STANDARD_PROJECT} == "openEuler:Mainline" ]];then - i=1 - for repo_url in ${OBS_STANDARD_REPO_URL[@]} - do - repo_name="obs_standard_${i}" - cat>>${OBS_REPO_CONF}<<-EOF - [${repo_name}] - name=${repo_name} - baseurl=${repo_url}/ - enabled=1 - gpgcheck=0 - EOF - let i+=1 - done -else - cat>>${OBS_REPO_CONF}<<-EOF - [obs-standard] - name=obs-standard - baseurl=${OBS_STANDARD_REPO_URL}/ - enabled=1 - gpgcheck=0 - EOF -fi -cat ${OBS_REPO_CONF} +cat ${REPO_CONF} diff --git a/script/step/get_epol_rpms.sh b/script/step/get_epol_rpms.sh index 358825d..e862595 100644 --- a/script/step/get_epol_rpms.sh +++ b/script/step/get_epol_rpms.sh @@ -3,110 +3,35 @@ # Copyright Huawei Technologies Co., Ltd. 2010-2018. All rights reserved. set -e -###################### -# 构建iso镜像 -# Globals: -# Arguments: -# Returns: -###################### -function get_epol_rpms() +function check_rpm_sign() { - chroot_run "cd /home; bash make_version.sh get_epol_rpms_inchroot" -} -###################### -# get excluded packages binary rpms -# Globals: -# Arguments: -# Returns: -###################### -function get_excluded_pkg_rpm() -{ - obs_project="${OBS_EPOL_PROJECT}" - osc prjresults ${obs_project} --csv 2>/dev/null | grep "excluded" > excluded_lst - > aarch64_epol_excluded_lst - > x86_64_epol_excluded_lst - excluded_msgs=`cat excluded_lst` - for excluded_msg in ${excluded_msgs} + > /tmp/not_sign_rpm + for pkg in `ls $1/Packages/*.rpm` do - pkg_name=$(echo $excluded_msg | awk -F";" '{print $1}') - a_status=$(echo $excluded_msg | awk -F";" '{print $2}') - x_status=$(echo $excluded_msg | awk -F";" '{print $3}') - if [[ ${a_status} == "excluded" ]];then - arch="x86_64" - elif [[ ${x_status} == "excluded" ]];then - arch="aarch64" - else - log_warning "${pkg_name} not excluded" - continue - fi - rpmlist=$(osc ls -b ${obs_project} ${pkg_name} standard_${arch} ${arch} 2>/dev/null | grep "\.src.rpm") - if [[ ${rpmlist} != "" ]];then - for rpm_name in ${rpmlist[@]} - do - tmp=${rpm_name%.*.rpm*} - echo $tmp >> ${arch}_epol_excluded_lst - done + rpm -Kv ${pkg} | grep "key ID [a-z0-9]*: OK" > /dev/null + if [[ $? -ne 0 ]];then + echo "${pkg}" >> /tmp/not_sign_rpm fi done - rm -rf excluded_lst + if [[ -s "/tmp/not_sign_rpm" ]];then + echo "[Error]: some rpm is not sign" + cat /tmp/not_sign_rpm + exit 1 + fi + rm -f /tmp/not_sign_rpm } + ###################### -# wait project build stable +# 构建iso镜像 # Globals: # Arguments: # Returns: ###################### -function wait_project_published() +function get_epol_rpms() { - set +e - waitime=3600 - obs_project="${OBS_EPOL_PROJECT}" - while [ $waitime -gt 0 ] - do - osc prjresults ${obs_project} --csv 2>/dev/null | grep "aarch64/published" | grep "x86_64/published" - if [ $? -eq 0 ];then - log_info "${obs_project} is published" - break - fi - let waitime=$waitime-5 - sleep 5 - done - if [ $waitime -eq 0 ];then - log_error "get_epol_rpms fail" - fi - get_excluded_pkg_rpm - temp_dir="/tmp/compare_list_dir" - rm -rf ${temp_dir} && mkdir -p ${temp_dir} - mv /etc/yum.repos.d /etc/yum.repos.d.bak - archs="aarch64 x86_64" - for arch in $archs - do - rm -rf /etc/yum.repos.d/* - yum-config-manager --add-repo "${OBS_EPOL_REPO_URL%/*}/standard_${arch}" - yum clean all - yum list --installroot="${temp_dir}" --available | awk '{print $1,$2}' | grep "\.src" > ${arch}_epol_lst - sed -i -e 's/\.src /-/' ${arch}_epol_lst - sort -u ${arch}_epol_lst -o ${arch}_epol_lst - excluded_rpms=`cat ${arch}_epol_excluded_lst` - for excluded_rpm in ${excluded_rpms} - do - sed -i "/^${excluded_rpm}/d" ${arch}_epol_lst - done - rm -rf ${arch}_epol_excluded_lst "${temp_dir}/var" - done - rm -rf ${temp_dir} - rm -rf /etc/yum.repos.d - mv /etc/yum.repos.d.bak /etc/yum.repos.d - diff -Nur aarch64_epol_lst x86_64_epol_lst - if [ $? -eq 0 ];then - rm -rf aarch64_epol_lst x86_64_epol_lst - log_info "obs repo two architecture rpms version or release are same" - else - rm -rf aarch64_epol_lst x86_64_epol_lst - log_error "obs repo two architecture rpms version or release are not same" - fi - set -e + chroot_run "cd /home; bash make_version.sh get_epol_rpms_inchroot" } + ###################### # 在chroot中构建iso镜像 # Globals: @@ -122,18 +47,15 @@ function get_epol_rpms_inchroot() TIME=${TIME#"${version}"-} CUSTOM_DIR="${TIME_DIR}" RELEASE_DIR="${release_dir}/EPOL" + set +e mv /etc/yum.repos.d /etc/yum.repos.d.bak mkdir -p /etc/yum.repos.d /tmp/EPOL/${ARCH}/Packages "/tmp/EPOL/source/Packages" - if [ -n "${OBS_BRINGINRELY_URL}" ];then - bringinrely_repo="--add-repo ${OBS_BRINGINRELY_URL}" - else - bringinrely_repo="" - fi - # main standard - yum-config-manager --add-repo "${OBS_EPOL_REPO_URL}" ${bringinrely_repo} + rpm --import /home/RPM-GPG-KEY-EBS + + yum-config-manager --add-repo ${EPOL_PROJECT_REPO} yum clean all - SSH_CMD="mkdir -p ${RELEASE_DIR}/main" + SSH_CMD="rm -rf ${RELEASE_DIR};mkdir -p ${RELEASE_DIR}/main" sshcmd "${SSH_CMD}" tmp_dir="/tmp/EPOL/main/${ARCH}" mkdir -p ${tmp_dir}/Packages @@ -141,15 +63,20 @@ function get_epol_rpms_inchroot() unrpms=`cat ${UNABLE_INSTALL_LIST}` for unrpm in ${unrpms} do - sed -i "/^${unrpm}\./d" ava_epol_lst + sed -i "/^${unrpm}\./d" ava_epol_lst done + if [ ! -s ava_epol_lst ];then + echo "There don't have some rpms in the repo" + exit 1 + fi yumdownloader --installroot="${tmp_dir}/Packages" --destdir="${tmp_dir}/Packages" $(cat ava_epol_lst | tr '\n' ' ') rm -rf ${tmp_dir}/Packages/var + check_rpm_sign ${tmp_dir} createrepo -d ${tmp_dir} sshscp "${tmp_dir}" "${RELEASE_DIR}/main/" if [[ "$ARCH" == "aarch64" ]];then rm -rf /etc/yum.repos.d/* - yum-config-manager --add-repo "${OBS_EPOL_REPO_URL%/*}/standard_aarch64" --add-repo "${OBS_EPOL_REPO_URL%/*}/standard_x86_64" + yum-config-manager --add-repo "${EPOL_PROJECT_REPO%/*}/aarch64" --add-repo "${EPOL_PROJECT_REPO%/*}/x86_64" yum clean all tmp_source="/tmp/EPOL/main/source" mkdir -p ${tmp_source}/Packages @@ -157,10 +84,15 @@ function get_epol_rpms_inchroot() unrpms=`cat ${UNABLE_INSTALL_SOURCE_LIST}` for unrpm in ${unrpms} do - sed -i "/^${unrpm}\./d" ava_epol_lst + sed -i "/^${unrpm}\./d" ava_epol_lst done + if [ ! -s ava_epol_lst ];then + echo "There don't have some rpms in the repo" + exit 1 + fi yumdownloader --installroot="${tmp_source}/Packages" --destdir="${tmp_source}/Packages" --source $(cat ava_epol_lst | tr '\n' ' ') rm -rf ${tmp_source}/Packages/var + check_rpm_sign ${tmp_source} createrepo -d ${tmp_source} sshscp "${tmp_source}" "${RELEASE_DIR}/main/" SSH_CMD="mkdir -p ${RELEASE_DIR}/update/main/source/Packages && createrepo -d ${RELEASE_DIR}/update/main/source" @@ -169,25 +101,59 @@ function get_epol_rpms_inchroot() sshcmd "${SSH_CMD}" fi # multi version - if [[ ${OBS_EPOL_MULTI_VERSION_LIST} != "" ]];then - for r in ${OBS_EPOL_MULTI_VERSION_LIST} + if [[ ${EPOL_MULTI_VERSION_LIST} != "" ]];then + for r in ${EPOL_MULTI_VERSION_LIST} do - SUB_EPOL_MULTI_REPO_URL="$(echo ${r//:/:\/})" TMP=`echo ${r%:*}` PKG=`echo ${TMP##*:}` VER=`echo ${r##*:}` tmp_dir="/tmp/EPOL/multi_version/${PKG}/${VER}/${ARCH}" mkdir -p ${tmp_dir}/Packages - repo_url="http://${OBS_SERVER_IP}:82/${SUB_EPOL_MULTI_REPO_URL}/standard_${ARCH}" + if [[ "${VER}" == "Train" ]];then + repo_name="${Train_repo}" + fi + if [[ "${VER}" == "Wallaby" ]];then + repo_name="${Wallaby_repo}" + fi + if [[ "${VER}" == "Antelope" ]];then + repo_name="${Antelope_repo}" + fi + if [[ "${VER}" == "humble" ]];then + repo_name="${humble_repo}" + fi + if [[ "${VER}" == "noetic" ]];then + repo_name="${noetic_repo}" + fi + if [[ "${VER}" == "For-Virt" ]];then + repo_name="${nestos_for_virt_repo}" + fi + if [[ "${VER}" == "For-Container" ]];then + repo_name="${nestos_for_container_repo}" + fi + if [[ "${PKG}" == "kubernetes" ]];then + repo_name="${kubernetes_repo}" + fi + if [[ "${PKG}" == "lustre" ]];then + repo_name="${lustre_repo}" + fi + repo_url="${repo_name}/${ARCH}" rm -rf /etc/yum.repos.d/* yum-config-manager --add-repo ${repo_url} yum clean all yum list --installroot="${tmp_dir}/Packages" --available | awk '{print $1}' | grep -E "noarch|${ARCH}" | grep -v "\.src" > ava_epol_lst - unrpms=`cat ${UNABLE_INSTALL_LIST}_${VER}` + if [[ "${PKG}" == "kubernetes" ]] || [[ "${PKG}" == "lustre" ]];then + unrpms=`cat ${UNABLE_INSTALL_LIST}_${PKG}` + else + unrpms=`cat ${UNABLE_INSTALL_LIST}_${VER}` + fi for unrpm in ${unrpms} do sed -i "/^${unrpm}\./d" ava_epol_lst done + if [ ! -s ava_epol_lst ];then + echo "[Warning]: ${repo_url} don't have rpm." + continue + fi yumdownloader --installroot="${tmp_dir}/Packages" --destdir="${tmp_dir}/Packages" $(cat ava_epol_lst | tr '\n' ' ') rm -rf ${tmp_dir}/Packages/var createrepo -d ${tmp_dir} @@ -197,9 +163,14 @@ function get_epol_rpms_inchroot() if [[ "${ARCH}" == "aarch64" ]];then tmp_source="/tmp/EPOL/multi_version/${PKG}/${VER}/source" mkdir -p ${tmp_source}/Packages - yum-config-manager --add-repo "http://${OBS_SERVER_IP}:82/${SUB_EPOL_MULTI_REPO_URL}/standard_x86_64" - yum list --installroot="${tmp_source}/Packages" --available | awk '{print $1}' | grep "\.src" > ava_epol_lst - unrpms=`cat ${UNABLE_INSTALL_SOURCE_LIST}_${VER}` + yum-config-manager --add-repo "${repo_name}/x86_64" + yum list --installroot="${tmp_source}/Packages" --available | awk '{print $1}' | grep -E "noarch|${ARCH}" | grep -v "\.src" > ava_epol_lst + + if [[ "${PKG}" == "kubernetes" ]] || [[ "${PKG}" == "lustre" ]];then + unrpms=`cat ${UNABLE_INSTALL_SOURCE_LIST}_${PKG}` + else + unrpms=`cat ${UNABLE_INSTALL_SOURCE_LIST}_${VER}` + fi for unrpm in ${unrpms} do sed -i "/^${unrpm}\./d" ava_epol_lst diff --git a/script/step/make_debug_everything.sh b/script/step/make_debug_everything.sh index a95cc95..e8eb40b 100644 --- a/script/step/make_debug_everything.sh +++ b/script/step/make_debug_everything.sh @@ -28,7 +28,7 @@ function make_debug_everything_inchroot() TIME=${TIME_DIR##*/} TIME=${TIME#"${version}"-} - yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" + yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" yum clean all -c "${yum_conf}" if rpm -q oemaker &> /dev/null; then yum remove oemaker -y @@ -44,7 +44,7 @@ function make_debug_everything_inchroot() num=0 while [ "${num}" -lt 3 ] do - bash -x oemaker -t everything_debug -p ${PRODUCTS} -v "${OS_VERSION}" -r "" -s "${OBS_STANDARD_REPO_URL}" + bash -x oemaker -t everything_debug -p ${PRODUCTS} -v "${OS_VERSION}" -r "" -s "${STANDARD_PROJECT_REPO}" if [ $? -eq 0 ];then break elif [ $? -eq 133 ]; then @@ -68,10 +68,6 @@ function make_debug_everything_inchroot() fi create_checksum "${TGZ_NAME}" - #TIME_DIR="${PRE_VERSION}/${VERSION}/${version}-${TIME}" - #log_info "${HTTP_DIR}/${TIME_DIR}" > "${WORK_DIR}"releasedir_info - CUSTOM_DIR="${TIME_DIR}" - #RELEASE_DIR="${HTTP_DIR}/${CUSTOM_DIR}" RELEASE_DIR="${release_dir}/ISO/$ARCH" MOUNT_DIR="${release_dir}/debuginfo/$ARCH" SSH_CMD="mkdir -p ${RELEASE_DIR} ${MOUNT_DIR}" diff --git a/script/step/make_desktop_iso.sh b/script/step/make_desktop_iso.sh index e78784e..c873882 100644 --- a/script/step/make_desktop_iso.sh +++ b/script/step/make_desktop_iso.sh @@ -26,7 +26,7 @@ function make_desktop_iso_inchroot() TIME_DIR="${release_dir#${HTTP_DIR}}" TIME=${TIME_DIR##*/} TIME=${TIME#"${version}"-} - yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" + yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" yum clean all -c "${yum_conf}" if rpm -q oemaker &> /dev/null; then yum remove oemaker -y @@ -36,7 +36,7 @@ function make_desktop_iso_inchroot() fi yum install oemaker lorax -y -c "${yum_conf}" cd /opt/oemaker - REPOS=`echo "${OBS_STANDARD_REPO_URL} ${OBS_EPOL_REPO_URL} ${OBS_STANDARD_THIRD_REPO_URL}" | sed 's/[ \t]*$//g'` + REPOS=`echo "${STANDARD_PROJECT_REPO} ${EPOL_PROJECT_REPO} ${THIRD_REPO}" | sed 's/[ \t]*$//g'` set +e num=0 set +u @@ -70,7 +70,6 @@ function make_desktop_iso_inchroot() cd ../.. umount temp [ -n temp ] && rm -rf temp - CUSTOM_DIR="${TIME_DIR}" RELEASE_DIR="${release_dir}/workstation/$ARCH" SSH_CMD="mkdir -p ${RELEASE_DIR}" sshcmd "${SSH_CMD}" diff --git a/script/step/make_docker_image.sh b/script/step/make_docker_image.sh index 0f25b22..6c16fe8 100644 --- a/script/step/make_docker_image.sh +++ b/script/step/make_docker_image.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" +yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" dogsheng_arch="$(uname -m)" function kiwi_init() { @@ -88,13 +88,14 @@ function make_docker_image_inchroot() # build sed -i "s#IMAGE_NAME#${version_time}#" "${BUILD_SCRIPT_DIR}"/config/docker_image/config.xml + sed -i 's/container=.*>/container=\"'${CONTAINER_NAME}'\">/g' "${BUILD_SCRIPT_DIR}"/config/docker_image/config.xml MOUNT_DIR=$(echo "${release_dir}" | cut -d '/' -f 5-9) mkdir -p /mnt/EulerOS TMPDIR=$(mktemp '/mnt/EulerOS/docker-XXXX')-$(date +%F-%T) mkdir -p "${TMPDIR}" - for repo_url in $(echo "${OBS_STANDARD_REPO_URL}" | xargs) + for repo_url in $(echo "${STANDARD_PROJECT_REPO}" | xargs) do sed -i "/obs_repo_here/a " "${BUILD_SCRIPT_DIR}"/config/docker_image/config.xml done diff --git a/script/step/make_edge_iso.sh b/script/step/make_edge_iso.sh index cb456ac..39a9330 100644 --- a/script/step/make_edge_iso.sh +++ b/script/step/make_edge_iso.sh @@ -26,7 +26,7 @@ function make_edge_iso_inchroot() TIME_DIR="${release_dir#${HTTP_DIR}}" TIME=${TIME_DIR##*/} TIME=${TIME#"${version}"-} - yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" + yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" yum clean all -c "${yum_conf}" if rpm -q oemaker &> /dev/null; then yum remove oemaker -y @@ -36,7 +36,7 @@ function make_edge_iso_inchroot() fi yum install oemaker lorax linux-firmware -y -c "${yum_conf}" cd /opt/oemaker - REPOS=`echo "${OBS_STANDARD_REPO_URL} ${OBS_EPOL_REPO_URL} ${OBS_STANDARD_THIRD_REPO_URL}" | sed 's/[ \t]*$//g'` + REPOS=`echo "${STANDARD_PROJECT_REPO} ${EPOL_PROJECT_REPO} ${THIRD_REPO}" | sed 's/[ \t]*$//g'` set +e num=0 set +u @@ -63,7 +63,6 @@ function make_edge_iso_inchroot() TGZ_NAME=$(ls *"${ARCH}"-dvd.iso) if [ x"${TGZ_NAME}" == x'' ]; then log_error "can not find iso";fi create_checksum "${TGZ_NAME}" - #log_info "${HTTP_DIR}/${TIME_DIR}" > "${WORK_DIR}"releasedir_info iso_rpmlist="${OS_NAME}-${OS_VERSION}-edge-${ARCH}.rpmlist" mkdir temp && mount *"${ARCH}"-dvd.iso temp cd temp/Packages @@ -71,7 +70,6 @@ function make_edge_iso_inchroot() cd ../.. umount temp [ -n temp ] && rm -rf temp - CUSTOM_DIR="${TIME_DIR}" RELEASE_DIR="${release_dir}/edge_img/$ARCH" SSH_CMD="mkdir -p ${RELEASE_DIR}" sshcmd "${SSH_CMD}" diff --git a/script/step/make_hmi.sh b/script/step/make_hmi.sh index d9d2ff8..21e1e87 100644 --- a/script/step/make_hmi.sh +++ b/script/step/make_hmi.sh @@ -11,13 +11,35 @@ function make_hmi_inchroot() local counter=0 version="openeuler" RESULT_HMI="/usr1/mkeuleros/result_hmi" + yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" [ -d "${RESULT_HMI}" ] && rm -rf "${RESULT_HMI}/*" mkdir -p "${RESULT_HMI}" + + HMI_REPOS="${STANDARD_PROJECT_REPO}" + yum clean all -c ${yum_conf} + yum install -y qemu-img bc sudo parted dosfstools e2fsprogs xz -c ${yum_conf} + + rm -rf CreateImage + git clone https://gitee.com/openeuler/CreateImage.git + if [ $? -ne 0 ];then + echo "[ERROR] git clone CreateImage failed" + exit 1 + fi + sed -i '/#disbale other repos/i \cp /etc/resolv.conf ${TARGET_ROOT}/etc/' CreateImage/hooks/root.d/01-create-root + sed -i '/most reliable/i \rm -f ${TARGET_ROOT}/etc/resolv.conf' CreateImage/hooks/root.d/01-create-root + sed -i '47a \cat ${repo_config}' CreateImage/hooks/root.d/01-create-root + + yumdownloader kernel -c ${yum_conf} + kernel_version=$(rpm -qi kernel-* | grep Version | awk '{print $NF}') + if [[ "${kernel_version}" > "6.1.0" ]] || [[ "${kernel_version}" == "6.1.0" ]];then + sed -i 's/nomodeset//g' CreateImage/hooks/finalise.d/50-bootloader + fi + rm -f kernel-*.rpm + rm -rf /usr/share/CreateImage && mkdir -p /usr/share/CreateImage + cp CreateImage/bin/* /usr/bin/ + cp -a CreateImage/lib CreateImage/hooks CreateImage/config /usr/share/CreateImage + rm -rf CreateImage - HMI_REPOS=$(echo "${OBS_STANDARD_REPO_URL}") - #HMI_REPOS=$(echo "${OBS_STANDARD_REPO_URL}" | sed 's/ / -r /g') - yum -c "${BUILD_SCRIPT_DIR}"/config/repo_conf/obs-repo.conf clean all - yum install -y CreateImage sudo parted dosfstools e2fsprogs -c "${BUILD_SCRIPT_DIR}"/config/repo_conf/obs-repo.conf pushd "${RESULT_HMI}" set +e chmod 755 /usr/bin/create-image diff --git a/script/step/make_iso.sh b/script/step/make_iso.sh index 419b22b..f7e7400 100644 --- a/script/step/make_iso.sh +++ b/script/step/make_iso.sh @@ -26,7 +26,7 @@ function make_iso_inchroot() TIME_DIR="${release_dir#${HTTP_DIR}}" TIME=${TIME_DIR##*/} TIME=${TIME#"${version}"-} - yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" + yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" yum clean all -c "${yum_conf}" if rpm -q oemaker &> /dev/null; then yum remove oemaker -y @@ -36,7 +36,7 @@ function make_iso_inchroot() fi yum install oemaker lorax -y -c "${yum_conf}" cd /opt/oemaker - REPOS=`echo "${OBS_STANDARD_REPO_URL} ${OBS_STANDARD_THIRD_REPO_URL}" | sed 's/[ \t]*$//g'` + REPOS=`echo "${STANDARD_PROJECT_REPO} ${THIRD_REPO}" | sed 's/[ \t]*$//g'` set +e num=0 set +u @@ -63,20 +63,29 @@ function make_iso_inchroot() TGZ_NAME=$(ls *"${ARCH}"-dvd.iso) if [ x"${TGZ_NAME}" == x'' ]; then log_error "can not find iso";fi create_checksum "${TGZ_NAME}" - #log_info "${HTTP_DIR}/${TIME_DIR}" > "${WORK_DIR}"releasedir_info iso_rpmlist="${OS_NAME}-${OS_VERSION}-${ARCH}.rpmlist" + iso_srclist="baseos_bin_2_src.csv" mkdir temp && mount *"${ARCH}"-dvd.iso temp cd temp/Packages ls *.rpm > "../../${iso_rpmlist}" + echo "bin,src,version,repo" > "../../${iso_srclist}" + for p in $(cat "../../${iso_rpmlist}") + do + bin=$(echo ${p%-*-*}) + src_rpm=$(rpm -qi $p | grep "Source RPM" | awk '{print $NF}') + src=$(echo ${src_rpm%-*-*}) + tmp_version=$(rpm -q $p --queryformat %{VERSION}-%{RELEASE}) + version=$(echo ${tmp_version%.oe*}) + echo "${bin},${src},${version},baseos" >> "../../${iso_srclist}" + done cd ../.. umount temp [ -n temp ] && rm -rf temp - CUSTOM_DIR="${TIME_DIR}" RELEASE_DIR="${release_dir}/ISO/$ARCH" MOUNT_DIR="${release_dir}/OS/$ARCH" SSH_CMD="mkdir -p ${RELEASE_DIR} ${MOUNT_DIR}" sshcmd "${SSH_CMD}" - sshscp "${TGZ_NAME} ${TGZ_NAME}${SHA256SUM} ${iso_rpmlist}" "${RELEASE_DIR}" + sshscp "${TGZ_NAME} ${TGZ_NAME}${SHA256SUM} ${iso_rpmlist} ${iso_srclist}" "${RELEASE_DIR}" set +e ret=$(get_repose ssh -i ~/.ssh/super_publish_rsa ${SSHPORT} root@${RELEASE_SERVER_IP} mount | grep ${RELEASE_VERSION_DIR} | grep OS/${ARCH} | grep -v test | awk '{print $3}') for mp in $ret diff --git a/script/step/make_iso_everysrc.sh b/script/step/make_iso_everysrc.sh index c0c4eb3..4678cc9 100644 --- a/script/step/make_iso_everysrc.sh +++ b/script/step/make_iso_everysrc.sh @@ -26,7 +26,7 @@ function make_iso_everysrc_inchroot() TIME=${TIME_DIR##*/} TIME=${TIME#"${version}"-} - yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" + yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" yum clean all -c "${yum_conf}" if rpm -q oemaker &> /dev/null; then yum remove oemaker -y @@ -48,8 +48,8 @@ function make_iso_everysrc_inchroot() sed -i '//a\ '$rpmsname'' config/rpmlist.xml done sed -i '/parse_rpmlist_xml \"conflict\"/d' rpm.sh - OBS_STANDARD_REPO_URL=${OBS_STANDARD_REPO_URL%/*} - REPOS=`echo "${OBS_STANDARD_REPO_URL}/standard_aarch64 ${OBS_STANDARD_REPO_URL}/standard_x86_64 ${OBS_STANDARD_THIRD_REPO_URL}" | sed 's/[ \t]*$//g'` + STANDARD_PROJECT_REPO=${STANDARD_PROJECT_REPO%/*} + REPOS=`echo "${STANDARD_PROJECT_REPO}/aarch64 ${STANDARD_PROJECT_REPO}/x86_64 ${THIRD_REPO}" | sed 's/[ \t]*$//g'` set +e num=0 while [ "${num}" -lt 3 ] @@ -78,9 +78,6 @@ function make_iso_everysrc_inchroot() fi create_checksum "${TGZ_NAME}" - #TIME_DIR="${PRE_VERSION}/${VERSION}/${version}-${TIME}" - #log_info "${HTTP_DIR}/${TIME_DIR}" > "${WORK_DIR}"releasedir_info - CUSTOM_DIR="${TIME_DIR}" RELEASE_DIR="${release_dir}/ISO/source" MOUNT_DIR="${release_dir}/source" SSH_CMD="mkdir -p ${RELEASE_DIR}" diff --git a/script/step/make_iso_everything.sh b/script/step/make_iso_everything.sh index e201481..195d0f8 100644 --- a/script/step/make_iso_everything.sh +++ b/script/step/make_iso_everything.sh @@ -28,7 +28,7 @@ function make_iso_everything_inchroot() TIME=${TIME_DIR##*/} TIME=${TIME#"${version}"-} - yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" + yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" yum clean all -c "${yum_conf}" yum install python3 -y -c "${yum_conf}" @@ -47,9 +47,10 @@ function make_iso_everything_inchroot() rpmsnames=`cat ${UNABLE_INSTALL_LIST}` for rpmsname in $rpmsnames do + sed -i "/>${rpmname}<\/packagereq>/d" config/rpmlist.xml sed -i '//a\ '$rpmsname'' config/rpmlist.xml done - REPOS=`echo "${OBS_STANDARD_REPO_URL} ${OBS_STANDARD_THIRD_REPO_URL}" | sed 's/[ \t]*$//g'` + REPOS=`echo "${STANDARD_PROJECT_REPO} ${THIRD_REPO}" | sed 's/[ \t]*$//g'` set +e num=0 while [ "${num}" -lt 3 ] @@ -78,10 +79,6 @@ function make_iso_everything_inchroot() fi create_checksum "${TGZ_NAME}" - #TIME_DIR="${PRE_VERSION}/${VERSION}/${version}-${TIME}" - #log_info "${HTTP_DIR}/${TIME_DIR}" > "${WORK_DIR}"releasedir_info - CUSTOM_DIR="${TIME_DIR}" - #RELEASE_DIR="${HTTP_DIR}/${CUSTOM_DIR}" RELEASE_DIR="${release_dir}/ISO/$ARCH" MOUNT_DIR="${release_dir}/everything/$ARCH" SSH_CMD="mkdir -p ${RELEASE_DIR} ${MOUNT_DIR}" diff --git a/script/step/make_microvm_image.sh b/script/step/make_microvm_image.sh index fd98bbb..7c44269 100644 --- a/script/step/make_microvm_image.sh +++ b/script/step/make_microvm_image.sh @@ -2,7 +2,7 @@ set -e arch="$(uname -m)" -yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" +yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" ERROR(){ echo `date` - ERROR, $* | tee -a ${log_dir}/${builddate}.log diff --git a/script/step/make_netinst_iso.sh b/script/step/make_netinst_iso.sh index e253dce..78d0746 100644 --- a/script/step/make_netinst_iso.sh +++ b/script/step/make_netinst_iso.sh @@ -28,7 +28,7 @@ function make_netinst_iso_inchroot() TIME=${TIME_DIR##*/} TIME=${TIME#"${version}"-} - yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" + yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" yum clean all -c "${yum_conf}" if rpm -q oemaker &> /dev/null; then yum remove oemaker -y @@ -44,7 +44,7 @@ function make_netinst_iso_inchroot() num=0 while [ "${num}" -lt 3 ] do - bash -x oemaker -t netinst -p ${PRODUCTS} -v "${OS_VERSION}" -r "" -s "${OBS_STANDARD_REPO_URL}" + bash -x oemaker -t netinst -p ${PRODUCTS} -v "${OS_VERSION}" -r "" -s "${STANDARD_PROJECT_REPO}" if [ $? -eq 0 ];then break elif [ $? -eq 133 ]; then @@ -68,8 +68,6 @@ function make_netinst_iso_inchroot() fi create_checksum "${TGZ_NAME}" - CUSTOM_DIR="${TIME_DIR}" - #RELEASE_DIR="${HTTP_DIR}/${CUSTOM_DIR}" RELEASE_DIR="${release_dir}/ISO/$ARCH" SSH_CMD="mkdir -p ${RELEASE_DIR}" sshcmd "${SSH_CMD}" diff --git a/script/step/make_raspi_image.sh b/script/step/make_raspi_image.sh index 2251905..958e447 100644 --- a/script/step/make_raspi_image.sh +++ b/script/step/make_raspi_image.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/obs-repo.conf" +yum_conf="${BUILD_SCRIPT_DIR}/config/repo_conf/repofile.conf" dogsheng_arch="$(uname -m)" ERROR(){ @@ -254,7 +254,7 @@ function make_raspi_image_inchroot() rootfs_dir=${workdir}/rootfs root_mnt=${workdir}/root boot_mnt=${workdir}/boot - euler_dir=${BUILD_SCRIPT_DIR}/config/raspi_image + euler_dir=/home/config/raspi_image CONFIG_RPM_LIST=${euler_dir}/rpmlist builddate=$(date +%Y%m%d) diff --git a/script/step/make_riscv64_qemu_image.sh b/script/step/make_riscv64_qemu_image.sh index da66363..c50a994 100644 --- a/script/step/make_riscv64_qemu_image.sh +++ b/script/step/make_riscv64_qemu_image.sh @@ -8,7 +8,7 @@ fi set -e export OPENEULER_CHROOT_PATH="/usr1/openeuler" -yum_conf="/home/config/repo_conf/obs-repo.conf" +yum_conf="/home/config/repo_conf/repofile.conf" hw_arch="$(uname -m)" IMAGE_SIZE_MB=16384 @@ -312,4 +312,4 @@ function make_riscv64_qemu_image_inchroot() IFS=$'\n' make_rootfs make_img -} \ No newline at end of file +} diff --git a/script/step/release_dir.sh b/script/step/release_dir.sh index 838e78d..fd28efe 100644 --- a/script/step/release_dir.sh +++ b/script/step/release_dir.sh @@ -8,10 +8,12 @@ function set_release_dir() version="openeuler" [ -z "${version}" ] && log_error "You must set version in config file or from CI web page" log_info "------------the version is ${version}-----------" - SSH_CMD="mkdir -p ${HTTP_DIR}/${PRE_VERSION}/${version}_ARM64; echo ${version} > ${HTTP_DIR}/${PRE_VERSION}/${version}_ARM64/version; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_ARM64/version" - sshcmd "${SSH_CMD}" - SSH_CMD="mkdir -p ${HTTP_DIR}/${PRE_VERSION}/${version}_X86; echo ${version} > ${HTTP_DIR}/${PRE_VERSION}/${version}_X86/version; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_X86/version" - sshcmd "${SSH_CMD}" + arch_list=(ARM64 X86) + for arch in ${arch_list[@]} + do + SSH_CMD="mkdir -p ${HTTP_DIR}/${PRE_VERSION}/${version}_${arch}; echo ${version} > ${HTTP_DIR}/${PRE_VERSION}/${version}_${arch}/version; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_${arch}/version" + sshcmd "${SSH_CMD}" + done if [ -f "${WORK_DIR}DEBUG" ] then @@ -19,19 +21,20 @@ function set_release_dir() else debug='nodebug' fi - - SSH_CMD="echo ${debug} > ${HTTP_DIR}/${PRE_VERSION}/${version}_ARM64/log_level; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_ARM64/log_level" - sshcmd "${SSH_CMD}" - SSH_CMD="echo ${debug} > ${HTTP_DIR}/${PRE_VERSION}/${version}_X86/log_level; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_X86/log_level" - sshcmd "${SSH_CMD}" + for arch in ${arch_list[@]} + do + SSH_CMD="echo ${debug} > ${HTTP_DIR}/${PRE_VERSION}/${version}_${arch}/log_level; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_${arch}/log_level" + sshcmd "${SSH_CMD}" + done release_tmp='release_tmp' TIME=$(date +%Y-%m-%d-%H-%M-%S) TIME_DIR="${PRE_VERSION}/${version}-${TIME}" - SSH_CMD="echo ${HTTP_DIR}/${TIME_DIR} > ${HTTP_DIR}/${PRE_VERSION}/${version}_ARM64/${release_tmp}; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_ARM64/${release_tmp}" - sshcmd "${SSH_CMD}" - SSH_CMD="echo ${HTTP_DIR}/${TIME_DIR} > ${HTTP_DIR}/${PRE_VERSION}/${version}_X86/${release_tmp}; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_X86/${release_tmp}" - sshcmd "${SSH_CMD}" + for arch in ${arch_list[@]} + do + SSH_CMD="echo ${HTTP_DIR}/${TIME_DIR} > ${HTTP_DIR}/${PRE_VERSION}/${version}_${arch}/${release_tmp}; chmod 644 ${HTTP_DIR}/${PRE_VERSION}/${version}_${arch}/${release_tmp}" + sshcmd "${SSH_CMD}" + done } # get release dir diff --git a/script/tools/chroot.sh b/script/tools/chroot.sh index 7a231d0..85ae669 100644 --- a/script/tools/chroot.sh +++ b/script/tools/chroot.sh @@ -14,17 +14,60 @@ function chroot_addbep() function chroot_init() { chroot_clean - rm -f openEuler_chroot-*.noarch.rpm - wget -q -r -l1 -nd -A 'openEuler_chroot-*.noarch.rpm' "${OBS_EXTRAS_REPO_URL}/noarch/" &> /dev/null - rpm -ivh openEuler_chroot-*.noarch.rpm - rm -f openEuler_chroot-*.noarch.rpm + pkglist="pigz expect wget bash vim grep patch tar gzip bzip2 openssh-clients yum perl createrepo_c dnf-utils git kpartx xz" + if [[ ${ARCH} == "ppc64le" ]];then + pkglist="${pkglist} grub2-tools grub2-ppc64le-modules" + fi + yum install -c ${REPO_CONF} --installroot=${OPENEULER_CHROOT_PATH} -y ${pkglist} -x glibc32 + mkdir -m 755 -p ${OPENEULER_CHROOT_PATH}/dev/pts + test -d ${OPENEULER_CHROOT_PATH}/dev/shm || rm -f ${OPENEULER_CHROOT_PATH}/dev/shm + mkdir -m 755 -p ${OPENEULER_CHROOT_PATH}/dev/shm + mount --bind /dev ${OPENEULER_CHROOT_PATH}/dev/ + while read com file mode arg + do + rm -f ${OPENEULER_CHROOT_PATH}/dev/$file + if [[ $com = ln ]];then + ln -s $arg ${OPENEULER_CHROOT_PATH}/dev/$file + continue + fi + $com -m $mode ${OPENEULER_CHROOT_PATH}/dev/$file $arg + done << DEVLIST + mknod null 666 c 1 3 + mknod zero 666 c 1 5 + mknod full 622 c 1 7 + mknod random 666 c 1 8 + mknod urandom 644 c 1 9 + mknod tty 666 c 5 0 + mknod ptmx 666 c 5 2 + mknod loop-control 600 c 10 237 + mknod loop0 640 b 7 0 + mknod loop1 640 b 7 1 + mknod loop2 640 b 7 2 + mknod loop3 640 b 7 3 + ln fd 777 /proc/self/fd + ln stdin 777 fd/0 + ln stdout 777 fd/1 + ln stderr 777 fd/2 +DEVLIST + mount -n -tdevpts -omode=0620,gid=5 none ${OPENEULER_CHROOT_PATH}/dev/pts + mkdir -p ${OPENEULER_CHROOT_PATH}/proc ${OPENEULER_CHROOT_PATH}/sys + mount --bind /sys ${OPENEULER_CHROOT_PATH}/sys + mount --bind /proc ${OPENEULER_CHROOT_PATH}/proc + + rm -f "${OPENEULER_CHROOT_PATH}/etc/yum.repos.d"/* + cp "${REPO_CONF}" "${OPENEULER_CHROOT_PATH}/etc/yum.repos.d/" cp -a "${BUILD_SCRIPT_DIR}"/* "${OPENEULER_CHROOT_PATH}/home" mkdir -p "${OPENEULER_CHROOT_PATH}/root/.ssh/" cp /root/.ssh/super_publish_rsa "${OPENEULER_CHROOT_PATH}/root/.ssh/" + cp /etc/resolv.conf "${OPENEULER_CHROOT_PATH}/etc/" if echo "${BUILD_SCRIPT_DIR}" | grep 'gcov-'; then sed -i '/VERSION=/ iexport CI_PROJECT="openeuler_ARM64_gcov"' "${OPENEULER_CHROOT_PATH}"/home/internal.sh fi - chroot "${OPENEULER_CHROOT_PATH}" /bin/bash -c "echo ${OBS_SERVER_IP} openeuler-obs-repo.huawei.com >> /etc/hosts" + if [[ ${STEP} == "get_epol_rpms" ]];then + git clone --depth=1 https://gitee.com/src-openeuler/openEuler-repos -b openEuler-23.03 + cp openEuler-repos/RPM-GPG-KEY-EBS "${OPENEULER_CHROOT_PATH}/home/" + rm -rf openEuler-repos + fi } function chroot_clean() -- Gitee