From 4f5cd78286ea94456d2e72cdbedf568785ec5484 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Thu, 10 Mar 2022 10:40:16 +0800 Subject: [PATCH 01/11] add src rpm package compare,daily build dir check --- script/tools/check_rpms_complete.py | 422 ++++++++++++++++++++++++++++ script/tools/daily_build_check.py | 48 +++- 2 files changed, 462 insertions(+), 8 deletions(-) create mode 100644 script/tools/check_rpms_complete.py diff --git a/script/tools/check_rpms_complete.py b/script/tools/check_rpms_complete.py new file mode 100644 index 0000000..6a9db4d --- /dev/null +++ b/script/tools/check_rpms_complete.py @@ -0,0 +1,422 @@ +#!/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologging.es Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: dongjie +# Create: 2022-03-28 +# ****************************************************************************** +import os +import sys +import re +import yaml +import xlwt +import shutil +import requests +import argparse +import subprocess +import logging +from bs4 import BeautifulSoup + +LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" +DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) + +class CheckRpmsComplete(object): + """ + The entrance check for rpms complete + """ + + def __init__(self, **kwargs): + """ + kawrgs: dict,init dict by 'a': 'A' style + rooturl: the daily build page url + main_branch:choose which branch you need to check + """ + self.kwargs = kwargs + self.rooturl = self.kwargs['daily_build_url'] + self.main_branch = self.kwargs['main_branch'] + self.dirflag = self.kwargs['dir_flag'] + self.datebranch = self.kwargs['date_branch'] + self.uploadflag = self.kwargs['uploadflag'] + self.obs = self.kwargs['obs'] + self.offical = self.kwargs['offical'] + + def html_downloader(self, url): + """ + download url html content + """ + user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' + headers = {'User-Agent': user_agent} + r = requests.get(url, headers=headers) + if r.status_code == 200: + r.encoding = 'utf-8' + logging.info("this html page download success！{}".format(url)) + return r.text + else: + logging.error("this html page download failed！{}".format(url)) + return '' + + def rpms_parser(self,html_content): + """ + parse html content and get rpm name + """ + rpms_list = [] + soup = BeautifulSoup(html_content, "html.parser") + tr_content = soup.find_all('tr') + for line in tr_content: + td_content = line.find_all('td') + if td_content: + dir_url = td_content[0].find('a', href=True) + size = td_content[1].text + if dir_url.get('title',''): + if dir_url['title'] != '../' and self.check_is_rpm(dir_url['title']): + rpms_list.append(dir_url['title']) + return rpms_list + + def check_is_rpm(self, filename): + """ + file type filter + """ + if filename.endswith(".rpm"): + return True + + def obs_rpms_parser(self,html_content): + rpms_list = [] + soup = BeautifulSoup(html_content, "html.parser") + for k in soup.find_all('a',href=True): + if k.get('href',''): + if '.rpm' in k['href']: + rpms_list.append(k['href']) + return rpms_list + + def get_everything_rpms(self): + trans_url = self.main_branch.replace('-',':/') + x86_64_dir = "{}/{}/standard_x86_64/src/".format(self.obs,trans_url) + x86_64_content = self.html_downloader(x86_64_dir) + x86_64_rpms = self.obs_rpms_parser(x86_64_content) + aarch64_dir = "{}/{}/standard_aarch64/src/".format(self.obs,trans_url) + aarch64_content = self.html_downloader(aarch64_dir) + aarch64_rpms = self.obs_rpms_parser(aarch64_content) + everything_data = { + "x86-64":x86_64_rpms, + "aarch64":aarch64_rpms + } + return everything_data + + + def get_rpms(self,epol_dir,source_dir,parser_type='general'): + ''' + get rpms name from epoldir or source_dir + ''' + repo_data = {} + epol_content = self.html_downloader(epol_dir) + source_content = self.html_downloader(source_dir) + if parser_type == 'general': + epol_rpms = self.rpms_parser(epol_content) + source_rpms = self.rpms_parser(source_content) + else: + epol_rpms = self.obs_rpms_parser(epol_content) + source_rpms = self.obs_rpms_parser(source_content) + repo_data = { + 'epol_rpms': epol_rpms, + 'source_rpms': source_rpms + } + return repo_data + + def compare_rpms(self,repo_data,daily_repo_data,obs_repo_data): + ''' + get epol and source rpms data and start compare + :param repo_data:dict of offical repo src repo rpms include epol + :param daily_repo_data:dict of daily build repo src repo rpms include epol + :param obs_repo_data:dict of obs repo src repo rpms include epol + ''' + offical_epol_rpms = repo_data.get("epol_rpms",[]) + daily_epol_rpms = daily_repo_data.get("epol_rpms",[]) + obs_epol_rpms = obs_repo_data.get("epol_rpms",[]) + offical_src_rpms = repo_data.get("source_rpms",[]) + daily_src_rpms = daily_repo_data.get("source_rpms",[]) + obs_src_rpms = obs_repo_data.get("source_rpms",[]) + epol_result = self.compare_detail_rpms(daily_epol_rpms,offical_epol_rpms,obs_epol_rpms) + self.write_file(epol_result) + source_result = self.compare_detail_rpms(daily_src_rpms,offical_src_rpms,obs_src_rpms,compare_type='source') + self.write_file(source_result,compare_type='source') + + + def compare_detail_rpms(self,daily,offical,obs,compare_type='epol'): + ''' + src rpms detail compare + :param daily:List of daily build repo rpms + :param offical:List of offical repo rpms + :param obs:List of obs repo rpms + :return:dict of compare result + ''' + datas = {} + if daily and offical: + # daily rpms compare with offical epol rpms + in_daily_offical = list(set(daily).difference(set(offical))) + in_offical_daily = list(set(offical).difference(set(daily))) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:offical_repo*********************************") + in_daily_offical,in_offical_daily = self.rpm_version_compare(in_daily_offical,in_offical_daily) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:offical_repo*********************************") + logging.info("{} rpms compare below rpms in daily repo not in offical repo：{}".format(compare_type,in_daily_offical)) + logging.info("{} rpms compare below rpms in offical repo not in daily repo：{}".format(compare_type,in_offical_daily)) + daily_offical = {"rpmname":[self.datebranch,"offical repo"]} + for rpm in in_daily_offical: + daily_offical[rpm] = [1,0] + for rpm in in_offical_daily: + daily_offical[rpm] = [0,1] + datas["daily_offical"] = daily_offical + else: + if not daily and not offical: + daily_offical = {"error info":["{}get repo rpms failed".format(self.datebranch),"offical repo get rpms failed"]} + elif not daily: + daily_offical = {"error info":["{}get repo rpms failed".format(self.datebranch),"offical repo"]} + else: + daily_offical = {"error info":[self.datebranch,"offical repo get rpms failed"]} + datas["daily_offical"] = daily_offical + if daily and obs: + # daily rpms compare with obs epol rpms + in_daily_obs = list(set(daily).difference(set(obs))) + in_obs_daily = list(set(obs).difference(set(daily))) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:obs_repo*********************************") + in_daily_obs,in_obs_daily = self.rpm_version_compare(in_daily_obs,in_obs_daily) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:obs_repo*********************************") + logging.info("{} rpms compare below rpms in daily repo not in obs repo：{}".format(compare_type,in_daily_obs)) + logging.info("{} rpms compare below rpms in obs repo not in daily repo：{}".format(compare_type,in_obs_daily)) + daily_obs = {"rpmname":[self.datebranch,"obs repo"]} + for rpm in in_daily_obs: + daily_obs[rpm] = [1,0] + for rpm in in_obs_daily: + daily_obs[rpm] = [0,1] + datas["daily_obs"] = daily_obs + else: + if not daily and not obs: + daily_obs = {"error info":["{}get repo rpms failed".format(self.datebranch),"obs repo get rpms failed"]} + elif not daily: + daily_obs = {"error info":["{}get repo rpms failed".format(self.datebranch),"obs repo"]} + else: + daily_obs = {"error info":[self.datebranch,"obs repo get rpms failed"]} + datas["daily_obs"] = daily_obs + if obs and offical: + # daily rpms compare with offical epol rpms + in_obs_offical = list(set(obs).difference(set(offical))) + in_offical_obs = list(set(offical).difference(set(obs))) + logging.info("***************************************SIDEA:obs_repo,SIDEB:offical_repo*********************************") + in_obs_offical,in_offical_obs = self.rpm_version_compare(in_obs_offical,in_offical_obs) + logging.info("***************************************SIDEA:obs_repo,SIDEB:offical_repo*********************************") + logging.info("{} rpms compare below rpms in obs repo not in offical repo：{}".format(compare_type,in_obs_offical)) + logging.info("{} rpms compare below rpms in offical repo not in obs repo：{}".format(compare_type,in_offical_obs)) + obs_offical = {"rpmname":["obs repo","offical repo"]} + for rpm in in_obs_offical: + obs_offical[rpm] = [1,0] + for rpm in in_offical_obs: + obs_offical[rpm] = [0,1] + datas["obs_offical"] = obs_offical + else: + if not obs and not offical: + obs_offical = {"error info":["obs get repo rpms failed","offical repo get rpms failed"]} + elif not obs: + obs_offical = {"error info":["obs get repo rpms failed","offical repo"]} + else: + obs_offical = {"error info":["obs repo","offical repo get rpms failed"]} + datas["obs_offical"] = obs_offical + return datas + + def compare_everything_rpms(self,data,source_data): + ''' + compare obs x86_64 and aarch64 with daily build source rpms + :param data:pre compare obs x86_64 and aarch64 rpms + :param source_data:daily build source rpms + ''' + datas = {} + if source_data.get('source_rpms',[]): + source_rpms = source_data['source_rpms'] + for key,value in data.items(): + if value: + in_value_source = list(set(value).difference(set(source_rpms))) + in_source_value = list(set(source_rpms).difference(set(value))) + in_value_source,in_source_value = self.rpm_version_compare(in_value_source,in_source_value) + value_source = {"rpmname":["obs-{}-src".format(key),"{}-source".format(self.datebranch)]} + for rpm in in_value_source: + value_source[rpm] = [1,0] + for rpm in in_source_value: + value_source[rpm] = [0,1] + now_arc = "{}-daily".format(key) + datas[now_arc] = value_source + return datas + + + def write_file(self,data,compare_type='epol'): + ''' + out put compare result to excel + :param data:pre compare result data + ''' + if data: + book = xlwt.Workbook(encoding='utf-8') + for key,value in data.items(): + sheet = book.add_sheet(key,cell_overwrite_ok=True) + r = 0 + for i, j in value.items(): + le = len(j) + sheet.write(r, 0, i,) + for c in range(1, le + 1): + sheet.write(r, c, j[c - 1]) + r += 1 + book.save("./{}-{}-src-rpm-compare-result.xls".format(self.main_branch,compare_type)) + + def _add_to_obs(self): + """ + add result file to obs project + """ + filenames = [] + path=os.getcwd() + f_list = os.listdir(path) + for xlsfile in f_list: + if os.path.splitext(xlsfile)[1] == '.xls': + filenames.append(xlsfile) + cmd = "cd %s && osc co home:Admin:ISO/%s" % (path,self.main_branch) + ret = os.popen(cmd).read() + targer_dir = os.path.join(path, "home:Admin:ISO/{}".format(self.main_branch)) + if os.path.exists(targer_dir): + for file in filenames: + src_file = os.path.join(path,file) + dist_file = os.path.join(path,targer_dir,file) + shutil.copyfile(src_file, dist_file) + cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir,file) + ret = os.popen(cmd).read() + logging.info("compare result already upload to OBS project:{}".format(self.main_branch)) + else: + logging.warning("this {} not in obs project".format(self.main_branch)) + + + def rpm_version_compare(self,paira,pairb): + ''' + check rpm version and compare + :param paira:List of sidea rpms that needs to be compared + :param pairb:List of sideb rpms that needs to be compared + ''' + for rpma in paira[:]: + rpm_namea = self.rpm_name(rpma) + for rpmb in pairb[:]: + rpm_nameb = self.rpm_name(rpmb) + if rpm_nameb == rpm_namea: + logging.info("SIDEA rpm version:{} SIDEB rpm version：{}".format(rpma,rpmb)) + pairb.remove(rpmb) + paira.remove(rpma) + break + return paira,pairb + + def rpm_n_v_r_d_a(self,rpm): + """ + parse rpm package name,version,release,publisher + :param rpm:complete rpm name + :return:split rpm + """ + # eg: grpc-1.31.0-6.oe1.x86_64.rpm + name = self.rpm_name(rpm) + + rpm_split = re.match(r"(.+)-(.+)\.(.+)?\.(.+)\.rpm", rpm.replace(name, "", 1)) + if rpm_split: + return name, rpm_split.group(1), rpm_split.group(2), rpm_split.group(3), rpm_split.group(4) + return name, rpm_split.group(1), rpm_split.group(2), None, rpm_split.group(3) + + def rpm_name(self,rpm): + """ + :param rpm:complete rpm name + :return:only rpm name + """ + m = re.match(r"^(.+)-.+-.+", rpm) + + if m: + return m.group(1) + else: + return rpm + + def shell_cmd(self,cmd_list): + """ + :param cmd_list: + :return: + """ + p = subprocess.Popen(cmd_list, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + return p.returncode, out.decode("utf-8", errors="ignore"), err.decode("utf-8", errors="ignore") + + + def _check_src_rpm(self): + """ + Check the src rpm entry function + """ + # repo_data = self.get_repo_rpms() + # daily_repo_data = self.get_dailybuild_repo_rpms() + # obs_repo_data = self.get_obs_repo_rpms() + offical_epol_dir = "{}/{}/EPOL/main/source/Packages/".format(self.offical,self.main_branch) + offical_source_dir = "{}/{}/source/Packages/".format(self.offical,self.main_branch) + repo_data = self.get_rpms(offical_epol_dir,offical_source_dir) + daily_epol_dir = "{}/{}/{}/EPOL/main/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) + daily_source_dir = "{}/{}/{}/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) + daily_repo_data = self.get_rpms(daily_epol_dir,daily_source_dir) + trans_url = self.main_branch.replace('-',':/') + obs_epol_dir = "{}/{}:/Epol/standard_x86_64/src/".format(self.obs,trans_url) + obs_source_dir = "{}/{}/standard_x86_64/src/".format(self.obs,trans_url) + obs_repo_data = self.get_rpms(obs_epol_dir,obs_source_dir,parser_type='obs') + self.compare_rpms(repo_data,daily_repo_data,obs_repo_data) + + def _check_everything_rpm(self): + """ + Check the everything x86-64 and aarch64 rpms entry function + """ + everything_data = self.get_everything_rpms() + daily_epol_dir = "{}/{}/{}/EPOL/main/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) + daily_source_dir = "{}/{}/{}/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) + source_data = self.get_rpms(daily_epol_dir,daily_source_dir) + result = self.compare_everything_rpms(everything_data,source_data) + self.write_file(result,compare_type='everything') + + def run(self): + self._check_src_rpm() + self._check_everything_rpm() + if self.uploadflag == '1': + self._add_to_obs() + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dirflag", + default="openeuler-2022-", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument( + "--rooturl", + help="daily build page root url,eg: http://ip/dailybuild") + parser.add_argument( + "--branch", + help="which branch you want to check,eg:openEuler-22.03-LTS") + parser.add_argument( + "--datebranch", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument( + "--obs", + help="obs repo ip") + parser.add_argument( + "--offical", + help="offical repo ip") + parser.add_argument("--uploadflag",default="1") + args = parser.parse_args() + kw = { + "daily_build_url": args.rooturl, + "main_branch": args.branch, + "date_branch": args.datebranch, + "dir_flag": args.dirflag, + "uploadflag": args.uploadflag, + "obs":args.obs, + "offical":args.offical + } + check = CheckRpmsComplete(**kw) + check.run() \ No newline at end of file diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index a5557b0..a9f78f9 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -15,7 +15,10 @@ # ****************************************************************************** import os import sys +import re import yaml +import xlwt +import shutil import requests import argparse import logging @@ -24,6 +27,7 @@ from bs4 import BeautifulSoup LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" DATE_FORMAT = "%Y-%m-%d %H:%M:%S" logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) +UNMOUNT_DIRS = ['debuginfo','OS','source'] class CheckDailyBuild(object): @@ -42,6 +46,7 @@ class CheckDailyBuild(object): self.main_branch = self.kwargs['main_branch'] self.dirflag = self.kwargs['dir_flag'] self.datebranch = self.kwargs['date_branch'] + self.mountflag = self.kwargs['mountflag'] self.standard_dir = self.load_standard() def html_downloader(self, url): @@ -92,8 +97,8 @@ class CheckDailyBuild(object): """ dir_map = {} html_content = self.html_downloader(main_branch_url) - if html_content: - current_dir = self.html_parser(main_branch_url, html_content) + current_dir = self.html_parser(main_branch_url, html_content) + if current_dir: dir_map[main_branch_url] = [] for first_dir in current_dir: dir_split = first_dir.split('/')[5] @@ -129,8 +134,8 @@ class CheckDailyBuild(object): if item.split('|')[-1] != 'ISFILE/': self.check_current_dir(item, origin_dir, temp_list) except Exception as e: - logging.info("error url can not open:{}".format(c_dir)) - logging.info("error url can not open:{}".format(e)) + logging.warning("error url can not open:{}".format(c_dir)) + # logging.info("error url can not open:{}".format(e)) return temp_list def load_standard(self): @@ -140,6 +145,9 @@ class CheckDailyBuild(object): try: with open('./standard.yaml', 'r', encoding='utf-8') as f: result = yaml.load(f, Loader=yaml.FullLoader) + if self.mountflag == '0': + for delitem in UNMOUNT_DIRS: + del result[delitem] return result except Exception as e: logging.info("error read standard.yaml,please check") @@ -148,15 +156,32 @@ class CheckDailyBuild(object): """ branch dir compare with standard dir """ + logging.info("********************************************CHECK RESULT*********************************************************") standard_dir = self.standard_dir + error_flag = False for key, c_standard in standard_dir.items(): for current_dir in c_standard: if '*' in current_dir: current_dir = current_dir.replace('*', current_branch) if current_dir not in dir_list: + error_flag = True logging.error( - 'this dir not found,link url:{}{}'.format( + 'this dir or file not found,link url:{}{}'.format( c_dir, current_dir)) + logging.info("********************************************CHECK RESULT*********************************************************") + if error_flag: + logging.error("some files or dirs not found in：{}".format(self.datebranch)) + raise SystemExit("*****************************************PLEASE CHECK*****************************************************") + + def check_input_args(self,complete_url): + html_content = self.html_downloader(complete_url) + current_dir = self.html_parser(complete_url, html_content) + if not current_dir: + logging.error("error url can not open,please check your input：{}".format(complete_url)) + raise SystemExit("*******PLEASE CHECK YOUR INPUT ARGS*******") + else: + return True + def _get_main_branch(self): if self.datebranch: @@ -164,11 +189,15 @@ class CheckDailyBuild(object): complete_key = "{}/{}/".format(self.rooturl, self.main_branch) complete_value = "{}/{}/{}/".format( self.rooturl, self.main_branch, self.datebranch) - dir_map[complete_key] = [complete_value] + check_args_result = self.check_input_args(complete_value) + if check_args_result: + dir_map[complete_key] = [complete_value] + self.check_every_dir(dir_map) else: main_branch_url = "{}/{}/".format(self.rooturl, self.main_branch) dir_map = self.start_check(main_branch_url) - self.check_every_dir(dir_map) + self.check_every_dir(dir_map) + def run(self): self._get_main_branch() @@ -189,11 +218,14 @@ if __name__ == "__main__": parser.add_argument( "--datebranch", help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument("--mountflag",default="1") + args = parser.parse_args() kw = { "daily_build_url": args.rooturl, "main_branch": args.branch, "date_branch": args.datebranch, - "dir_flag": args.dirflag + "dir_flag": args.dirflag, + "mountflag": args.mountflag } check = CheckDailyBuild(**kw) check.run() -- Gitee From eb7cab08b3ce619457f3b8dfbac2d7e5564e2fa7 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 19 Aug 2022 16:02:57 +0800 Subject: [PATCH 02/11] sda --- script/tools/daily_build_check.py | 69 +++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index a9f78f9..08cb28e 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -17,7 +17,6 @@ import os import sys import re import yaml -import xlwt import shutil import requests import argparse @@ -152,28 +151,59 @@ class CheckDailyBuild(object): except Exception as e: logging.info("error read standard.yaml,please check") + # def compare_standard(self, dir_list, current_branch, c_dir): + # """ + # branch dir compare with standard dir + # """ + # logging.info("********************************************CHECK RESULT*********************************************************") + # standard_dir = self.standard_dir + # error_flag = False + # for key, c_standard in standard_dir.items(): + # for current_dir in c_standard: + # if '*' in current_dir: + # current_dir = current_dir.replace('*', current_branch) + # if current_dir not in dir_list: + # error_flag = True + # logging.error( + # 'this dir or file not found,link url:{}{}'.format( + # c_dir, current_dir)) + # logging.info("********************************************CHECK RESULT*********************************************************") + # if error_flag: + # logging.error("some files or dirs not found in：{}".format(self.datebranch)) + # raise SystemExit("*****************************************PLEASE CHECK*****************************************************") + def compare_standard(self, dir_list, current_branch, c_dir): """ branch dir compare with standard dir """ logging.info("********************************************CHECK RESULT*********************************************************") + compare_results = {"compare_results":[]} standard_dir = self.standard_dir error_flag = False for key, c_standard in standard_dir.items(): for current_dir in c_standard: - if '*' in current_dir: - current_dir = current_dir.replace('*', current_branch) + # if '*' in current_dir: + current_dir = current_dir.replace('*', current_branch) + temp_dic = {} if current_dir not in dir_list: error_flag = True + temp_dic[current_dir] = False logging.error( 'this dir or file not found,link url:{}{}'.format( c_dir, current_dir)) + else: + temp_dic[current_dir] = True + compare_results["compare_results"].append(temp_dic) + # logging.info("********************************************CHECK RESULT*********************************************************") + # if error_flag: + # logging.error("some files or dirs not found in：{}".format(self.datebranch)) + # raise SystemExit("*****************************************PLEASE CHECK*****************************************************") + self.write_yaml(compare_results) logging.info("********************************************CHECK RESULT*********************************************************") - if error_flag: - logging.error("some files or dirs not found in：{}".format(self.datebranch)) - raise SystemExit("*****************************************PLEASE CHECK*****************************************************") + self._add_to_obs() + - def check_input_args(self,complete_url): + def check_input_args(self, complete_url): html_content = self.html_downloader(complete_url) current_dir = self.html_parser(complete_url, html_content) if not current_dir: @@ -182,6 +212,26 @@ class CheckDailyBuild(object): else: return True + def _add_to_obs(self): + """ + add result file to obs project + """ + # src_file = os.path.join(os.getcwd(),"{}.yaml".format(self.datebranch)) + file = "{}.yaml".format(self.datebranch) + path=os.getcwd() + cmd = "cd %s && osc co home:Admin:ISO/%s" % (path,self.main_branch) + ret = os.popen(cmd).read() + targer_dir = os.path.join(path, "home:Admin:ISO/{}".format(self.main_branch)) + if os.path.exists(targer_dir): + src_file = os.path.join(path, file) + dist_file = os.path.join(path, targer_dir, file) + shutil.copyfile(src_file, dist_file) + cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir, file) + ret = os.popen(cmd).read() + logging.info("compare result already upload to OBS project:{}".format(self.main_branch)) + else: + logging.warning("this {} not in obs project".format(self.main_branch)) + def _get_main_branch(self): if self.datebranch: @@ -198,6 +248,11 @@ class CheckDailyBuild(object): dir_map = self.start_check(main_branch_url) self.check_every_dir(dir_map) + def write_yaml(self, dict_msg): + file_path = os.path.join(os.getcwd(),"{}.yaml".format(self.datebranch)) + with open(file_path, "w", encoding='utf-8') as f: + yaml.dump(dict_msg, f, default_flow_style=False, sort_keys=False) + def run(self): self._get_main_branch() -- Gitee From 160b4422c82a67d5dde2e94e8998e00314ce0810 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 9 Sep 2022 16:05:05 +0800 Subject: [PATCH 03/11] 3 --- script/tools/check_rpms_complete.py | 38 +- script/tools/daily_build_check.py | 8 +- script/tools/standard.yaml | 570 ++++++++++++++++++---------- script/tools/test-2209.yaml | 27 ++ script/tools/test.py | 61 +++ script/tools/test.yaml | 31 ++ script/tools/test1.yaml | 164 ++++++++ script/tools/test2.yaml | 22 ++ script/tools/xnio | 74 ++++ 9 files changed, 798 insertions(+), 197 deletions(-) create mode 100644 script/tools/test-2209.yaml create mode 100644 script/tools/test.py create mode 100644 script/tools/test.yaml create mode 100644 script/tools/test1.yaml create mode 100644 script/tools/test2.yaml create mode 100644 script/tools/xnio diff --git a/script/tools/check_rpms_complete.py b/script/tools/check_rpms_complete.py index 6a9db4d..b10c510 100644 --- a/script/tools/check_rpms_complete.py +++ b/script/tools/check_rpms_complete.py @@ -131,6 +131,35 @@ class CheckRpmsComplete(object): } return repo_data + def get_rpms_from_yaml(self, branch): + """ + branch: branch name + return: branch pkgs dict + """ + # all_branch_pkgs = [] + repo_data = {'epol_rpms':[], 'source_rpms':[]} + if os.path.exists(os.path.join(self.release_management_path, branch)): + standard_dirs = os.listdir(os.path.join(self.release_management_path, branch)) + for standard_dir in standard_dirs: + file_path = os.path.join(self.release_management_path, branch, standard_dir) + if not os.path.isdir(file_path) or standard_dir == 'delete': + standard_dirs.remove(standard_dir) + for c_dir in standard_dirs: + release_path = os.path.join(self.release_management_path, branch, c_dir, 'pckg-mgmt.yaml') + if os.path.exists(release_path): + with open(release_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + all_branch_pkgs.extend(result['packages']) + for pkg in all_branch_pkgs: + # project_pkgs.setdefault(pkg['obs_to'], []).append(pkg['name']) + if 'Epol' in pkg['obs_to']: + repo_data['epol_rpms'].append(pkg['name']) + else: + repo_data['source_rpms'].append(pkg['name']) + else: + log.error("this branch {} not exist in repo release_management".format(branch)) + return repo_data + def compare_rpms(self,repo_data,daily_repo_data,obs_repo_data): ''' get epol and source rpms data and start compare @@ -253,6 +282,11 @@ class CheckRpmsComplete(object): datas[now_arc] = value_source return datas + def compare_release_rpms(self, release_data, daily_data): + for arch, datas in daily_data.items(): + release_pkgs = release_data.get(arch, []) + + def write_file(self,data,compare_type='epol'): ''' @@ -374,10 +408,12 @@ class CheckRpmsComplete(object): Check the everything x86-64 and aarch64 rpms entry function """ everything_data = self.get_everything_rpms() + release_data = self.get_rpms_from_yaml(self.main_branch) daily_epol_dir = "{}/{}/{}/EPOL/main/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) daily_source_dir = "{}/{}/{}/source/Packages/".format(self.rooturl,self.main_branch,self.datebranch) source_data = self.get_rpms(daily_epol_dir,daily_source_dir) - result = self.compare_everything_rpms(everything_data,source_data) + release_result = self.compare_release_rpms(release_data, source_data) + result = self.compare_everything_rpms(everything_data, source_data) self.write_file(result,compare_type='everything') def run(self): diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 08cb28e..ebd0a87 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -143,12 +143,17 @@ class CheckDailyBuild(object): """ try: with open('./standard.yaml', 'r', encoding='utf-8') as f: - result = yaml.load(f, Loader=yaml.FullLoader) + yaml_result = yaml.load(f, Loader=yaml.FullLoader) + if yaml_result.get(self.main_branch,''): + result = yaml_result[self.main_branch] + else: + result = yaml_result['standard'] if self.mountflag == '0': for delitem in UNMOUNT_DIRS: del result[delitem] return result except Exception as e: + logging.info(e) logging.info("error read standard.yaml,please check") # def compare_standard(self, dir_list, current_branch, c_dir): @@ -177,6 +182,7 @@ class CheckDailyBuild(object): branch dir compare with standard dir """ logging.info("********************************************CHECK RESULT*********************************************************") + print (dir_list) compare_results = {"compare_results":[]} standard_dir = self.standard_dir error_flag = False diff --git a/script/tools/standard.yaml b/script/tools/standard.yaml index 6c5fdbc..50ad513 100644 --- a/script/tools/standard.yaml +++ b/script/tools/standard.yaml @@ -1,196 +1,376 @@ -MAIN_DIR: - - EPOL/ - - ISO/ - - OS/ - - debuginfo/ - - docker_img/ - - everything/ - - raspi_img/ - - source/ - - stratovirt_img/ - - update/ - - virtual_machine_img/ -EPOL: - - EPOL/main/ - - EPOL/update/ - - EPOL/multi_version/ - - EPOL/multi_version/OpenStack/ - - EPOL/main/source/ - - EPOL/main/aarch64/ - - EPOL/main/x86_64/ - - EPOL/update/main/ - - EPOL/update/multi_version/ - - EPOL/update/main/aarch64/ - - EPOL/main/source/Packages/ - - EPOL/main/source/repodata/ - - EPOL/main/aarch64/Packages/ - - EPOL/main/aarch64/repodata/ - - EPOL/main/x86_64/repodata/ - - EPOL/update/multi_version/source/ - - EPOL/update/main/aarch64/Packages/ - - EPOL/update/main/aarch64/repodata/ - - EPOL/update/main/source/Packages/ - - EPOL/update/main/source/repodata/ - - EPOL/update/main/x86_64/Packages/ - - EPOL/update/main/x86_64/repodata/ - - EPOL/multi_version/OpenStack/Train/ - - EPOL/multi_version/OpenStack/Wallaby/ - - EPOL/multi_version/OpenStack/Train/aarch64/ - - EPOL/multi_version/OpenStack/Train/source/ - - EPOL/multi_version/OpenStack/Train/x86_64/ - - EPOL/multi_version/OpenStack/Train/aarch64/Packages/ - - EPOL/multi_version/OpenStack/Train/aarch64/repodata/ - - EPOL/multi_version/OpenStack/Train/source/Packages/ - - EPOL/multi_version/OpenStack/Train/source/repodata/ - - EPOL/multi_version/OpenStack/Train/x86_64/Packages/ - - EPOL/multi_version/OpenStack/Train/x86_64/repodata/ - - EPOL/multi_version/OpenStack/Wallaby/aarch64/ - - EPOL/multi_version/OpenStack/Wallaby/source/ - - EPOL/multi_version/OpenStack/Wallaby/x86_64/ - - EPOL/multi_version/OpenStack/Wallaby/aarch64/Packages/ - - EPOL/multi_version/OpenStack/Wallaby/aarch64/repodata/ - - EPOL/multi_version/OpenStack/Wallaby/source/Packages/ - - EPOL/multi_version/OpenStack/Wallaby/source/repodata/ - - EPOL/multi_version/OpenStack/Wallaby/x86_64/Packages/ - - EPOL/multi_version/OpenStack/Wallaby/x86_64/repodata/ - - EPOL/update/multi_version/source/Packages/ - - EPOL/update/multi_version/source/repodata/ - - EPOL/update/multi_version/x86_64/Packages/ - - EPOL/update/multi_version/x86_64/repodata/ - - EPOL/update/multi_version/aarch64/Packages/ - - EPOL/update/multi_version/aarch64/repodata/ -ISO: - - ISO/aarch64/ - - ISO/x86_64/ - - ISO/source/ - - ISO/aarch64/*-aarch64-dvd.iso - - ISO/aarch64/*-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-aarch64.rpmlist - - ISO/aarch64/*-everything-aarch64-dvd.iso - - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-everything-debug-aarch64-dvd.iso - - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum - - ISO/aarch64/*-netinst-aarch64-dvd.iso - - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum - - ISO/source/*-source-dvd.iso - - ISO/source/*-source-dvd.iso.sha256sum - - ISO/x86_64/*-everything-debug-x86_64-dvd.iso - - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-netinst-x86_64-dvd.iso - - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-x86_64-dvd.iso - - ISO/x86_64/*-x86_64-dvd.iso.sha256sum - - ISO/x86_64/*-x86_64.rpmlist -OS: - - OS/aarch64/ - - OS/x86_64/ - - OS/aarch64/EFI/ - - OS/aarch64/Packages/ - - OS/aarch64/docs/ - - OS/aarch64/images/ - - OS/aarch64/repodata/ - - OS/aarch64/RPM-GPG-KEY-openEuler/ - - OS/aarch64/TRANS.TBL - - OS/aarch64/boot.catalog - - OS/x86_64/EFI/ - - OS/x86_64/Packages/ - - OS/x86_64/docs/ - - OS/x86_64/images/ - - OS/x86_64/repodata/ - - OS/x86_64/RPM-GPG-KEY-openEuler/ - - OS/x86_64/TRANS.TBL - - OS/x86_64/boot.catalog -debuginfo: - - debuginfo/aarch64/ - - debuginfo/x86_64/ - - debuginfo/aarch64/EFI/ - - debuginfo/aarch64/Packages/ - - debuginfo/aarch64/docs/ - - debuginfo/aarch64/images/ - - debuginfo/aarch64/repodata/ - - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ - - debuginfo/aarch64/TRANS.TBL - - debuginfo/aarch64/boot.catalog - - debuginfo/x86_64/EFI/ - - debuginfo/x86_64/Packages/ - - debuginfo/x86_64/docs/ - - debuginfo/x86_64/images/ - - debuginfo/x86_64/repodata/ - - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ - - debuginfo/x86_64/TRANS.TBL - - debuginfo/x86_64/boot.catalog -docker_img: - - docker_img/aarch64/ - - docker_img/x86_64/ - - docker_img/aarch64/openEuler-docker.aarch64.tar.xz - - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum - - docker_img/x86_64/openEuler-docker.x86_64.tar.xz - - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum -everything: - - everything/aarch64/ - - everything/x86_64/ - - everything/aarch64/EFI/ - - everything/aarch64/Packages/ - - everything/aarch64/docs/ - - everything/aarch64/images/ - - everything/aarch64/repodata/ - - everything/aarch64/RPM-GPG-KEY-openEuler/ - - everything/aarch64/TRANS.TBL - - everything/aarch64/boot.catalog - - everything/x86_64/EFI/ - - everything/x86_64/Packages/ - - everything/x86_64/docs/ - - everything/x86_64/images/ - - everything/x86_64/repodata/ - - everything/x86_64/RPM-GPG-KEY-openEuler/ - - everything/x86_64/TRANS.TBL - - everything/x86_64/boot.catalog -raspi_img: - - raspi_img/*-raspi-aarch64.img - - raspi_img/*-raspi-aarch64.img.sha256sum - - raspi_img/*-raspi-aarch64.img.xz - - raspi_img/*-raspi-aarch64.img.xz.sha256sum -source: - - source/aarch64/EFI/ - - source/aarch64/Packages/ - - source/aarch64/docs/ - - source/aarch64/images/ - - source/aarch64/repodata/ - - source/aarch64/RPM-GPG-KEY-openEuler/ - - source/aarch64/TRANS.TBL - - source/aarch64/boot.catalog -stratovirt_img: - - stratovirt_img/aarch64/ - - stratovirt_img/x86_64/ - - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz - - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum - - stratovirt_img/aarch64/EFI/ - - stratovirt_img/aarch64/std-vmlinux.bin - - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum - - stratovirt_img/aarch64/vmlinux.bin - - stratovirt_img/aarch64/vmlinux.bin.sha256sum - - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz - - stratovirt_img/x86_64/EFI/ - - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum - - stratovirt_img/x86_64/std-vmlinux.bin - - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum - - stratovirt_img/x86_64/vmlinux.bin - - stratovirt_img/x86_64/vmlinux.bin.sha256sum -update: - - update/aarch64/ - - update/source/ - - update/x86_64/ - - update/aarch64/Packages/ - - update/aarch64/repodata/ - - update/source/Packages/ - - update/source/repodata/ - - update/x86_64/Packages/ - - update/x86_64/repodata/ +standard: + MAIN_DIR: + - EPOL/ + - ISO/ + - OS/ + - debuginfo/ + - docker_img/ + - everything/ + - raspi_img/ + - source/ + - stratovirt_img/ + - update/ + - virtual_machine_img/ + EPOL: + - EPOL/main/ + - EPOL/update/ + - EPOL/multi_version/ + - EPOL/multi_version/OpenStack/ + - EPOL/main/source/ + - EPOL/main/aarch64/ + - EPOL/main/x86_64/ + - EPOL/update/main/ + - EPOL/update/multi_version/ + - EPOL/update/main/aarch64/ + - EPOL/update/main/source/ + - EPOL/update/main/x86_64/ + - EPOL/main/source/Packages/ + - EPOL/main/source/repodata/ + - EPOL/main/aarch64/Packages/ + - EPOL/main/aarch64/repodata/ + - EPOL/main/x86_64/repodata/ + - EPOL/update/multi_version/source/ + - EPOL/update/main/aarch64/Packages/ + - EPOL/update/main/aarch64/repodata/ + - EPOL/update/main/source/Packages/ + - EPOL/update/main/source/repodata/ + - EPOL/update/main/x86_64/Packages/ + - EPOL/update/main/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Train/ + - EPOL/multi_version/OpenStack/Wallaby/ + - EPOL/multi_version/OpenStack/Train/aarch64/ + - EPOL/multi_version/OpenStack/Train/source/ + - EPOL/multi_version/OpenStack/Train/x86_64/ + - EPOL/multi_version/OpenStack/Train/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Train/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Train/source/Packages/ + - EPOL/multi_version/OpenStack/Train/source/repodata/ + - EPOL/multi_version/OpenStack/Train/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Train/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/ + - EPOL/multi_version/OpenStack/Wallaby/source/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/source/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/source/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/repodata/ + - EPOL/update/multi_version/source/Packages/ + - EPOL/update/multi_version/source/repodata/ + - EPOL/update/multi_version/x86_64/Packages/ + - EPOL/update/multi_version/x86_64/repodata/ + - EPOL/update/multi_version/aarch64/Packages/ + - EPOL/update/multi_version/aarch64/repodata/ + ISO: + - ISO/aarch64/ + - ISO/x86_64/ + - ISO/source/ + - ISO/aarch64/*-aarch64-dvd.iso + - ISO/aarch64/*-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-aarch64.rpmlist + - ISO/aarch64/*-everything-aarch64-dvd.iso + - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-netinst-aarch64-dvd.iso + - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum + - ISO/source/*-source-dvd.iso + - ISO/source/*-source-dvd.iso.sha256sum + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-netinst-x86_64-dvd.iso + - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64-dvd.iso + - ISO/x86_64/*-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64.rpmlist + OS: + - OS/aarch64/ + - OS/x86_64/ + - OS/aarch64/EFI/ + - OS/aarch64/Packages/ + - OS/aarch64/docs/ + - OS/aarch64/images/ + - OS/aarch64/repodata/ + - OS/aarch64/RPM-GPG-KEY-openEuler/ + - OS/aarch64/TRANS.TBL + - OS/aarch64/boot.catalog + - OS/x86_64/EFI/ + - OS/x86_64/Packages/ + - OS/x86_64/docs/ + - OS/x86_64/images/ + - OS/x86_64/repodata/ + - OS/x86_64/RPM-GPG-KEY-openEuler/ + - OS/x86_64/TRANS.TBL + - OS/x86_64/boot.catalog + debuginfo: + - debuginfo/aarch64/ + - debuginfo/x86_64/ + - debuginfo/aarch64/EFI/ + - debuginfo/aarch64/Packages/ + - debuginfo/aarch64/docs/ + - debuginfo/aarch64/images/ + - debuginfo/aarch64/repodata/ + - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ + - debuginfo/aarch64/TRANS.TBL + - debuginfo/aarch64/boot.catalog + - debuginfo/x86_64/EFI/ + - debuginfo/x86_64/Packages/ + - debuginfo/x86_64/docs/ + - debuginfo/x86_64/images/ + - debuginfo/x86_64/repodata/ + - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ + - debuginfo/x86_64/TRANS.TBL + - debuginfo/x86_64/boot.catalog + docker_img: + - docker_img/aarch64/ + - docker_img/x86_64/ + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum + everything: + - everything/aarch64/ + - everything/x86_64/ + - everything/aarch64/EFI/ + - everything/aarch64/Packages/ + - everything/aarch64/docs/ + - everything/aarch64/images/ + - everything/aarch64/repodata/ + - everything/aarch64/RPM-GPG-KEY-openEuler/ + - everything/aarch64/TRANS.TBL + - everything/aarch64/boot.catalog + - everything/x86_64/EFI/ + - everything/x86_64/Packages/ + - everything/x86_64/docs/ + - everything/x86_64/images/ + - everything/x86_64/repodata/ + - everything/x86_64/RPM-GPG-KEY-openEuler/ + - everything/x86_64/TRANS.TBL + - everything/x86_64/boot.catalog + raspi_img: + - raspi_img/*-raspi-aarch64.img + - raspi_img/*-raspi-aarch64.img.sha256sum + - raspi_img/*-raspi-aarch64.img.xz + - raspi_img/*-raspi-aarch64.img.xz.sha256sum + source: + - source/aarch64/EFI/ + - source/aarch64/Packages/ + - source/aarch64/docs/ + - source/aarch64/images/ + - source/aarch64/repodata/ + - source/aarch64/RPM-GPG-KEY-openEuler/ + - source/aarch64/TRANS.TBL + - source/aarch64/boot.catalog + stratovirt_img: + - stratovirt_img/aarch64/ + - stratovirt_img/x86_64/ + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum + - stratovirt_img/aarch64/EFI/ + - stratovirt_img/aarch64/std-vmlinux.bin + - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum + - stratovirt_img/aarch64/vmlinux.bin + - stratovirt_img/aarch64/vmlinux.bin.sha256sum + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz + - stratovirt_img/x86_64/EFI/ + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum + - stratovirt_img/x86_64/std-vmlinux.bin + - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum + - stratovirt_img/x86_64/vmlinux.bin + - stratovirt_img/x86_64/vmlinux.bin.sha256sum + update: + - update/aarch64/ + - update/source/ + - update/x86_64/ + - update/aarch64/Packages/ + - update/aarch64/repodata/ + - update/source/Packages/ + - update/source/repodata/ + - update/x86_64/Packages/ + - update/x86_64/repodata/ -virtual_machine_img: - - virtual_machine_img/aarch64/ - - virtual_machine_img/x86_64/ - - virtual_machine_img/aarch64/*-aarch64.qcow2.xz - - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum - - virtual_machine_img/x86_64/*-x86_64.qcow2.xz - - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum \ No newline at end of file + virtual_machine_img: + - virtual_machine_img/aarch64/ + - virtual_machine_img/x86_64/ + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum +openEuler-22.09-1: + MAIN_DIR: + - EPOL/ + - ISO/ + - OS/ + - debuginfo/ + - docker_img/ + - edge_img/ + - embedded_img/ + - everything/ + - raspi_img/ + - source/ + - stratovirt_img/ + - update/ + - virtual_machine_img/ + EPOL: + - EPOL/main/ + - EPOL/update/ + - EPOL/main/source/ + - EPOL/main/aarch64/ + - EPOL/main/x86_64/ + - EPOL/update/main/ + - EPOL/update/multi_version/ + - EPOL/update/main/aarch64/ + - EPOL/main/source/Packages/ + - EPOL/main/source/repodata/ + - EPOL/main/aarch64/Packages/ + - EPOL/main/aarch64/repodata/ + - EPOL/main/x86_64/repodata/ + - EPOL/update/multi_version/source/ + - EPOL/update/main/aarch64/Packages/ + - EPOL/update/main/aarch64/repodata/ + - EPOL/update/main/source/Packages/ + - EPOL/update/main/source/repodata/ + - EPOL/update/main/x86_64/Packages/ + - EPOL/update/main/x86_64/repodata/ + - EPOL/update/multi_version/source/Packages/ + - EPOL/update/multi_version/source/repodata/ + - EPOL/update/multi_version/x86_64/Packages/ + - EPOL/update/multi_version/x86_64/repodata/ + - EPOL/update/multi_version/aarch64/Packages/ + - EPOL/update/multi_version/aarch64/repodata/ + ISO: + - ISO/aarch64/ + - ISO/x86_64/ + - ISO/source/ + - ISO/aarch64/*-aarch64-dvd.iso + - ISO/aarch64/*-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-aarch64.rpmlist + - ISO/aarch64/*-everything-aarch64-dvd.iso + - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-netinst-aarch64-dvd.iso + - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum + - ISO/source/*-source-dvd.iso + - ISO/source/*-source-dvd.iso.sha256sum + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-netinst-x86_64-dvd.iso + - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64-dvd.iso + - ISO/x86_64/*-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64.rpmlist + OS: + - OS/aarch64/ + - OS/x86_64/ + - OS/aarch64/EFI/ + - OS/aarch64/Packages/ + - OS/aarch64/docs/ + - OS/aarch64/images/ + - OS/aarch64/repodata/ + - OS/aarch64/RPM-GPG-KEY-openEuler/ + - OS/aarch64/TRANS.TBL + - OS/aarch64/boot.catalog + - OS/x86_64/EFI/ + - OS/x86_64/Packages/ + - OS/x86_64/docs/ + - OS/x86_64/images/ + - OS/x86_64/repodata/ + - OS/x86_64/RPM-GPG-KEY-openEuler/ + - OS/x86_64/TRANS.TBL + - OS/x86_64/boot.catalog + debuginfo: + - debuginfo/aarch64/ + - debuginfo/x86_64/ + - debuginfo/aarch64/EFI/ + - debuginfo/aarch64/Packages/ + - debuginfo/aarch64/docs/ + - debuginfo/aarch64/images/ + - debuginfo/aarch64/repodata/ + - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ + - debuginfo/aarch64/TRANS.TBL + - debuginfo/aarch64/boot.catalog + - debuginfo/x86_64/EFI/ + - debuginfo/x86_64/Packages/ + - debuginfo/x86_64/docs/ + - debuginfo/x86_64/images/ + - debuginfo/x86_64/repodata/ + - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ + - debuginfo/x86_64/TRANS.TBL + - debuginfo/x86_64/boot.catalog + docker_img: + - docker_img/aarch64/ + - docker_img/x86_64/ + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum + everything: + - everything/aarch64/ + - everything/x86_64/ + - everything/aarch64/EFI/ + - everything/aarch64/Packages/ + - everything/aarch64/docs/ + - everything/aarch64/images/ + - everything/aarch64/repodata/ + - everything/aarch64/RPM-GPG-KEY-openEuler/ + - everything/aarch64/TRANS.TBL + - everything/aarch64/boot.catalog + - everything/x86_64/EFI/ + - everything/x86_64/Packages/ + - everything/x86_64/docs/ + - everything/x86_64/images/ + - everything/x86_64/repodata/ + - everything/x86_64/RPM-GPG-KEY-openEuler/ + - everything/x86_64/TRANS.TBL + - everything/x86_64/boot.catalog + raspi_img: + - raspi_img/*-raspi-aarch64.img + - raspi_img/*-raspi-aarch64.img.sha256sum + - raspi_img/*-raspi-aarch64.img.xz + - raspi_img/*-raspi-aarch64.img.xz.sha256sum + source: + - source/aarch64/EFI/ + - source/aarch64/Packages/ + - source/aarch64/docs/ + - source/aarch64/images/ + - source/aarch64/repodata/ + - source/aarch64/RPM-GPG-KEY-openEuler/ + - source/aarch64/TRANS.TBL + - source/aarch64/boot.catalog + stratovirt_img: + - stratovirt_img/aarch64/ + - stratovirt_img/x86_64/ + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum + - stratovirt_img/aarch64/EFI/ + - stratovirt_img/aarch64/std-vmlinux.bin + - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum + - stratovirt_img/aarch64/vmlinux.bin + - stratovirt_img/aarch64/vmlinux.bin.sha256sum + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz + - stratovirt_img/x86_64/EFI/ + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum + - stratovirt_img/x86_64/std-vmlinux.bin + - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum + - stratovirt_img/x86_64/vmlinux.bin + - stratovirt_img/x86_64/vmlinux.bin.sha256sum + update: + - update/aarch64/ + - update/source/ + - update/x86_64/ + - update/aarch64/Packages/ + - update/aarch64/repodata/ + - update/source/Packages/ + - update/source/repodata/ + - update/x86_64/Packages/ + - update/x86_64/repodata/ + + virtual_machine_img: + - virtual_machine_img/aarch64/ + - virtual_machine_img/x86_64/ + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum \ No newline at end of file diff --git a/script/tools/test-2209.yaml b/script/tools/test-2209.yaml new file mode 100644 index 0000000..516e4ea --- /dev/null +++ b/script/tools/test-2209.yaml @@ -0,0 +1,27 @@ +EPOL: + - value: true + - children: + - main: + - children: + - value: true + - source: + - children: + - value: true + - Packages: + - value: true + - repodata: + - value: true + - aarch64: + - children: + - value: true + - Packages: + - value: true + - repodata: + - value: true + - x86_64: + - children: + - value: true + - Packages: + - value: true + - repodata: + - value: true \ No newline at end of file diff --git a/script/tools/test.py b/script/tools/test.py new file mode 100644 index 0000000..5a9444c --- /dev/null +++ b/script/tools/test.py @@ -0,0 +1,61 @@ +import os +import sys +import re +import yaml +import shutil +import requests +import argparse +import logging +import json +from bs4 import BeautifulSoup + +def read_yaml(): + try: + with open('./test.yaml', 'r', encoding='utf-8') as f: + yaml_result = yaml.load(f, Loader=yaml.FullLoader) + # print (yaml_result['standard']) + return yaml_result['standard'] + except Exception as e: + print (e) + logging.info("error read standard.yaml,please check") + +def deal_result(result): + # print (result) + generate_json = json.dumps(result,sort_keys=False,indent=4,separators=(',',': ')) + print(generate_json) + all_result = [] + for key,value in result.items(): + deal_result_j(key,value,key) + + +def deal_result_j(ke,result,complete_key): + print (complete_key) + for line in result: + if isinstance(line, dict): + for key,value in line.items(): + complete_key = complete_key + '/' + key + deal_result_j(key,value,complete_key) + else: + temp_key = complete_key + '/' + line + print (temp_key) + +def write_to_result(result): + # guess_layer = {1:result[temp[0]],2:result[temp[0]][temp[1]]} + list1 = ['EPOL','EPOL/main','EPOL/main/source','EPOL/main/source/aarch64/Packages'] + for line in list1: + temp = line.split('/') + print (temp) + num = len(temp) + if num == 1: + result[temp[0]].append({'value':True}) + elif num == 2: + # result['Epol']['main'] + print (temp[0]) + print (result['EPOL']['main']) + # result[temp[0]][temp[1]].append({'value':True}) + print (result) + +if __name__ == '__main__': + result = read_yaml() + deal_result(result) + write_to_result(result) diff --git a/script/tools/test.yaml b/script/tools/test.yaml new file mode 100644 index 0000000..b2c87ad --- /dev/null +++ b/script/tools/test.yaml @@ -0,0 +1,31 @@ +standard: + EPOL: + - main: + - source: + - Packages + - repodata + - aarch64: + - Packages + - repodata + - x86_64: + - Packages + - repodata + OS: + - aarch64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog + - x86_64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog diff --git a/script/tools/test1.yaml b/script/tools/test1.yaml new file mode 100644 index 0000000..3e838f8 --- /dev/null +++ b/script/tools/test1.yaml @@ -0,0 +1,164 @@ +standard: + EPOL: + - value: true + - children: + - main: + - children: + - value: true + - source: + - children: + - value: true + - Packages: + - value: true + - repodata: + - value: true + - aarch64: + - children: + - value: true + - Packages: + - value: true + - repodata: + - value: true + - x86_64: + - children: + - value: true + - Packages: + - value: true + - repodata: + - value: true + ISO: + - aarch64: + - $-aarch64-dvd.iso + - $-aarch64-dvd.iso.sha256sum + - $-aarch64.rpmlist + - $-everything-aarch64-dvd.iso + - $-everything-aarch64-dvd.iso.sha256sum + - $-everything-debug-aarch64-dvd.iso + - $-everything-debug-aarch64-dvd.iso.sha256sum + - $-netinst-aarch64-dvd.iso + - $-netinst-aarch64-dvd.iso.sha256sum + - x86_64: + - $-everything-debug-x86_64-dvd.iso + - $-everything-debug-x86_64-dvd.iso.sha256sum + - $-netinst-x86_64-dvd.iso + - $-netinst-x86_64-dvd.iso.sha256sum + - $-x86_64-dvd.iso + - $-x86_64-dvd.iso.sha256sum + - $-x86_64.rpmlist + - source: + - $-source-dvd.iso + - $-source-dvd.iso.sha256sum + OS: + - aarch64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog + - x86_64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog + debuginfo: + - aarch64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog + - x86_64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog + docker_img: + - aarch64: + - openEuler-docker.aarch64.tar.xz + - openEuler-docker.aarch64.tar.xz.sha256sum + - x86_64: + - openEuler-docker.x86_64.tar.xz + - openEuler-docker.x86_64.tar.xz.sha256sum + everything: + - aarch64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog + - x86_64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog + raspi_img: + - $-raspi-aarch64.img + - $-raspi-aarch64.img.sha256sum + - $-raspi-aarch64.img.xz + - $-raspi-aarch64.img.xz.sha256sum + source: + - aarch64: + - EFI + - Packages + - docs + - images + - repodata + - RPM-GPG-KEY-openEuler + - TRANS.TBL + - boot.catalog + stratovirt_img: + - aarch64: + - $-stratovirt-aarch64.img.xz + - $-stratovirt-aarch64.img.xz.sha256sum + - EFI + - std-vmlinux.bin + - std-vmlinux.bin.sha256sum + - vmlinux.bin + - vmlinux.bin.sha256sum + - x86_64: + - $-stratovirt-x86_64.img.xz + - EFI + - $-stratovirt-x86_64.img.xz.sha256sum + - std-vmlinux.bin + - std-vmlinux.bin.sha256sum + - vmlinux.bin + - vmlinux.bin.sha256sum + update: + - aarch64: + - Packages + - repodata + - x86_64: + - Packages + - repodata + - source: + - Packages + - repodata + virtual_machine_img: + - aarch64: + - $-aarch64.qcow2.xz + - $-aarch64.qcow2.xz.sha256sum + - x86_64: + - $-x86_64.qcow2.xz + - $-x86_64.qcow2.xz.sha256sum + diff --git a/script/tools/test2.yaml b/script/tools/test2.yaml new file mode 100644 index 0000000..f7b0aba --- /dev/null +++ b/script/tools/test2.yaml @@ -0,0 +1,22 @@ +standard: + EPOL: + - value: true + - main: + - value: true + - source: + - Packages: + - value: false + - repodata: + - value: true + - aarch64: + - value: true + - Packages: + - value: true + - repodata: + - value: true + - x86_64: + - value: true + - Packages: + - value: true + - repodata: + - value: true \ No newline at end of file diff --git a/script/tools/xnio b/script/tools/xnio new file mode 100644 index 0000000..1a1dc93 --- /dev/null +++ b/script/tools/xnio @@ -0,0 +1,74 @@ + + + + +window.gon = {};gon.locale="zh-CN";gon.sentry_dsn=null;gon.baidu_register_hm_push=null;gon.sensor={"server_url":"https://haveaniceday.gitee.com:3443/sa?project=production","sdk_url":"https://cn-assets.gitee.com/assets/static/sensors-sdk-2f850fa5b654ad55ac0993fda2f37ba5.js","page_type":"其他"};gon.info={"controller_path":"tree","action_name":"show","current_user":false};gon.tour_env={"current_user":null,"action_name":"show","original_url":"https://gitee.com/src-openeuler/xnio/tree/openEuler-21.09","controller_path":"tree"};gon.http_clone="https://gitee.com/src-openeuler/xnio.git";gon.user_project="src-openeuler/xnio";gon.manage_branch="管理分支";gon.manage_tag="管理标签";gon.enterprise_id=5292411;gon.create_reaction_path="/src-openeuler/xnio/reactions";gon.ipipe_base_url="https://go-api.gitee.com";gon.artifact_base_url="https://go-repo.gitee.com";gon.gitee_go_remote_url="https://go.gitee.com/assets";gon.gitee_go_active=false;gon.current_project_is_mirror=false;gon.ref="openEuler-21.09"; + link: "https://gitee.com/src-openeuler/xnio/tree/openEuler-21.09", // 分享链接，该链接域名或路径必须与当前页面对应的公众号JS安全域名一致 + link: "https://gitee.com/src-openeuler/xnio/tree/openEuler-21.09", // 分享链接，该链接域名或路径必须与当前页面对应的公众号JS安全域名一致 + reponame = "src-openeuler/xnio"; + + + + +6 +0 +9 + src-openEuler / xnio