diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index a5557b0bc19f8564985bd5c88ac8522512fba097..3440fb1bdda222d84c53ad23f6cae7e66e78959a 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -15,7 +15,10 @@ # ****************************************************************************** import os import sys +import re import yaml +import xlwt +import shutil import requests import argparse import logging @@ -24,6 +27,7 @@ from bs4 import BeautifulSoup LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" DATE_FORMAT = "%Y-%m-%d %H:%M:%S" logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) +UNMOUNT_DIRS = ['debuginfo','OS','source'] class CheckDailyBuild(object): @@ -42,6 +46,7 @@ class CheckDailyBuild(object): self.main_branch = self.kwargs['main_branch'] self.dirflag = self.kwargs['dir_flag'] self.datebranch = self.kwargs['date_branch'] + self.mountflag = self.kwargs['mountflag'] self.standard_dir = self.load_standard() def html_downloader(self, url): @@ -92,8 +97,8 @@ class CheckDailyBuild(object): """ dir_map = {} html_content = self.html_downloader(main_branch_url) - if html_content: - current_dir = self.html_parser(main_branch_url, html_content) + current_dir = self.html_parser(main_branch_url, html_content) + if current_dir: dir_map[main_branch_url] = [] for first_dir in current_dir: dir_split = first_dir.split('/')[5] @@ -140,14 +145,19 @@ class CheckDailyBuild(object): try: with open('./standard.yaml', 'r', encoding='utf-8') as f: result = yaml.load(f, Loader=yaml.FullLoader) + if self.mountflag == '0': + for delitem in UNMOUNT_DIRS: + del result[delitem] return result except Exception as e: + print (e) logging.info("error read standard.yaml,please check") def compare_standard(self, dir_list, current_branch, c_dir): """ branch dir compare with standard dir """ + logging.info("********************************************CHECK RESULT*********************************************************") standard_dir = self.standard_dir for key, c_standard in standard_dir.items(): for current_dir in c_standard: @@ -155,8 +165,19 @@ class CheckDailyBuild(object): current_dir = current_dir.replace('*', current_branch) if current_dir not in dir_list: logging.error( - 'this dir not found,link url:{}{}'.format( + 'this dir or file not found,link url:{}{}'.format( c_dir, current_dir)) + logging.info("********************************************CHECK RESULT*********************************************************") + + def check_input_args(self,complete_url): + html_content = self.html_downloader(complete_url) + current_dir = self.html_parser(complete_url, html_content) + if not current_dir: + logging.error("error url can not open,please check your input：{}".format(complete_url)) + raise SystemExit("*******PLEASE CHECK YOUR INPUT ARGS*******") + else: + return True + def _get_main_branch(self): if self.datebranch: @@ -164,14 +185,265 @@ class CheckDailyBuild(object): complete_key = "{}/{}/".format(self.rooturl, self.main_branch) complete_value = "{}/{}/{}/".format( self.rooturl, self.main_branch, self.datebranch) - dir_map[complete_key] = [complete_value] + check_args_result = self.check_input_args(complete_value) + if check_args_result: + dir_map[complete_key] = [complete_value] + self.check_every_dir(dir_map) else: main_branch_url = "{}/{}/".format(self.rooturl, self.main_branch) dir_map = self.start_check(main_branch_url) - self.check_every_dir(dir_map) + self.check_every_dir(dir_map) + + def rpms_parser(self,html_content): + rpms_list = [] + soup = BeautifulSoup(html_content, "html.parser") + tr_content = soup.find_all('tr') + for line in tr_content: + td_content = line.find_all('td') + if td_content: + dir_url = td_content[0].find('a', href=True) + size = td_content[1].text + if dir_url.get('href',''): + if dir_url['href'] != '../' and self.check_is_rpm(dir_url['href']): + rpms_list.append(dir_url['href']) + return rpms_list + + def obs_rpms_parser(self,html_content): + rpms_list = [] + soup = BeautifulSoup(html_content, "html.parser") + for k in soup.find_all('a',href=True): + if k.get('href',''): + if '.rpm' in k['href']: + rpms_list.append(k['href']) + return rpms_list + + def get_repo_rpms(self): + ''' + get rpms from offical repo + ''' + repo_data = {} + epol_rpms_dir = "https://repo.openeuler.org/{}/EPOL/main/source/Packages/".format(self.main_branch) + epol_content = self.html_downloader(epol_rpms_dir) + epol_rpms = self.rpms_parser(epol_content) + source_rpms_dir = "https://repo.openeuler.org/{}/source/Packages/".format(self.main_branch) + source_content = self.html_downloader(source_rpms_dir) + source_rpms = self.rpms_parser(source_content) + repo_data = { + 'epol_rpms': epol_rpms, + 'source_rpms': source_rpms + } + return repo_data + + def get_dailybuild_repo_rpms(self): + ''' + get rpms from daily build repo + ''' + daily_build_repo_data = {} + epol_rpms_dir = "http://121.36.84.172/dailybuild/{}/{}/EPOL/main/source/Packages/".format(self.main_branch,self.datebranch) + epol_content = self.html_downloader(epol_rpms_dir) + epol_rpms = self.rpms_parser(epol_content) + source_rpms_dir = "http://121.36.84.172/dailybuild/{}/{}/source/Packages/".format(self.main_branch,self.datebranch) + source_content = self.html_downloader(source_rpms_dir) + source_rpms = self.rpms_parser(source_content) + daily_build_repo_data = { + 'epol_rpms': epol_rpms, + 'source_rpms': source_rpms + } + return daily_build_repo_data + + def get_obs_repo_rpms(self): + ''' + get rpms from obs repo + ''' + obs_repo_data = {} + trans_url = self.main_branch.replace('-',':/') + epol_obs_url = "http://119.3.219.20:82/{}:/Epol/standard_x86_64/src/".format(trans_url) + epol_content = self.html_downloader(epol_obs_url) + epol_obs_rpms = self.obs_rpms_parser(epol_content) + src_obs_url = "http://119.3.219.20:82/{}/standard_x86_64/src/".format(trans_url) + src_content = self.html_downloader(src_obs_url) + src_obs_rpms = self.obs_rpms_parser(src_content) + obs_repo_data = { + 'epol_rpms': epol_obs_rpms, + 'source_rpms': src_obs_rpms + } + return obs_repo_data + + def compare_rpms(self,repo_data,daily_repo_data,obs_repo_data): + ''' + get epol and source rpms data and start compare + :param repo_data:dict of offical repo src repo rpms include epol + :param daily_repo_data:dict of daily build repo src repo rpms include epol + :param obs_repo_data:dict of obs repo src repo rpms include epol + ''' + offical_epol_rpms = repo_data.get("epol_rpms",[]) + daily_epol_rpms = daily_repo_data.get("epol_rpms",[]) + obs_epol_rpms = obs_repo_data.get("epol_rpms",[]) + offical_src_rpms = repo_data.get("source_rpms",[]) + daily_src_rpms = daily_repo_data.get("source_rpms",[]) + obs_src_rpms = obs_repo_data.get("source_rpms",[]) + epol_result = self.compare_detail_rpms(daily_epol_rpms,offical_epol_rpms,obs_epol_rpms) + self.write_file(epol_result) + source_result = self.compare_detail_rpms(daily_src_rpms,offical_src_rpms,obs_src_rpms,compare_type='source') + self.write_file(source_result,compare_type='source') + + def compare_detail_rpms(self,daily,offical,obs,compare_type='epol'): + ''' + src rpms detail compare + :param daily:List of daily build repo rpms + :param offical:List of offical repo rpms + :param obs:List of obs repo rpms + :return:dict of compare result + ''' + datas = {} + if daily and offical: + # daily rpms compare with offical epol rpms + in_daily_offical = list(set(daily).difference(set(offical))) + in_offical_daily = list(set(offical).difference(set(daily))) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:offical_repo*********************************") + in_daily_offical,in_offical_daily = self.rpm_version_compare(in_daily_offical,in_offical_daily) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:offical_repo*********************************") + logging.info("{} rpms compare below rpms in daily repo not in offical repo：{}".format(compare_type,in_daily_offical)) + logging.info("{} rpms compare below rpms in offical repo not in daily repo：{}".format(compare_type,in_offical_daily)) + daily_offical = {"rpmname":[self.datebranch,"offical repo"]} + for rpm in in_daily_offical: + daily_offical[rpm] = [1,0] + for rpm in in_offical_daily: + daily_offical[rpm] = [0,1] + datas["daily_offical"] = daily_offical + if daily and obs: + # daily rpms compare with obs epol rpms + in_daily_obs = list(set(daily).difference(set(obs))) + in_obs_daily = list(set(obs).difference(set(daily))) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:obs_repo*********************************") + in_daily_obs,in_obs_daily = self.rpm_version_compare(in_daily_obs,in_obs_daily) + logging.info("***************************************SIDEA:daily_build_repo,SIDEB:obs_repo*********************************") + logging.info("{} rpms compare below rpms in daily repo not in obs repo：{}".format(compare_type,in_daily_obs)) + logging.info("{} rpms compare below rpms in obs repo not in daily repo：{}".format(compare_type,in_obs_daily)) + daily_obs = {"rpmname":[self.datebranch,"obs repo"]} + for rpm in in_daily_obs: + daily_obs[rpm] = [1,0] + for rpm in in_obs_daily: + daily_obs[rpm] = [0,1] + datas["daily_obs"] = daily_obs + if obs and offical: + # daily rpms compare with offical epol rpms + in_obs_offical = list(set(obs).difference(set(offical))) + in_offical_obs = list(set(offical).difference(set(obs))) + logging.info("***************************************SIDEA:obs_repo,SIDEB:offical_repo*********************************") + in_obs_offical,in_offical_obs = self.rpm_version_compare(in_obs_offical,in_offical_obs) + logging.info("***************************************SIDEA:obs_repo,SIDEB:offical_repo*********************************") + logging.info("{} rpms compare below rpms in obs repo not in offical repo：{}".format(compare_type,in_obs_offical)) + logging.info("{} rpms compare below rpms in offical repo not in obs repo：{}".format(compare_type,in_offical_obs)) + obs_offical = {"rpmname":["obs repo","offical repo"]} + for rpm in in_obs_offical: + obs_offical[rpm] = [1,0] + for rpm in in_offical_obs: + obs_offical[rpm] = [0,1] + datas["obs_offical"] = obs_offical + return datas + + def write_file(self,data,compare_type='epol'): + ''' + out put compare result to excel + :param data:pre compare result data + ''' + if data: + book = xlwt.Workbook(encoding='utf-8') + for key,value in data.items(): + sheet = book.add_sheet(key,cell_overwrite_ok=True) + r = 0 + for i, j in value.items(): + le = len(j) + sheet.write(r, 0, i,) + for c in range(1, le + 1): + sheet.write(r, c, j[c - 1]) + r += 1 # 行数 + book.save("./{}-{}-src-rpm-compare-result.xls".format(self.main_branch,compare_type)) + # self._add_to_obs(compare_type) + + def _add_to_obs(self): + """ + add result file to obs project + """ + filenames = [] + path=os.getcwd() + f_list = os.listdir(path) + for xlsfile in f_list: + if os.path.splitext(xlsfile)[1] == '.xls': + filenames.append(xlsfile) + cmd = "cd %s && osc co home:Admin:ISO/%s" % (path,self.main_branch) + ret = os.popen(cmd).read() + targer_dir = os.path.join(path, "home:Admin:ISO/{}".format(self.main_branch)) + if os.path.exists(targer_dir): + for file in filenames: + src_file = os.path.join(path,file) + dist_file = os.path.join(path,targer_dir,file) + shutil.copyfile(src_file, dist_file) + cmd = "cd %s && osc add %s && osc ci -m1" % (targer_dir,file) + ret = os.popen(cmd).read() + else: + logging.info("this {} not in obs project".format(self.main_branch)) + + + def rpm_version_compare(self,paira,pairb): + ''' + check rpm version and compare + :param paira:List of sidea rpms that needs to be compared + :param pairb:List of sideb rpms that needs to be compared + ''' + for rpma in paira[:]: + rpm_namea = self.rpm_name(rpma) + for rpmb in pairb[:]: + rpm_nameb = self.rpm_name(rpmb) + if rpm_nameb == rpm_namea: + logging.info("SIDEA rpm version:{} SIDEB rpm version：{}".format(rpma,rpmb)) + pairb.remove(rpmb) + paira.remove(rpma) + break + return paira,pairb + + def rpm_n_v_r_d_a(self,rpm): + """ + parse rpm package name,version,release,publisher + :param rpm:complete rpm name + :return:split rpm + """ + # eg: grpc-1.31.0-6.oe1.x86_64.rpm + name = self.rpm_name(rpm) + + rpm_split = re.match(r"(.+)-(.+)\.(.+)?\.(.+)\.rpm", rpm.replace(name, "", 1)) + if rpm_split: + return name, rpm_split.group(1), rpm_split.group(2), rpm_split.group(3), rpm_split.group(4) + return name, rpm_split.group(1), rpm_split.group(2), None, rpm_split.group(3) + + def rpm_name(self,rpm): + """ + :param rpm:complete rpm name + :return:only rpm name + """ + m = re.match(r"^(.+)-.+-.+", rpm) + + if m: + return m.group(1) + else: + return rpm + + + def _check_src_rpm(self): + """ + Check the src rpm entry function + """ + repo_data = self.get_repo_rpms() + daily_repo_data = self.get_dailybuild_repo_rpms() + obs_repo_data = self.get_obs_repo_rpms() + self.compare_rpms(repo_data,daily_repo_data,obs_repo_data) + def run(self): self._get_main_branch() + self._check_src_rpm() + self._add_to_obs() if __name__ == "__main__": @@ -189,11 +461,14 @@ if __name__ == "__main__": parser.add_argument( "--datebranch", help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument("--mountflag",default="1") + args = parser.parse_args() kw = { "daily_build_url": args.rooturl, "main_branch": args.branch, "date_branch": args.datebranch, - "dir_flag": args.dirflag + "dir_flag": args.dirflag, + "mountflag": args.mountflag } check = CheckDailyBuild(**kw) check.run()