From f043fa2ddfafbe7bfa05f639affbe9eddf28c3ab Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Mon, 28 Feb 2022 19:23:52 +0800 Subject: [PATCH 1/8] =?UTF-8?q?daily=20build=20=E7=9B=AE=E5=BD=95=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E5=B7=A5=E5=85=B7=E4=BB=A3=E7=A0=81=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log.py | 6 ++ script/config/constant.py | 19 +++++ script/tools/daily_build_check.py | 126 ++++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 script/config/constant.py create mode 100644 script/tools/daily_build_check.py diff --git a/log.py b/log.py index 2079bca..ecfecd5 100644 --- a/log.py +++ b/log.py @@ -46,3 +46,9 @@ def log_init(): return log +log = log_init() + + +if __name__ == "__main__": + log.info("test") + diff --git a/script/config/constant.py b/script/config/constant.py new file mode 100644 index 0000000..8209b9f --- /dev/null +++ b/script/config/constant.py @@ -0,0 +1,19 @@ +class Constant(object): + """ + class Constant standard dir + """ + + STANDARD_DIR = { + 'MAIN_DIR': {'EPOL/','ISO/','OS/','debuginfo/','docker_img/','everything/','raspi_img/','source/','stratovirt_img/','update/','virtual_machine_img/'}, + 'EPOL': {'EPOL/main/aarch64/Packages/', 'EPOL/main/aarch64/repodata/', 'EPOL/main/source/', 'EPOL/main/source/Packages/', 'EPOL/main/source/repodata/', 'EPOL/main/x86_64/repodata/', 'EPOL/update/main/aarch64/Packages/', 'EPOL/update/main/aarch64/repodata/', 'EPOL/update/main/source/Packages/', 'EPOL/update/main/source/repodata/', 'EPOL/update/main/x86_64/Packages/', 'EPOL/update/main/x86_64/repodata/','EPOL/update/multi_version/aarch64/Packages/', 'EPOL/update/multi_version/aarch64/repodata/', 'EPOL/update/multi_version/source/Packages/', 'EPOL/update/multi_version/source/repodata/', 'EPOL/update/multi_version/x86_64/Packages/', 'EPOL/update/multi_version/x86_64/repodata/'}, + 'ISO': {'ISO/aarch64/*-aarch64-dvd.iso', 'ISO/aarch64/*-aarch64-dvd.iso.sha256sum', 'ISO/aarch64/*-aarch64.rpmlist', 'ISO/aarch64/*-everything-aarch64-dvd.iso', 'ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum', 'ISO/aarch64/*-everything-debug-aarch64-dvd.iso', 'ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum', 'ISO/aarch64/*-netinst-aarch64-dvd.iso', 'ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum', 'ISO/source/*-source-dvd.iso', 'ISO/source/*-source-dvd.iso.sha256sum', 'ISO/x86_64/*-everything-debug-x86_64-dvd.iso', 'ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum', 'ISO/x86_64/*-netinst-x86_64-dvd.iso', 'ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum', 'ISO/x86_64/*-x86_64-dvd.iso', 'ISO/x86_64/*-x86_64-dvd.iso.sha256sum', 'ISO/x86_64/*-x86_64.rpmlist'}, + 'OS': {'OS/aarch64/EFI/', 'OS/aarch64/Packages/', 'OS/aarch64/docs/', 'OS/aarch64/images/', 'OS/aarch64/repodata/', 'OS/aarch64/RPM-GPG-KEY-openEuler/', 'OS/aarch64/TRANS.TBL', 'OS/aarch64/boot.catalog', 'OS/x86_64/EFI/', 'OS/x86_64/Packages/', 'OS/x86_64/docs/', 'OS/x86_64/images/', 'OS/x86_64/repodata/', 'OS/x86_64/RPM-GPG-KEY-openEuler/', 'OS/x86_64/TRANS.TBL', 'OS/x86_64/boot.catalog'}, + 'debuginfo': {'debuginfo/aarch64/EFI/', 'debuginfo/aarch64/Packages/', 'debuginfo/aarch64/docs/', 'debuginfo/aarch64/images/', 'debuginfo/aarch64/repodata/', 'debuginfo/aarch64/RPM-GPG-KEY-openEuler/', 'debuginfo/aarch64/TRANS.TBL', 'debuginfo/aarch64/boot.catalog', 'debuginfo/x86_64/EFI/', 'debuginfo/x86_64/Packages/', 'debuginfo/x86_64/docs/', 'debuginfo/x86_64/images/', 'debuginfo/x86_64/repodata/', 'debuginfo/x86_64/RPM-GPG-KEY-openEuler/', 'debuginfo/x86_64/TRANS.TBL', 'debuginfo/x86_64/boot.catalog'}, + 'docker_img': {'docker_img/aarch64/openEuler-docker.aarch64.tar.xz', 'docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum', 'docker_img/x86_64/openEuler-docker.x86_64.tar.xz', 'docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum'}, + 'everything': {'everything/aarch64/EFI/', 'everything/aarch64/Packages/', 'everything/aarch64/docs/', 'everything/aarch64/images/', 'everything/aarch64/repodata/', 'everything/aarch64/RPM-GPG-KEY-openEuler/', 'everything/aarch64/TRANS.TBL', 'everything/aarch64/boot.catalog', 'everything/x86_64/EFI/', 'everything/x86_64/Packages/', 'everything/x86_64/docs/', 'everything/x86_64/images/', 'everything/x86_64/repodata/', 'everything/x86_64/RPM-GPG-KEY-openEuler/', 'everything/x86_64/TRANS.TBL', 'everything/x86_64/boot.catalog'}, + 'raspi_img': {'raspi_img/*-raspi-aarch64.img', 'raspi_img/*-raspi-aarch64.img.sha256sum', 'raspi_img/*-raspi-aarch64.img.xz', 'raspi_img/*-raspi-aarch64.img.xz.sha256sum'}, + 'source': {'source/aarch64/EFI/', 'source/aarch64/Packages/', 'source/aarch64/docs/', 'source/aarch64/images/', 'source/aarch64/repodata/', 'source/aarch64/RPM-GPG-KEY-openEuler/', 'source/aarch64/TRANS.TBL', 'source/aarch64/boot.catalog'}, + 'stratovirt_img': {'stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz', 'stratovirt_img/aarch64/EFI/', 'stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum', 'stratovirt_img/aarch64/std-vmlinux.bin', 'stratovirt_img/aarch64/std-vmlinux.bin.sha256sum', 'stratovirt_img/aarch64/vmlinux.bin', 'stratovirt_img/aarch64/vmlinux.bin.sha256sum', 'stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz', 'stratovirt_img/x86_64/EFI/', 'stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum', 'stratovirt_img/x86_64/std-vmlinux.bin', 'stratovirt_img/x86_64/std-vmlinux.bin.sha256sum', 'stratovirt_img/x86_64/vmlinux.bin', 'stratovirt_img/x86_64/vmlinux.bin.sha256sum'}, + 'update': {'update/aarch64/Packages/', 'update/aarch64/repodata/', 'update/source/Packages/', 'update/source/repodata/', 'update/x86_64/Packages/', 'update/x86_64/repodata/'}, + 'virtual_machine_img': {'virtual_machine_img/aarch64/*-aarch64.qcow2.xz', 'virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum', 'virtual_machine_img/x86_64/*-x86_64.qcow2.xz', 'virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum'} + } \ No newline at end of file diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py new file mode 100644 index 0000000..ba385eb --- /dev/null +++ b/script/tools/daily_build_check.py @@ -0,0 +1,126 @@ +import os +import sys +import requests +from bs4 import BeautifulSoup +CURRENT_DIR = os.path.split(os.path.abspath(__file__))[0] # 当前目录 +config_path = CURRENT_DIR.rsplit('/', 2)[0] # 上三级目录 +sys.path.append(config_path) +from script.config.constant import Constant +from log import log_init + +class CheckDailyBuild(object): + """ + The entrance check for daily build + """ + def __init__(self, **kwargs): + """ + kawrgs: dict,init dict by 'a': 'A' style + rooturl: the daily build page url + main_branch:choose which branch you need to check + main_branch_flag : if this value is True only check all main_branch + """ + self.kwargs = kwargs + self.rooturl = self.kwargs['daily_build_url'] + self.main_branch = self.kwargs['main_branch'] + self.main_branch_flag = self.kwargs['main_branch_flag'] + + def html_downloader(self, url): + if url is None: + return None + user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' + headers = {'User-Agent': user_agent} + r = requests.get(url, headers=headers) + if r.status_code == 200: + r.encoding = 'utf-8' + log.info("this html page download success！{}".format(url)) + return r.text + else: + log.error("this html page download failed！{}".format(url)) + return True + + def html_parser(self, url, html_content): + url_list = [] + soup = BeautifulSoup(html_content, "html.parser") + tr_content = soup.find_all('tr') + for line in tr_content: + td_content = line.find_all('td') + if td_content: + dir_url = td_content[0].find('a', href=True) + size = td_content[1].text + if dir_url: + if dir_url['href'] != '../' and size == '-': + url_list.append(url + dir_url['href']) + elif dir_url['href'] != '../' and size != '-': + url_list.append(url + dir_url['href'] + '|ISFILE/') + return url_list + + def start_check(self, main_branch_url): + dir_map = {} + for main in main_branch_url: + html_content = self.html_downloader(main) + current_dir = self.html_parser(main, html_content) + dir_map[main] = [] + for dir in current_dir: + dir_split = dir.split('/')[5] + if dir_split.startswith('openeuler-2022-'): + dir_map[main].append(dir) + return dir_map + + def check_every_dir(self, dir_map): + for dir_url, dir in dir_map.items(): + if dir: + c_branch=dir_url.split("/")[-2] + for c_dir in dir: + origin_dir = c_dir + dir_list = self.check_current_dir(c_dir, origin_dir, []) + self.compare_standard(dir_list, c_branch, c_dir) + return True + + def check_current_dir(self, dir, origin_dir, temp_list): + try: + this_page = self.html_downloader(dir) + page_dir = self.html_parser(dir, this_page) + for item in page_dir: + item_name = item.replace(origin_dir, '') + if 'ISFILE/' in item_name: + item_name = item_name.replace('|ISFILE/', '') + temp_list.append(item_name) + if item.split('|')[-1] != 'ISFILE/': + self.check_current_dir(item, origin_dir, temp_list) + except Exception as e: + log.info("error url can not open:{}".format(dir)) + return temp_list + + def compare_standard(self, dir_list, current_branch, c_dir): + standard_dir = Constant.STANDARD_DIR + for key, c_standard in standard_dir.items(): + for current_dir in c_standard: + if '*' in current_dir: + current_dir = current_dir.replace('*', current_branch) + if current_dir not in dir_list: + log.error('this dir not found,link url:{}{}'.format(c_dir, current_dir)) + + + + def _get_main_branch(self): + if not self.main_branch_flag: + html_content = self.html_downloader(self.rooturl) + main_branch_url = self.html_parser(self.rooturl, html_content) + else: + main_branch_url = [self.rooturl + self.main_branch] + dir_map = self.start_check(main_branch_url) + self.check_every_dir(dir_map) + + def run(self): + self._get_main_branch() + + + + +if __name__ == "__main__": + kw = {"daily_build_url": "http://121.36.84.172/dailybuild/", + "main_branch": "openEuler-22.03-LTS/", + "main_branch_flag": True} + check = CheckDailyBuild(**kw) + log = log_init() + check.run() \ No newline at end of file -- Gitee From fa3fcbf1328b0e118407719f36f047d36a92c4b4 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 1 Mar 2022 14:34:56 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=BC=BA=E9=99=B7?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log.py | 3 +- script/config/constant.py | 2 ++ script/tools/daily_build_check.py | 53 ++++++++++++++++--------------- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/log.py b/log.py index ecfecd5..6683452 100644 --- a/log.py +++ b/log.py @@ -46,9 +46,8 @@ def log_init(): return log -log = log_init() - if __name__ == "__main__": + log = log_init() log.info("test") diff --git a/script/config/constant.py b/script/config/constant.py index 8209b9f..c4b11f8 100644 --- a/script/config/constant.py +++ b/script/config/constant.py @@ -1,3 +1,5 @@ +#!/bin/env python3 +# -*- encoding=utf8 -*- class Constant(object): """ class Constant standard dir diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index ba385eb..fe36584 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -1,23 +1,27 @@ +#!/bin/env python3 +# -*- encoding=utf8 -*- +from log import log_init +from script.config.constant import Constant import os import sys import requests from bs4 import BeautifulSoup -CURRENT_DIR = os.path.split(os.path.abspath(__file__))[0] # 当前目录 -config_path = CURRENT_DIR.rsplit('/', 2)[0] # 上三级目录 +CURRENT_DIR = os.path.split(os.path.abspath(__file__))[0] +config_path = CURRENT_DIR.rsplit('/', 2)[0] sys.path.append(config_path) -from script.config.constant import Constant -from log import log_init + class CheckDailyBuild(object): """ The entrance check for daily build """ + def __init__(self, **kwargs): """ kawrgs: dict,init dict by 'a': 'A' style rooturl: the daily build page url - main_branch:choose which branch you need to check - main_branch_flag : if this value is True only check all main_branch + main_branch:choose which branch you need to check + main_branch_flag : if this value is True only check all main_branch """ self.kwargs = kwargs self.rooturl = self.kwargs['daily_build_url'] @@ -60,26 +64,25 @@ class CheckDailyBuild(object): html_content = self.html_downloader(main) current_dir = self.html_parser(main, html_content) dir_map[main] = [] - for dir in current_dir: - dir_split = dir.split('/')[5] + for first_dir in current_dir: + dir_split = first_dir.split('/')[5] if dir_split.startswith('openeuler-2022-'): - dir_map[main].append(dir) + dir_map[main].append(first_dir) return dir_map def check_every_dir(self, dir_map): - for dir_url, dir in dir_map.items(): - if dir: - c_branch=dir_url.split("/")[-2] - for c_dir in dir: + for dir_url, dir_list in dir_map.items(): + if dir_list: + c_branch = dir_url.split("/")[-2] + for c_dir in dir_list: origin_dir = c_dir - dir_list = self.check_current_dir(c_dir, origin_dir, []) - self.compare_standard(dir_list, c_branch, c_dir) - return True + dir_result = self.check_current_dir(c_dir, origin_dir, []) + self.compare_standard(dir_result, c_branch, c_dir) - def check_current_dir(self, dir, origin_dir, temp_list): + def check_current_dir(self, c_dir, origin_dir, temp_list): try: - this_page = self.html_downloader(dir) - page_dir = self.html_parser(dir, this_page) + this_page = self.html_downloader(c_dir) + page_dir = self.html_parser(c_dir, this_page) for item in page_dir: item_name = item.replace(origin_dir, '') if 'ISFILE/' in item_name: @@ -88,7 +91,7 @@ class CheckDailyBuild(object): if item.split('|')[-1] != 'ISFILE/': self.check_current_dir(item, origin_dir, temp_list) except Exception as e: - log.info("error url can not open:{}".format(dir)) + log.info("error url can not open:{}".format(c_dir)) return temp_list def compare_standard(self, dir_list, current_branch, c_dir): @@ -98,9 +101,9 @@ class CheckDailyBuild(object): if '*' in current_dir: current_dir = current_dir.replace('*', current_branch) if current_dir not in dir_list: - log.error('this dir not found,link url:{}{}'.format(c_dir, current_dir)) - - + log.error( + 'this dir not found,link url:{}{}'.format( + c_dir, current_dir)) def _get_main_branch(self): if not self.main_branch_flag: @@ -115,12 +118,10 @@ class CheckDailyBuild(object): self._get_main_branch() - - if __name__ == "__main__": kw = {"daily_build_url": "http://121.36.84.172/dailybuild/", "main_branch": "openEuler-22.03-LTS/", "main_branch_flag": True} check = CheckDailyBuild(**kw) log = log_init() - check.run() \ No newline at end of file + check.run() -- Gitee From 004ba60eba3bab80c4ae6b416a2692b6076f4c8a Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 2 Mar 2022 17:59:01 +0800 Subject: [PATCH 3/8] code review fix --- log.py | 5 ----- script/{config => tools}/constant.py | 0 script/tools/daily_build_check.py | 4 ++-- 3 files changed, 2 insertions(+), 7 deletions(-) rename script/{config => tools}/constant.py (100%) diff --git a/log.py b/log.py index 6683452..2079bca 100644 --- a/log.py +++ b/log.py @@ -46,8 +46,3 @@ def log_init(): return log - -if __name__ == "__main__": - log = log_init() - log.info("test") - diff --git a/script/config/constant.py b/script/tools/constant.py similarity index 100% rename from script/config/constant.py rename to script/tools/constant.py diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index fe36584..1861e0c 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -1,7 +1,6 @@ #!/bin/env python3 # -*- encoding=utf8 -*- -from log import log_init -from script.config.constant import Constant +from constant import Constant import os import sys import requests @@ -9,6 +8,7 @@ from bs4 import BeautifulSoup CURRENT_DIR = os.path.split(os.path.abspath(__file__))[0] config_path = CURRENT_DIR.rsplit('/', 2)[0] sys.path.append(config_path) +from log import log_init class CheckDailyBuild(object): -- Gitee From 73019677efc62cc7aac5bf7214dda50b6ca0cbcb Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 2 Mar 2022 18:46:33 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=8C=E4=BF=AE=E6=94=B9log=E5=BC=95?= =?UTF-8?q?=E5=85=A5=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log.py | 1 + script/tools/daily_build_check.py | 22 ++++++++++------------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/log.py b/log.py index 2079bca..b4f5d76 100644 --- a/log.py +++ b/log.py @@ -46,3 +46,4 @@ def log_init(): return log +log = log_init() \ No newline at end of file diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 1861e0c..31b6d32 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -4,11 +4,12 @@ from constant import Constant import os import sys import requests +import argparse from bs4 import BeautifulSoup CURRENT_DIR = os.path.split(os.path.abspath(__file__))[0] config_path = CURRENT_DIR.rsplit('/', 2)[0] sys.path.append(config_path) -from log import log_init +from log import log class CheckDailyBuild(object): @@ -21,12 +22,10 @@ class CheckDailyBuild(object): kawrgs: dict,init dict by 'a': 'A' style rooturl: the daily build page url main_branch:choose which branch you need to check - main_branch_flag : if this value is True only check all main_branch """ self.kwargs = kwargs self.rooturl = self.kwargs['daily_build_url'] self.main_branch = self.kwargs['main_branch'] - self.main_branch_flag = self.kwargs['main_branch_flag'] def html_downloader(self, url): if url is None: @@ -106,11 +105,7 @@ class CheckDailyBuild(object): c_dir, current_dir)) def _get_main_branch(self): - if not self.main_branch_flag: - html_content = self.html_downloader(self.rooturl) - main_branch_url = self.html_parser(self.rooturl, html_content) - else: - main_branch_url = [self.rooturl + self.main_branch] + main_branch_url = [self.rooturl + self.main_branch] dir_map = self.start_check(main_branch_url) self.check_every_dir(dir_map) @@ -119,9 +114,12 @@ class CheckDailyBuild(object): if __name__ == "__main__": - kw = {"daily_build_url": "http://121.36.84.172/dailybuild/", - "main_branch": "openEuler-22.03-LTS/", - "main_branch_flag": True} + parser = argparse.ArgumentParser() + parser.add_argument("--rooturl", help="please input daily build page root url,eg: http://ip/dailybuild/") + parser.add_argument("--branch", help="please input which branch you want check,eg:openEuler-22.03-LTS/") + args = parser.parse_args() + parser.print_help() + kw = {"daily_build_url": args.rooturl, + "main_branch": args.branch} check = CheckDailyBuild(**kw) - log = log_init() check.run() -- Gitee From 677b288eac63d9da777a71338c7d1fcfca015b2a Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 4 Mar 2022 16:32:29 +0800 Subject: [PATCH 5/8] fix code --- script/tools/constant.py | 21 ---- script/tools/daily_build_check.py | 99 ++++++++++++--- script/tools/standard.yaml | 196 ++++++++++++++++++++++++++++++ 3 files changed, 277 insertions(+), 39 deletions(-) delete mode 100644 script/tools/constant.py create mode 100644 script/tools/standard.yaml diff --git a/script/tools/constant.py b/script/tools/constant.py deleted file mode 100644 index c4b11f8..0000000 --- a/script/tools/constant.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/env python3 -# -*- encoding=utf8 -*- -class Constant(object): - """ - class Constant standard dir - """ - - STANDARD_DIR = { - 'MAIN_DIR': {'EPOL/','ISO/','OS/','debuginfo/','docker_img/','everything/','raspi_img/','source/','stratovirt_img/','update/','virtual_machine_img/'}, - 'EPOL': {'EPOL/main/aarch64/Packages/', 'EPOL/main/aarch64/repodata/', 'EPOL/main/source/', 'EPOL/main/source/Packages/', 'EPOL/main/source/repodata/', 'EPOL/main/x86_64/repodata/', 'EPOL/update/main/aarch64/Packages/', 'EPOL/update/main/aarch64/repodata/', 'EPOL/update/main/source/Packages/', 'EPOL/update/main/source/repodata/', 'EPOL/update/main/x86_64/Packages/', 'EPOL/update/main/x86_64/repodata/','EPOL/update/multi_version/aarch64/Packages/', 'EPOL/update/multi_version/aarch64/repodata/', 'EPOL/update/multi_version/source/Packages/', 'EPOL/update/multi_version/source/repodata/', 'EPOL/update/multi_version/x86_64/Packages/', 'EPOL/update/multi_version/x86_64/repodata/'}, - 'ISO': {'ISO/aarch64/*-aarch64-dvd.iso', 'ISO/aarch64/*-aarch64-dvd.iso.sha256sum', 'ISO/aarch64/*-aarch64.rpmlist', 'ISO/aarch64/*-everything-aarch64-dvd.iso', 'ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum', 'ISO/aarch64/*-everything-debug-aarch64-dvd.iso', 'ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum', 'ISO/aarch64/*-netinst-aarch64-dvd.iso', 'ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum', 'ISO/source/*-source-dvd.iso', 'ISO/source/*-source-dvd.iso.sha256sum', 'ISO/x86_64/*-everything-debug-x86_64-dvd.iso', 'ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum', 'ISO/x86_64/*-netinst-x86_64-dvd.iso', 'ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum', 'ISO/x86_64/*-x86_64-dvd.iso', 'ISO/x86_64/*-x86_64-dvd.iso.sha256sum', 'ISO/x86_64/*-x86_64.rpmlist'}, - 'OS': {'OS/aarch64/EFI/', 'OS/aarch64/Packages/', 'OS/aarch64/docs/', 'OS/aarch64/images/', 'OS/aarch64/repodata/', 'OS/aarch64/RPM-GPG-KEY-openEuler/', 'OS/aarch64/TRANS.TBL', 'OS/aarch64/boot.catalog', 'OS/x86_64/EFI/', 'OS/x86_64/Packages/', 'OS/x86_64/docs/', 'OS/x86_64/images/', 'OS/x86_64/repodata/', 'OS/x86_64/RPM-GPG-KEY-openEuler/', 'OS/x86_64/TRANS.TBL', 'OS/x86_64/boot.catalog'}, - 'debuginfo': {'debuginfo/aarch64/EFI/', 'debuginfo/aarch64/Packages/', 'debuginfo/aarch64/docs/', 'debuginfo/aarch64/images/', 'debuginfo/aarch64/repodata/', 'debuginfo/aarch64/RPM-GPG-KEY-openEuler/', 'debuginfo/aarch64/TRANS.TBL', 'debuginfo/aarch64/boot.catalog', 'debuginfo/x86_64/EFI/', 'debuginfo/x86_64/Packages/', 'debuginfo/x86_64/docs/', 'debuginfo/x86_64/images/', 'debuginfo/x86_64/repodata/', 'debuginfo/x86_64/RPM-GPG-KEY-openEuler/', 'debuginfo/x86_64/TRANS.TBL', 'debuginfo/x86_64/boot.catalog'}, - 'docker_img': {'docker_img/aarch64/openEuler-docker.aarch64.tar.xz', 'docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum', 'docker_img/x86_64/openEuler-docker.x86_64.tar.xz', 'docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum'}, - 'everything': {'everything/aarch64/EFI/', 'everything/aarch64/Packages/', 'everything/aarch64/docs/', 'everything/aarch64/images/', 'everything/aarch64/repodata/', 'everything/aarch64/RPM-GPG-KEY-openEuler/', 'everything/aarch64/TRANS.TBL', 'everything/aarch64/boot.catalog', 'everything/x86_64/EFI/', 'everything/x86_64/Packages/', 'everything/x86_64/docs/', 'everything/x86_64/images/', 'everything/x86_64/repodata/', 'everything/x86_64/RPM-GPG-KEY-openEuler/', 'everything/x86_64/TRANS.TBL', 'everything/x86_64/boot.catalog'}, - 'raspi_img': {'raspi_img/*-raspi-aarch64.img', 'raspi_img/*-raspi-aarch64.img.sha256sum', 'raspi_img/*-raspi-aarch64.img.xz', 'raspi_img/*-raspi-aarch64.img.xz.sha256sum'}, - 'source': {'source/aarch64/EFI/', 'source/aarch64/Packages/', 'source/aarch64/docs/', 'source/aarch64/images/', 'source/aarch64/repodata/', 'source/aarch64/RPM-GPG-KEY-openEuler/', 'source/aarch64/TRANS.TBL', 'source/aarch64/boot.catalog'}, - 'stratovirt_img': {'stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz', 'stratovirt_img/aarch64/EFI/', 'stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum', 'stratovirt_img/aarch64/std-vmlinux.bin', 'stratovirt_img/aarch64/std-vmlinux.bin.sha256sum', 'stratovirt_img/aarch64/vmlinux.bin', 'stratovirt_img/aarch64/vmlinux.bin.sha256sum', 'stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz', 'stratovirt_img/x86_64/EFI/', 'stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum', 'stratovirt_img/x86_64/std-vmlinux.bin', 'stratovirt_img/x86_64/std-vmlinux.bin.sha256sum', 'stratovirt_img/x86_64/vmlinux.bin', 'stratovirt_img/x86_64/vmlinux.bin.sha256sum'}, - 'update': {'update/aarch64/Packages/', 'update/aarch64/repodata/', 'update/source/Packages/', 'update/source/repodata/', 'update/x86_64/Packages/', 'update/x86_64/repodata/'}, - 'virtual_machine_img': {'virtual_machine_img/aarch64/*-aarch64.qcow2.xz', 'virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum', 'virtual_machine_img/x86_64/*-x86_64.qcow2.xz', 'virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum'} - } \ No newline at end of file diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 31b6d32..95da3d2 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -1,8 +1,8 @@ #!/bin/env python3 # -*- encoding=utf8 -*- -from constant import Constant import os import sys +import yaml import requests import argparse from bs4 import BeautifulSoup @@ -26,10 +26,14 @@ class CheckDailyBuild(object): self.kwargs = kwargs self.rooturl = self.kwargs['daily_build_url'] self.main_branch = self.kwargs['main_branch'] + self.dirflag = self.kwargs['dir_flag'] + self.datebranch = self.kwargs['date_branch'] + self.standard_dir = self.load_standard() def html_downloader(self, url): - if url is None: - return None + """ + download url html content + """ user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' headers = {'User-Agent': user_agent} r = requests.get(url, headers=headers) @@ -39,9 +43,12 @@ class CheckDailyBuild(object): return r.text else: log.error("this html page download failed！{}".format(url)) - return True + return '' def html_parser(self, url, html_content): + """ + parse html content + """ url_list = [] soup = BeautifulSoup(html_content, "html.parser") tr_content = soup.find_all('tr') @@ -54,22 +61,40 @@ class CheckDailyBuild(object): if dir_url['href'] != '../' and size == '-': url_list.append(url + dir_url['href']) elif dir_url['href'] != '../' and size != '-': - url_list.append(url + dir_url['href'] + '|ISFILE/') + if not self.check_is_rpm(dir_url['href']): + url_list.append(url + dir_url['href'] + '|ISFILE/') return url_list + def check_is_rpm(self, filename): + """ + file type filter + """ + if filename.endswith(".rpm"): + return True + def start_check(self, main_branch_url): + """ + get choose branch need check dir + """ dir_map = {} - for main in main_branch_url: - html_content = self.html_downloader(main) - current_dir = self.html_parser(main, html_content) - dir_map[main] = [] + html_content = self.html_downloader(main_branch_url) + if html_content: + current_dir = self.html_parser(main_branch_url, html_content) + dir_map[main_branch_url] = [] for first_dir in current_dir: dir_split = first_dir.split('/')[5] - if dir_split.startswith('openeuler-2022-'): - dir_map[main].append(first_dir) + if dir_split.startswith(self.dirflag): + dir_map[main_branch_url].append(first_dir) + else: + log.error( + "error url can not open,please check your input：{}".format(main_branch_url)) + raise SystemExit("*******PLEASE CHECK YOUR INPUT ARGS*******") return dir_map def check_every_dir(self, dir_map): + """ + check choose branch and get every dir compare + """ for dir_url, dir_list in dir_map.items(): if dir_list: c_branch = dir_url.split("/")[-2] @@ -91,10 +116,25 @@ class CheckDailyBuild(object): self.check_current_dir(item, origin_dir, temp_list) except Exception as e: log.info("error url can not open:{}".format(c_dir)) + log.info("error url can not open:{}".format(e)) return temp_list + def load_standard(self): + """ + read config standard openeuler dir yaml file + """ + try: + with open('./standard.yaml', 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + return result + except Exception as e: + log.info("error read standard.yaml,please check:{}".format(c_dir)) + def compare_standard(self, dir_list, current_branch, c_dir): - standard_dir = Constant.STANDARD_DIR + """ + branch dir compare with standard dir + """ + standard_dir = self.standard_dir for key, c_standard in standard_dir.items(): for current_dir in c_standard: if '*' in current_dir: @@ -105,8 +145,15 @@ class CheckDailyBuild(object): c_dir, current_dir)) def _get_main_branch(self): - main_branch_url = [self.rooturl + self.main_branch] - dir_map = self.start_check(main_branch_url) + if self.datebranch: + dir_map = {} + complete_key = "{}/{}/".format(self.rooturl, self.main_branch) + complete_value = "{}/{}/{}/".format( + self.rooturl, self.main_branch, self.datebranch) + dir_map[complete_key] = [complete_value] + else: + main_branch_url = "{}/{}/".format(self.rooturl, self.main_branch) + dir_map = self.start_check(main_branch_url) self.check_every_dir(dir_map) def run(self): @@ -115,11 +162,27 @@ class CheckDailyBuild(object): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--rooturl", help="please input daily build page root url,eg: http://ip/dailybuild/") - parser.add_argument("--branch", help="please input which branch you want check,eg:openEuler-22.03-LTS/") + parser.add_argument( + "--dirflag", + default="openeuler-2022-", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument( + "--rooturl", + help="daily build page root url,eg: http://ip/dailybuild") + parser.add_argument( + "--branch", + help="which branch you want to check,eg:openEuler-22.03-LTS") + parser.add_argument( + "--datebranch", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") args = parser.parse_args() parser.print_help() - kw = {"daily_build_url": args.rooturl, - "main_branch": args.branch} + kw = { + "daily_build_url": args.rooturl, + "main_branch": args.branch, + "date_branch": args.datebranch, + "dir_flag": args.dirflag + } + print(kw) check = CheckDailyBuild(**kw) check.run() diff --git a/script/tools/standard.yaml b/script/tools/standard.yaml new file mode 100644 index 0000000..6c5fdbc --- /dev/null +++ b/script/tools/standard.yaml @@ -0,0 +1,196 @@ +MAIN_DIR: + - EPOL/ + - ISO/ + - OS/ + - debuginfo/ + - docker_img/ + - everything/ + - raspi_img/ + - source/ + - stratovirt_img/ + - update/ + - virtual_machine_img/ +EPOL: + - EPOL/main/ + - EPOL/update/ + - EPOL/multi_version/ + - EPOL/multi_version/OpenStack/ + - EPOL/main/source/ + - EPOL/main/aarch64/ + - EPOL/main/x86_64/ + - EPOL/update/main/ + - EPOL/update/multi_version/ + - EPOL/update/main/aarch64/ + - EPOL/main/source/Packages/ + - EPOL/main/source/repodata/ + - EPOL/main/aarch64/Packages/ + - EPOL/main/aarch64/repodata/ + - EPOL/main/x86_64/repodata/ + - EPOL/update/multi_version/source/ + - EPOL/update/main/aarch64/Packages/ + - EPOL/update/main/aarch64/repodata/ + - EPOL/update/main/source/Packages/ + - EPOL/update/main/source/repodata/ + - EPOL/update/main/x86_64/Packages/ + - EPOL/update/main/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Train/ + - EPOL/multi_version/OpenStack/Wallaby/ + - EPOL/multi_version/OpenStack/Train/aarch64/ + - EPOL/multi_version/OpenStack/Train/source/ + - EPOL/multi_version/OpenStack/Train/x86_64/ + - EPOL/multi_version/OpenStack/Train/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Train/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Train/source/Packages/ + - EPOL/multi_version/OpenStack/Train/source/repodata/ + - EPOL/multi_version/OpenStack/Train/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Train/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/ + - EPOL/multi_version/OpenStack/Wallaby/source/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/source/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/source/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/repodata/ + - EPOL/update/multi_version/source/Packages/ + - EPOL/update/multi_version/source/repodata/ + - EPOL/update/multi_version/x86_64/Packages/ + - EPOL/update/multi_version/x86_64/repodata/ + - EPOL/update/multi_version/aarch64/Packages/ + - EPOL/update/multi_version/aarch64/repodata/ +ISO: + - ISO/aarch64/ + - ISO/x86_64/ + - ISO/source/ + - ISO/aarch64/*-aarch64-dvd.iso + - ISO/aarch64/*-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-aarch64.rpmlist + - ISO/aarch64/*-everything-aarch64-dvd.iso + - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-netinst-aarch64-dvd.iso + - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum + - ISO/source/*-source-dvd.iso + - ISO/source/*-source-dvd.iso.sha256sum + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-netinst-x86_64-dvd.iso + - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64-dvd.iso + - ISO/x86_64/*-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64.rpmlist +OS: + - OS/aarch64/ + - OS/x86_64/ + - OS/aarch64/EFI/ + - OS/aarch64/Packages/ + - OS/aarch64/docs/ + - OS/aarch64/images/ + - OS/aarch64/repodata/ + - OS/aarch64/RPM-GPG-KEY-openEuler/ + - OS/aarch64/TRANS.TBL + - OS/aarch64/boot.catalog + - OS/x86_64/EFI/ + - OS/x86_64/Packages/ + - OS/x86_64/docs/ + - OS/x86_64/images/ + - OS/x86_64/repodata/ + - OS/x86_64/RPM-GPG-KEY-openEuler/ + - OS/x86_64/TRANS.TBL + - OS/x86_64/boot.catalog +debuginfo: + - debuginfo/aarch64/ + - debuginfo/x86_64/ + - debuginfo/aarch64/EFI/ + - debuginfo/aarch64/Packages/ + - debuginfo/aarch64/docs/ + - debuginfo/aarch64/images/ + - debuginfo/aarch64/repodata/ + - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ + - debuginfo/aarch64/TRANS.TBL + - debuginfo/aarch64/boot.catalog + - debuginfo/x86_64/EFI/ + - debuginfo/x86_64/Packages/ + - debuginfo/x86_64/docs/ + - debuginfo/x86_64/images/ + - debuginfo/x86_64/repodata/ + - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ + - debuginfo/x86_64/TRANS.TBL + - debuginfo/x86_64/boot.catalog +docker_img: + - docker_img/aarch64/ + - docker_img/x86_64/ + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum +everything: + - everything/aarch64/ + - everything/x86_64/ + - everything/aarch64/EFI/ + - everything/aarch64/Packages/ + - everything/aarch64/docs/ + - everything/aarch64/images/ + - everything/aarch64/repodata/ + - everything/aarch64/RPM-GPG-KEY-openEuler/ + - everything/aarch64/TRANS.TBL + - everything/aarch64/boot.catalog + - everything/x86_64/EFI/ + - everything/x86_64/Packages/ + - everything/x86_64/docs/ + - everything/x86_64/images/ + - everything/x86_64/repodata/ + - everything/x86_64/RPM-GPG-KEY-openEuler/ + - everything/x86_64/TRANS.TBL + - everything/x86_64/boot.catalog +raspi_img: + - raspi_img/*-raspi-aarch64.img + - raspi_img/*-raspi-aarch64.img.sha256sum + - raspi_img/*-raspi-aarch64.img.xz + - raspi_img/*-raspi-aarch64.img.xz.sha256sum +source: + - source/aarch64/EFI/ + - source/aarch64/Packages/ + - source/aarch64/docs/ + - source/aarch64/images/ + - source/aarch64/repodata/ + - source/aarch64/RPM-GPG-KEY-openEuler/ + - source/aarch64/TRANS.TBL + - source/aarch64/boot.catalog +stratovirt_img: + - stratovirt_img/aarch64/ + - stratovirt_img/x86_64/ + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum + - stratovirt_img/aarch64/EFI/ + - stratovirt_img/aarch64/std-vmlinux.bin + - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum + - stratovirt_img/aarch64/vmlinux.bin + - stratovirt_img/aarch64/vmlinux.bin.sha256sum + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz + - stratovirt_img/x86_64/EFI/ + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum + - stratovirt_img/x86_64/std-vmlinux.bin + - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum + - stratovirt_img/x86_64/vmlinux.bin + - stratovirt_img/x86_64/vmlinux.bin.sha256sum +update: + - update/aarch64/ + - update/source/ + - update/x86_64/ + - update/aarch64/Packages/ + - update/aarch64/repodata/ + - update/source/Packages/ + - update/source/repodata/ + - update/x86_64/Packages/ + - update/x86_64/repodata/ + +virtual_machine_img: + - virtual_machine_img/aarch64/ + - virtual_machine_img/x86_64/ + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum \ No newline at end of file -- Gitee From 5f8e17203cf2603d31331f3d026ffb345d5e2090 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 4 Mar 2022 16:38:43 +0800 Subject: [PATCH 6/8] fix code --- script/tools/daily_build_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index 95da3d2..a10d739 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -128,7 +128,7 @@ class CheckDailyBuild(object): result = yaml.load(f, Loader=yaml.FullLoader) return result except Exception as e: - log.info("error read standard.yaml,please check:{}".format(c_dir)) + log.info("error read standard.yaml,please check") def compare_standard(self, dir_list, current_branch, c_dir): """ -- Gitee From 35fb767e9759d7bf97e39be5a39bfcb3ad390f1c Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 8 Mar 2022 15:34:44 +0800 Subject: [PATCH 7/8] code fix --- log.py | 2 -- script/tools/daily_build_check.py | 39 ++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/log.py b/log.py index b4f5d76..1db712a 100644 --- a/log.py +++ b/log.py @@ -45,5 +45,3 @@ def log_init(): log.addHandler(ch) return log - -log = log_init() \ No newline at end of file diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py index a10d739..0448dc6 100644 --- a/script/tools/daily_build_check.py +++ b/script/tools/daily_build_check.py @@ -1,15 +1,29 @@ #!/bin/env python3 # -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologging.es Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: dongjie +# Create: 2022-03-02 +# ****************************************************************************** import os import sys import yaml import requests import argparse +import logging from bs4 import BeautifulSoup -CURRENT_DIR = os.path.split(os.path.abspath(__file__))[0] -config_path = CURRENT_DIR.rsplit('/', 2)[0] -sys.path.append(config_path) -from log import log + +LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" +DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) class CheckDailyBuild(object): @@ -39,10 +53,10 @@ class CheckDailyBuild(object): r = requests.get(url, headers=headers) if r.status_code == 200: r.encoding = 'utf-8' - log.info("this html page download success！{}".format(url)) + logging.info("this html page download success！{}".format(url)) return r.text else: - log.error("this html page download failed！{}".format(url)) + logging.error("this html page download failed！{}".format(url)) return '' def html_parser(self, url, html_content): @@ -86,7 +100,7 @@ class CheckDailyBuild(object): if dir_split.startswith(self.dirflag): dir_map[main_branch_url].append(first_dir) else: - log.error( + logging.error( "error url can not open,please check your input：{}".format(main_branch_url)) raise SystemExit("*******PLEASE CHECK YOUR INPUT ARGS*******") return dir_map @@ -115,8 +129,8 @@ class CheckDailyBuild(object): if item.split('|')[-1] != 'ISFILE/': self.check_current_dir(item, origin_dir, temp_list) except Exception as e: - log.info("error url can not open:{}".format(c_dir)) - log.info("error url can not open:{}".format(e)) + logging.info("error url can not open:{}".format(c_dir)) + logging.info("error url can not open:{}".format(e)) return temp_list def load_standard(self): @@ -128,7 +142,7 @@ class CheckDailyBuild(object): result = yaml.load(f, Loader=yaml.FullLoader) return result except Exception as e: - log.info("error read standard.yaml,please check") + logging.info("error read standard.yaml,please check") def compare_standard(self, dir_list, current_branch, c_dir): """ @@ -140,7 +154,7 @@ class CheckDailyBuild(object): if '*' in current_dir: current_dir = current_dir.replace('*', current_branch) if current_dir not in dir_list: - log.error( + logging.error( 'this dir not found,link url:{}{}'.format( c_dir, current_dir)) @@ -168,7 +182,7 @@ if __name__ == "__main__": help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") parser.add_argument( "--rooturl", - help="daily build page root url,eg: http://ip/dailybuild") + help="daily build page root url,eg: http://ip/dailybuild") parser.add_argument( "--branch", help="which branch you want to check,eg:openEuler-22.03-LTS") @@ -183,6 +197,5 @@ if __name__ == "__main__": "date_branch": args.datebranch, "dir_flag": args.dirflag } - print(kw) check = CheckDailyBuild(**kw) check.run() -- Gitee From 084db1fe73cc4fe2430db59ddf045cd1c57a79f6 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Mon, 28 Feb 2022 19:23:52 +0800 Subject: [PATCH 8/8] =?UTF-8?q?daily=20build=20=E7=9B=AE=E5=BD=95=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log.py | 1 - script/tools/daily_build_check.py | 201 ++++++++++++++++++++++++++++++ script/tools/standard.yaml | 196 +++++++++++++++++++++++++++++ 3 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 script/tools/daily_build_check.py create mode 100644 script/tools/standard.yaml diff --git a/log.py b/log.py index 2079bca..1db712a 100644 --- a/log.py +++ b/log.py @@ -45,4 +45,3 @@ def log_init(): log.addHandler(ch) return log - diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py new file mode 100644 index 0000000..0448dc6 --- /dev/null +++ b/script/tools/daily_build_check.py @@ -0,0 +1,201 @@ +#!/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologging.es Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: dongjie +# Create: 2022-03-02 +# ****************************************************************************** +import os +import sys +import yaml +import requests +import argparse +import logging +from bs4 import BeautifulSoup + +LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" +DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) + + +class CheckDailyBuild(object): + """ + The entrance check for daily build + """ + + def __init__(self, **kwargs): + """ + kawrgs: dict,init dict by 'a': 'A' style + rooturl: the daily build page url + main_branch:choose which branch you need to check + """ + self.kwargs = kwargs + self.rooturl = self.kwargs['daily_build_url'] + self.main_branch = self.kwargs['main_branch'] + self.dirflag = self.kwargs['dir_flag'] + self.datebranch = self.kwargs['date_branch'] + self.standard_dir = self.load_standard() + + def html_downloader(self, url): + """ + download url html content + """ + user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' + headers = {'User-Agent': user_agent} + r = requests.get(url, headers=headers) + if r.status_code == 200: + r.encoding = 'utf-8' + logging.info("this html page download success！{}".format(url)) + return r.text + else: + logging.error("this html page download failed！{}".format(url)) + return '' + + def html_parser(self, url, html_content): + """ + parse html content + """ + url_list = [] + soup = BeautifulSoup(html_content, "html.parser") + tr_content = soup.find_all('tr') + for line in tr_content: + td_content = line.find_all('td') + if td_content: + dir_url = td_content[0].find('a', href=True) + size = td_content[1].text + if dir_url: + if dir_url['href'] != '../' and size == '-': + url_list.append(url + dir_url['href']) + elif dir_url['href'] != '../' and size != '-': + if not self.check_is_rpm(dir_url['href']): + url_list.append(url + dir_url['href'] + '|ISFILE/') + return url_list + + def check_is_rpm(self, filename): + """ + file type filter + """ + if filename.endswith(".rpm"): + return True + + def start_check(self, main_branch_url): + """ + get choose branch need check dir + """ + dir_map = {} + html_content = self.html_downloader(main_branch_url) + if html_content: + current_dir = self.html_parser(main_branch_url, html_content) + dir_map[main_branch_url] = [] + for first_dir in current_dir: + dir_split = first_dir.split('/')[5] + if dir_split.startswith(self.dirflag): + dir_map[main_branch_url].append(first_dir) + else: + logging.error( + "error url can not open,please check your input：{}".format(main_branch_url)) + raise SystemExit("*******PLEASE CHECK YOUR INPUT ARGS*******") + return dir_map + + def check_every_dir(self, dir_map): + """ + check choose branch and get every dir compare + """ + for dir_url, dir_list in dir_map.items(): + if dir_list: + c_branch = dir_url.split("/")[-2] + for c_dir in dir_list: + origin_dir = c_dir + dir_result = self.check_current_dir(c_dir, origin_dir, []) + self.compare_standard(dir_result, c_branch, c_dir) + + def check_current_dir(self, c_dir, origin_dir, temp_list): + try: + this_page = self.html_downloader(c_dir) + page_dir = self.html_parser(c_dir, this_page) + for item in page_dir: + item_name = item.replace(origin_dir, '') + if 'ISFILE/' in item_name: + item_name = item_name.replace('|ISFILE/', '') + temp_list.append(item_name) + if item.split('|')[-1] != 'ISFILE/': + self.check_current_dir(item, origin_dir, temp_list) + except Exception as e: + logging.info("error url can not open:{}".format(c_dir)) + logging.info("error url can not open:{}".format(e)) + return temp_list + + def load_standard(self): + """ + read config standard openeuler dir yaml file + """ + try: + with open('./standard.yaml', 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + return result + except Exception as e: + logging.info("error read standard.yaml,please check") + + def compare_standard(self, dir_list, current_branch, c_dir): + """ + branch dir compare with standard dir + """ + standard_dir = self.standard_dir + for key, c_standard in standard_dir.items(): + for current_dir in c_standard: + if '*' in current_dir: + current_dir = current_dir.replace('*', current_branch) + if current_dir not in dir_list: + logging.error( + 'this dir not found,link url:{}{}'.format( + c_dir, current_dir)) + + def _get_main_branch(self): + if self.datebranch: + dir_map = {} + complete_key = "{}/{}/".format(self.rooturl, self.main_branch) + complete_value = "{}/{}/{}/".format( + self.rooturl, self.main_branch, self.datebranch) + dir_map[complete_key] = [complete_value] + else: + main_branch_url = "{}/{}/".format(self.rooturl, self.main_branch) + dir_map = self.start_check(main_branch_url) + self.check_every_dir(dir_map) + + def run(self): + self._get_main_branch() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dirflag", + default="openeuler-2022-", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument( + "--rooturl", + help="daily build page root url,eg: http://ip/dailybuild") + parser.add_argument( + "--branch", + help="which branch you want to check,eg:openEuler-22.03-LTS") + parser.add_argument( + "--datebranch", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + args = parser.parse_args() + parser.print_help() + kw = { + "daily_build_url": args.rooturl, + "main_branch": args.branch, + "date_branch": args.datebranch, + "dir_flag": args.dirflag + } + check = CheckDailyBuild(**kw) + check.run() diff --git a/script/tools/standard.yaml b/script/tools/standard.yaml new file mode 100644 index 0000000..6c5fdbc --- /dev/null +++ b/script/tools/standard.yaml @@ -0,0 +1,196 @@ +MAIN_DIR: + - EPOL/ + - ISO/ + - OS/ + - debuginfo/ + - docker_img/ + - everything/ + - raspi_img/ + - source/ + - stratovirt_img/ + - update/ + - virtual_machine_img/ +EPOL: + - EPOL/main/ + - EPOL/update/ + - EPOL/multi_version/ + - EPOL/multi_version/OpenStack/ + - EPOL/main/source/ + - EPOL/main/aarch64/ + - EPOL/main/x86_64/ + - EPOL/update/main/ + - EPOL/update/multi_version/ + - EPOL/update/main/aarch64/ + - EPOL/main/source/Packages/ + - EPOL/main/source/repodata/ + - EPOL/main/aarch64/Packages/ + - EPOL/main/aarch64/repodata/ + - EPOL/main/x86_64/repodata/ + - EPOL/update/multi_version/source/ + - EPOL/update/main/aarch64/Packages/ + - EPOL/update/main/aarch64/repodata/ + - EPOL/update/main/source/Packages/ + - EPOL/update/main/source/repodata/ + - EPOL/update/main/x86_64/Packages/ + - EPOL/update/main/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Train/ + - EPOL/multi_version/OpenStack/Wallaby/ + - EPOL/multi_version/OpenStack/Train/aarch64/ + - EPOL/multi_version/OpenStack/Train/source/ + - EPOL/multi_version/OpenStack/Train/x86_64/ + - EPOL/multi_version/OpenStack/Train/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Train/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Train/source/Packages/ + - EPOL/multi_version/OpenStack/Train/source/repodata/ + - EPOL/multi_version/OpenStack/Train/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Train/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/ + - EPOL/multi_version/OpenStack/Wallaby/source/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/source/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/source/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/repodata/ + - EPOL/update/multi_version/source/Packages/ + - EPOL/update/multi_version/source/repodata/ + - EPOL/update/multi_version/x86_64/Packages/ + - EPOL/update/multi_version/x86_64/repodata/ + - EPOL/update/multi_version/aarch64/Packages/ + - EPOL/update/multi_version/aarch64/repodata/ +ISO: + - ISO/aarch64/ + - ISO/x86_64/ + - ISO/source/ + - ISO/aarch64/*-aarch64-dvd.iso + - ISO/aarch64/*-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-aarch64.rpmlist + - ISO/aarch64/*-everything-aarch64-dvd.iso + - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-netinst-aarch64-dvd.iso + - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum + - ISO/source/*-source-dvd.iso + - ISO/source/*-source-dvd.iso.sha256sum + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-netinst-x86_64-dvd.iso + - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64-dvd.iso + - ISO/x86_64/*-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64.rpmlist +OS: + - OS/aarch64/ + - OS/x86_64/ + - OS/aarch64/EFI/ + - OS/aarch64/Packages/ + - OS/aarch64/docs/ + - OS/aarch64/images/ + - OS/aarch64/repodata/ + - OS/aarch64/RPM-GPG-KEY-openEuler/ + - OS/aarch64/TRANS.TBL + - OS/aarch64/boot.catalog + - OS/x86_64/EFI/ + - OS/x86_64/Packages/ + - OS/x86_64/docs/ + - OS/x86_64/images/ + - OS/x86_64/repodata/ + - OS/x86_64/RPM-GPG-KEY-openEuler/ + - OS/x86_64/TRANS.TBL + - OS/x86_64/boot.catalog +debuginfo: + - debuginfo/aarch64/ + - debuginfo/x86_64/ + - debuginfo/aarch64/EFI/ + - debuginfo/aarch64/Packages/ + - debuginfo/aarch64/docs/ + - debuginfo/aarch64/images/ + - debuginfo/aarch64/repodata/ + - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ + - debuginfo/aarch64/TRANS.TBL + - debuginfo/aarch64/boot.catalog + - debuginfo/x86_64/EFI/ + - debuginfo/x86_64/Packages/ + - debuginfo/x86_64/docs/ + - debuginfo/x86_64/images/ + - debuginfo/x86_64/repodata/ + - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ + - debuginfo/x86_64/TRANS.TBL + - debuginfo/x86_64/boot.catalog +docker_img: + - docker_img/aarch64/ + - docker_img/x86_64/ + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum +everything: + - everything/aarch64/ + - everything/x86_64/ + - everything/aarch64/EFI/ + - everything/aarch64/Packages/ + - everything/aarch64/docs/ + - everything/aarch64/images/ + - everything/aarch64/repodata/ + - everything/aarch64/RPM-GPG-KEY-openEuler/ + - everything/aarch64/TRANS.TBL + - everything/aarch64/boot.catalog + - everything/x86_64/EFI/ + - everything/x86_64/Packages/ + - everything/x86_64/docs/ + - everything/x86_64/images/ + - everything/x86_64/repodata/ + - everything/x86_64/RPM-GPG-KEY-openEuler/ + - everything/x86_64/TRANS.TBL + - everything/x86_64/boot.catalog +raspi_img: + - raspi_img/*-raspi-aarch64.img + - raspi_img/*-raspi-aarch64.img.sha256sum + - raspi_img/*-raspi-aarch64.img.xz + - raspi_img/*-raspi-aarch64.img.xz.sha256sum +source: + - source/aarch64/EFI/ + - source/aarch64/Packages/ + - source/aarch64/docs/ + - source/aarch64/images/ + - source/aarch64/repodata/ + - source/aarch64/RPM-GPG-KEY-openEuler/ + - source/aarch64/TRANS.TBL + - source/aarch64/boot.catalog +stratovirt_img: + - stratovirt_img/aarch64/ + - stratovirt_img/x86_64/ + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum + - stratovirt_img/aarch64/EFI/ + - stratovirt_img/aarch64/std-vmlinux.bin + - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum + - stratovirt_img/aarch64/vmlinux.bin + - stratovirt_img/aarch64/vmlinux.bin.sha256sum + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz + - stratovirt_img/x86_64/EFI/ + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum + - stratovirt_img/x86_64/std-vmlinux.bin + - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum + - stratovirt_img/x86_64/vmlinux.bin + - stratovirt_img/x86_64/vmlinux.bin.sha256sum +update: + - update/aarch64/ + - update/source/ + - update/x86_64/ + - update/aarch64/Packages/ + - update/aarch64/repodata/ + - update/source/Packages/ + - update/source/repodata/ + - update/x86_64/Packages/ + - update/x86_64/repodata/ + +virtual_machine_img: + - virtual_machine_img/aarch64/ + - virtual_machine_img/x86_64/ + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum \ No newline at end of file -- Gitee