diff --git a/log.py b/log.py index 2079bca8ea3ba89f3a7b723dd69aa6f1478efe22..1db712aa97b17ba9f9954d56e6a5d62747e5fd87 100644 --- a/log.py +++ b/log.py @@ -45,4 +45,3 @@ def log_init(): log.addHandler(ch) return log - diff --git a/script/tools/daily_build_check.py b/script/tools/daily_build_check.py new file mode 100644 index 0000000000000000000000000000000000000000..0448dc62ead1c4c23760f9bc43c0600db8ab964b --- /dev/null +++ b/script/tools/daily_build_check.py @@ -0,0 +1,201 @@ +#!/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologging.es Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: dongjie +# Create: 2022-03-02 +# ****************************************************************************** +import os +import sys +import yaml +import requests +import argparse +import logging +from bs4 import BeautifulSoup + +LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" +DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) + + +class CheckDailyBuild(object): + """ + The entrance check for daily build + """ + + def __init__(self, **kwargs): + """ + kawrgs: dict,init dict by 'a': 'A' style + rooturl: the daily build page url + main_branch:choose which branch you need to check + """ + self.kwargs = kwargs + self.rooturl = self.kwargs['daily_build_url'] + self.main_branch = self.kwargs['main_branch'] + self.dirflag = self.kwargs['dir_flag'] + self.datebranch = self.kwargs['date_branch'] + self.standard_dir = self.load_standard() + + def html_downloader(self, url): + """ + download url html content + """ + user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' + headers = {'User-Agent': user_agent} + r = requests.get(url, headers=headers) + if r.status_code == 200: + r.encoding = 'utf-8' + logging.info("this html page download success！{}".format(url)) + return r.text + else: + logging.error("this html page download failed！{}".format(url)) + return '' + + def html_parser(self, url, html_content): + """ + parse html content + """ + url_list = [] + soup = BeautifulSoup(html_content, "html.parser") + tr_content = soup.find_all('tr') + for line in tr_content: + td_content = line.find_all('td') + if td_content: + dir_url = td_content[0].find('a', href=True) + size = td_content[1].text + if dir_url: + if dir_url['href'] != '../' and size == '-': + url_list.append(url + dir_url['href']) + elif dir_url['href'] != '../' and size != '-': + if not self.check_is_rpm(dir_url['href']): + url_list.append(url + dir_url['href'] + '|ISFILE/') + return url_list + + def check_is_rpm(self, filename): + """ + file type filter + """ + if filename.endswith(".rpm"): + return True + + def start_check(self, main_branch_url): + """ + get choose branch need check dir + """ + dir_map = {} + html_content = self.html_downloader(main_branch_url) + if html_content: + current_dir = self.html_parser(main_branch_url, html_content) + dir_map[main_branch_url] = [] + for first_dir in current_dir: + dir_split = first_dir.split('/')[5] + if dir_split.startswith(self.dirflag): + dir_map[main_branch_url].append(first_dir) + else: + logging.error( + "error url can not open,please check your input：{}".format(main_branch_url)) + raise SystemExit("*******PLEASE CHECK YOUR INPUT ARGS*******") + return dir_map + + def check_every_dir(self, dir_map): + """ + check choose branch and get every dir compare + """ + for dir_url, dir_list in dir_map.items(): + if dir_list: + c_branch = dir_url.split("/")[-2] + for c_dir in dir_list: + origin_dir = c_dir + dir_result = self.check_current_dir(c_dir, origin_dir, []) + self.compare_standard(dir_result, c_branch, c_dir) + + def check_current_dir(self, c_dir, origin_dir, temp_list): + try: + this_page = self.html_downloader(c_dir) + page_dir = self.html_parser(c_dir, this_page) + for item in page_dir: + item_name = item.replace(origin_dir, '') + if 'ISFILE/' in item_name: + item_name = item_name.replace('|ISFILE/', '') + temp_list.append(item_name) + if item.split('|')[-1] != 'ISFILE/': + self.check_current_dir(item, origin_dir, temp_list) + except Exception as e: + logging.info("error url can not open:{}".format(c_dir)) + logging.info("error url can not open:{}".format(e)) + return temp_list + + def load_standard(self): + """ + read config standard openeuler dir yaml file + """ + try: + with open('./standard.yaml', 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + return result + except Exception as e: + logging.info("error read standard.yaml,please check") + + def compare_standard(self, dir_list, current_branch, c_dir): + """ + branch dir compare with standard dir + """ + standard_dir = self.standard_dir + for key, c_standard in standard_dir.items(): + for current_dir in c_standard: + if '*' in current_dir: + current_dir = current_dir.replace('*', current_branch) + if current_dir not in dir_list: + logging.error( + 'this dir not found,link url:{}{}'.format( + c_dir, current_dir)) + + def _get_main_branch(self): + if self.datebranch: + dir_map = {} + complete_key = "{}/{}/".format(self.rooturl, self.main_branch) + complete_value = "{}/{}/{}/".format( + self.rooturl, self.main_branch, self.datebranch) + dir_map[complete_key] = [complete_value] + else: + main_branch_url = "{}/{}/".format(self.rooturl, self.main_branch) + dir_map = self.start_check(main_branch_url) + self.check_every_dir(dir_map) + + def run(self): + self._get_main_branch() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dirflag", + default="openeuler-2022-", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + parser.add_argument( + "--rooturl", + help="daily build page root url,eg: http://ip/dailybuild") + parser.add_argument( + "--branch", + help="which branch you want to check,eg:openEuler-22.03-LTS") + parser.add_argument( + "--datebranch", + help="which date branch you want to check,eg:openeuler-2022-03-04-09-22-07") + args = parser.parse_args() + parser.print_help() + kw = { + "daily_build_url": args.rooturl, + "main_branch": args.branch, + "date_branch": args.datebranch, + "dir_flag": args.dirflag + } + check = CheckDailyBuild(**kw) + check.run() diff --git a/script/tools/standard.yaml b/script/tools/standard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c5fdbcc4c8a3f9981d3c321b9faa6bbe4e6353d --- /dev/null +++ b/script/tools/standard.yaml @@ -0,0 +1,196 @@ +MAIN_DIR: + - EPOL/ + - ISO/ + - OS/ + - debuginfo/ + - docker_img/ + - everything/ + - raspi_img/ + - source/ + - stratovirt_img/ + - update/ + - virtual_machine_img/ +EPOL: + - EPOL/main/ + - EPOL/update/ + - EPOL/multi_version/ + - EPOL/multi_version/OpenStack/ + - EPOL/main/source/ + - EPOL/main/aarch64/ + - EPOL/main/x86_64/ + - EPOL/update/main/ + - EPOL/update/multi_version/ + - EPOL/update/main/aarch64/ + - EPOL/main/source/Packages/ + - EPOL/main/source/repodata/ + - EPOL/main/aarch64/Packages/ + - EPOL/main/aarch64/repodata/ + - EPOL/main/x86_64/repodata/ + - EPOL/update/multi_version/source/ + - EPOL/update/main/aarch64/Packages/ + - EPOL/update/main/aarch64/repodata/ + - EPOL/update/main/source/Packages/ + - EPOL/update/main/source/repodata/ + - EPOL/update/main/x86_64/Packages/ + - EPOL/update/main/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Train/ + - EPOL/multi_version/OpenStack/Wallaby/ + - EPOL/multi_version/OpenStack/Train/aarch64/ + - EPOL/multi_version/OpenStack/Train/source/ + - EPOL/multi_version/OpenStack/Train/x86_64/ + - EPOL/multi_version/OpenStack/Train/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Train/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Train/source/Packages/ + - EPOL/multi_version/OpenStack/Train/source/repodata/ + - EPOL/multi_version/OpenStack/Train/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Train/x86_64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/ + - EPOL/multi_version/OpenStack/Wallaby/source/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/aarch64/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/source/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/source/repodata/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/Packages/ + - EPOL/multi_version/OpenStack/Wallaby/x86_64/repodata/ + - EPOL/update/multi_version/source/Packages/ + - EPOL/update/multi_version/source/repodata/ + - EPOL/update/multi_version/x86_64/Packages/ + - EPOL/update/multi_version/x86_64/repodata/ + - EPOL/update/multi_version/aarch64/Packages/ + - EPOL/update/multi_version/aarch64/repodata/ +ISO: + - ISO/aarch64/ + - ISO/x86_64/ + - ISO/source/ + - ISO/aarch64/*-aarch64-dvd.iso + - ISO/aarch64/*-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-aarch64.rpmlist + - ISO/aarch64/*-everything-aarch64-dvd.iso + - ISO/aarch64/*-everything-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso + - ISO/aarch64/*-everything-debug-aarch64-dvd.iso.sha256sum + - ISO/aarch64/*-netinst-aarch64-dvd.iso + - ISO/aarch64/*-netinst-aarch64-dvd.iso.sha256sum + - ISO/source/*-source-dvd.iso + - ISO/source/*-source-dvd.iso.sha256sum + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso + - ISO/x86_64/*-everything-debug-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-netinst-x86_64-dvd.iso + - ISO/x86_64/*-netinst-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64-dvd.iso + - ISO/x86_64/*-x86_64-dvd.iso.sha256sum + - ISO/x86_64/*-x86_64.rpmlist +OS: + - OS/aarch64/ + - OS/x86_64/ + - OS/aarch64/EFI/ + - OS/aarch64/Packages/ + - OS/aarch64/docs/ + - OS/aarch64/images/ + - OS/aarch64/repodata/ + - OS/aarch64/RPM-GPG-KEY-openEuler/ + - OS/aarch64/TRANS.TBL + - OS/aarch64/boot.catalog + - OS/x86_64/EFI/ + - OS/x86_64/Packages/ + - OS/x86_64/docs/ + - OS/x86_64/images/ + - OS/x86_64/repodata/ + - OS/x86_64/RPM-GPG-KEY-openEuler/ + - OS/x86_64/TRANS.TBL + - OS/x86_64/boot.catalog +debuginfo: + - debuginfo/aarch64/ + - debuginfo/x86_64/ + - debuginfo/aarch64/EFI/ + - debuginfo/aarch64/Packages/ + - debuginfo/aarch64/docs/ + - debuginfo/aarch64/images/ + - debuginfo/aarch64/repodata/ + - debuginfo/aarch64/RPM-GPG-KEY-openEuler/ + - debuginfo/aarch64/TRANS.TBL + - debuginfo/aarch64/boot.catalog + - debuginfo/x86_64/EFI/ + - debuginfo/x86_64/Packages/ + - debuginfo/x86_64/docs/ + - debuginfo/x86_64/images/ + - debuginfo/x86_64/repodata/ + - debuginfo/x86_64/RPM-GPG-KEY-openEuler/ + - debuginfo/x86_64/TRANS.TBL + - debuginfo/x86_64/boot.catalog +docker_img: + - docker_img/aarch64/ + - docker_img/x86_64/ + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz + - docker_img/aarch64/openEuler-docker.aarch64.tar.xz.sha256sum + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz + - docker_img/x86_64/openEuler-docker.x86_64.tar.xz.sha256sum +everything: + - everything/aarch64/ + - everything/x86_64/ + - everything/aarch64/EFI/ + - everything/aarch64/Packages/ + - everything/aarch64/docs/ + - everything/aarch64/images/ + - everything/aarch64/repodata/ + - everything/aarch64/RPM-GPG-KEY-openEuler/ + - everything/aarch64/TRANS.TBL + - everything/aarch64/boot.catalog + - everything/x86_64/EFI/ + - everything/x86_64/Packages/ + - everything/x86_64/docs/ + - everything/x86_64/images/ + - everything/x86_64/repodata/ + - everything/x86_64/RPM-GPG-KEY-openEuler/ + - everything/x86_64/TRANS.TBL + - everything/x86_64/boot.catalog +raspi_img: + - raspi_img/*-raspi-aarch64.img + - raspi_img/*-raspi-aarch64.img.sha256sum + - raspi_img/*-raspi-aarch64.img.xz + - raspi_img/*-raspi-aarch64.img.xz.sha256sum +source: + - source/aarch64/EFI/ + - source/aarch64/Packages/ + - source/aarch64/docs/ + - source/aarch64/images/ + - source/aarch64/repodata/ + - source/aarch64/RPM-GPG-KEY-openEuler/ + - source/aarch64/TRANS.TBL + - source/aarch64/boot.catalog +stratovirt_img: + - stratovirt_img/aarch64/ + - stratovirt_img/x86_64/ + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz + - stratovirt_img/aarch64/*-stratovirt-aarch64.img.xz.sha256sum + - stratovirt_img/aarch64/EFI/ + - stratovirt_img/aarch64/std-vmlinux.bin + - stratovirt_img/aarch64/std-vmlinux.bin.sha256sum + - stratovirt_img/aarch64/vmlinux.bin + - stratovirt_img/aarch64/vmlinux.bin.sha256sum + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz + - stratovirt_img/x86_64/EFI/ + - stratovirt_img/x86_64/*-stratovirt-x86_64.img.xz.sha256sum + - stratovirt_img/x86_64/std-vmlinux.bin + - stratovirt_img/x86_64/std-vmlinux.bin.sha256sum + - stratovirt_img/x86_64/vmlinux.bin + - stratovirt_img/x86_64/vmlinux.bin.sha256sum +update: + - update/aarch64/ + - update/source/ + - update/x86_64/ + - update/aarch64/Packages/ + - update/aarch64/repodata/ + - update/source/Packages/ + - update/source/repodata/ + - update/x86_64/Packages/ + - update/x86_64/repodata/ + +virtual_machine_img: + - virtual_machine_img/aarch64/ + - virtual_machine_img/x86_64/ + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz + - virtual_machine_img/aarch64/*-aarch64.qcow2.xz.sha256sum + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz + - virtual_machine_img/x86_64/*-x86_64.qcow2.xz.sha256sum \ No newline at end of file