diff --git a/Helper/package_helper.py b/Helper/package_helper.py index e4e41dda07ce5615aeef0602743ff89f486a7f4a..2376171e01bcc7990108bef2ab4fcc5b06426510 100644 --- a/Helper/package_helper.py +++ b/Helper/package_helper.py @@ -18,7 +18,7 @@ This is a wiki bot tool for assisting community governance # ******************************************************************************/ import urllib.request -import bs4 +#import bs4 from model.Issue import Issue from model.PullRequest import PullRequest diff --git a/Helper/project_helper.py b/Helper/project_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..296b392ff199f6d0cf563b1e6da1435530d5136f --- /dev/null +++ b/Helper/project_helper.py @@ -0,0 +1,36 @@ +#!/usr/bin/python3 +""" +This is a wiki bot tool for assisting community governance +""" +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: Sinever +# Create: 2022-05-16 +# Description: This is a wiki bot tool for assisting community governance +# ******************************************************************************/ +from utils.log import logger +from utils.weblib import get_all_pattern_strings_from_a_url +from utils.constant import SRC_OPENEULER + + +FORMAT_ISSUE = r"(src-openeuler/{}/issues/.*\?from=project-issue)" +PATTERN_PAGE_NUM = r"/src-openeuler/tensorflow/issues?page=\d" + + +class ProjectHelp(object): + + @staticmethod + def get_all_issues(project): + logger.info("Start to get all issues of a project.") + PATTERN_ISSUE = FORMAT_ISSUE.format(project.get_name()) + issue_strings = get_all_pattern_strings_from_a_url(SRC_OPENEULER + project.get_name() + "/issues", PATTERN_ISSUE) + print(issue_strings) + logger.info("End to get all issues of a project.") diff --git a/Helper/sig_helper.py b/Helper/sig_helper.py index dae043ae05a0524a57a7368c54c691ce96366f04..26b194b528bf435226e5774929126b519b0007db 100644 --- a/Helper/sig_helper.py +++ b/Helper/sig_helper.py @@ -27,6 +27,8 @@ from model.Project import Project from Helper.package_helper import PackageHelper from utils.log import logger +from utils.weblib import get_all_pattern_strings_from_a_url +from Helper.project_helper import ProjectHelp URL_SIG = "https://gitee.com/openeuler/community/tree/master/sig/" PATH_MAINTAINER = "/OWNERS" @@ -53,12 +55,15 @@ class SigHelper(object): # »ñȡÿ¸ösigµÄprojectÐÅÏ¢ SigHelper.get_all_project_names(sig_list) + for sig in sig_list: + for project in sig.get_projects(): + ProjectHelp.get_all_issues(project) logger.info("End to get sig info.") @staticmethod def get_all_sig_names(sig_list): logger.info("Start to get all sig names.") - names = SigHelper.get_all_object_names_from_a_url(URL_SIG, PATTERN_SIG) + names = get_all_pattern_strings_from_a_url(URL_SIG, PATTERN_SIG) for name in names[1:]: sig = Sig(name) sig_list.append(sig) @@ -69,32 +74,18 @@ class SigHelper(object): logger.info("Start to get all maintainer names.") for sig in sig_list: url = URL_SIG + sig.get_name() + PATH_MAINTAINER - names = SigHelper.get_all_object_names_from_a_url(url, PATTERN_MAINTAINER) + names = get_all_pattern_strings_from_a_url(url, PATTERN_MAINTAINER) for name in names: maintainer = Developer(name) sig.add_maintainer(maintainer) logger.info("End to get all maintainer names.") - @staticmethod - def get_all_object_names_from_a_url(url, pattern): - logger.info("Start to get all object names from a url.") - names = [] - try: - page = urllib.request.urlopen(url) - contents = page.read().decode('utf-8') - compiled_pattern = re.compile(pattern) - names = compiled_pattern.findall(contents) - except urllib.error.URLError as e: - logger.warning(e) - logger.info("End to get all maintainer names.") - return names - @staticmethod def get_all_project_names(sig_list): logger.info("Start to get all project names.") for sig in sig_list: url = URL_SIG + sig.get_name() + PATH_PROJECT - names = SigHelper.get_all_object_names_from_a_url(url, PATTERN_PROJECT) + names = get_all_pattern_strings_from_a_url(url, PATTERN_PROJECT) print(sig.get_name(), names) for name in names: project = Project(name) diff --git a/model/Project.py b/model/Project.py index 6f1b2fad0be0296beda66aba90b42968e7b37b96..476e501ddc2b4dabf5915d97a7d07282c092629f 100644 --- a/model/Project.py +++ b/model/Project.py @@ -16,6 +16,8 @@ This is a wiki bot tool for assisting community governance # Create: 2021-01-12 # Description: This is a wiki bot tool for assisting community governance # ******************************************************************************/ +from utils.constant import SRC_OPENEULER + class Project(object): @@ -35,7 +37,7 @@ class Project(object): return self.__name def get_url(self): - return "https://gitee.com/" + self.__name + return SRC_OPENEULER + self.__name def get_issues(self): return self.__issues diff --git a/openeuler-wiki-bot.py b/openeuler-wiki-bot.py index c2478abc484c1a236b9c5c5def0dcf3e3871532b..3b4cf1c0ffcb35cbb1401206869f38733f012404 100755 --- a/openeuler-wiki-bot.py +++ b/openeuler-wiki-bot.py @@ -19,9 +19,7 @@ This is a wiki bot tool for assisting community governance import argparse from Helper.sig_helper import SigHelper -from Helper.report_helper import ReportHelper - -from utils.log import logger +from report.excel import ReportHelper sig_list = [] diff --git a/report/excel.py b/report/excel.py new file mode 100644 index 0000000000000000000000000000000000000000..e6b3f6f952bb845d45f72a8413e860c35e83d1a5 --- /dev/null +++ b/report/excel.py @@ -0,0 +1,110 @@ +#!/usr/bin/python3 +""" +This is a wiki bot tool for assisting community governance +""" +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: Hubble_Zhu +# Create: 2021-01-12 +# Description: This is a wiki bot tool for assisting community governance +# ******************************************************************************/ + +import xlsxwriter + + +class ReportHelper(object): + + @staticmethod + def generate_sheet_sig(sig_sheet, sig_list): + sig_sheet.write('A1', 'Sig') + sig_sheet.write('B1', '软件包') + sig_sheet.write('C1', '软件包URL') + + row = 1 + col = 0 + for sig in sig_list: + for package in sig.get_package_list(): + sig_sheet.write(row, col, sig.get_sig_name()) + sig_sheet.write(row, col + 1, package.get_package_name()) + sig_sheet.write(row, col + 2, package.get_url()) + row = row + 1 + + @staticmethod + def generate_sheet_pr(pr_sheet, sig_list): + pr_sheet.write('A1', 'Sig') + pr_sheet.write('B1', '软件包') + pr_sheet.write('C1', '软件包URL') + pr_sheet.write('D1', 'PR title') + pr_sheet.write('E1', 'PR url') + + row = 1 + col = 0 + for sig in sig_list: + for package in sig.get_package_list(): + for pr in package.get_pr_list(): + pr_sheet.write(row, col, sig.get_sig_name()) + pr_sheet.write(row, col + 1, package.get_package_name()) + pr_sheet.write(row, col + 2, package.get_url()) + pr_sheet.write(row, col + 3, pr.get_title()) + pr_sheet.write(row, col + 4, pr.get_url()) + row = row + 1 + + @staticmethod + def generate_sheet_issue(issue_sheet, sig_list): + issue_sheet.write('A1', 'Sig') + issue_sheet.write('B1', '软件包') + issue_sheet.write('C1', '软件包URL') + issue_sheet.write('D1', 'Issue title') + issue_sheet.write('E1', 'Issue url') + + row = 1 + col = 0 + for sig in sig_list: + for package in sig.get_package_list(): + for issue in package.get_issue_list(): + issue_sheet.write(row, col, sig.get_sig_name()) + issue_sheet.write(row, col + 1, package.get_package_name()) + issue_sheet.write(row, col + 2, package.get_url()) + issue_sheet.write(row, col + 3, issue.get_title()) + issue_sheet.write(row, col + 4, issue.get_url()) + row = row + 1 + + @staticmethod + def generate_report(sig_list, file_path='sig_info.xlsx'): + print("===== Start Generate Report =====") + workbook = xlsxwriter.Workbook(file_path) + sig_sheet = workbook.add_worksheet(name="sig_info") + ReportHelper.generate_sheet_sig(sig_sheet, sig_list) + + pr_sheet = workbook.add_worksheet(name="pr_info") + ReportHelper.generate_sheet_pr(pr_sheet, sig_list) + + issue_sheet = workbook.add_worksheet(name="issue_info") + ReportHelper.generate_sheet_issue(issue_sheet, sig_list) + + workbook.close() + print("===== Generate Report Done =====") + print("Report file: ", file_path) + + @staticmethod + def print_sig_list(sig_list): + print("===== Start Print Sig List =====") + print("Total sig num: ", len(sig_list)) + for sig in sig_list: + print(sig.get_name()) + print("maintainers:") + for maintainer in sig.get_maintainers(): + print(maintainer.get_name()) + print("projects:") + for project in sig.get_projects(): + print(project.get_name()) + print("===== Print Sig List Done =====") + diff --git a/utils/constant.py b/utils/constant.py new file mode 100644 index 0000000000000000000000000000000000000000..7e7310a50eedb8f81e083db681c15fd8598dabe9 --- /dev/null +++ b/utils/constant.py @@ -0,0 +1,4 @@ +GITEE = "https://gitee.com/" +SRC_OPENEULER = "https://gitee.com/src-openeuler/" +OPENEULER = "https://gitee.com/openeuler/" +ISSUE = "issues" diff --git a/utils/log.py b/utils/log.py index 93d17c26dc291bc1d0a9f8cf19a1cd1b04a19d70..1eb8c7e562bf2f4306d1d0f98cf69b66f842b27b 100644 --- a/utils/log.py +++ b/utils/log.py @@ -27,21 +27,21 @@ class CompressedRotatingFileHandler(logging.handlers.RotatingFileHandler): self.stream = self._open() -def logging_init(logFileDir, logFileName, maxBytes, backupCount): - if not os.path.exists(logFileDir): - os.makedirs(logFileDir, mode=0o644) +def logging_init(logdir, filename, maxbytes, backupcount): + if not os.path.exists(logdir): + os.makedirs(logdir, mode=0o644) filename = os.path.join( - logFileDir, - logFileName + "_" + "runtime" + ".log" + logdir, + filename + "_" + "runtime" + ".log" ) - log_object = logging.getLogger(logFileName) + log_object = logging.getLogger(filename) log_object.propagate = False log_object.setLevel(logging.DEBUG) file_handler = CompressedRotatingFileHandler(filename=filename, - maxBytes=int(maxBytes), - backupCount=int(backupCount)) + maxBytes=int(maxbytes), + backupCount=int(backupcount)) formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s -" diff --git a/utils/weblib.py b/utils/weblib.py new file mode 100644 index 0000000000000000000000000000000000000000..48583bd4a2f5580b2108bca14b8161a9acad62e8 --- /dev/null +++ b/utils/weblib.py @@ -0,0 +1,43 @@ +#!/usr/bin/python3 +""" +This is a wiki bot tool for assisting community governance +""" +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: Sinever +# Create: 2022-05-16 +# Description: This is a wiki bot tool for assisting community governance +# ******************************************************************************/ + +import urllib.request +import re + +from utils.log import logger + + +def get_all_pattern_strings_from_a_url(url, pattern): + logger.info("Start to get all object names from a url.") + result = [] + try: + page = urllib.request.urlopen(url) + contents = page.read().decode('utf-8') + compiled_pattern = re.compile(pattern) + result = compiled_pattern.findall(contents) + except urllib.error.URLError as e: + logger.warning(e) + logger.info("End to get all object names.") + return result + + +FORMAT_ISSUE = r"(src-openeuler/tensorflow/issues/.*\?from=project-issue)" +text = "https://gitee.com/src-openeuler/tensorflow/issues" +if __name__ == '__main__': + print(get_all_pattern_strings_from_a_url(text, FORMAT_ISSUE))