diff --git a/Helper/package_helper.py b/Helper/package_helper.py index e4e41dda07ce5615aeef0602743ff89f486a7f4a..ed415266fc4ad51886bd6f36d1abe9848d2a3989 100644 --- a/Helper/package_helper.py +++ b/Helper/package_helper.py @@ -18,7 +18,6 @@ This is a wiki bot tool for assisting community governance # ******************************************************************************/ import urllib.request -import bs4 from model.Issue import Issue from model.PullRequest import PullRequest diff --git a/Helper/report_helper.py b/Helper/report_helper.py index 9904ffbc98e41a9930f890b0b84d41e3dbe912f0..e6b3f6f952bb845d45f72a8413e860c35e83d1a5 100644 --- a/Helper/report_helper.py +++ b/Helper/report_helper.py @@ -99,8 +99,12 @@ class ReportHelper(object): print("===== Start Print Sig List =====") print("Total sig num: ", len(sig_list)) for sig in sig_list: - print(sig.get_sig_name()) + print(sig.get_name()) + print("maintainers:") for maintainer in sig.get_maintainers(): print(maintainer.get_name()) + print("projects:") + for project in sig.get_projects(): + print(project.get_name()) print("===== Print Sig List Done =====") diff --git a/Helper/sig_helper.py b/Helper/sig_helper.py index f342c36b5c1aa7d06f9e16747997e81e372ec758..dae043ae05a0524a57a7368c54c691ce96366f04 100644 --- a/Helper/sig_helper.py +++ b/Helper/sig_helper.py @@ -23,13 +23,17 @@ import re from model.Sig import Sig from model.Maintainer import Maintainer from model.Developer import Developer +from model.Project import Project from Helper.package_helper import PackageHelper - +from utils.log import logger URL_SIG = "https://gitee.com/openeuler/community/tree/master/sig/" +PATH_MAINTAINER = "/OWNERS" +PATH_PROJECT = "/sig-info.yaml" PATTERN_SIG = r'"/openeuler/community/tree/master/sig/(.*)"' PATTERN_MAINTAINER = r"- (.*?) " +PATTERN_PROJECT = r"repo: src-openeuler/(.*?) " BlockList = [ "openeuler/blog", @@ -40,44 +44,75 @@ class SigHelper(object): @staticmethod def init_sig_info(sig_list): - print("===== Start Init Sig Info... =====") - tmp_sig = None - + logger.info("Start to get sig info.") # 获取sig列表 - page = urllib.request.urlopen(URL_SIG) - contents = page.read().decode('utf-8') - pattern = re.compile(PATTERN_SIG) - signames = pattern.findall(contents) - for signame in signames[1:]: - sig = Sig(signame) + SigHelper.get_all_sig_names(sig_list) + + # 获取每个sig的maintainer信息 + # SigHelper.get_all_maintainer_names(sig_list) + + # 获取每个sig的project信息 + SigHelper.get_all_project_names(sig_list) + logger.info("End to get sig info.") + + @staticmethod + def get_all_sig_names(sig_list): + logger.info("Start to get all sig names.") + names = SigHelper.get_all_object_names_from_a_url(URL_SIG, PATTERN_SIG) + for name in names[1:]: + sig = Sig(name) sig_list.append(sig) + logger.info("End to get all sig names.") - # 获取每个sig的各种信息 + @staticmethod + def get_all_maintainer_names(sig_list): + logger.info("Start to get all maintainer names.") for sig in sig_list: - url_maintainer = URL_SIG + sig.get_sig_name() + "/OWNERS" - page = urllib.request.urlopen(url_maintainer) - contents = page.read().decode('utf-8') - pattern = re.compile(PATTERN_MAINTAINER) - maintainer_names = pattern.findall(contents) - print(sig.get_sig_name(), maintainer_names) - for maintainer_name in maintainer_names: - maintainer = Developer(maintainer_name) + url = URL_SIG + sig.get_name() + PATH_MAINTAINER + names = SigHelper.get_all_object_names_from_a_url(url, PATTERN_MAINTAINER) + for name in names: + maintainer = Developer(name) sig.add_maintainer(maintainer) + logger.info("End to get all maintainer names.") + + @staticmethod + def get_all_object_names_from_a_url(url, pattern): + logger.info("Start to get all object names from a url.") + names = [] + try: + page = urllib.request.urlopen(url) + contents = page.read().decode('utf-8') + compiled_pattern = re.compile(pattern) + names = compiled_pattern.findall(contents) + except urllib.error.URLError as e: + logger.warning(e) + logger.info("End to get all maintainer names.") + return names - print("===== Init Sig Info Done =====") + @staticmethod + def get_all_project_names(sig_list): + logger.info("Start to get all project names.") + for sig in sig_list: + url = URL_SIG + sig.get_name() + PATH_PROJECT + names = SigHelper.get_all_object_names_from_a_url(url, PATTERN_PROJECT) + print(sig.get_name(), names) + for name in names: + project = Project(name) + sig.add_project(project) + logger.info("End to get all project names.") @staticmethod def process_sig_issue(sig_list): - print("===== Start Process Sig Issue =====") + logger.info("Start to process get issue.") for sig in sig_list: for package in sig.get_package_list(): PackageHelper.generate_issue_info(package) - print("===== Process Sig Issue Done =====") + logger.info("End to process get issue.") @staticmethod def process_sig_pr(sig_list): - print("===== Start Process Sig Pull Request =====") + logger.info("Start to process sig pull request.") for sig in sig_list: for package in sig.get_package_list(): PackageHelper.generate_pr_info(package) - print("===== Process Sig Pull Request Done =====") + logger.info("End to process sig pull request.") diff --git a/model/Package.py b/model/Project.py similarity index 92% rename from model/Package.py rename to model/Project.py index 1e130ac00453373d28869e78e87a8806a7087c20..6f1b2fad0be0296beda66aba90b42968e7b37b96 100644 --- a/model/Package.py +++ b/model/Project.py @@ -18,7 +18,7 @@ This is a wiki bot tool for assisting community governance # ******************************************************************************/ -class Package(object): +class Project(object): def __init__(self, name): self.__name = name @@ -31,14 +31,14 @@ class Package(object): def add_pr(self, pr): self.__pull_requests.append(pr) - def get_package_name(self): + def get_name(self): return self.__name def get_url(self): return "https://gitee.com/" + self.__name - def get_issue_list(self): + def get_issues(self): return self.__issues - def get_pr_list(self): + def get_pull_requests(self): return self.__pull_requests diff --git a/model/Sig.py b/model/Sig.py index 958ca26215a1a1ecd795241b634d9fdac460594b..5194913171f33335f389c5501b7132d399d480a4 100644 --- a/model/Sig.py +++ b/model/Sig.py @@ -20,22 +20,22 @@ This is a wiki bot tool for assisting community governance class Sig(object): - def __init__(self, sigName): - self.__sigName = sigName - self.__packages = [] + def __init__(self, name): + self.__name = name + self.__projects = [] self.__maintainers = [] - def add_software(self, software): - self.__packages.append(software) + def add_project(self, project): + self.__projects.append(project) def add_maintainer(self, maintainer): self.__maintainers.append(maintainer) - def get_sig_name(self): - return self.__sigName + def get_name(self): + return self.__name - def get_package_list(self): - return self.__packages + def get_projects(self): + return self.__projects def get_maintainers(self): return self.__maintainers diff --git a/openeuler-wiki-bot.py b/openeuler-wiki-bot.py index bd54a8351fb5e7bb584eab5a0d49abb6c097442c..c2478abc484c1a236b9c5c5def0dcf3e3871532b 100755 --- a/openeuler-wiki-bot.py +++ b/openeuler-wiki-bot.py @@ -21,6 +21,7 @@ import argparse from Helper.sig_helper import SigHelper from Helper.report_helper import ReportHelper +from utils.log import logger sig_list = [] diff --git a/utils/conf.py b/utils/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..511c2340b22c9d68881ed75f818b604278ced91d --- /dev/null +++ b/utils/conf.py @@ -0,0 +1,15 @@ +import os +import configparser + +CONFIG_PATH_FILE = "/etc/openEuler-wiki-bot/wiki-bot.conf" + +configParser = configparser.ConfigParser() +if not os.path.exists(CONFIG_PATH_FILE): + CONFIG_PATH_FILE = "../wiki-bot.conf" + +configParser.read(CONFIG_PATH_FILE, encoding="utf-8") + +LOG_FILE_DIR = configParser.get("log", "log_file_dir", fallback='/var/log/openEuler-wiki-bot') +LOG_FILE_NAME = configParser.get("log", "log_file_name", fallback='openEuler-wiki-bot') +MAX_BYTES = configParser.get("log", "max_bytes", fallback=10 * 1024 * 1024) +BACKUP_COUNT = configParser.get("log", "backup_count", fallback=100) diff --git a/utils/log.py b/utils/log.py new file mode 100644 index 0000000000000000000000000000000000000000..93d17c26dc291bc1d0a9f8cf19a1cd1b04a19d70 --- /dev/null +++ b/utils/log.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import logging.handlers +from datetime import datetime +import shutil +import gzip +import os + +from utils.conf import LOG_FILE_DIR +from utils.conf import LOG_FILE_NAME +from utils.conf import MAX_BYTES +from utils.conf import BACKUP_COUNT + + +class CompressedRotatingFileHandler(logging.handlers.RotatingFileHandler): + def do_rollover(self): + if self.stream: + self.stream.close() + if self.backupCount > 0: + now = datetime.utcnow() + dfn = self.baseFilename + now.strftime('%Y-%m-%d_%H_%M') + ".gz" + with open(self.baseFilename, 'rb') \ + as f_in, gzip.open(dfn, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + self.mode = 'w' + self.stream = self._open() + + +def logging_init(logFileDir, logFileName, maxBytes, backupCount): + if not os.path.exists(logFileDir): + os.makedirs(logFileDir, mode=0o644) + + filename = os.path.join( + logFileDir, + logFileName + "_" + "runtime" + ".log" + ) + log_object = logging.getLogger(logFileName) + log_object.propagate = False + log_object.setLevel(logging.DEBUG) + + file_handler = CompressedRotatingFileHandler(filename=filename, + maxBytes=int(maxBytes), + backupCount=int(backupCount)) + + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s -" + " %(filename)s:%(lineno)d - %(message)s") + file_handler.setFormatter(formatter) + log_object.addHandler(file_handler) + return log_object + + +logger = logging_init(LOG_FILE_DIR, LOG_FILE_NAME, MAX_BYTES, BACKUP_COUNT) \ No newline at end of file diff --git a/wiki-bot.conf b/wiki-bot.conf new file mode 100644 index 0000000000000000000000000000000000000000..de169b4b1158596be122a8ec94045db013270776 --- /dev/null +++ b/wiki-bot.conf @@ -0,0 +1,5 @@ +[log] +log_file_dir = /var/log/openEuler-wiki-bot +log_file_name = openEuler-wiki-bot +max_bytes = 10000 +backup_count = 100 \ No newline at end of file