From e171b5a2b0771a489866e3297f00d1cecc8538b1 Mon Sep 17 00:00:00 2001 From: Xinwei Hu Date: Mon, 11 Sep 2023 16:46:01 +0800 Subject: [PATCH] Various fixes --- advisors/advisor_common.py | 133 ++++++++++++++++++++++++ advisors/check_missing_file.py | 15 ++- advisors/check_upstream.py | 33 +++--- advisors/check_version.py | 18 ++-- advisors/gitee.py | 16 --- advisors/helper/reviewer_checklist.yaml | 28 +++-- advisors/review_tool.py | 12 +-- advisors/yaml2url.py | 2 +- develop_env.fish | 17 ++- tests/test_yaml2url.py | 2 +- 10 files changed, 206 insertions(+), 70 deletions(-) create mode 100755 advisors/advisor_common.py diff --git a/advisors/advisor_common.py b/advisors/advisor_common.py new file mode 100755 index 00000000..9efb2367 --- /dev/null +++ b/advisors/advisor_common.py @@ -0,0 +1,133 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +Common functions for openEuler-Advisor tools +""" +import os +import re +import sys +import argparse +import subprocess +import shutil +import urllib +import yaml + +from advisors import gitee + +def exec_cmd(cmd, retry_times=0): + subp = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + check=False) + if subp.returncode != 0 and retry_times > 0: + for i in range(1, retry_times + 1): + print("cmd:%s execute failed,retry:%d" % (cmd, i)) + subp = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + check=False) + if subp.returncode == 0: + break + print(subp.stdout) + return subp.returncode + +def prepare_gitee_repo(work_dir, reuse, group, repo_name, branch): + """ + prepare local repository + """ + if not os.path.exists(work_dir): + os.makedirs(work_dir) + + repo = group + "/" + repo_name + + gitee_url = "https://gitee.com/{repo}.git".format(repo=repo) + + local_path = os.path.join(work_dir, "{}_{}".format(group, repo_name)) + + if os.path.exists(local_path) and not reuse: + print("%s already exist, delete as we dont reuse" % local_path) + shutil.rmtree(local_path) + + if not os.path.exists(local_path): + if exec_cmd(["git", "clone", gitee_url, local_path]) != 0: + print("ERROR: failed to git clone {}".format(gitee_url)) + return None + + previous_dir = os.getcwd() + os.chdir(local_path) + + if exec_cmd(["git", "checkout", branch]) != 0: + print("ERROR: failed to git checkout %s branch " % branch) + os.chdir(previous_dir) + return None + if exec_cmd(["git", "pull"]) != 0: + print("ERROR: failed to update to latest commit in %s branch" % branch) + os.chdir(previous_dir) + return None + + os.chdir(previous_dir) + return local_path + + +def get_openeuler_sigs(): + """ + return a list of openeuler sigs + """ + + exception_list = ["README.md", "create_sig_info_template.py", "sig-template"] + exp_set = set(exception_list) + ipath = prepare_gitee_repo(".", True, "openeuler", "community", "master") + if ipath == None: + return None + + sigs = [] + spath = os.path.join(ipath, "sig") + for f in os.listdir(spath): + if f in exp_set: + continue + sigs.append(f) + + return sigs + +def get_repos_by_openeuler_sig(sig): + """ + return a list of repos maintained by given openeuler sig + """ + ipath = prepare_gitee_repo(".", True, "openeuler", "community", "master") + if ipath == None: + return None + + repos = [] + spath = os.path.join(ipath, "sig", sig) + + spath_openeuler = os.path.join(spath, "openeuler") + for r, d, f in os.walk(spath_openeuler): + for file in f: + if file.endswith(".yaml"): + repos.append("openeuler/"+file[:-5]) + + spath_srcopeneuler = os.path.join(spath, "src-openeuler") + for r, d, f in os.walk(spath_srcopeneuler): + for file in f: + if file.endswith(".yaml"): + repos.append("src-openeuler/"+file[:-5]) + + return repos + + +if __name__ == "__main__": + print(get_openeuler_sigs()) + print(get_repos_by_openeuler_sig('sig-rfo')) + print(get_repos_by_openeuler_sig('sig-sw-arch')) diff --git a/advisors/check_missing_file.py b/advisors/check_missing_file.py index aec1e333..e7dd9b95 100755 --- a/advisors/check_missing_file.py +++ b/advisors/check_missing_file.py @@ -21,6 +21,7 @@ import argparse from datetime import datetime from advisors import gitee +from advisors import advisor_common NEW_SPEC_ISSUE_BODY = """Dear {repo} maintainer: 亲爱的 {repo} 维护者： @@ -72,9 +73,6 @@ Yours openEuler Advisor. """ - - - def main_process(repo, push, check_file): """ Main process for command line @@ -147,12 +145,11 @@ def main(): print("ERROR: Not support {file} check".format(file=args.file)) return - try: - user_gitee = gitee.Gitee() - except NameError: - sys.exit(1) - repos = user_gitee.get_repos_by_sig(args.sig) - print(repos) + repos = advisor_common.get_repos_by_openeuler_sig(args.sig) + if not repos: + print("ERROR: no repos find is {sig} sig, please check the sig name.".format( + sig=args.sig)) + return fail_list = [] total = len(repos) diff --git a/advisors/check_upstream.py b/advisors/check_upstream.py index 0ac0abf6..a90c9e36 100755 --- a/advisors/check_upstream.py +++ b/advisors/check_upstream.py @@ -248,27 +248,22 @@ def check_metacpan(info, clean_tag=True): resp = resp.text - tag_list = resp.splitlines() - condition = "value=\"/release" - - len_tag_list = len(tag_list) - 1 - for index in range(len_tag_list): - if condition in tag_list[index]: - tag = tag_list[index + 1] - print(tag) - index = index + 1 - if 'DEV' in tag: - continue - tag = tag.lstrip() - tag = tag.rstrip() - tags[tag] = None - - if not tags: - eprint("{repo} found unsorted on cpan.metacpan.org".format(repo=info["src_repo"])) - return tags - last_query = {"time_stamp": datetime.now(), "raw_data": resp} info["last_query"] = last_query + + tags = [] + tag = None + tags_json = json.loads(resp) + if "version" in tags_json: + tag = tags_json["version"] + elif "version" in tags.json["metadata"]: + tag = tags_json["metadata"]["version"] + elif "version_numified" in tags_json: + tag = str(tags_json["version_numified"]) + if tag: + tag = tag.lstrip('0').rstrip('0') + tags.append(tag) + if clean_tag: tags = clean_tags(tags, info) return tags diff --git a/advisors/check_version.py b/advisors/check_version.py index 8a51b41a..1cc5aaae 100755 --- a/advisors/check_version.py +++ b/advisors/check_version.py @@ -20,7 +20,8 @@ import argparse import time from advisors import gitee from advisors.oa_upgradable import main_process - +from advisors import advisor_common +import time def main(): @@ -34,20 +35,17 @@ def main(): parameters.add_argument("-d", "--default", type=str, default=os.getcwd(), help="The fallback place to look for YAML information") - parameters.add_argument("-s", "--sig", required=True, + parameters.add_argument("-s", "--sig", type=str, default="sig-recycle", help="Check yaml by Sig") args = parameters.parse_args() + if args.sig: + sig = args.sig + else: + sig = 'sig-recycle' - sig = args.sig - - try: - user_gitee = gitee.Gitee() - except NameError: - sys.exit(1) + repos = advisor_common.get_repos_by_openeuler_sig(sig) - repos = user_gitee.get_repos_by_sig(sig) - print(repos) total = len(repos) index = 0 upgrade_list = [] diff --git a/advisors/gitee.py b/advisors/gitee.py index e72e1173..8ffa0149 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -336,22 +336,6 @@ class Gitee(): return my_dir['sha'] return '' - def get_sigs(self): - """ - Get upstream sigs tree info - """ - sigs = {} - master_tree = self.__get_community_tree('master') - for my_dir in master_tree: - if my_dir['type'] == 'tree' and my_dir['path'] == 'sig': - sig_sha = my_dir['sha'] - - sig_tree = self.__get_community_tree(sig_sha) - for my_dir in sig_tree: - if my_dir['type'] == 'tree': - sigs[my_dir['path']] = my_dir['sha'] - - return sigs def get_repos_by_sig(self, sig): """ diff --git a/advisors/helper/reviewer_checklist.yaml b/advisors/helper/reviewer_checklist.yaml index 9e2c22d4..50e81261 100644 --- a/advisors/helper/reviewer_checklist.yaml +++ b/advisors/helper/reviewer_checklist.yaml @@ -15,16 +15,21 @@ basic: condition: null claim: PR和实际代码修改和内容描述一致 explain: 提交的说明文字应该和实际代码修改内容保持一致。 + - + name: PR-clean-submission + condition: null + claim: 应当一个 PR 完成一件事情 + explain: PR 应当完成一件有意义的事情，不应该把一个修改目标拆成多个 PR，也不应该把太多修改目标整合在一个 PR 中。不建议对过于简单的工作单独发起 PR。 - name: PR-standard-check condition: null claim: PR符合gitee的规范检查要求 explain: 码云对提交的缺陷扫描、规范扫描告警每一条都需要确认。 - - name: PR-one-submission + name: PR-CI-result condition: null - claim: 建议PR中只有一次提交 - explain: 如果PR中包含多次提交，建议整合成一个，保持提交记录整洁。[[Gitee work flow](https://gitee.com/openeuler/community/blob/master/zh/contributors/Gitee-workflow.md)] + claim: PR应该通过所有CI测试，包括不同架构的 check_install 测试 + explain: check_consistency 由于基础设施问题可忽略，其他CI项目如失败必须给出说明。 - name: PR-temp-check condition: new-file-add @@ -67,6 +72,12 @@ src-openeuler: claim: 提交到 master 分支的新版本，是否已经是主线最新的 explain: 原则上 master 分支要求必须是最新的社区上游版本 + - + name: PR-upgrade-check + condition: null + claim: 版本升级中的差异应当分析说明 + explain: compare_package的结果应当给出分析，保证版本差异的可理解。 + - name: PR-increamental-rls condition: null @@ -76,8 +87,8 @@ src-openeuler: - name: PR-license condition: null - claim: License域设置是否正确 - explain: 原则上 License 应是 SPDX 中已知标准 + claim: CI License检查是否正确 + explain: License错误必须解决 - name: PR-source-integerity @@ -112,7 +123,7 @@ src-openeuler: name: PR-changelog condition: null claim: changelog 记录是否合理 - explain: Changelog 格式要符合要求，变更记录易于阅读理解 + explain: Changelog 格式要符合要求，变更记录易于阅读理解。如果是版本升级，需要简单摘要说明升级带来的变化。 customization: community: @@ -134,6 +145,11 @@ customization: dlt-chk: claim: "已确认删除 {repo} " explain: "为了保证兼容性，技术委员会建议将 {repo} 移动到 sig-recycle 管理一段时间，给用户切换时间。" + - + name: single-submission + condition: null + claim: 建议PR中只有一次提交 + explain: 如果PR中包含多次提交，建议整合成一个，保持提交记录整洁。[[Gitee work flow](https://gitee.com/openeuler/community/blob/master/zh/contributors/Gitee-workflow.md)] - name: maintainer-add-explain condition: maintainer-change diff --git a/advisors/review_tool.py b/advisors/review_tool.py index 7d57e53d..12f2920a 100755 --- a/advisors/review_tool.py +++ b/advisors/review_tool.py @@ -518,7 +518,7 @@ def src_openeuler_review(cklist, branch): review_body = "" for key1, value1 in cklist['src-openeuler'].items(): for value2 in value1: - if value2['name'] == 'PR-latest-version' and branch == 'master': + if value2['name'] == 'PR-latest-version' and branch != 'master': continue item = join_check_item(categorizer[key1], value2['claim'], value2['explain']) @@ -615,7 +615,7 @@ def check_pr_url(url): check whether the URL of Pull Request is valid """ if url: - pattern = re.compile(r'https://gitee.com/(open_euler/dashboard/projects/)?' + pattern = re.compile(r'https://(e.)?gitee.com/(open_euler/repos/)?' + r'(openeuler|src-openeuler)/([A-Za-z0-9-_]*)/pulls/(\d+$)') return pattern.match(url) return None @@ -628,10 +628,10 @@ def extract_params(args): if args.url and len(args.url) > 0: res = check_pr_url(args.url) if res: - group = res.group(2) - repo_name = res.group(3) - pull_id = res.group(4) - return group, repo_name, pull_id + group = res.group(3) + repo_name = res.group(4) + pull_id = res.group(5) + return (group, repo_name, pull_id) print("ERROR: URL is wrong, please check!") return () if args.repo and args.pull and len(args.repo) > 0 and len(args.pull) > 0: diff --git a/advisors/yaml2url.py b/advisors/yaml2url.py index 7dd16fff..1cda9b8f 100755 --- a/advisors/yaml2url.py +++ b/advisors/yaml2url.py @@ -75,7 +75,7 @@ def __get_metacpan_url(pkg_info): """ Get metacpan repo url of package """ - url = urljoin("https://metacpan.org/release/", pkg_info["src_repo"]) + url = urljoin("https://fastapi.metacpan.org/v1/release/", pkg_info["src_repo"]) return url diff --git a/develop_env.fish b/develop_env.fish index 305083f1..1f7025e5 100755 --- a/develop_env.fish +++ b/develop_env.fish @@ -5,8 +5,8 @@ set -e fish_trace set advisor_path (cd (dirname (status -f)); and pwd) set existed 0 -for path in $PYTHONPATH - if [ $advisor_path = $path ] +for p in $PYTHONPATH + if [ $advisor_path = $p ] set existed 1 break end @@ -16,3 +16,16 @@ if [ $existed -eq 0 ] set -x PYTHONPATH $PYTHONPATH $advisor_path end echo "PYTHONPATH=$PYTHONPATH" + +set existed 0 +for p in $PATH + if [ "$advisor_path/advisors" = $p ] + set existed 1 + break + end +end +if [ $existed -eq 0 ] + set -x PATH $PATH $advisor_path/advisors $advisor_path/command +end +echo "PATH=$PATH" + diff --git a/tests/test_yaml2url.py b/tests/test_yaml2url.py index ac43c210..09c4fd77 100644 --- a/tests/test_yaml2url.py +++ b/tests/test_yaml2url.py @@ -106,7 +106,7 @@ def test_get_metacpan_url(): pkg_info = yaml.load(doc, Loader=yaml.Loader) url = yaml2url.yaml2url(pkg_info) - assert url == "https://metacpan.org/release/File-Which" + assert url == "https://fastapi.metacpan.org/v1/release/File-Which" def test_get_rubygem_url(): -- Gitee