diff --git a/advisors/check_abi.py b/advisors/check_abi.py index 45d9e6be1952f9e796b3ee93e0e739193722a8f7..afe4b4c8aaab68166f03bf33240715f9f90aa1c6 100755 --- a/advisors/check_abi.py +++ b/advisors/check_abi.py @@ -14,9 +14,10 @@ # ******************************************************************************/ """ -(1) This script is used to check the ABI changes between the old +(1) This script is used to check the ABI changes between the old and new versions of dynamic libraries. - The merged result on difference is saved in the xxx_all_abidiff.out file in the working directory + The merged result on difference is saved in the xxx_all_abidiff.out file in the working + directory. default path: /var/tmp/xxx_all_abidiff.out (2) This script depends on abidiff from libabigail package. @@ -31,7 +32,6 @@ import subprocess import sys import os import logging -import io import shutil import tempfile @@ -42,41 +42,44 @@ def parse_command_line(): parser.add_argument("-d", "--work_path", default="/var/tmp", nargs="?", help="The work path to put rpm2cpio files and results" " (e.g. /home/tmp_abidiff default: /var/tmp/)") - parser.add_argument("-a", "--show_all_info", action="store_true", default=False, + parser.add_argument("-a", "--show_all_info", action="store_true", default=False, help="show all infos includ changes in member name") parser.add_argument("-v", "--verbose", action="store_true", default=False, - help="Show additional information") + help="Show additional information") subparser = parser.add_subparsers(dest='command_name', - help="Compare between two RPMs or two .so files") + help="Compare between two RPMs or two .so files") rpm_parser = subparser.add_parser('compare_rpm', help="Compare between two RPMs") rpm_parser.add_argument("-r", "--rpms", required=True, nargs=2, - metavar=('old_rpm', 'new_rpm'), - help="Path or URL of both the old and new RPMs") - rpm_parser.add_argument("-d", "--debuginfo_rpm", nargs=2, - metavar=('old_debuginfo_rpm', 'new_debuginfo_rpm'), required=False, - help = "Path or URL of both the old and new debuginfo RPMs, corresponding to compared RPMs.") + metavar=('old_rpm', 'new_rpm'), + help="Path or URL of both the old and new RPMs") + rpm_parser.add_argument("-d", "--debuginfo_rpm", nargs=2, + metavar=('old_debuginfo_rpm', 'new_debuginfo_rpm'), + required=False, + help="Path or URL of both the old and new debuginfo RPMs," + "corresponding to compared RPMs.") rpm_parser.set_defaults(func=process_with_rpm) so_parser = subparser.add_parser('compare_so', help="Compare between two .so files") so_parser.add_argument("-s", "--sos", required=True, nargs=2, - metavar=('old_so', 'new_so'), - help="Path or URL of both the old and new .so files") + metavar=('old_so', 'new_so'), + help="Path or URL of both the old and new .so files") so_parser.add_argument("-f", "--debuginfo_path", nargs=2, required=False, - metavar=('old_debuginfo_path', 'new_debuginfo_path'), - help = "Path or URL of both the old and new debuginfo files, corresponding to compared .so files.") + metavar=('old_debuginfo_path', 'new_debuginfo_path'), + help="Path or URL of both the old and new debuginfo files," + "corresponding to compared .so files.") so_parser.set_defaults(func=process_with_so) - + config = parser.parse_args() - if config.command_name == None: + if config.command_name is None: parser.print_help() sys.exit(0) else: return config - + def list_so_files(path): """ Generate a list of all .so files in the directory. @@ -85,8 +88,8 @@ def list_so_files(path): # we cannot rely on number suffix for some .so files use complex version scheme. exception_list = ["hmac"] so_files = set() - for dirpath, dirnames, files in os.walk(path): - for filename in files: + for dirpath, _, files in os.walk(path): + for filename in files: fp = os.path.join(dirpath, filename) if os.path.islink(fp): continue @@ -124,18 +127,19 @@ def find_all_so_file(path1, path2): prev_left = previous_sos - prev_matched curr_left = current_sos - curr_matched - if len(prev_left) != 0: + if prev_left: logging.info("Unmatched .so file in previous version") logging.info("Usually means deleted .so in current version") logging.info("%s\n", prev_left) - if len(curr_left) != 0: + if curr_left: logging.info("Unmatched .so file in current version") logging.info("Usually means newly added .so in current version") logging.info("%s\n", curr_left) logging.debug("mapping of .so files:%s\n", all_so_pair) return all_so_pair - + + def make_abi_path(work_path, abipath): """ Get the path to put so file from rpm @@ -149,20 +153,20 @@ def make_abi_path(work_path, abipath): def get_rpm_path(rpm_url, dest): - """Get the path of rpm package""" + """Get the path of rpm package""" + rpm_path = "" if os.path.isfile(rpm_url): - abs_rpmpath = os.path.abspath(rpm_url) - logging.debug("rpm exists:%s", abs_rpmpath) - return abs_rpmpath + rpm_path = os.path.abspath(rpm_url) + logging.debug("rpm exists:%s", rpm_path) else: rpm_name = os.path.basename(rpm_url) rpm_path = os.path.join(dest, rpm_name) logging.debug("downloading %s......", rpm_name) - subprocess.call(["curl", rpm_url, "-L", - "--connect-timeout", "10", - "--max-time", "600", - "-sS", "-o", rpm_path]) - return rpm_path + subprocess.call(["curl", rpm_url, "-L", + "--connect-timeout", "10", + "--max-time", "600", + "-sS", "-o", rpm_path]) + return rpm_path def do_rpm2cpio(rpm2cpio_path, rpm_file): """ @@ -184,7 +188,7 @@ def merge_all_abidiff_files(all_abidiff_files, work_path, rpm_base_name): if os.path.exists(merged_file): subprocess.run("rm -rf {}".format(merged_file), shell=True) - ofile = open(merged_file, "a+") + ofile = open(merged_file, "a+") for diff_file in all_abidiff_files: diff_file_name = os.path.basename(diff_file) ofile.write("---------------diffs in {}:----------------\n".format(diff_file_name)) @@ -197,14 +201,14 @@ def do_abidiff(config, all_so_pair, work_path, base_name, debuginfo_path): """ Exec the abidiff and write result to files. return the abidiff returncode. - """ - if len(all_so_pair) == 0: + """ + if not all_so_pair: logging.info("There are no .so files to compare") sys.exit(0) if debuginfo_path: - logging.debug("old_debuginfo_path:%s\nnew_debuginfo_path:%s", - debuginfo_path[0], debuginfo_path[1]) + logging.debug("old_debuginfo_path:%s\nnew_debuginfo_path:%s", + debuginfo_path[0], debuginfo_path[1]) with_debuginfo = True else: with_debuginfo = False @@ -215,8 +219,9 @@ def do_abidiff(config, all_so_pair, work_path, base_name, debuginfo_path): new_so_file = all_so_pair[old_so_file] logging.debug("begin abidiff between %s and %s", old_so_file, new_so_file) - abidiff_file = os.path.join(work_path, - "{}_{}_abidiff.out".format(base_name, os.path.basename(new_so_file))) + abidiff_file = os.path.join(work_path, + "{}_{}_abidiff.out".format(base_name, + os.path.basename(new_so_file))) so_options = "{} {}".format(old_so_file, new_so_file) @@ -230,43 +235,43 @@ def do_abidiff(config, all_so_pair, work_path, base_name, debuginfo_path): else: debug_options = "" - abidiff_cmd = "abidiff {so_options} {debug_options} {additional_options} > {difffile}".format( - so_options=so_options, - debug_options=debug_options, - additional_options=additional_options, - difffile=abidiff_file) + abidiff_template = "abidiff {so_options} {debug_options} {additional_options} > {difffile}" + abidiff_cmd = abidiff_template.format(so_options=so_options, + debug_options=debug_options, + additional_options=additional_options, + difffile=abidiff_file) ret = subprocess.run(abidiff_cmd, shell=True) - + all_abidiff_files.append(abidiff_file) logging.info("result write in: %s", abidiff_file) return_code |= ret.returncode merged_file = merge_all_abidiff_files(all_abidiff_files, work_path, base_name) logging.info("all results writed in: %s", merged_file) - return return_code + return return_code + - def validate_sos(config): """ Validate the command arguments - """ + """ for so in config.sos: if not os.path.isfile(so) or ".so" not in so: logging.error(f"{so} not exists or not a .so file") - sys.exit(0) + sys.exit(0) if config.debuginfo_path: for d in config.debuginfo_path: if not os.path.exists(d): logging.error(f"{d} not exists") - sys.exit(0) - + sys.exit(0) + def check_result(returncode): """ Check the result of abidiff - """ + """ ABIDIFF_ERROR_BIT = 1 if returncode == 0: logging.info("No ABI differences found.") @@ -275,11 +280,11 @@ def check_result(returncode): else: logging.info("ABI differences found.") - + def process_with_rpm(config): """ Process the file with type of rpm. - """ + """ work_path = config.work_path temp_path = os.path.abspath(tempfile.mkdtemp(dir=work_path)) @@ -289,23 +294,24 @@ def process_with_rpm(config): rpm_path = [get_rpm_path(x[0], x[1]) for x in zip(config.rpms, abi_paths)] logging.debug("rpm_path:%s\n", rpm_path) - [do_rpm2cpio(x[0], x[1]) for x in zip(abi_paths, rpm_path)] + _ = [do_rpm2cpio(x[0], x[1]) for x in zip(abi_paths, rpm_path)] if config.debuginfo_rpm: - debuginfo_rpm_path = [get_rpm_path(x[0], x[1]) for x in zip(config.debuginfo_rpm, abi_paths)] + debuginfo_rpm_path = [get_rpm_path(x[0], x[1]) + for x in zip(config.debuginfo_rpm, abi_paths)] logging.debug("debuginfo_rpm_path:%s\n", debuginfo_rpm_path) - - [do_rpm2cpio(x[0], x[1]) for x in zip(abi_paths, debuginfo_rpm_path)] - + + _ = [do_rpm2cpio(x[0], x[1]) for x in zip(abi_paths, debuginfo_rpm_path)] + os.chdir(temp_path) logging.debug("\n----begin abidiff working in path:%s----", os.getcwd()) - - so_paths = [ os.path.join(x, "usr/lib64") for x in abi_paths ] + + so_paths = [os.path.join(x, "usr/lib64") for x in abi_paths] all_so_pairs = find_all_so_file(so_paths[0], so_paths[1]) - debuginfo_paths = [ os.path.join(x, "usr/lib/debug") for x in abi_paths ] + debuginfo_paths = [os.path.join(x, "usr/lib/debug") for x in abi_paths] rpm_base_name = os.path.basename(rpm_path[0]).split('.')[0] @@ -316,10 +322,10 @@ def process_with_rpm(config): return returncode -def process_with_so(config): +def process_with_so(config): """ Process the file with type of .so. - """ + """ validate_sos(config) work_path = config.work_path all_so_pair = {} @@ -327,7 +333,7 @@ def process_with_so(config): all_so_pair[so_path[0]] = so_path[1] os.chdir(work_path) logging.debug("\n----begin abidiff with .so working in path:%s----", os.getcwd()) - + so_base_name = os.path.basename(so_path[0]).split('.')[0] if config.debuginfo_path: debuginfo_path = list(map(os.path.abspath, config.debuginfo_path)) @@ -338,7 +344,7 @@ def process_with_so(config): check_result(returncode) return returncode - + def main(): """Entry point for check_abi""" config = parse_command_line() @@ -349,6 +355,6 @@ def main(): ret = config.func(config) sys.exit(ret) - + if __name__ == "__main__": main() diff --git a/advisors/check_upstream.py b/advisors/check_upstream.py index 22e3ccc58437cc6ae02223581ea1da67148128d5..873c614bc68bc1d7e3a6442592b677e776bb4763 100755 --- a/advisors/check_upstream.py +++ b/advisors/check_upstream.py @@ -60,7 +60,7 @@ def clean_tags(tags, info): separator_regex = re.compile(info["seperator"]) result_list = [separator_regex.sub(".", x) for x in result_list] - result_list = [x for x in result_list if x[0].isdigit()] + result_list = [x for x in result_list if len(x) > 0 and x[0].isdigit()] return result_list diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py old mode 100644 new mode 100755 index 781044377bbe187228550440c285d8163eb32abd..5447e0943df926b015ab853154f537c2653aa5a2 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -2,16 +2,13 @@ """ This is a script to check upgradable information against upstream """ -from pyrpm.spec import Spec, replace_macros - -import yaml -import json -import datetime -import sys import os +import sys import argparse -import urllib.error +from pyrpm.spec import Spec, replace_macros + +import yaml import gitee import check_upstream @@ -22,7 +19,8 @@ def _get_rec_excpt(): """ Get except case of version recommend """ - y_file = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "helper/ver_rec_excpt.yaml")) + y_file = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), + "helper/ver_rec_excpt.yaml")) excpt = yaml.load(y_file, Loader=yaml.Loader) return excpt @@ -31,10 +29,10 @@ def _filter_except(excpts, sources): """ Filter except case in sources """ - for e in excpts: - sources = [s for s in sources if e not in s] + for exp in excpts: + sources = [s for s in sources if exp not in s] return sources - + def get_ver_tags(gt, repo, cwd_path=None): """ @@ -44,7 +42,8 @@ def get_ver_tags(gt, repo, cwd_path=None): try: repo_yaml = open(os.path.join(cwd_path, repo + ".yaml")).read() except FileNotFoundError: - print("WARNING: {pkg}.yaml can't be found in local path: {path}.".format(pkg=repo, path=cwd_path)) + print("WARNING: {pkg}.yaml can't be found in local path: {path}.".format(pkg=repo, + path=cwd_path)) repo_yaml = gt.get_yaml(repo) else: repo_yaml = gt.get_yaml(repo) @@ -84,18 +83,21 @@ def get_ver_tags(gt, repo, cwd_path=None): excpt_list = _get_rec_excpt() if repo in excpt_list: - tags = _filter_except(excpt_list[repo], tags) + tags = _filter_except(excpt_list[repo], tags) return tags -if __name__ == "__main__": +def main(): + """ + Main entrance of the functionality + """ parameters = argparse.ArgumentParser() parameters.add_argument("-p", "--push", action="store_true", - help="Push the version bump as an issue to src-openeuler repository") + help="Push the version bump as an issue to src-openeuler repository") parameters.add_argument("-d", "--default", type=str, default=os.getcwd(), - help="The fallback place to look for YAML information") + help="The fallback place to look for YAML information") parameters.add_argument("repo", type=str, - help="Repository to be checked for upstream version info") + help="Repository to be checked for upstream version info") args = parameters.parse_args() @@ -140,4 +142,10 @@ Please consider upgrading. Yours openEuler Advisor. If you think this is not proper issue, Please visit https://gitee.com/openeuler/openEuler-Advisor. -Issues and feedbacks are welcome.""".format(repo=args.repo, ver=ver_rec.latest_version, cur_ver=cur_version)) +Issues and feedbacks are welcome.""".format(repo=args.repo, + ver=ver_rec.latest_version, + cur_ver=cur_version)) + + +if __name__ == "__main__": + main() diff --git a/advisors/oe_review b/advisors/oe_review new file mode 100755 index 0000000000000000000000000000000000000000..5ceb938e278d24363a6c9b9705b951b7f63de0b1 --- /dev/null +++ b/advisors/oe_review @@ -0,0 +1,45 @@ +#!/usr/bin/python3 +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +Review tool for openEuler submission +""" +import os +import argparse +import subprocess + +def main(): + """ + Main entrance of the functionality + """ + pars = argparse.ArgumentParser() + pars.add_argument("-p", "--pull", type=str, help="Number ID of Pull Request", required=True) + pars.add_argument("repo", help="Repository to be reviewed") + pars.add_argument("-r", "--reuse", help="Reuse current local git dirctory", action="store_true") + + args = pars.parse_args() + + gitee_url_prefix = "git@gitee.com:src-openeuler/" + + if not args.reuse: + subprocess.call(["git", "clone", gitee_url_prefix + args.repo]) + os.chdir(args.repo.split('/')[1]) + + subprocess.call(["git", "fetch", + gitee_url_prefix + args.repo, + "pull/{n}/head:pr_{n}".format(n=args.pull)]) + + print("You are reviewing {repo} pull {n}".format(repo=args.repo, n=args.pull)) + print("Don't forget to try to merge master branch") + +if __name__ == "__main__": + main() diff --git a/advisors/psrtool.py b/advisors/psrtool.py new file mode 100755 index 0000000000000000000000000000000000000000..84dcbd17d99062e65c3b33bfda68d3acddf8383d --- /dev/null +++ b/advisors/psrtool.py @@ -0,0 +1,108 @@ +#!/usr/bin/python3 +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# +# ******************************************************************************/ +""" +This is a helper script provides a way to query or list packages in specific SIG. +This script was inspired by previous work from @love_hangzhou +""" + +import re +import argparse +import yaml + +def list_packages(sigs, sig_name): + """ + List all packages managed by specific SIG + """ + for sig in sigs: + if sig['name'].lower() == sig_name.lower(): + return sig['repositories'] + return [] + + +def list_sigs(sigs): + """ + List all current SIGs + """ + result = [] + for sig in sigs: + result.append(sig['name']) + return result + + +def package_to_sigs(sigs, pkg_names): + """ + Query which SIG manages the packages. + """ + result = {} + for pkg in pkg_names: + for sig in sigs: + repos = sig['repositories'] + for repo in repos: + search_obj = re.search(pkg.lower(), repo.lower(), 0) + if search_obj: + result[repo] = sig['name'] + return result + + +def print_list(lista): + """ + Helper for print list + """ + for i in lista: + print(i) + + +def print_dict(dicta): + """ + Helper for print dictionary + """ + for k in dicta.keys(): + print(k + ": " + dicta[k]) + + +def main(): + """ + Main entrance of functionality + """ + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-l", "--list", default=False, nargs='?', + help="List packages managed by the specific SIG, or list all SIGs instead") + group.add_argument("-q", "--query_packages", default=False, nargs='+', + help="Query which SIG manage the specific package") + parser.add_argument("-f", "--yaml", default="sig/sigs.yaml", + help="Local path of sigs.yaml file") + + args = parser.parse_args() + + try: + with open(args.yaml, 'r') as yaml_file: + sigs = yaml.load(yaml_file, Loader=yaml.Loader)['sigs'] + except IOError: + print("Failed to load information from %s" % args.yaml) + parser.print_help() + exit(1) + + if args.list: + print_list(list_packages(sigs, args.list)) + elif args.list is None: + print_list(list_sigs(sigs)) + elif args.query_packages: + print_dict(package_to_sigs(sigs, args.query_packages)) + else: + pass + exit(0) + +if __name__ == "__main__": + main() diff --git a/advisors/tc_statistic.py b/advisors/tc_statistic.py index 89ec1854742b8726261faa8c34ead8f6147decb5..c0b9d5a94c78f5324775918951fc99cbb3943770 100755 --- a/advisors/tc_statistic.py +++ b/advisors/tc_statistic.py @@ -76,7 +76,7 @@ class Advisor(object): if num <= 100: list_all_url = list_all_url + "per_page={num}&page=1".format(num=num) - return self.get_json(url) + return self.get_json(list_all_url) list_all_url = list_all_url + "per_page=100&page=" @@ -128,7 +128,8 @@ if __name__ == "__main__": for t in tc_members: tc_statistic[t] = 0 PRs = adv.get_recent_prs(int(args.number)) - print("Statistic of recent {num} PRs".format(len(PRs)) + print("Statistic of recent {num} PRs".format(num=len(PRs))) + for pr in PRs: commenter = pr["user"]["login"] if commenter in tc_members: @@ -140,5 +141,5 @@ if __name__ == "__main__": tc_statistic[commenter] += 1 for tc in tc_statistic.keys(): - print("{tc} mades {num} comments".format(tc=tc, num=tc_statistic[tc])) + print("{tc} made {num} comments".format(tc=tc, num=tc_statistic[tc])) diff --git a/upstream-info/automake.yaml b/upstream-info/automake.yaml index d2b11114b12a68cf1b59d47930a8c2b80bb2df1f..9893a86f005ec0e90f5d33eb4c69acbe45367313 100644 --- a/upstream-info/automake.yaml +++ b/upstream-info/automake.yaml @@ -1,4 +1,4 @@ version_control: git -src_repo: https://git.savannah.gnu.org/git/automake.git +src_repo: https://git.savannah.gnu.org/git/automake.git tag_prefix: "^v" separator: "." diff --git a/upstream-info/netpbm.yaml b/upstream-info/netpbm.yaml index f5c5e4fcfe04b30c11adb0672ed7fab825117dcb..fc6523cf801d70603a9fc93e9548d335f6464a9b 100644 --- a/upstream-info/netpbm.yaml +++ b/upstream-info/netpbm.yaml @@ -1,6 +1,5 @@ version_control: svn -src_repo: http://svn.code.sf.net/p/netpbm/code/release_number +src_repo: https://svn.code.sf.net/p/netpbm/code tag_dir: release_number tag_prefix: ^v separator: . - \ No newline at end of file diff --git a/upstream-info/unixODBC.yaml b/upstream-info/unixODBC.yaml index a724d4d52f6f2d5434cc6503e993328adde588a2..b6ec59343366843f3970fee0ddcc440a61410eeb 100644 --- a/upstream-info/unixODBC.yaml +++ b/upstream-info/unixODBC.yaml @@ -1,4 +1,4 @@ version_control: svn -src_repo: https://svn.code.sf.net/p/unixodbc/code/ -tag_prefix: 2.3.0 +src_repo: https://svn.code.sf.net/p/unixodbc/code +tag_prefix: "" seperator: .