From 936c54c663b74a2843f4c2837fb9a0ea4bfe087a Mon Sep 17 00:00:00 2001 From: licihua Date: Sat, 19 Feb 2022 17:58:37 +0800 Subject: [PATCH] adapt sigs directory change --- advisors/check_missing_file.py | 36 +++++---------------- advisors/check_version.py | 43 ++++++++++++------------- advisors/gitee.py | 59 ++++++++++++++++++++++++++++++---- 3 files changed, 81 insertions(+), 57 deletions(-) diff --git a/advisors/check_missing_file.py b/advisors/check_missing_file.py index 92df7626..1060c00b 100755 --- a/advisors/check_missing_file.py +++ b/advisors/check_missing_file.py @@ -72,24 +72,7 @@ Yours openEuler Advisor. """ -def get_repos_by_sig(sig): - """ - Get repos by sig - """ - try: - user_gitee = gitee.Gitee() - except NameError: - sys.exit(1) - yaml_data = user_gitee.get_sigs() - repo_list = [] - for sigs in yaml_data['sigs']: - if sig not in (sigs['name'], 'all'): - continue - repo_name = sigs['repositories'] - repo_list.append(repo_name) - repo_list = [i for item in repo_list for i in item] - return repo_list def main_process(repo, push, check_file): @@ -148,7 +131,7 @@ def main(): Main process for command line """ pars = argparse.ArgumentParser() - pars.add_argument("-s", "--sig", type=str, default='all', help="Sig name") + pars.add_argument("-s", "--sig", type=str, required=True, help="Sig name") pars.add_argument("-p", "--push", help="Push the advise to gitee.com/src-openeuler", action="store_true") @@ -160,21 +143,18 @@ def main(): print("ERROR: Not support {file} check".format(file=args.file)) return - repos = get_repos_by_sig(args.sig) - if not repos: - print("ERROR: no repos find is {sig} sig, please check the sig name.".format( - sig=args.sig)) - return + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) + repos = user_gitee.get_repos_by_sig(args.sig) + print(repos) fail_list = [] total = len(repos) index = 0 - for repo in repos: + for check_repo in repos: index = index + 1 - url = repo.split('/')[0] - if url == 'openeuler': - continue - check_repo = repo.split('/')[1] result = main_process(check_repo, args.push, args.file) if result != 'OK': fail_list.append(check_repo) diff --git a/advisors/check_version.py b/advisors/check_version.py index f46cd95e..2cd0a203 100755 --- a/advisors/check_version.py +++ b/advisors/check_version.py @@ -17,10 +17,10 @@ This is an automatic script for checking source url of package import os import sys import argparse +import time from advisors import gitee from advisors.oa_upgradable import main_process -from advisors.check_missing_file import get_repos_by_sig -import time + def main(): @@ -34,37 +34,36 @@ def main(): parameters.add_argument("-d", "--default", type=str, default=os.getcwd(), help="The fallback place to look for YAML information") - parameters.add_argument("-s", "--sig", + parameters.add_argument("-s", "--sig", required=True, help="Check yaml by Sig") args = parameters.parse_args() - if args.sig: - sig = args.sig - else: - sig = 'all' - repos = get_repos_by_sig(sig) + sig = args.sig + + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) + + repos = user_gitee.get_repos_by_sig(sig) print(repos) total = len(repos) index = 0 upgrade_list = [] - for repo in repos: - index = index + 1 - url = repo.split('/')[0] - if url == 'openeuler': - continue - check_repo = repo.split('/')[1] + for check_repo in repos: # sleep 10 second, avoid limited by github\gitlab + index = index + 1 time.sleep(10) result = main_process(args.push, args.default, check_repo) if result: if result[1] != result[2]: print('''INFO: {index} in {total} check {repo} need upgrade \ from {current} to {latest}'''.format(index=index, - total=total, - repo=result[0], - current=result[1], - latest=result[2])) + total=total, + repo=result[0], + current=result[1], + latest=result[2])) result.append('Y') upgrade_list.append(result) else: @@ -72,14 +71,14 @@ from {current} to {latest}'''.format(index=index, upgrade_list.append(result) print('''INFO: {index} in {total} check {repo} not need \ upgrade'''.format(index=index, - total=total, - repo=check_repo)) + total=total, + repo=check_repo)) else: upgrade_list.append([check_repo, '-', '-', '-']) print('''INFO: {index} in {total} check {repo} \ latest version failed.'''.format(index=index, - total=total, - repo=check_repo)) + total=total, + repo=check_repo)) if upgrade_list: print("Repo upgrade check result:") diff --git a/advisors/gitee.py b/advisors/gitee.py index 1c057b0a..d78f1d18 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -101,7 +101,7 @@ class Gitee(): return result.read().decode("utf-8") except urllib.error.HTTPError as err: print("ERROR: error occurred.\nerrcode: %d\nreason: %s\nheaders:\n%s" - % (err.code, err.reason, str(err.headers))) + % (err.code, err.reason, str(err.headers))) return False def __patch_gitee(self, url, values, headers=None): @@ -117,7 +117,7 @@ class Gitee(): return result.read().decode("utf-8") except urllib.error.HTTPError as err: print("ERROR: error occurred.\nerrcode: %d\nreason: %s\nheaders:\n%s" - % (err.code, err.reason, str(err.headers))) + % (err.code, err.reason, str(err.headers))) return False def fork_repo(self, repo, owner="src-openeuler"): @@ -318,16 +318,61 @@ class Gitee(): return '' return base64.b64decode(resp["content"]).decode("utf-8") + def __get_community_tree(self, sha): + """ + Get openeuler/community tree + """ + url = "https://gitee.com/api/v5/repos/openeuler/community/git/trees/{sha}?".format(sha=sha) + dirs = self.__get_gitee_json(url) + return dirs['tree'] + + def __get_community_sha(self, sha, path): + """ + Get openeuler/community sha by sha and file path + """ + tree = self.__get_community_tree(sha) + for my_dir in tree: + if my_dir['path'] == path: + return my_dir['sha'] + return '' + def get_sigs(self): """ - Get upstream sigs + Get upstream sigs tree info """ - sigs_url = "https://gitee.com/openeuler/community/raw/master/sig/sigs.yaml" - req = urllib.request.Request(url=sigs_url, headers=self.headers) - data = urllib.request.urlopen(req) - sigs = yaml.load(data.read().decode("utf-8"), Loader=yaml.Loader) + sigs = {} + master_tree = self.__get_community_tree('master') + for my_dir in master_tree: + if my_dir['type'] == 'tree' and my_dir['path'] == 'sig': + sig_sha = my_dir['sha'] + + sig_tree = self.__get_community_tree(sig_sha) + for my_dir in sig_tree: + if my_dir['type'] == 'tree': + sigs[my_dir['path']] = my_dir['sha'] + return sigs + def get_repos_by_sig(self, sig): + """ + Get repos list by sig + """ + sigs = self.get_sigs() + if sig not in sigs.keys(): + return '' + repo_list = [] + + openeuler_sha = self.__get_community_sha(sigs[sig], 'src-openeuler') + initials_tree = self.__get_community_tree(openeuler_sha) + for initials_dir in initials_tree: + openeuler_repo_tree = self.__get_community_tree(initials_dir['sha']) + for my_repo_dir in openeuler_repo_tree: + repo_name = my_repo_dir['path'] + repo_name = repo_name[:-5] + repo_list.append(repo_name) + return repo_list + + def get_community(self, repo): """ Get yaml data from community repo -- Gitee