diff --git a/advisors/check_upstream.py b/advisors/check_upstream.py index 954099bc1972de9c624858f34c118e0aaad39086..7b91b8db1809c4241aa0bc268d6b36c76e4c7379 100755 --- a/advisors/check_upstream.py +++ b/advisors/check_upstream.py @@ -36,8 +36,8 @@ def load_last_query_result(info, force_reload=False): else: return "" -def clean_tags(tags, info): +def clean_tags(tags, info): if info.get("tag_pattern", "") != "" and info.get("tag_pattern", "") is not None: pattern_regex = re.compile(info["tag_pattern"]) result_list = [pattern_regex.sub("\\1", x) for x in tags] @@ -50,7 +50,7 @@ def clean_tags(tags, info): if info.get("separator", ".") != "." and info.get("separator", ".") is not None: separator_regex = re.compile(info["separator"]) result_list = [separator_regex.sub(".", x) for x in result_list] - + # Xinwei used to mis-spell 'separator'. # Followings are kept for compatability until all yaml files are fixed. if info.get("seperator", ".") != "." and info.get("seperator", ".") is not None: @@ -58,7 +58,7 @@ def clean_tags(tags, info): result_list = [separator_regex.sub(".", x) for x in result_list] result_list = [x for x in result_list if x[0].isdigit()] - + return result_list diff --git a/advisors/gitee.py b/advisors/gitee.py index 6925c2eaa62f17ea61edb2709cd41871de7fceb2..557f72bc6aeda3c9d4528789adc36990d78155fd 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -26,12 +26,13 @@ class Gitee(object): self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW 64; rv:50.0) Gecko/20100101 Firefox/50.0'} self.gitee_url = "https://gitee.com/" - self.src_openeuler_url = self.gitee_url + "src-openeuler/{package}/raw/master/" + self.src_openeuler_url = self.gitee_url + "src-openeuler/{package}/raw/{branch}/" self.advisor_url = self.gitee_url + "openeuler/openEuler-Advisor/raw/master/" self.specfile_url_template = self.src_openeuler_url + "{specfile}" self.yamlfile_url_template = self.src_openeuler_url + "{package}.yaml" #self.advisor_url_template = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/{package}.yaml" self.advisor_url_template = self.advisor_url + "upstream-info/{package}.yaml" + self.community_url_template = self.gitee_url + "openeuler/community/raw/master/repository/{repository}.yaml" #self.specfile_exception_url = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/helper/specfile_exceptions.yaml" self.specfile_exception_url = self.advisor_url + "advisors/helper/specfile_exceptions.yaml" self.time_format = "%Y-%m-%dT%H:%M:%S%z" @@ -96,7 +97,7 @@ Yours openEuler-Advisor. def get_gitee_json(self, url): """ - get and load gitee json response + Get and load gitee json response """ headers = self.headers.copy() #headers = {} @@ -106,33 +107,31 @@ Yours openEuler-Advisor. def get_spec_exception(self): """ - get well known spec file exceptions + Get well known spec file exceptions """ resp = self.get_gitee(self.specfile_exception_url) exps = yaml.load(resp, Loader=yaml.Loader) return exps - def get_spec(self, pkg): + def get_spec(self, pkg, br="master"): """ - get openeuler spec file for specific package + Get openeuler spec file for specific package """ - specurl = self.specfile_url_template.format(package=pkg, specfile=pkg + ".spec") + specurl = self.specfile_url_template.format(branch=br, package=pkg, specfile=pkg + ".spec") exp = self.get_spec_exception() if pkg in exp: dir_name = exp[pkg]["dir"] file_name = exp[pkg]["file"] specurl = urllib.parse.urljoin(specurl, os.path.join(dir_name, file_name)) - try: resp = self.get_gitee(specurl) except urllib.error.HTTPError: resp = "" - return resp - def get_yaml(self, pkg): + def get_yaml(self, pkg, br="master"): """ - get upstream yaml metadata for specific package + Get upstream yaml metadata for specific package """ yamlurl = self.advisor_url_template.format(package=pkg) try: @@ -140,19 +139,30 @@ Yours openEuler-Advisor. except urllib.error.HTTPError: resp = "Not found" if re.match("Not found", resp): - yamlurl = self.yamlfile_url_template.format(package=pkg) + yamlurl = self.yamlfile_url_template.format(branch=br, package=pkg) try: resp = self.get_gitee(yamlurl) except urllib.error.HTTPError: resp = "Not found" if re.match("Not found", resp): - print("Cannot find upstream metadata") + print("Cann't find yaml metadata for {package} from upstream-info.".format(package=pkg)) return False else: return resp else: return resp + def get_community(self, repo): + """ + Get yaml data from community repo + """ + yamlurl = self.community_url_template.format(repository=repo) + try: + resp = self.get_gitee(yamlurl) + except urllib.error.HTTPError: + resp = "" + return resp + def get_issues(self, pkg, prj="src-openeuler"): """ List all open issues of pkg diff --git a/advisors/helper/ver_rec_excpt.yaml b/advisors/helper/ver_rec_excpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e0a0d962aa9925c3a411959994fda7ea92306e7 --- /dev/null +++ b/advisors/helper/ver_rec_excpt.yaml @@ -0,0 +1,10 @@ +--- +# version recommend exception list +gegl04: + - '20001120.v002' +gimp: + - '19990910' +nss: + - '334.20030307' +glibc: + - '9000' diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py index 949f55b17227195a43c2c18e4c8db87ac43c0bf9..df9c8afe16a03b49d6a4d1c99ae21af38871ca74 100755 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -12,10 +12,72 @@ import os import argparse import urllib.error + import gitee import check_upstream import version_recommend + +def _get_rec_excpt(): + """ + Get except case of version recommend + """ + y_file = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "helper/ver_rec_excpt.yaml")) + excpt = yaml.load(y_file, Loader=yaml.Loader) + return excpt + + +def _filter_except(excpts, sources): + """ + Filter except case in sources + """ + for e in excpts: + sources = [s for s in sources if e not in s] + return sources + + +def get_ver_tags(gt, repo, cwd_path=None): + """ + Get version tags of given package + """ + if cwd_path: + try: + repo_yaml = open(os.path.join(cwd_path, repo + ".yaml")).read() + except FileNotFoundError: + print("Cann't find yaml metadata for {pkg} from current working directory.".format(pkg=repo)) + repo_yaml = gt.get_yaml(repo) + else: + repo_yaml = gt.get_yaml(repo) + + if repo_yaml: + pkg_info = yaml.load(repo_yaml, Loader=yaml.Loader) + else: + return None + + vc_type = pkg_info["version_control"] + if vc_type == "hg": + tags = check_upstream.check_hg(pkg_info) + elif vc_type == "github": + tags = check_upstream.check_github(pkg_info) + elif vc_type == "git": + tags = check_upstream.check_git(pkg_info) + elif vc_type == "gitlab.gnome": + tags = check_upstream.check_gnome(pkg_info) + elif vc_type == "svn": + tags = check_upstream.check_svn(pkg_info) + elif vc_type == "metacpan": + tags = check_upstream.check_metacpan(pkg_info) + elif vc_type == "pypi": + tags = check_upstream.check_pypi(pkg_info) + else: + print("Unsupport version control method {vc}".format(vc=vc_type)) + + excpt_list = _get_rec_excpt() + if repo in excpt_list: + tags = _filter_except(excpt_list[repo], tags) + return tags + + if __name__ == "__main__": parameters = argparse.ArgumentParser() parameters.add_argument("-p", "--push", action="store_true", @@ -27,56 +89,24 @@ if __name__ == "__main__": args = parameters.parse_args() - gitee = gitee.Gitee() - prj_name = args.repo - spec_string = gitee.get_spec(prj_name) + user_gitee = gitee.Gitee() + spec_string = user_gitee.get_spec(args.repo) if not spec_string: - print("{repo} seems to be an empty repository".format(repo=args.repo)) + print("{pkg}.spec can't be found on the master branch".format(pkg=args.repo)) sys.exit(1) - s_spec = Spec.from_string(spec_string) - - current_version = replace_macros(s_spec.version, s_spec) - - print("Checking ", prj_name) - print("current version is ", current_version) - - try: - prj_info_string = open(os.path.join(args.default, prj_name + ".yaml")).read() - except FileNotFoundError: - prj_info_string = "" - - if not prj_info_string: - print("Get YAML info from gitee") - try: - prj_info_string = gitee.get_yaml(prj_name) - except urllib.error.HTTPError: - print("Failed to get YAML info for {pkg}".format(pkg=prj_name)) - sys.exit(1) + spec_file = Spec.from_string(spec_string) + cur_version = replace_macros(spec_file.version, spec_file) + print("Checking ", args.repo) + print("current version is ", cur_version) - prj_info = yaml.load(prj_info_string, Loader=yaml.Loader) + pkg_tags = get_ver_tags(user_gitee, args.repo, args.default) + print("known release tags:", pkg_tags) - vc_type = prj_info["version_control"] - if vc_type == "hg": - tags = check_upstream.check_hg(prj_info) - elif vc_type == "github": - tags = check_upstream.check_github(prj_info) - elif vc_type == "git": - tags = check_upstream.check_git(prj_info) - elif vc_type == "gitlab.gnome": - tags = check_upstream.check_gnome(prj_info) - elif vc_type == "svn": - tags = check_upstream.check_svn(prj_info) - elif vc_type == "metacpan": - tags = check_upstream.check_metacpan(prj_info) - elif vc_type == "pypi": - tags = check_upstream.check_pypi(prj_info) - else: - print("Unsupport version control method {vc}".format(vc=vc_type)) + if pkg_tags is None: sys.exit(1) + ver_rec = version_recommend.VersionRecommend(pkg_tags, cur_version, 0) - print("known release tags :", tags) - v = version_recommend.VersionRecommend(tags, current_version, 0) - print("Latest version is ", v.latest_version) - print("Maintain version is", v.maintain_version) + print("Latest version is", ver_rec.latest_version) + print("Maintain version is", ver_rec.maintain_version) diff --git a/advisors/simple-update-robot.py b/advisors/simple-update-robot.py index 054ddd1f961bc7655956543cfcaaf3bd0efa3b3e..1ee8b1f41e5a1e859b8ea6c39fbbde1e266491f6 100755 --- a/advisors/simple-update-robot.py +++ b/advisors/simple-update-robot.py @@ -21,8 +21,11 @@ import subprocess import os.path import re import datetime + +import oa_upgradable import version_recommend + def download_source_url(spec, o_ver, n_ver): """ Download source file from Source or Source0 URL @@ -59,10 +62,61 @@ def download_upstream_url(gt, repo, n_ver): return False +def update_check(spec, o_ver, n_ver): + """ + Requirements check for upgraded package + """ + if len(spec.patches) >= 1: + print("I'm too naive to handle complicated package.") + print("This package has multiple in-house patches.") + return False + + ver_type = version_recommend.VersionType() + if(ver_type.compare(n_ver, o_ver) == 1): + return True + else: + print("Update failed >> [{pkg}: current_ver:{cur_ver}, upgraded_ver:{upd_ver}]".format( + pkg=spec.name, cur_ver=o_ver, upd_ver=n_ver)) + return False + + +def fork_clone_repo(gt, repo): + """ + Fork repo from src-openEuler to private repository, then clone it to local + """ + if not gt.fork_repo(repo): + print("The repo of {pkg} seems to have been forked.".format(pkg=repo)) + + name = gt.token["user"] + subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=name, pkg=repo)]) + os.chdir(repo) + + +def download_src(gt, spec, o_ver, n_ver): + """ + Download source code for upgraded package + """ + source_file = download_source_url(spec, o_ver, n_ver) + if source_file: + print(source_file) + return True + else: + source_file = download_upstream_url(gt, spec.name, n_ver) + if source_file: + print(source_file) + return True + else: + print("Failed to download the latest source code.") + return False + + def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None): """ Create new spec file for upgraded package """ + fn = open(repo + "_old.spec", "w") + fn.write(spec_str) + fn.close() fn = open(repo + ".spec", "w") in_changelog = False for l in spec_str.splitlines(): @@ -89,55 +143,133 @@ def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None): fn.write("\n") fn.close() + +def auto_update_pkg(gt, u_branch, u_pkg): + """ + Auto upgrade based on given branch for single package + """ + spec_str = gt.get_spec(u_pkg, u_branch) + if not spec_str: + print("{pkg}.spec can't be found on the {br} branch. ".format( + pkg=u_pkg, br=u_branch)) + sys.exit(1) + pkg_spec = Spec.from_string(spec_str) + pkg_ver = replace_macros(pkg_spec.version, pkg_spec) + + pkg_tags = oa_upgradable.get_ver_tags(gt, u_pkg) + if pkg_tags is None: + sys.exit(1) + ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0) + rec_up_ver = pkg_ver + if re.search("master", u_branch): + rec_up_ver = ver_rec.latest_version + elif re.search("LTS", u_branch): + rec_up_ver = ver_rec.maintain_version + else: + print("Only support master and LTS version upgrade.") + sys.exit(1) + + fork_clone_repo(gt, u_pkg) + + if not update_check(pkg_spec, pkg_ver, rec_up_ver): + sys.exit(1) + + if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver): + sys.exit(1) + + create_spec(u_pkg, spec_str, pkg_ver, rec_up_ver) + + +def auto_update_repo(gt, u_branch, u_repo): + """ + Auto upgrade based on given branch for packages in given repository + """ + repo_yaml = gt.get_community(u_repo) + if not repo_yaml: + print("{repo}.yaml in community is empty.".format(repo=u_repo)) + sys.exit(1) + + pkg_info = yaml.load(repo_yaml, Loader=yaml.loader) + pkg_list = pkg_info.get("repositories") + for pkg in pkg_list: + pkg_name = pkg.get("name") + spec_str = gt.get_spec(pkg_name, u_branch) + if not spec_str: + print("{pkg}.spec can't be found on the {br} branch. ".format( + pkg=pkg_name, br=u_branch)) + continue + pkg_spec = Spec.from_string(spec_str) + pkg_ver = replace_macros(pkg_spec.version, pkg_spec) + + pkg_tags = oa_upgradable.get_ver_tags(gt, pkg_name) + if pkg_tags is None: + continue + ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0) + rec_up_ver = pkg_ver + if re.search("master", u_branch): + rec_up_ver = ver_rec.latest_version + elif re.search("LTS", u_branch): + rec_up_ver = ver_rec.maintain_version + else: + print("Only support master and LTS version upgrade.") + sys.exit(1) + + fork_clone_repo(gt, pkg_name) + + if not update_check(pkg_spec, pkg_ver, rec_up_ver): + continue + + if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver): + continue + + create_spec(pkg_name, spec_str, pkg_ver, rec_up_ver) + + if __name__ == "__main__": pars = argparse.ArgumentParser() - pars.add_argument("pkg", type=str, help="The package to be upgraded") + pars.add_argument("repo_pkg", type=str, help="The repository or package to be upgraded") + pars.add_argument("branch", type=str, help="The branch that upgrade based") + pars.add_argument("-u", "--update", type=str, help="Auto upgrade for packages in repository or single package", + choices=["repo", "pkg"]) pars.add_argument("-n", "--new_version", type=str, help="New upstream version of package will be upgrade to") pars.add_argument("-s", "--create_spec", help="Create spec file", action="store_true") pars.add_argument("-d", "--download", help="Download upstream source code", action="store_true") - pars.add_argument("-f", "--fork", help="fork src-openeuler repo into users", action="store_true") - pars.add_argument("-c", "--clone", help="clone privatge repo to local", action="store_true") + pars.add_argument("-fc", "--fork_then_clone", help="Fork src-openeuler repo into users, then clone to local", + action="store_true") pars.add_argument("-p", "--PR", help="Create upgrade PR", action="store_true") args = pars.parse_args() + + user_gitee = gitee.Gitee() - my_gitee = gitee.Gitee() - my_version = version_recommend.VersionType() - spec_string= my_gitee.get_spec(args.pkg) - - s_spec = Spec.from_string(spec_string) - cur_ver = replace_macros(s_spec.version, s_spec) + if args.update: + if args.update == "repo": + auto_update_repo(user_gitee, args.branch, args.repo_pkg) + else: + auto_update_pkg(user_gitee, args.branch, args.repo_pkg) + else: + spec_string = user_gitee.get_spec(args.repo_pkg, args.branch) + if not spec_string: + print("{pkg}.spec can't be found on the {br} branch. ".format(pkg=args.repo_pkg, br=args.branch)) + sys.exit(1) + spec_file = Spec.from_string(spec_string) + cur_version = replace_macros(spec_file.version, spec_file) - if args.fork: - if not my_gitee.fork_repo(args.pkg): - print("The repo of {pkg} seems to have been forked.".format(pkg=args.pkg)) + if args.fork_then_clone: + fork_clone_repo(user_gitee, args.repo_pkg) - if args.clone: - user=my_gitee.token["user"] - subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=user, pkg=args.pkg)]) - os.chdir(args.pkg) + if args.download or args.create_spec: + if not args.new_version: + print("Please specify the upgraded version of the {repo}".format(repo=args.repo_pkg)) + sys.exit(1) + elif not update_check(spec_file, cur_version, args.new_version): + sys.exit(1) - if args.download: - source_file = download_source_url(s_spec, cur_ver, args.new_version) - if source_file: - print(source_file) - else: - source_file = download_upstream_url(my_gitee, args.pkg, args.new_version) - if source_file: - print(source_file) - else: - print("Failed to download the latest source code.") + if args.download: + if not download_src(user_gitee, spec_file, cur_version, args.new_version): sys.exit(1) - if args.create_spec: - if len(s_spec.patches) >= 1: - print("I'm too naive to handle complicated package.") - print("This package has multiple in-house patches.") - sys.exit(1) - if(my_version.compare(args.new_version, cur_ver) ==1): - create_spec(args.pkg, spec_string, cur_ver, args.new_version) - else: - print("Please check version of {pkg} will upgrade to, it's current version is {version}.".format( - pkg=args.pkg, version=cur_ver)) + if args.create_spec: + create_spec(args.repo_pkg, spec_string, cur_version, args.new_version) - if args.PR: - my_gitee.create_pr(my_gitee.token["user"], args.pkg) + if args.PR: + user_gitee.create_pr(user_gitee.token["user"], args.repo_pkg) diff --git a/advisors/version_recommend.py b/advisors/version_recommend.py index d4b0d7c848acac74864a2bc9f4e96efe708dbf94..47468630018c7a2cc0ad37e3a0f25e782d04c73d 100755 --- a/advisors/version_recommend.py +++ b/advisors/version_recommend.py @@ -72,7 +72,7 @@ class VersionType(object): :return 0: z1 equal then z2 :raises: None """ - return self._compare(self, z1, z2) + return self._compare(z1, z2) def _compare(self, z1, z2): """ @@ -91,11 +91,15 @@ class VersionType(object): len2 = len(d2) length = min(len1, len2) for index in range(length): - if d1[index].isdigit() and d1[index].isdigit(): + if d1[index].isdigit() and d2[index].isdigit(): if int(d1[index]) > int(d2[index]): return 1 elif int(d1[index]) < int(d2[index]): return -1 + elif d1[index].isdigit(): + return 1 + elif d2[index].isdigit(): + return -1 else: if d1[index] > d2[index]: return 1 @@ -126,7 +130,7 @@ class VersionType(object): :returns: The split result :raises: None """ - for f, s in re.findall(r'([\d]+)|([^\d.]+)', x): + for f, s in re.findall(r'([\d]+)|([^\d.-]+)', x): if f: float(f) yield f @@ -1085,6 +1089,10 @@ class VersionRecommend(object): if m is None: # 版本号应该是数字开头 return False + m = re.search(r'[ab]\d', version) + if not m is None: + return False + if 'rc' in version \ or 'RC' in version \ or 'dev' in version \ diff --git a/upstream-info/python2.yaml b/upstream-info/python2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97065dcc6d23e040835cd7cd0015adb6ab747068 --- /dev/null +++ b/upstream-info/python2.yaml @@ -0,0 +1,6 @@ +--- +version_control: github +src_repo: python/cpython +tag_prefix: "^v?3*|^v" +seperator: "." +url: https://github.com/python/cpython.git diff --git a/upstream-info/python3.yaml b/upstream-info/python3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1ad3009dd64034ed7b97d345bfab9f0241792d6 --- /dev/null +++ b/upstream-info/python3.yaml @@ -0,0 +1,6 @@ +--- +version_control: github +src_repo: python/cpython +tag_prefix: "^v" +seperator: "." +url: https://github.com/python/cpython.git