diff --git a/advisors/check_missing_file.py b/advisors/check_missing_file.py index ae69f5aee49212155299f51c7ad4ebb91a3027e0..92df7626f84f33a718b99554130e8d278a2372a3 100755 --- a/advisors/check_missing_file.py +++ b/advisors/check_missing_file.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#****************************************************************************** +# ****************************************************************************** # Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. # licensed under the Mulan PSL v2. # You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -22,7 +22,6 @@ from datetime import datetime from advisors import gitee - NEW_SPEC_ISSUE_BODY = """Dear {repo} maintainer: 亲爱的 {repo} 维护者： @@ -72,6 +71,7 @@ This is a automatic advise from openEuler-Advisor. If you think the advise is no Yours openEuler Advisor. """ + def get_repos_by_sig(sig): """ Get repos by sig @@ -91,6 +91,7 @@ def get_repos_by_sig(sig): repo_list = [i for item in repo_list for i in item] return repo_list + def main_process(repo, push, check_file): """ Main process for command line @@ -108,22 +109,24 @@ def main_process(repo, push, check_file): return 'NOK' if not file: + need_push_issue = True print("no {file} file found for {repo} project".format(file=check_file, repo=repo)) if push: issues = my_gitee.get_issues(repo) for issue in issues: if issue["title"] == "Submit {file} file into this repository".format( file=check_file): + need_push_issue = False ages = datetime.now() - my_gitee.get_gitee_datetime(issue["created_at"]) if ages.days <= 10: print("Advise has been issues only %d days ago" % ages.days) print("Give developers more time to handle it.") break my_gitee.post_issue_comment(repo, issue["number"], NEW_COMMENT.format( - repo=repo, + repo=repo, days=ages.days)) break - else: + if need_push_issue: if check_file == 'spec': my_gitee.post_issue(repo, "Submit {file} file into this repository".format( @@ -186,5 +189,6 @@ def main(): for repo in fail_list: print(repo) + if __name__ == "__main__": main() diff --git a/advisors/check_upstream.py b/advisors/check_upstream.py index 8145380f8c22f769c5d5450a8349a9b4cc78c801..58fe743c7554469a4c0fe19400a979d3754f2bee 100755 --- a/advisors/check_upstream.py +++ b/advisors/check_upstream.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#****************************************************************************** +# ****************************************************************************** # Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. # licensed under the Mulan PSL v2. # You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -21,7 +21,6 @@ import subprocess from datetime import datetime from urllib.parse import urljoin import requests - from advisors import yaml2url TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" @@ -32,6 +31,23 @@ def eprint(*args, **kwargs): print("DEBUG: ", *args, file=sys.stderr, **kwargs) +def get_resp(url, **kwargs): + r"""Sends a GET request. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :return: :class:`Response ` object + :rtype: requests.Response + """ + try: + resp = requests.get(url, **kwargs) + except requests.RequestException as e: + eprint("{url} > requests.get return error: {error}.".format(url=url, error=e)) + return "" + + return resp + + def load_last_query_result(info, force_reload=False): """ If there's last_query stored in yaml, load it @@ -43,7 +59,7 @@ def load_last_query_result(info, force_reload=False): if "last_query" in info.keys(): last_query = info.pop("last_query") - #age = datetime.now() - datetime.strptime(last_query["time_stamp"], TIME_FORMAT) + # age = datetime.now() - datetime.strptime(last_query["time_stamp"], TIME_FORMAT) age = datetime.now() - last_query["time_stamp"].replace(tzinfo=None) if age.days < 7: eprint("{repo} > Reuse Last Query".format(repo=info["src_repo"])) @@ -114,14 +130,16 @@ def check_hg_raw(info, clean_tag=True): """ Check hg version info via raw-tags """ - eprint("{repo} > Using hg raw-tags".format(repo=info["src_repo"]+"/raw-tags")) + eprint("{repo} > Using hg raw-tags".format(repo=info["src_repo"] + "/raw-tags")) resp = load_last_query_result(info) if resp == "": headers = { - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' - } + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' + } url = yaml2url.yaml2url(info) - resp = requests.get(url, headers=headers) + resp = get_resp(url, headers=headers) + if not resp: + return "" resp = resp.text need_trick, url, cookies = dirty_redirect_tricks(url, resp) if need_trick: @@ -131,12 +149,13 @@ def check_hg_raw(info, clean_tag=True): for cookie in cookies: key, value = cookie.split('=') c_dict[key] = value - resp = requests.get(url, headers=headers, cookies=c_dict) - resp = resp.text - last_query = {} - last_query["time_stamp"] = datetime.now() - last_query["raw_data"] = resp + resp = get_resp(url, headers=headers, cookies=c_dict) + if not resp: + return "" + + resp = resp.text + last_query = {"time_stamp": datetime.now(), "raw_data": resp} info["last_query"] = last_query tags = [] for line in resp.splitlines(): @@ -150,14 +169,17 @@ def check_hg(info, clean_tag=True): """ Check hg version info via json """ - eprint("{repo} > Using hg json-tags".format(repo=info["src_repo"]+"/json-tags")) + eprint("{repo} > Using hg json-tags".format(repo=info["src_repo"] + "/json-tags")) resp = load_last_query_result(info) if resp == "": headers = { - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' - } + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' + } url = yaml2url.yaml2url(info) - resp = requests.get(url, headers=headers) + resp = get_resp(url, headers=headers) + if not resp: + return "" + resp = resp.text need_trick, url, cookies = dirty_redirect_tricks(url, resp) if need_trick: @@ -167,12 +189,13 @@ def check_hg(info, clean_tag=True): for cookie in cookies: key, value = cookie.split('=') c_dict[key] = value - resp = requests.get(url, headers=headers, cookies=c_dict) - resp = resp.text - last_query = {} - last_query["time_stamp"] = datetime.now() - last_query["raw_data"] = resp + resp = get_resp(url, headers=headers, cookies=c_dict) + if not resp: + return "" + + resp = resp.text + last_query = {"time_stamp": datetime.now(), "raw_data": resp} info["last_query"] = last_query # try and except ? tags_json = json.loads(resp) @@ -191,17 +214,15 @@ def check_metacpan(info, clean_tag=True): resp = load_last_query_result(info) if resp == "": headers = { - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' - } + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' + } url = yaml2url.yaml2url(info) - try: - print(url) - resp = requests.get(url, headers=headers) - except requests.RequestException as e: - eprint("{repo} > requests.get return error: {error}.".format(repo=info["src_repo"], error=e)) + print(url) + resp = get_resp(url, headers=headers) + if not resp: return "" - resp = resp.text + resp = resp.text tags = [] tag_list = resp.splitlines() condition = "value=\"/release" @@ -219,10 +240,9 @@ def check_metacpan(info, clean_tag=True): if not tags: eprint("{repo} found unsorted on cpan.metacpan.org".format(repo=info["src_repo"])) - sys.exit(1) - last_query = {} - last_query["time_stamp"] = datetime.now() - last_query["raw_data"] = resp + return "" + + last_query = {"time_stamp": datetime.now(), "raw_data": resp} info["last_query"] = last_query if clean_tag: tags = clean_tags(tags, info) @@ -237,17 +257,19 @@ def check_pypi(info, clean_tag=True): tags = [] if resp == "": headers = { - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' - } + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' + } url = yaml2url.yaml2url(info) - resp = requests.get(url, headers=headers) + resp = get_resp(url, headers=headers) + if not resp: + return "" data = resp.json() for key in data["releases"].keys(): tags.append(key) if not tags: eprint("{repo} > No Response or JSON parse failed".format(repo=info["src_repo"])) - sys.exit(1) + return "" if clean_tag: tags = clean_tags(tags, info) return tags @@ -261,17 +283,19 @@ def check_rubygem(info, clean_tag=True): tags = [] if resp == "": headers = { - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' - } + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' + } url = yaml2url.yaml2url(info) - resp = requests.get(url, headers=headers) + resp = get_resp(url, headers=headers) + if not resp: + return "" data = resp.json() for release in data: tags.append(release["number"]) if not tags: eprint("{repo} > No Response or JSON parse failed".format(repo=info["src_repo"])) - sys.exit(1) + return "" if clean_tag: tags = clean_tags(tags, info) return tags @@ -383,11 +407,13 @@ def check_gnu_ftp(info, clean_tag=True): Check version info via compare ftp release tar file for gnu """ headers = { - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' - } + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' + } url = yaml2url.yaml2url(info) eprint("{repo} > List ftp directory".format(repo=url)) - resp = requests.get(url, headers=headers) + resp = get_resp(url, headers=headers) + if not resp: + return "" resp = resp.text re_pattern = re.compile("href=\"(.*)\">(\\1)") tags = [] @@ -405,11 +431,15 @@ def check_ftp(info, clean_tag=True): Check version info via compare ftp release tar file """ headers = { - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' - } + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' + } url = yaml2url.yaml2url(info) eprint("{repo} > List ftp directory".format(repo=url)) - resp = requests.get(url, headers=headers) + + resp = get_resp.get(url, headers=headers) + if not resp: + return "" + resp = resp.text re_pattern = re.compile("href=\"(.*)\">(.*)") tags = [] @@ -488,15 +518,13 @@ def check_sourceforge(info, clean_tag=True): tags = [] if resp == "": headers = { - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' } url = yaml2url.yaml2url(info) print("check_sourceforge, url = " + url) - try: - resp = requests.get(url, headers=headers) - except ConnectionError as err: - print("ERROR: connect {} error.".format(url), err) - return '' + resp = get_resp(url, headers=headers) + if not resp: + return "" data = resp.text lines = data.splitlines() diff --git a/advisors/check_version.py b/advisors/check_version.py index 33c8c56f3c19e3bee4ca9a1f170e66c8ba04c67c..57e04a25643fc05dc369d03ccbe47264b05f6803 100755 --- a/advisors/check_version.py +++ b/advisors/check_version.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#****************************************************************************** +# ****************************************************************************** # Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. # licensed under the Mulan PSL v2. # You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -20,6 +20,7 @@ import argparse from advisors import gitee from advisors.oa_upgradable import main_process from advisors.check_missing_file import get_repos_by_sig +import time def main(): @@ -43,6 +44,7 @@ def main(): sig = 'all' repos = get_repos_by_sig(sig) + print(repos) total = len(repos) index = 0 upgrade_list = [] @@ -52,6 +54,8 @@ def main(): if url == 'openeuler': continue check_repo = repo.split('/')[1] + # sleep 10 second, avoid limited by github\gitlab + time.sleep(10) result = main_process(args.push, args.default, check_repo) if result: print('''INFO: {index} in {total} check {repo} need upgrade \ @@ -74,5 +78,6 @@ upgrade'''.format(index=index, current=upgrade_repo[1], latest=upgrade_repo[2])) + if __name__ == "__main__": main() diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py index af7abe5a740fc443ec7184af60ff331e48e480d6..80624bd53e6b1b7ddcb9c0a971823ec86eb8ce22 100755 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -24,6 +24,22 @@ import yaml from advisors import gitee from advisors import check_upstream from advisors import version_recommend +from datetime import datetime + +NEW_COMMENT = """Dear {repo} maintainer: + +We found this issue has been open for {days} days. + +If you have any problems during implementation, please let the community known. + +We'll try our best to help. + +This is a automatic recommendation from openEuler-Advisor. If you think the suggestion is incorrect, + +please fill an issue at https://gitee.com/openeuler/openEuler-Advisor to help us improve. + +Yours openEuler Advisor. +""" def _filter_except(excpts, sources): @@ -142,9 +158,24 @@ def main_process(push, default, repo): print("Latest version is", ver_rec.latest_version) print("Maintain version is", ver_rec.maintain_version) + need_push_issue = True if cur_version != ver_rec.latest_version: if push: - user_gitee.post_issue(repo, "Upgrade to latest release", """Dear {repo} maintainer: + issues = user_gitee.get_issues(repo) + for issue in issues: + if issue["title"] == "Upgrade to latest release": + need_push_issue = False + ages = datetime.now() - user_gitee.get_gitee_datetime(issue["created_at"]) + if ages.days <= 30: + print("Advise has been issues only %d days ago" % ages.days) + print("Give developers more time to handle it.") + break + user_gitee.post_issue_comment(repo, issue["number"], NEW_COMMENT.format( + repo=repo, + days=ages.days)) + break + if need_push_issue: + user_gitee.post_issue(repo, "Upgrade to latest release", """Dear {repo} maintainer: We found the latest version of {repo} is {ver}, while the current version in openEuler mainline is {cur_ver}. @@ -158,5 +189,7 @@ Issues and feedbacks are welcome.""".format(repo=repo, cur_ver=cur_version)) return repo, cur_version, ver_rec.latest_version return None + + if __name__ == "__main__": main()