From ea1cf8900fd2686aa7553b1c7e556dd3c8e8a39b Mon Sep 17 00:00:00 2001 From: gongzt Date: Sat, 22 Aug 2020 09:54:10 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=8C=85=E7=94=9F=E5=91=BD=E5=91=A8?= =?UTF-8?q?=E6=9C=9F=E7=9A=84=E6=A0=B8=E5=BF=83=E6=A1=86=E6=9E=B6=E3=80=81?= =?UTF-8?q?=E5=8C=85=E7=9B=B8=E5=85=B3=E4=BF=A1=E6=81=AF=E7=9A=84=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../packageship/application/__init__.py | 43 +++- .../packageship/application/app_global.py | 4 +- .../packageship/application/apps/__init__.py | 8 +- .../application/apps/lifecycle/__init__.py | 20 ++ .../apps/lifecycle/function/__init__.py | 4 + .../apps/lifecycle/function/base.py | 17 ++ .../apps/lifecycle/function/download_yaml.py | 212 +++++++++++++++++ .../application/apps/lifecycle/serialize.py | 29 +++ .../application/apps/lifecycle/url.py | 14 ++ .../application/apps/lifecycle/view.py | 223 ++++++++++++++++++ .../packageship/application/models/package.py | 67 +++++- .../packageship/application/settings.py | 18 +- packageship/packageship/manage.py | 6 +- packageship/packageship/package.ini | 38 +++ packageship/packageship/selfpkg.py | 6 +- packageship/setup.py | 14 +- 16 files changed, 680 insertions(+), 43 deletions(-) create mode 100644 packageship/packageship/application/apps/lifecycle/__init__.py create mode 100644 packageship/packageship/application/apps/lifecycle/function/__init__.py create mode 100644 packageship/packageship/application/apps/lifecycle/function/base.py create mode 100644 packageship/packageship/application/apps/lifecycle/function/download_yaml.py create mode 100644 packageship/packageship/application/apps/lifecycle/serialize.py create mode 100644 packageship/packageship/application/apps/lifecycle/url.py create mode 100644 packageship/packageship/application/apps/lifecycle/view.py diff --git a/packageship/packageship/application/__init__.py b/packageship/packageship/application/__init__.py index bc3a6316..810d6ce5 100644 --- a/packageship/packageship/application/__init__.py +++ b/packageship/packageship/application/__init__.py @@ -2,14 +2,44 @@ """ Initial operation and configuration of the flask project """ +import sys +import threading from flask import Flask from flask_session import Session +from flask_apscheduler import APScheduler +from packageship import system_config from packageship.application.settings import Config from packageship.libs.log import setup_log +from packageship.libs.configutils.readconfig import ReadConfig OPERATION = None +def _timed_task(app): + """ + + """ + from .apps.lifecycle.function.download_yaml import update_pkg_info # pylint: disable=import-outside-toplevel + + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + + _hour = _readconfig.get_config('TIMEDTASK', 'hour') + if not _hour or not isinstance(_hour, int) or _hour < 0 or _hour > 23: + _hour = 3 + _minute = _readconfig.get_config('TIMEDTASK', 'minute') + if not _hour or not isinstance(_hour, int) or _hour < 0 or _hour > 59: + _minute = 0 + app.apscheduler.add_job( # pylint: disable=no-member + func=update_pkg_info, id="update_package_data", trigger="cron", hour=_hour, minute=_minute) + app.apscheduler.add_job( # pylint: disable=no-member + func=update_pkg_info, + id="issue_catch", + trigger="cron", + hour=_hour, + minute=_minute, + args=(False,)) + + def init_app(operation): """ Project initialization function @@ -23,16 +53,23 @@ def init_app(operation): app.config.from_object(Config) + # Register a scheduled task + scheduler = APScheduler() + scheduler.init_app(app) + scheduler.start() + # Open session function Session(app) - global OPERATION + global OPERATION # pylint: disable=global-statement OPERATION = operation # Register Blueprint - from packageship.application.apps import blue_point - for blue, api in blue_point: + from packageship.application import apps # pylint: disable=import-outside-toplevel + for blue, api in apps.blue_point: api.init_app(app) app.register_blueprint(blue) + _timed_task(app) + return app diff --git a/packageship/packageship/application/app_global.py b/packageship/packageship/application/app_global.py index 25d9dbe0..611f309e 100644 --- a/packageship/packageship/application/app_global.py +++ b/packageship/packageship/application/app_global.py @@ -19,8 +19,8 @@ def identity_verification(): """ if request.url_rule: url_rule = request.url_rule.rule - for view, url, authentication in urls: - if url == url_rule and application.OPERATION in authentication.keys(): + for _view, url, authentication in urls: + if url.lower() == url_rule.lower() and application.OPERATION in authentication.keys(): if request.method not in authentication.get(application.OPERATION): return False break diff --git a/packageship/packageship/application/apps/__init__.py b/packageship/packageship/application/apps/__init__.py index 0cb8d57f..e1d108ef 100644 --- a/packageship/packageship/application/apps/__init__.py +++ b/packageship/packageship/application/apps/__init__.py @@ -2,10 +2,12 @@ """ Blueprint collection trying to page """ -from packageship.application.apps.package import package, api as package_api +from packageship.application.apps import package +from packageship.application.apps import lifecycle -blue_point = [ - (package, package_api), +blue_point = [ # pylint: disable=invalid-name + (package.package, package.api), + (lifecycle.lifecycle, lifecycle.api) ] __all__ = ['blue_point'] diff --git a/packageship/packageship/application/apps/lifecycle/__init__.py b/packageship/packageship/application/apps/lifecycle/__init__.py new file mode 100644 index 00000000..d17a06a5 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 +""" + Blueprint registration for life cycle +""" +from flask.blueprints import Blueprint +from flask_restful import Api +from packageship.application.apps.lifecycle.url import urls +from packageship import application + +lifecycle = Blueprint('lifecycle', __name__) + +# init restapi +api = Api() + +for view, url, operation in urls: + if application.OPERATION and application.OPERATION in operation.keys(): + api.add_resource(view, url) + + +__all__ = ['lifecycle', 'api'] diff --git a/packageship/packageship/application/apps/lifecycle/function/__init__.py b/packageship/packageship/application/apps/lifecycle/function/__init__.py new file mode 100644 index 00000000..1d9852d5 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/function/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/python3 +""" + Business approach to package life cycle +""" diff --git a/packageship/packageship/application/apps/lifecycle/function/base.py b/packageship/packageship/application/apps/lifecycle/function/base.py new file mode 100644 index 00000000..3631dde4 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/function/base.py @@ -0,0 +1,17 @@ +#!/usr/bin/python3 +""" +General approach to version control tools +""" +from packageship.libs.log import Log + + +class Base(): + """ + Public method to get project tags and download yaml file + """ + + def __init__(self): + self.headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW 64; rv:50.0) Gecko/20100101 \ + Firefox / 50.0 '} + self.log = Log(__name__) diff --git a/packageship/packageship/application/apps/lifecycle/function/download_yaml.py b/packageship/packageship/application/apps/lifecycle/function/download_yaml.py new file mode 100644 index 00000000..68c1c042 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/function/download_yaml.py @@ -0,0 +1,212 @@ +#!/usr/bin/python3 +""" +Dynamically obtain the content of the yaml file \ +that saves the package information, periodically \ +obtain the content and save it in the database +""" +import copy +from concurrent.futures import ThreadPoolExecutor +import datetime as date +import requests +import yaml +from sqlalchemy.exc import SQLAlchemyError +from requests.exceptions import HTTPError +from packageship import system_config +from packageship.application.models.package import packages +from packageship.application.models.package import packages_maintainer +from packageship.libs.dbutils import DBHelper +from packageship.libs.exception import Error, ContentNoneException +from packageship.libs.configutils.readconfig import ReadConfig + +from .base import Base +from .gitee import Gitee + + +class ParseYaml(): + """ + Description: Download the contents of the yaml file + + Attributes: + base: base class instance + pkg: Specific package data + _table_name: The name of the data table to be operated + openeuler_advisor_url: Get the warehouse address of the yaml file + _yaml_content: The content of the yaml file + """ + + def __init__(self, pkg_info, base, table_name): + self.base = base + self.pkg = pkg_info + self._table_name = table_name + self.openeuler_advisor_url = self._path_stitching(pkg_info.name) + self._yaml_content = None + self.timed_task_open = self._timed_task_status() + + def _timed_task_status(self): + """ + The open state of information such as the maintainer in the scheduled task + """ + _timed_task_status = True + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + open_status = _readconfig.get_config('TIMEDTASK', 'open') + if open_status not in ('True', 'False'): + self.base.log.logger.error( + 'Wrong setting of the open state value of the scheduled task') + if open_status == 'False': + self.timed_task_open = False + return _timed_task_status + + def _path_stitching(self, pkg_name): + """ + The path of the remote service call + """ + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + _remote_url = _readconfig.get_config('LIFECYCLE', 'warehouse_remote') + if _remote_url is None: + _remote_url = 'https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/' + return _remote_url + '{pkg_name}.yaml'.format(pkg_name=pkg_name) + + def update_pkg_info(self): + """ + Download the contents of the yaml file + + """ + if self._openeuler_advisor_exists_yaml(): + self._save_to_database() + else: + msg = "The yaml information of the %s package has not been\ + obtained yet" % self.pkg.name + self.base.log.logger.warning(msg) + + def _read_yaml_content(self, url): + """ + + """ + try: + response = requests.get( + url, headers=self.base.headers) + if response.status_code == 200: + self._yaml_content = yaml.safe_load(response.content) + + except HTTPError as error: + self.base.log.logger.error(error) + + def _openeuler_advisor_exists_yaml(self): + """ + Determine whether there is a yaml file with the current \ + package name under the openeuler-advisor project + + """ + self._read_yaml_content(self.openeuler_advisor_url) + if self._yaml_content: + return True + return False + + def _save_to_database(self): + """ + Save the acquired yaml file information to the database + + Raises: + ContentNoneException: The added entity content is empty + Error: An error occurred during data addition + """ + self._parse_warehouse_info() + tags = self._yaml_content.get('git_tag', None) + self._parse_tags_content(tags) + if self.timed_task_open: + _maintainer = self._yaml_content.get('maintainer') + if _maintainer and isinstance(_maintainer, list): + self.pkg.maintainer = _maintainer[0] + self.pkg.maintainlevel = self._yaml_content.get('maintainlevel') + try: + with DBHelper(db_name="lifecycle") as database: + database.session.begin(subtransactions=True) + database.add(self.pkg) + if self.timed_task_open and self.pkg.maintainer: + _packages_maintainer = database.session.query( + packages_maintainer).filter( + packages_maintainer.name == self.pkg.name).first() + if _packages_maintainer: + _packages_maintainer.name = self.pkg.name + _packages_maintainer.maintainer = self.pkg.maintainer + _packages_maintainer.maintainlevel = self.pkg.maintainlevel + else: + database.add(packages_maintainer( + name=self.pkg.name, maintainer=self.pkg.maintainer, + maintainlevel=self.pkg.maintainlevel)) + database.session.commit() + except (Error, ContentNoneException, SQLAlchemyError) as error: + self.base.log.logger.error(error) + + def _parse_warehouse_info(self): + """ + Parse the warehouse information in the yaml file + + """ + if self._yaml_content: + self.pkg.version_control = self._yaml_content.get( + 'version_control') + self.pkg.src_repo = self._yaml_content.get('src_repo') + self.pkg.tag_prefix = self._yaml_content.get('tag_prefix') + + def _parse_tags_content(self, tags): + """ + Parse the obtained tags content + + """ + try: + # Integrate tags information into key-value pairs + _tags = [(tag.split()[0], tag.split()[1]) for tag in tags] + _tags = sorted(_tags, key=lambda x: x[0], reverse=True) + self.pkg.latest_version = _tags[0][1] + self.pkg.latest_version_time = _tags[0][0] + _end_time = date.datetime.strptime( + self.pkg.latest_version_time, '%Y-%m-%d') + if self.pkg.latest_version != self.pkg.version: + for _version in _tags: + if _version[1] == self.pkg.version: + _end_time = date.datetime.strptime( + _version[0], '%Y-%m-%d') + self.pkg.used_time = (date.datetime.now() - _end_time).days + + except (IndexError, Error) as index_error: + self.base.log.logger.error(index_error) + + +def update_pkg_info(pkg_info_update=True): + """ + Update the information of the upstream warehouse in the source package + + """ + try: + base_control = Base() + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') + _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse') + if _warehouse is None: + _warehouse = 'src-openeuler' + if not isinstance(pool_workers, int): + pool_workers = 10 + # Open thread pool + pool = ThreadPoolExecutor(max_workers=pool_workers) + with DBHelper(db_name="lifecycle") as database: + for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer'], + database.engine.table_names()): + + cls_model = packages.package_meta(table_name) + # Query a specific table + for package_item in database.session.query(cls_model).all(): + if pkg_info_update: + parse_yaml = ParseYaml( + pkg_info=copy.deepcopy(package_item), + base=base_control, + table_name=table_name) + pool.submit(parse_yaml.update_pkg_info) + else: + # Get the issue of each warehouse and save it + gitee_issue = Gitee( + package_item, _warehouse, package_item.name, table_name) + pool.submit(gitee_issue.query_issues_info) + pool.shutdown() + except SQLAlchemyError as error_msg: + base_control.log.logger.error(error_msg) diff --git a/packageship/packageship/application/apps/lifecycle/serialize.py b/packageship/packageship/application/apps/lifecycle/serialize.py new file mode 100644 index 00000000..1a03b936 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/serialize.py @@ -0,0 +1,29 @@ +#!/usr/bin/python3 +""" +Description: marshmallow serialize +""" +from marshmallow import Schema +from packageship.application.models.package import packages_issue, packages + + +class IssueDownloadSchema(Schema): + """ + Field serialization for issue file download + """ + + class Meta: # pylint: disable=missing-class-docstring + model = packages_issue + fields = ('issue_id', 'issue_url', 'issue_content', + 'issue_title', 'issue_status', 'pkg_name', 'issue_type', 'related_release') + + +class PackagesDownloadSchema(Schema): + """ + Field serialization for package file download + """ + + class Meta: # pylint: disable=missing-class-docstring + model = packages + fields = ('name', 'url', 'rpm_license', 'version', 'release', 'release_time', + 'used_time', 'latest_version', 'latest_version_time', + 'feature', 'cve', 'defect', 'maintainer', 'maintainlevel') diff --git a/packageship/packageship/application/apps/lifecycle/url.py b/packageship/packageship/application/apps/lifecycle/url.py new file mode 100644 index 00000000..ea08d589 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/url.py @@ -0,0 +1,14 @@ +#!/usr/bin/python3 +""" +Life cycle of url giant whale collection +""" +from . import view + +urls = [ # pylint: disable=invalid-name + # Download package data or iSsue data + (view.DownloadFile, '/lifeCycle/download/', {'query': ('GET')}), + # Get a collection of maintainers list + (view.MaintainerView, '/lifeCycle/maintainer', {'query': ('GET')}), + # Get the columns that need to be displayed by default in the package + (view.TableColView, '/packages/tablecol', {'query': ('GET')}), +] diff --git a/packageship/packageship/application/apps/lifecycle/view.py b/packageship/packageship/application/apps/lifecycle/view.py new file mode 100644 index 00000000..832d2bf6 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/view.py @@ -0,0 +1,223 @@ +#!/usr/bin/python3 +""" +Life cycle related api interface +""" +import io +import json +import math +import os +from concurrent.futures import ThreadPoolExecutor + +import pandas as pd +import yaml +from flask import request +from flask import jsonify, make_response +from flask import current_app +from flask_restful import Resource +from marshmallow import ValidationError +from sqlalchemy.exc import DisconnectionError, SQLAlchemyError + +from packageship import system_config +from packageship.application.apps.package.serialize import DataFormatVerfi, UpdatePackagesSchema +from packageship.libs.configutils.readconfig import ReadConfig +from packageship.libs.exception import Error +from packageship.application.apps.package.function.constants import ResponseCode +from packageship.libs.dbutils.sqlalchemy_helper import DBHelper +from packageship.application.models.package import packages_issue +from packageship.application.models.package import packages +from packageship.application.models.package import packages_maintainer +from packageship.libs.log import Log +from .serialize import IssueDownloadSchema, PackagesDownloadSchema +from .function.gitee import Gitee as gitee + +LOGGER = Log(__name__) + + +# pylint: disable = no-self-use + +class DownloadFile(Resource): + """ + Download the content of the issue or the excel file of the package content + """ + + def _download_excel(self, file_type, table_name=None): + """ + Download excel file + """ + file_name = 'packages.xlsx' + if file_type == 'packages': + download_content = self.__get_packages_content(table_name) + else: + file_name = 'issues.xlsx' + download_content = self.__get_issues_content() + if download_content is None: + return jsonify( + ResponseCode.response_json( + ResponseCode.SERVICE_ERROR)) + pd_dataframe = self.__to_dataframe(download_content) + + _response = self.__bytes_save(pd_dataframe) + return self.__set_response_header(_response, file_name) + + def __bytes_save(self, data_frame): + """ + Save the file content in the form of a binary file stream + """ + try: + bytes_io = io.BytesIO() + writer = pd.ExcelWriter( # pylint: disable=abstract-class-instantiated + bytes_io, engine='xlsxwriter') + data_frame.to_excel(writer, sheet_name='Summary', index=False) + writer.save() + writer.close() + bytes_io.seek(0) + _response = make_response(bytes_io.getvalue()) + bytes_io.close() + return _response + except (IOError, Error) as io_error: + current_app.logger.error(io_error) + return make_response() + + def __set_response_header(self, response, file_name): + """ + Set http response header information + """ + response.headers['Content-Type'] = \ + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + response.headers["Cache-Control"] = "no-cache" + response.headers['Content-Disposition'] = 'attachment; filename={file_name}'.format( + file_name=file_name) + return response + + def __get_packages_content(self, table_name): + """ + Get package list information + """ + try: + with DBHelper(db_name='lifecycle') as database: + # Query all package data in the specified table + _model = packages.package_meta(table_name) + _packageinfos = database.session.query(_model).all() + packages_dicts = PackagesDownloadSchema( + many=True).dump(_packageinfos) + return packages_dicts + + except (SQLAlchemyError, DisconnectionError) as error: + current_app.logger.error(error) + return None + + def __get_issues_content(self): + """ + Get the list of issues + """ + try: + with DBHelper(db_name='lifecycle') as database: + _issues = database.session.query(packages_issue).all() + issues_dicts = IssueDownloadSchema(many=True).dump(_issues) + return issues_dicts + except (SQLAlchemyError, DisconnectionError) as error: + current_app.logger.error(error) + return None + + def __to_dataframe(self, datas): + """ + Convert the obtained information into pandas content format + """ + + data_frame = pd.DataFrame(datas) + return data_frame + + def get(self, file_type): + """ + Download package collection information and isse list information + + """ + if file_type not in ['packages', 'issues']: + return jsonify( + ResponseCode.response_json( + ResponseCode.PARAM_ERROR)) + + table_name = request.args.get('table_name', None) + response = self._download_excel(file_type, table_name) + return response + + +class MaintainerView(Resource): + """ + Maintainer name collection + """ + + def __query_maintainers(self): + """ + Query the names of all maintainers in the specified table + """ + try: + with DBHelper(db_name='lifecycle') as database: + maintainers = database.session.query( + packages_maintainer.maintainer).group_by(packages_maintainer.maintainer).all() + return [maintainer_item[0] + for maintainer_item in maintainers if maintainer_item[0]] + except (SQLAlchemyError, DisconnectionError) as error: + current_app.logger.error(error) + return [] + + def get(self): + """ + Get the list of maintainers + """ + # Group query of the names of all maintainers in the current table + maintainers = self.__query_maintainers() + return jsonify(ResponseCode.response_json( + ResponseCode.SUCCESS, + maintainers)) + + +class TableColView(Resource): + """ + The default column of the package shows the interface + """ + + def __columns_names(self): + """ + Mapping of column name and title + """ + columns = [ + ('name', 'Name', True), + ('version', 'Version', True), + ('release', 'Release', True), + ('url', 'Url', True), + ('linense', 'License', False), + ('feature', 'Feature', False), + ('maintainer', 'Maintainer', True), + ('maintainlevel', 'Maintenance Level', True), + ('release_time', 'Release Time', False), + ('end_of_lifecycle', 'End of Life Cycle', True), + ('maintainer_status', 'Maintain Status', True), + ('latest_version', 'Latest Version', False), + ('latest_version_time', 'Latest Version Release Time', False), + ('issue', 'Issue', True)] + return columns + + def __columns_mapping(self): + """ + + """ + columns = list() + for column in self.__columns_names(): + columns.append({ + 'column_name': column[0], + 'label': column[1], + 'default_selected': column[2] + }) + return columns + + def get(self): + """ + Get the default display column of the package + + """ + table_mapping_columns = self.__columns_mapping() + return jsonify( + ResponseCode.response_json( + ResponseCode.SUCCESS, + table_mapping_columns)) diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index 725f4af7..78132bc9 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -2,7 +2,8 @@ """ Description: Database entity model mapping """ -from sqlalchemy import Column, Integer, String +import uuid +from sqlalchemy import Column, Integer, String, Text from packageship.libs.dbutils.sqlalchemy_helper import DBHelper @@ -126,18 +127,66 @@ class bin_provides(DBHelper.BASE): # pylint: disable=C0103,R0903 pkgKey = Column(Integer, nullable=True) -class maintenance_info(DBHelper.BASE): # pylint: disable=C0103,R0903 +class packages(): # pylint: disable=C0103,R0903 """ - Maintain data related to person information + Source code package version, issuer and other information """ - __tablename__ = 'maintenance_info' - + __table_args__ = {'extend_existing': True} id = Column(Integer, primary_key=True) - name = Column(String(500), nullable=True) + url = Column(String(500), nullable=True) + rpm_license = Column(String(500), nullable=True) + version = Column(String(200), nullable=True) + release = Column(String(200), nullable=True) + release_time = Column(String(50), nullable=True) + used_time = Column(Integer, default=0) + latest_version = Column(String(200), nullable=True) + latest_version_time = Column(String(50), nullable=True) + feature = Column(Integer, default=0) + cve = Column(Integer, default=0) + defect = Column(Integer, default=0) + maintainer = Column(String(200), nullable=True) + maintainlevel = Column(Integer, nullable=True) + version_control = Column(String(50), nullable=True) + src_repo = Column(String(500), nullable=True) + tag_prefix = Column(String(20), nullable=True) + summary = Column(String(500), nullable=True) + description = Column(String(500), nullable=True) - version = Column(String(500), nullable=True) + @classmethod + def package_meta(cls, table_name): + """ - maintaniner = Column(String(100), nullable=True) + """ + _uuid = str(uuid.uuid1()) + model = type(_uuid, (cls, DBHelper.BASE), { + '__tablename__': table_name}) + return model - maintainlevel = Column(String(100), nullable=True) + +class packages_issue(DBHelper.BASE): # pylint: disable=C0103,R0903 + """ + Source package issue + """ + __tablename__ = "packages_issue" + id = Column(Integer, primary_key=True) + issue_id = Column(String(50), nullable=True) + issue_url = Column(String(500), nullable=True) + issue_content = Column(Text, nullable=True) + issue_title = Column(String(1000), nullable=True) + issue_status = Column(String(20), nullable=True) + pkg_name = Column(String(500), nullable=False) + issue_download = Column(String(500), nullable=False) + issue_type = Column(String(50), nullable=True) + related_release = Column(String(500), nullable=True) + + +class packages_maintainer(DBHelper.BASE): # pylint: disable=C0103,R0903 + """ + Correspondence between source code package and maintainer + """ + __tablename__ = 'packages_maintainer' + id = Column(Integer, primary_key=True) + name = Column(String(200), nullable=True) + maintainer = Column(String(200), nullable=True) + maintainlevel = Column(Integer, nullable=True) diff --git a/packageship/packageship/application/settings.py b/packageship/packageship/application/settings.py index bc090439..12d561cb 100644 --- a/packageship/packageship/application/settings.py +++ b/packageship/packageship/application/settings.py @@ -3,6 +3,7 @@ Description: Basic configuration of flask framework """ import random +from packageship import system_config from packageship.libs.configutils.readconfig import ReadConfig @@ -19,9 +20,11 @@ class Config(): LOG_LEVEL = 'INFO' + SCHEDULER_API_ENABLED = True + def __init__(self): - self._read_config = ReadConfig() + self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH) self.set_config_val() @@ -33,14 +36,6 @@ class Config(): cls.SECRET_KEY = ''.join( [random.choice('abcdefghijklmnopqrstuvwxyz!@#$%^&*()') for index in range(random_len)]) - @classmethod - def _set_debug(cls, debug): - """ - Description: Set the debugging mode - """ - if debug == 'true': - cls.DEBUG = True - @classmethod def _set_log_level(cls, log_level): """ @@ -57,11 +52,6 @@ class Config(): """ Config._random_secret_key() - debug = self._read_config.get_system('debug') - - if debug: - Config._set_debug(debug) - log_level = self._read_config.get_config('LOG', 'log_level') if log_level: diff --git a/packageship/packageship/manage.py b/packageship/packageship/manage.py index dc79873e..25d8aa77 100644 --- a/packageship/packageship/manage.py +++ b/packageship/packageship/manage.py @@ -5,8 +5,8 @@ Description: Entry for project initialization and service startupc import os from packageship.libs.exception import Error try: - from packageship.system_config import SYS_CONFIG_PATH - if not os.path.exists(SYS_CONFIG_PATH): + from packageship import system_config + if not os.path.exists(system_config.SYS_CONFIG_PATH): raise FileNotFoundError( 'the system configuration file does not exist and the log cannot be started') except FileNotFoundError as file_not_found: @@ -36,7 +36,7 @@ def before_request(): if __name__ == "__main__": - _readconfig = ReadConfig() + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) port = _readconfig.get_system('write_port') addr = _readconfig.get_system('write_ip_addr') app.run(port=port, host=addr) diff --git a/packageship/packageship/package.ini b/packageship/packageship/package.ini index c562d9ef..e0f4f472 100644 --- a/packageship/packageship/package.ini +++ b/packageship/packageship/package.ini @@ -55,4 +55,42 @@ buffer-size=65536 http-timeout=600 ; Server response time harakiri=600 +; Allow multiple threads to run in the system +enable-threads=true + + +[TIMEDTASK] +; Execution frequency and switch of timing tasks +; Whether to execute the switch for batch update of information such +; as the maintainer during initialization. When set to True, the maintainer +; and other information will be updated when the scheduled task starts +; to execute. When it is set to False, it will be updated when the scheduled +; task is executed. , Does not update information such as maintainers and maintenance levels + +open=True + +; The time point at which the timing task is executed in a cycle. +; Every day is a cycle, and the time value can only be any integer period between 0 and 23 +hour=3 + +; Recurring timing task starts to execute the task at the current time point. +; The value range of this configuration item is an integer between 0 and 59 +minute=0 + +[LIFECYCLE] +; Configuration during the life cycle for tag information, issue and other information acquisition + +; The yaml address of each package is stored in the remote address, which can be +; a remote warehouse address or the address of a static resource service +warehouse_remote=https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/ + +; When performing timing tasks, you can open multi-threaded execution, and you can set +; the number of threads in the thread pool according to the configuration of the server + +pool_workers=10 + + +; The main body of the warehouse, the owner of the warehouse +; When this value is not set, the system will default to src-openeuler +warehouse=src-openeuler diff --git a/packageship/packageship/selfpkg.py b/packageship/packageship/selfpkg.py index f748eb3f..b8895531 100644 --- a/packageship/packageship/selfpkg.py +++ b/packageship/packageship/selfpkg.py @@ -7,8 +7,8 @@ from packageship.libs.exception import Error from packageship.libs.configutils.readconfig import ReadConfig try: - from packageship.system_config import SYS_CONFIG_PATH - if not os.path.exists(SYS_CONFIG_PATH): + from packageship import system_config + if not os.path.exists(system_config.SYS_CONFIG_PATH): raise FileNotFoundError( 'the system configuration file does not exist and the log cannot be started') except FileNotFoundError as file_not_found: @@ -36,7 +36,7 @@ def before_request(): if __name__ == "__main__": - _readconfig = ReadConfig() + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) port = _readconfig.get_system('query_port') addr = _readconfig.get_system('query_ip_addr') app.run(port=port, host=addr) diff --git a/packageship/setup.py b/packageship/setup.py index b10dc708..08488d7d 100644 --- a/packageship/setup.py +++ b/packageship/setup.py @@ -4,13 +4,9 @@ Package management program installation configuration file for software packaging """ from distutils.core import setup -import os -BASE_PATH = os.path.dirname(__file__) -path = os.path.join(BASE_PATH, 'Lib', 'site-packages', 'package') - -configpath = "/etc/pkgship/" +config_path = "/etc/pkgship/" # pylint: disable=invalid-name setup( name='packageship', @@ -29,6 +25,12 @@ setup( 'packageship.application.apps.package.function.packages', 'packageship.application.apps.package.function.searchdb', 'packageship.application.apps.package.function.self_depend', + 'packageship.application.apps.lifecycle.function.base', + 'packageship.application.apps.lifecycle.function.download_yaml', + 'packageship.application.apps.lifecycle.function.gitee', + 'packageship.application.apps.lifecycle.serialize', + 'packageship.application.apps.lifecycle.url', + 'packageship.application.apps.lifecycle.view', 'packageship.application.initsystem.data_import', 'packageship.application.models.package', 'packageship.application.settings', @@ -56,6 +58,6 @@ setup( long_description=open('README.md', encoding='utf-8').read(), author='gongzt', data_files=[ - (configpath, ['packageship/package.ini']), + (config_path, ['packageship/package.ini']), ('/usr/bin', ['packageship/pkgshipd'])] ) -- Gitee From bebeee24f48e41ee14f0204ec27bbaf05ffb1c2a Mon Sep 17 00:00:00 2001 From: gongzt Date: Sat, 22 Aug 2020 10:19:22 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=AF=B9package=5Fmeta=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packageship/packageship/application/models/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index 78132bc9..3c7b8564 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -156,7 +156,7 @@ class packages(): # pylint: disable=C0103,R0903 @classmethod def package_meta(cls, table_name): """ - + Dynamically generate different classes through metaclasses """ _uuid = str(uuid.uuid1()) model = type(_uuid, (cls, DBHelper.BASE), { -- Gitee From 897ed9e8602b8e2c5715117c6669fdda5ad9674a Mon Sep 17 00:00:00 2001 From: gongzt Date: Wed, 26 Aug 2020 16:00:32 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=B1=BB=E7=9A=84?= =?UTF-8?q?=E5=A4=A7=E9=A9=BC=E5=B3=B0=E5=91=BD=E5=90=8D=E8=A7=84=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apps/lifecycle/function/download_yaml.py | 14 +++---- .../application/apps/lifecycle/serialize.py | 12 +++--- .../application/apps/lifecycle/view.py | 12 +++--- .../apps/package/function/be_depend.py | 4 +- .../apps/package/function/packages.py | 38 ++++++++--------- .../apps/package/function/searchdb.py | 42 ++++++++++++------- .../application/initsystem/data_import.py | 30 ++++++------- .../packageship/application/models/package.py | 16 +++---- .../dependent_query_tests/test_be_depend.py | 2 + 9 files changed, 92 insertions(+), 78 deletions(-) diff --git a/packageship/packageship/application/apps/lifecycle/function/download_yaml.py b/packageship/packageship/application/apps/lifecycle/function/download_yaml.py index 68c1c042..c2173841 100644 --- a/packageship/packageship/application/apps/lifecycle/function/download_yaml.py +++ b/packageship/packageship/application/apps/lifecycle/function/download_yaml.py @@ -12,8 +12,8 @@ import yaml from sqlalchemy.exc import SQLAlchemyError from requests.exceptions import HTTPError from packageship import system_config -from packageship.application.models.package import packages -from packageship.application.models.package import packages_maintainer +from packageship.application.models.package import Packages +from packageship.application.models.package import PackagesMaintainer from packageship.libs.dbutils import DBHelper from packageship.libs.exception import Error, ContentNoneException from packageship.libs.configutils.readconfig import ReadConfig @@ -124,14 +124,14 @@ class ParseYaml(): database.add(self.pkg) if self.timed_task_open and self.pkg.maintainer: _packages_maintainer = database.session.query( - packages_maintainer).filter( - packages_maintainer.name == self.pkg.name).first() + PackagesMaintainer).filter( + PackagesMaintainer.name == self.pkg.name).first() if _packages_maintainer: _packages_maintainer.name = self.pkg.name _packages_maintainer.maintainer = self.pkg.maintainer _packages_maintainer.maintainlevel = self.pkg.maintainlevel else: - database.add(packages_maintainer( + database.add(PackagesMaintainer( name=self.pkg.name, maintainer=self.pkg.maintainer, maintainlevel=self.pkg.maintainlevel)) database.session.commit() @@ -190,10 +190,10 @@ def update_pkg_info(pkg_info_update=True): # Open thread pool pool = ThreadPoolExecutor(max_workers=pool_workers) with DBHelper(db_name="lifecycle") as database: - for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer'], + for table_name in filter(lambda x: x not in ['packages_issue', 'PackagesMaintainer'], database.engine.table_names()): - cls_model = packages.package_meta(table_name) + cls_model = Packages.package_meta(table_name) # Query a specific table for package_item in database.session.query(cls_model).all(): if pkg_info_update: diff --git a/packageship/packageship/application/apps/lifecycle/serialize.py b/packageship/packageship/application/apps/lifecycle/serialize.py index 1a03b936..a9ccbda3 100644 --- a/packageship/packageship/application/apps/lifecycle/serialize.py +++ b/packageship/packageship/application/apps/lifecycle/serialize.py @@ -3,7 +3,7 @@ Description: marshmallow serialize """ from marshmallow import Schema -from packageship.application.models.package import packages_issue, packages +from packageship.application.models.package import PackagesIssue, Packages class IssueDownloadSchema(Schema): @@ -11,8 +11,9 @@ class IssueDownloadSchema(Schema): Field serialization for issue file download """ - class Meta: # pylint: disable=missing-class-docstring - model = packages_issue + class Meta: + """Model mapping serialized fields""" + model = PackagesIssue fields = ('issue_id', 'issue_url', 'issue_content', 'issue_title', 'issue_status', 'pkg_name', 'issue_type', 'related_release') @@ -22,8 +23,9 @@ class PackagesDownloadSchema(Schema): Field serialization for package file download """ - class Meta: # pylint: disable=missing-class-docstring - model = packages + class Meta: + """Model mapping serialized fields""" + model = Packages fields = ('name', 'url', 'rpm_license', 'version', 'release', 'release_time', 'used_time', 'latest_version', 'latest_version_time', 'feature', 'cve', 'defect', 'maintainer', 'maintainlevel') diff --git a/packageship/packageship/application/apps/lifecycle/view.py b/packageship/packageship/application/apps/lifecycle/view.py index 832d2bf6..8ec3f21e 100644 --- a/packageship/packageship/application/apps/lifecycle/view.py +++ b/packageship/packageship/application/apps/lifecycle/view.py @@ -23,9 +23,9 @@ from packageship.libs.configutils.readconfig import ReadConfig from packageship.libs.exception import Error from packageship.application.apps.package.function.constants import ResponseCode from packageship.libs.dbutils.sqlalchemy_helper import DBHelper -from packageship.application.models.package import packages_issue -from packageship.application.models.package import packages -from packageship.application.models.package import packages_maintainer +from packageship.application.models.package import PackagesIssue +from packageship.application.models.package import Packages +from packageship.application.models.package import PackagesMaintainer from packageship.libs.log import Log from .serialize import IssueDownloadSchema, PackagesDownloadSchema from .function.gitee import Gitee as gitee @@ -96,7 +96,7 @@ class DownloadFile(Resource): try: with DBHelper(db_name='lifecycle') as database: # Query all package data in the specified table - _model = packages.package_meta(table_name) + _model = Packages.package_meta(table_name) _packageinfos = database.session.query(_model).all() packages_dicts = PackagesDownloadSchema( many=True).dump(_packageinfos) @@ -112,7 +112,7 @@ class DownloadFile(Resource): """ try: with DBHelper(db_name='lifecycle') as database: - _issues = database.session.query(packages_issue).all() + _issues = database.session.query(PackagesIssue).all() issues_dicts = IssueDownloadSchema(many=True).dump(_issues) return issues_dicts except (SQLAlchemyError, DisconnectionError) as error: @@ -154,7 +154,7 @@ class MaintainerView(Resource): try: with DBHelper(db_name='lifecycle') as database: maintainers = database.session.query( - packages_maintainer.maintainer).group_by(packages_maintainer.maintainer).all() + PackagesMaintainer.maintainer).group_by(PackagesMaintainer.maintainer).all() return [maintainer_item[0] for maintainer_item in maintainers if maintainer_item[0]] except (SQLAlchemyError, DisconnectionError) as error: diff --git a/packageship/packageship/application/apps/package/function/be_depend.py b/packageship/packageship/application/apps/package/function/be_depend.py index da50f0d4..a9f9769c 100644 --- a/packageship/packageship/application/apps/package/function/be_depend.py +++ b/packageship/packageship/application/apps/package/function/be_depend.py @@ -10,7 +10,7 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.sql import literal_column from flask import current_app from packageship.libs.dbutils import DBHelper -from packageship.application.models.package import src_pack +from packageship.application.models.package import SrcPack from packageship.application.apps.package.function.constants import ResponseCode @@ -59,7 +59,7 @@ class BeDepend(): """ with DBHelper(db_name=self.db_name) as data_base: src_obj = data_base.session.query( - src_pack).filter_by(name=self.source_name).first() + SrcPack).filter_by(name=self.source_name).first() if src_obj: # spell dictionary self.result_dict[self.source_name + "_src"] = [ diff --git a/packageship/packageship/application/apps/package/function/packages.py b/packageship/packageship/application/apps/package/function/packages.py index 5bf347d6..b5b5139b 100644 --- a/packageship/packageship/application/apps/package/function/packages.py +++ b/packageship/packageship/application/apps/package/function/packages.py @@ -12,12 +12,12 @@ from packageship.application.apps.package.function.constants import ResponseCode from packageship.application.apps.package.function.searchdb import db_priority from packageship.libs.dbutils import DBHelper -from packageship.application.models.package import src_pack -from packageship.application.models.package import src_requires -from packageship.application.models.package import bin_pack +from packageship.application.models.package import SrcPack +from packageship.application.models.package import SrcRequires +from packageship.application.models.package import BinPack from packageship.application.models.package import maintenance_info -from packageship.application.models.package import bin_requires -from packageship.application.models.package import bin_provides +from packageship.application.models.package import BinRequires +from packageship.application.models.package import BinProvides from packageship.libs.exception import Error @@ -34,7 +34,7 @@ def get_packages(dbname): Error: Abnormal error """ with DBHelper(db_name=dbname) as db_name: - src_pack_queryset = db_name.session.query(src_pack).all() + src_pack_queryset = db_name.session.query(SrcPack).all() if src_pack_queryset is None: return None resp_list = list() @@ -108,20 +108,20 @@ def buildep_packages(dbname, src_pack_pkgkey): with DBHelper(db_name=dbname) as db_name: # srcpack's pkgkey to src_requires find pkgkey s_pack_requires_set = db_name.session.query( - src_requires).filter_by(pkgKey=src_pack_pkgkey).all() + SrcRequires).filter_by(pkgKey=src_pack_pkgkey).all() # src_requires pkykey to find the name of the dependent component s_pack_requires_names = [ s_pack_requires_obj.name for s_pack_requires_obj in s_pack_requires_set] # Find pkgkey in bin_provides by the name of the dependent component - b_pack_provides_set = db_name.session.query(bin_provides).filter( - bin_provides.name.in_(s_pack_requires_names)).all() + b_pack_provides_set = db_name.session.query(BinProvides).filter( + BinProvides.name.in_(s_pack_requires_names)).all() b_pack_provides_pkg_list = [ b_pack_provides_obj.pkgKey for b_pack_provides_obj in b_pack_provides_set] # Go to bin_pack to find the name by pkgkey of bin_provides - b_bin_pack_set = db_name.session.query(bin_pack).filter( - bin_pack.pkgKey.in_(b_pack_provides_pkg_list)).all() + b_bin_pack_set = db_name.session.query(BinPack).filter( + BinPack.pkgKey.in_(b_pack_provides_pkg_list)).all() builddep = [b_bin_pack_obj.name for b_bin_pack_obj in b_bin_pack_set] return builddep @@ -143,7 +143,7 @@ def sub_packages(dbname, sourcename): subpack = dict() # The name of src_pack finds the sub-package bin_pack query set i_bin_pack_set = db_name.session.query( - bin_pack).filter_by(src_name=sourcename).all() + BinPack).filter_by(src_name=sourcename).all() if i_bin_pack_set is None: return subpack # Find the objects of each sub-package @@ -153,18 +153,18 @@ def sub_packages(dbname, sourcename): # Find the names of the components required to install bin_requires # dependencies i_bin_requires_set = db_name.session.query( - bin_requires).filter_by(pkgKey=i_bin_pack_pkgkey).all() + BinRequires).filter_by(pkgKey=i_bin_pack_pkgkey).all() i_bin_requires_names = [ b_bin_requires_obj.name for b_bin_requires_obj in i_bin_requires_set] # Find pkykey in bin_provides by the name of the dependent # component - i_bin_provides_set = db_name.session.query(bin_provides).filter( - bin_provides.name.in_(i_bin_requires_names)) + i_bin_provides_set = db_name.session.query(BinProvides).filter( + BinProvides.name.in_(i_bin_requires_names)) i_bin_provides_pkg_list = [ i_bin_provides_obj.pkgKey for i_bin_provides_obj in i_bin_provides_set] # Find the name in bin_pack by pkgkey - i_bin_pack_set = db_name.session.query(bin_pack).filter( - bin_pack.pkgKey.in_(i_bin_provides_pkg_list)) + i_bin_pack_set = db_name.session.query(BinPack).filter( + BinPack.pkgKey.in_(i_bin_provides_pkg_list)) i_bin_pack_names = [ in_bin_pack_obj.name for in_bin_pack_obj in i_bin_pack_set] subpack[i_bin_pack_name] = i_bin_pack_names @@ -185,7 +185,7 @@ def get_single_package(dbname, sourcename): """ with DBHelper(db_name=dbname) as db_name: package = dict() - src_pack_obj = db_name.session.query(src_pack).filter_by( + src_pack_obj = db_name.session.query(SrcPack).filter_by( name=sourcename).first() if src_pack_obj is None: return None @@ -273,7 +273,7 @@ def _update_package_info( result_data = True with DBHelper(db_name=dbname) as data_name: update_obj = data_name.session.query( - src_pack).filter_by(name=package_name).first() + SrcPack).filter_by(name=package_name).first() if update_obj is None: return False update_obj.maintaniner = maintainer diff --git a/packageship/packageship/application/apps/package/function/searchdb.py b/packageship/packageship/application/apps/package/function/searchdb.py index b8a91112..fed2b753 100644 --- a/packageship/packageship/application/apps/package/function/searchdb.py +++ b/packageship/packageship/application/apps/package/function/searchdb.py @@ -15,7 +15,7 @@ from sqlalchemy import exists from packageship.libs.dbutils import DBHelper from packageship.libs.log import Log -from packageship.application.models.package import bin_pack +from packageship.application.models.package import BinPack from packageship.libs.exception import ContentNoneException, Error from packageship.system_config import DATABASE_FILE_INFO from .constants import ResponseCode @@ -106,9 +106,11 @@ class SearchDB(): get_list.append(result.search_name) if not result.depend_name and result.req_name: if result.req_name in provides_not_found: - provides_not_found[result.req_name].append([result.search_name, result.search_src_name, result.search_version, db_name]) + provides_not_found[result.req_name].append( + [result.search_name, result.search_src_name, result.search_version, db_name]) else: - provides_not_found[result.req_name] = [[result.search_name, result.search_src_name, result.search_version, db_name]] + provides_not_found[result.req_name] = [ + [result.search_name, result.search_src_name, result.search_version, db_name]] else: obj = return_tuple( result.depend_name, @@ -123,7 +125,8 @@ class SearchDB(): get_list.clear() search_set.symmetric_difference_update(get_set) if not search_set: - install_result = self._get_install_pro_in_other_database(provides_not_found) + install_result = self._get_install_pro_in_other_database( + provides_not_found) result_list.extend(install_result) return result_list else: @@ -132,7 +135,8 @@ class SearchDB(): LOGGER.logger.error(error_msg) except SQLAlchemyError as error_msg: LOGGER.logger.error(error_msg) - install_result = self._get_install_pro_in_other_database(provides_not_found) + install_result = self._get_install_pro_in_other_database( + provides_not_found) result_list.extend(install_result) for binary_name in search_set: result_list.append((return_tuple(None, None, None, @@ -155,14 +159,14 @@ class SearchDB(): """ for db_name, data_base in self.db_object_dict.items(): try: - bin_obj = data_base.session.query(bin_pack).filter_by( + bin_obj = data_base.session.query(BinPack).filter_by( name=binary_name ).first() source_name = bin_obj.src_name source_version = bin_obj.version if source_name is not None: return ResponseCode.SUCCESS, db_name, \ - source_name, source_version + source_name, source_version except AttributeError as error_msg: LOGGER.logger.error(error_msg) except SQLAlchemyError as error_msg: @@ -283,7 +287,8 @@ class SearchDB(): if bin_set: for result in bin_set: if result.req_name not in not_found_binary: - LOGGER.logger.warning(result.req_name + " contains in two rpm packages!!!") + LOGGER.logger.warning( + result.req_name + " contains in two rpm packages!!!") else: for source_info in not_found_binary[result.req_name]: obj = return_tuple( @@ -305,7 +310,8 @@ class SearchDB(): if not_found_binary: for key, values in not_found_binary.items(): - LOGGER.logger.warning("CANNOT FOUND THE component" + key + " in all database") + LOGGER.logger.warning( + "CANNOT FOUND THE component" + key + " in all database") return result_list def _get_install_pro_in_other_database(self, not_found_binary): @@ -317,7 +323,7 @@ class SearchDB(): search_list = [] result_list = [] for db_name, data_base in self.db_object_dict.items(): - for key,values in not_found_binary.items(): + for key, values in not_found_binary.items(): search_list.append(key) search_set = set(search_list) search_list.clear() @@ -336,11 +342,12 @@ class SearchDB(): """.format(literal_column('name').in_(search_set))) bin_set = data_base.session. \ execute(sql_string, {'name_{}'.format(i): v - for i, v in enumerate(search_set, 1)}).fetchall() + for i, v in enumerate(search_set, 1)}).fetchall() if bin_set: for result in bin_set: if result.req_name not in not_found_binary: - LOGGER.logger.warning(result.req_name + " contains in two rpm packages!!!") + LOGGER.logger.warning( + result.req_name + " contains in two rpm packages!!!") else: for binary_info in not_found_binary[result.req_name]: obj = return_tuple( @@ -455,15 +462,18 @@ class SearchDB(): get_list.clear() s_name_set.symmetric_difference_update(get_set) if not s_name_set: - build_result = self._get_binary_in_other_database(provides_not_found) + build_result = self._get_binary_in_other_database( + provides_not_found) build_list.extend(build_result) return ResponseCode.SUCCESS, build_list if s_name_set: - build_result = self._get_binary_in_other_database(provides_not_found) + build_result = self._get_binary_in_other_database( + provides_not_found) build_list.extend(build_result) for source in s_name_set: - LOGGER.logger.warning("CANNOT FOUND THE source " + source + " in all database") + LOGGER.logger.warning( + "CANNOT FOUND THE source " + source + " in all database") return ResponseCode.SUCCESS, build_list def binary_search_database_for_first_time(self, binary_name): @@ -480,7 +490,7 @@ class SearchDB(): try: for db_name, data_base in self.db_object_dict.items(): if data_base.session.query( - exists().where(bin_pack.name == binary_name) + exists().where(BinPack.name == binary_name) ).scalar(): return db_name except AttributeError as attr_err: diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index a34cf9be..4dbcb9e5 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -14,12 +14,12 @@ from packageship.libs.exception import DatabaseRepeatException from packageship.libs.exception import Error from packageship.libs.configutils.readconfig import ReadConfig from packageship.libs.log import Log -from packageship.application.models.package import src_pack -from packageship.application.models.package import bin_pack -from packageship.application.models.package import bin_requires -from packageship.application.models.package import src_requires -from packageship.application.models.package import bin_provides -from packageship.application.models.package import packages +from packageship.application.models.package import SrcPack +from packageship.application.models.package import BinPack +from packageship.application.models.package import BinRequires +from packageship.application.models.package import SrcRequires +from packageship.application.models.package import BinProvides +from packageship.application.models.package import Packages from packageship import system_config LOGGER = Log(__name__) @@ -58,11 +58,11 @@ class InitDataBase(): 'mysql': MysqlDatabaseOperations } self.database_name = None - self._tables = ['src_pack', 'bin_pack', - 'bin_requires', 'src_requires', 'bin_provides'] + self._tables = ['SrcPack', 'BinPack', + 'BinRequires', 'SrcRequires', 'BinProvides'] # Create life cycle related databases and tables if not self.create_database(db_name='lifecycle', - tables=['packages_issue'], + tables=['PackagesIssue'], storage=True): raise SQLAlchemyError( 'Failed to create the specified database and table：lifecycle') @@ -309,7 +309,7 @@ class InitDataBase(): '{db_name}:There is no relevant data in the source \ package provided '.format(db_name=db_name)) with DBHelper(db_name=db_name) as database: - database.batch_add(packages_datas, src_pack) + database.batch_add(packages_datas, SrcPack) self._storage_packages(table_name, packages_datas) @@ -318,7 +318,7 @@ class InitDataBase(): """ The mapping relationship of the orm model """ - model = type("packages", (packages, DBHelper.BASE), { + model = type("packages", (Packages, DBHelper.BASE), { '__tablename__': table_name}) return model @@ -372,7 +372,7 @@ class InitDataBase(): raise ContentNoneException('{db_name}: The package data that the source package \ depends on is empty'.format(db_name=db_name)) with DBHelper(db_name=db_name) as database: - database.batch_add(requires_datas, src_requires) + database.batch_add(requires_datas, SrcRequires) def _save_bin_packages(self, db_name): """ @@ -402,7 +402,7 @@ class InitDataBase(): bin_packaegs[index]['src_name'] = src_package_name with DBHelper(db_name=db_name) as database: - database.batch_add(bin_packaegs, bin_pack) + database.batch_add(bin_packaegs, BinPack) def _save_bin_requires(self, db_name): """ @@ -423,7 +423,7 @@ class InitDataBase(): dependency package'.format(db_name=db_name)) with DBHelper(db_name=db_name) as database: - database.batch_add(requires_datas, bin_requires) + database.batch_add(requires_datas, BinRequires) def _save_bin_provides(self, db_name): """ @@ -444,7 +444,7 @@ class InitDataBase(): binary component '.format(db_name=db_name)) with DBHelper(db_name=db_name) as database: - database.batch_add(provides_datas, bin_provides) + database.batch_add(provides_datas, BinProvides) def __exists_repeat_database(self): """ diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index 3c7b8564..509bbd57 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -7,7 +7,7 @@ from sqlalchemy import Column, Integer, String, Text from packageship.libs.dbutils.sqlalchemy_helper import DBHelper -class src_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 +class SrcPack(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Source package model """ @@ -44,7 +44,7 @@ class src_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 maintainlevel = Column(String(100), nullable=True) -class bin_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 +class BinPack(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Description: functional description:Binary package data """ @@ -79,7 +79,7 @@ class bin_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 src_name = Column(String(500), nullable=True) -class bin_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 +class BinRequires(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Binary package dependent package entity model """ @@ -96,7 +96,7 @@ class bin_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 pre = Column(String(20), nullable=True) -class src_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 +class SrcRequires(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Source entity package dependent package entity model """ @@ -112,7 +112,7 @@ class src_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 pre = Column(String(20), nullable=True) -class bin_provides(DBHelper.BASE): # pylint: disable=C0103,R0903 +class BinProvides(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Component entity model provided by binary package """ @@ -127,7 +127,7 @@ class bin_provides(DBHelper.BASE): # pylint: disable=C0103,R0903 pkgKey = Column(Integer, nullable=True) -class packages(): # pylint: disable=C0103,R0903 +class Packages(): # pylint: disable=C0103,R0903 """ Source code package version, issuer and other information """ @@ -164,7 +164,7 @@ class packages(): # pylint: disable=C0103,R0903 return model -class packages_issue(DBHelper.BASE): # pylint: disable=C0103,R0903 +class PackagesIssue(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Source package issue """ @@ -181,7 +181,7 @@ class packages_issue(DBHelper.BASE): # pylint: disable=C0103,R0903 related_release = Column(String(500), nullable=True) -class packages_maintainer(DBHelper.BASE): # pylint: disable=C0103,R0903 +class PackagesMaintainer(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Correspondence between source code package and maintainer """ diff --git a/packageship/test/test_module/dependent_query_tests/test_be_depend.py b/packageship/test/test_module/dependent_query_tests/test_be_depend.py index fec663f7..8fcf171a 100644 --- a/packageship/test/test_module/dependent_query_tests/test_be_depend.py +++ b/packageship/test/test_module/dependent_query_tests/test_be_depend.py @@ -10,11 +10,13 @@ from test.base_code.common_test_code import compare_two_values, get_correct_json from packageship.application.apps.package.function.constants import ResponseCode from packageship.application.apps.package.function.searchdb import db_priority + class TestBeDepend(ReadTestBase): """ The dependencies of the package are tested """ db_name = db_priority()[0] + def test_lack_parameter(self): """ Less transmission is always parameter transmission -- Gitee From 6d013b9a7024aebef6d572feed33fea78513a129 Mon Sep 17 00:00:00 2001 From: gongzt Date: Wed, 26 Aug 2020 16:05:46 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E5=88=97=E8=A1=A8=E6=8E=A8=E5=AF=BC?= =?UTF-8?q?=E5=BC=8F=E4=B8=ADif=E5=88=A4=E6=96=AD=E5=8D=95=E7=8B=AC?= =?UTF-8?q?=E5=86=99=E4=B8=80=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packageship/packageship/application/apps/lifecycle/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packageship/packageship/application/apps/lifecycle/view.py b/packageship/packageship/application/apps/lifecycle/view.py index 8ec3f21e..eb171314 100644 --- a/packageship/packageship/application/apps/lifecycle/view.py +++ b/packageship/packageship/application/apps/lifecycle/view.py @@ -155,8 +155,8 @@ class MaintainerView(Resource): with DBHelper(db_name='lifecycle') as database: maintainers = database.session.query( PackagesMaintainer.maintainer).group_by(PackagesMaintainer.maintainer).all() - return [maintainer_item[0] - for maintainer_item in maintainers if maintainer_item[0]] + return [maintainer_item[0] for maintainer_item in maintainers + if maintainer_item[0]] except (SQLAlchemyError, DisconnectionError) as error: current_app.logger.error(error) return [] -- Gitee From dc4b4e6867c10d9106dfade24ecad57c7ae83919 Mon Sep 17 00:00:00 2001 From: gongzt Date: Wed, 26 Aug 2020 21:28:30 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E5=91=BD=E5=90=8D=E3=80=81=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E7=9A=84=E6=8F=8F=E8=BF=B0=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apps/lifecycle/function/download_yaml.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packageship/packageship/application/apps/lifecycle/function/download_yaml.py b/packageship/packageship/application/apps/lifecycle/function/download_yaml.py index c2173841..63c1751e 100644 --- a/packageship/packageship/application/apps/lifecycle/function/download_yaml.py +++ b/packageship/packageship/application/apps/lifecycle/function/download_yaml.py @@ -24,7 +24,9 @@ from .gitee import Gitee class ParseYaml(): """ - Description: Download the contents of the yaml file + Description: Analyze the downloaded remote yaml file, obtain the tags + and maintainer information in the yaml file, and save the obtained + relevant information into the database Attributes: base: base class instance @@ -66,9 +68,11 @@ class ParseYaml(): _remote_url = 'https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/' return _remote_url + '{pkg_name}.yaml'.format(pkg_name=pkg_name) - def update_pkg_info(self): + def update_database(self): """ - Download the contents of the yaml file + For the current package, determine whether the specific yaml file exists, parse + the data in it and save it in the database if it exists, and record the relevant + log if it does not exist """ if self._openeuler_advisor_exists_yaml(): @@ -78,7 +82,7 @@ class ParseYaml(): obtained yet" % self.pkg.name self.base.log.logger.warning(msg) - def _read_yaml_content(self, url): + def _get_yaml_content(self, url): """ """ @@ -97,7 +101,7 @@ class ParseYaml(): package name under the openeuler-advisor project """ - self._read_yaml_content(self.openeuler_advisor_url) + self._get_yaml_content(self.openeuler_advisor_url) if self._yaml_content: return True return False @@ -201,7 +205,7 @@ def update_pkg_info(pkg_info_update=True): pkg_info=copy.deepcopy(package_item), base=base_control, table_name=table_name) - pool.submit(parse_yaml.update_pkg_info) + pool.submit(parse_yaml.update_database) else: # Get the issue of each warehouse and save it gitee_issue = Gitee( -- Gitee