diff --git a/packageship/packageship/application/__init__.py b/packageship/packageship/application/__init__.py index bc3a6316a89f8f9f41d91387a43444657aab3f5b..810d6ce59ab6936a9c982d1fb8e8a3dcdfe778f4 100644 --- a/packageship/packageship/application/__init__.py +++ b/packageship/packageship/application/__init__.py @@ -2,14 +2,44 @@ """ Initial operation and configuration of the flask project """ +import sys +import threading from flask import Flask from flask_session import Session +from flask_apscheduler import APScheduler +from packageship import system_config from packageship.application.settings import Config from packageship.libs.log import setup_log +from packageship.libs.configutils.readconfig import ReadConfig OPERATION = None +def _timed_task(app): + """ + + """ + from .apps.lifecycle.function.download_yaml import update_pkg_info # pylint: disable=import-outside-toplevel + + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + + _hour = _readconfig.get_config('TIMEDTASK', 'hour') + if not _hour or not isinstance(_hour, int) or _hour < 0 or _hour > 23: + _hour = 3 + _minute = _readconfig.get_config('TIMEDTASK', 'minute') + if not _hour or not isinstance(_hour, int) or _hour < 0 or _hour > 59: + _minute = 0 + app.apscheduler.add_job( # pylint: disable=no-member + func=update_pkg_info, id="update_package_data", trigger="cron", hour=_hour, minute=_minute) + app.apscheduler.add_job( # pylint: disable=no-member + func=update_pkg_info, + id="issue_catch", + trigger="cron", + hour=_hour, + minute=_minute, + args=(False,)) + + def init_app(operation): """ Project initialization function @@ -23,16 +53,23 @@ def init_app(operation): app.config.from_object(Config) + # Register a scheduled task + scheduler = APScheduler() + scheduler.init_app(app) + scheduler.start() + # Open session function Session(app) - global OPERATION + global OPERATION # pylint: disable=global-statement OPERATION = operation # Register Blueprint - from packageship.application.apps import blue_point - for blue, api in blue_point: + from packageship.application import apps # pylint: disable=import-outside-toplevel + for blue, api in apps.blue_point: api.init_app(app) app.register_blueprint(blue) + _timed_task(app) + return app diff --git a/packageship/packageship/application/app_global.py b/packageship/packageship/application/app_global.py index 25d9dbe0301c55e2856f9fca10b15f143a7f33b1..611f309e9cab2895f7462fd45acacbff727ff468 100644 --- a/packageship/packageship/application/app_global.py +++ b/packageship/packageship/application/app_global.py @@ -19,8 +19,8 @@ def identity_verification(): """ if request.url_rule: url_rule = request.url_rule.rule - for view, url, authentication in urls: - if url == url_rule and application.OPERATION in authentication.keys(): + for _view, url, authentication in urls: + if url.lower() == url_rule.lower() and application.OPERATION in authentication.keys(): if request.method not in authentication.get(application.OPERATION): return False break diff --git a/packageship/packageship/application/apps/__init__.py b/packageship/packageship/application/apps/__init__.py index 0cb8d57f664444dcde627a8b0ec155954847abd0..e1d108ef25d45e5f5933389e5bc42705ade2bf24 100644 --- a/packageship/packageship/application/apps/__init__.py +++ b/packageship/packageship/application/apps/__init__.py @@ -2,10 +2,12 @@ """ Blueprint collection trying to page """ -from packageship.application.apps.package import package, api as package_api +from packageship.application.apps import package +from packageship.application.apps import lifecycle -blue_point = [ - (package, package_api), +blue_point = [ # pylint: disable=invalid-name + (package.package, package.api), + (lifecycle.lifecycle, lifecycle.api) ] __all__ = ['blue_point'] diff --git a/packageship/packageship/application/apps/lifecycle/__init__.py b/packageship/packageship/application/apps/lifecycle/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d17a06a54ae6322f651c9e9f125b645e31696989 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 +""" + Blueprint registration for life cycle +""" +from flask.blueprints import Blueprint +from flask_restful import Api +from packageship.application.apps.lifecycle.url import urls +from packageship import application + +lifecycle = Blueprint('lifecycle', __name__) + +# init restapi +api = Api() + +for view, url, operation in urls: + if application.OPERATION and application.OPERATION in operation.keys(): + api.add_resource(view, url) + + +__all__ = ['lifecycle', 'api'] diff --git a/packageship/packageship/application/apps/lifecycle/function/__init__.py b/packageship/packageship/application/apps/lifecycle/function/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1d9852d5d5ea7bd61eb19baf148cb26b4ea326a0 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/function/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/python3 +""" + Business approach to package life cycle +""" diff --git a/packageship/packageship/application/apps/lifecycle/function/base.py b/packageship/packageship/application/apps/lifecycle/function/base.py new file mode 100644 index 0000000000000000000000000000000000000000..3631dde4aae9dba270c91c60f08d4e66511fb7e1 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/function/base.py @@ -0,0 +1,17 @@ +#!/usr/bin/python3 +""" +General approach to version control tools +""" +from packageship.libs.log import Log + + +class Base(): + """ + Public method to get project tags and download yaml file + """ + + def __init__(self): + self.headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW 64; rv:50.0) Gecko/20100101 \ + Firefox / 50.0 '} + self.log = Log(__name__) diff --git a/packageship/packageship/application/apps/lifecycle/function/download_yaml.py b/packageship/packageship/application/apps/lifecycle/function/download_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..63c1751eb6e9a1e16f786fc381bd175c2d479ae4 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/function/download_yaml.py @@ -0,0 +1,216 @@ +#!/usr/bin/python3 +""" +Dynamically obtain the content of the yaml file \ +that saves the package information, periodically \ +obtain the content and save it in the database +""" +import copy +from concurrent.futures import ThreadPoolExecutor +import datetime as date +import requests +import yaml +from sqlalchemy.exc import SQLAlchemyError +from requests.exceptions import HTTPError +from packageship import system_config +from packageship.application.models.package import Packages +from packageship.application.models.package import PackagesMaintainer +from packageship.libs.dbutils import DBHelper +from packageship.libs.exception import Error, ContentNoneException +from packageship.libs.configutils.readconfig import ReadConfig + +from .base import Base +from .gitee import Gitee + + +class ParseYaml(): + """ + Description: Analyze the downloaded remote yaml file, obtain the tags + and maintainer information in the yaml file, and save the obtained + relevant information into the database + + Attributes: + base: base class instance + pkg: Specific package data + _table_name: The name of the data table to be operated + openeuler_advisor_url: Get the warehouse address of the yaml file + _yaml_content: The content of the yaml file + """ + + def __init__(self, pkg_info, base, table_name): + self.base = base + self.pkg = pkg_info + self._table_name = table_name + self.openeuler_advisor_url = self._path_stitching(pkg_info.name) + self._yaml_content = None + self.timed_task_open = self._timed_task_status() + + def _timed_task_status(self): + """ + The open state of information such as the maintainer in the scheduled task + """ + _timed_task_status = True + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + open_status = _readconfig.get_config('TIMEDTASK', 'open') + if open_status not in ('True', 'False'): + self.base.log.logger.error( + 'Wrong setting of the open state value of the scheduled task') + if open_status == 'False': + self.timed_task_open = False + return _timed_task_status + + def _path_stitching(self, pkg_name): + """ + The path of the remote service call + """ + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + _remote_url = _readconfig.get_config('LIFECYCLE', 'warehouse_remote') + if _remote_url is None: + _remote_url = 'https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/' + return _remote_url + '{pkg_name}.yaml'.format(pkg_name=pkg_name) + + def update_database(self): + """ + For the current package, determine whether the specific yaml file exists, parse + the data in it and save it in the database if it exists, and record the relevant + log if it does not exist + + """ + if self._openeuler_advisor_exists_yaml(): + self._save_to_database() + else: + msg = "The yaml information of the %s package has not been\ + obtained yet" % self.pkg.name + self.base.log.logger.warning(msg) + + def _get_yaml_content(self, url): + """ + + """ + try: + response = requests.get( + url, headers=self.base.headers) + if response.status_code == 200: + self._yaml_content = yaml.safe_load(response.content) + + except HTTPError as error: + self.base.log.logger.error(error) + + def _openeuler_advisor_exists_yaml(self): + """ + Determine whether there is a yaml file with the current \ + package name under the openeuler-advisor project + + """ + self._get_yaml_content(self.openeuler_advisor_url) + if self._yaml_content: + return True + return False + + def _save_to_database(self): + """ + Save the acquired yaml file information to the database + + Raises: + ContentNoneException: The added entity content is empty + Error: An error occurred during data addition + """ + self._parse_warehouse_info() + tags = self._yaml_content.get('git_tag', None) + self._parse_tags_content(tags) + if self.timed_task_open: + _maintainer = self._yaml_content.get('maintainer') + if _maintainer and isinstance(_maintainer, list): + self.pkg.maintainer = _maintainer[0] + self.pkg.maintainlevel = self._yaml_content.get('maintainlevel') + try: + with DBHelper(db_name="lifecycle") as database: + database.session.begin(subtransactions=True) + database.add(self.pkg) + if self.timed_task_open and self.pkg.maintainer: + _packages_maintainer = database.session.query( + PackagesMaintainer).filter( + PackagesMaintainer.name == self.pkg.name).first() + if _packages_maintainer: + _packages_maintainer.name = self.pkg.name + _packages_maintainer.maintainer = self.pkg.maintainer + _packages_maintainer.maintainlevel = self.pkg.maintainlevel + else: + database.add(PackagesMaintainer( + name=self.pkg.name, maintainer=self.pkg.maintainer, + maintainlevel=self.pkg.maintainlevel)) + database.session.commit() + except (Error, ContentNoneException, SQLAlchemyError) as error: + self.base.log.logger.error(error) + + def _parse_warehouse_info(self): + """ + Parse the warehouse information in the yaml file + + """ + if self._yaml_content: + self.pkg.version_control = self._yaml_content.get( + 'version_control') + self.pkg.src_repo = self._yaml_content.get('src_repo') + self.pkg.tag_prefix = self._yaml_content.get('tag_prefix') + + def _parse_tags_content(self, tags): + """ + Parse the obtained tags content + + """ + try: + # Integrate tags information into key-value pairs + _tags = [(tag.split()[0], tag.split()[1]) for tag in tags] + _tags = sorted(_tags, key=lambda x: x[0], reverse=True) + self.pkg.latest_version = _tags[0][1] + self.pkg.latest_version_time = _tags[0][0] + _end_time = date.datetime.strptime( + self.pkg.latest_version_time, '%Y-%m-%d') + if self.pkg.latest_version != self.pkg.version: + for _version in _tags: + if _version[1] == self.pkg.version: + _end_time = date.datetime.strptime( + _version[0], '%Y-%m-%d') + self.pkg.used_time = (date.datetime.now() - _end_time).days + + except (IndexError, Error) as index_error: + self.base.log.logger.error(index_error) + + +def update_pkg_info(pkg_info_update=True): + """ + Update the information of the upstream warehouse in the source package + + """ + try: + base_control = Base() + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') + _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse') + if _warehouse is None: + _warehouse = 'src-openeuler' + if not isinstance(pool_workers, int): + pool_workers = 10 + # Open thread pool + pool = ThreadPoolExecutor(max_workers=pool_workers) + with DBHelper(db_name="lifecycle") as database: + for table_name in filter(lambda x: x not in ['packages_issue', 'PackagesMaintainer'], + database.engine.table_names()): + + cls_model = Packages.package_meta(table_name) + # Query a specific table + for package_item in database.session.query(cls_model).all(): + if pkg_info_update: + parse_yaml = ParseYaml( + pkg_info=copy.deepcopy(package_item), + base=base_control, + table_name=table_name) + pool.submit(parse_yaml.update_database) + else: + # Get the issue of each warehouse and save it + gitee_issue = Gitee( + package_item, _warehouse, package_item.name, table_name) + pool.submit(gitee_issue.query_issues_info) + pool.shutdown() + except SQLAlchemyError as error_msg: + base_control.log.logger.error(error_msg) diff --git a/packageship/packageship/application/apps/lifecycle/serialize.py b/packageship/packageship/application/apps/lifecycle/serialize.py new file mode 100644 index 0000000000000000000000000000000000000000..a9ccbda3471dc81748169fc3377e127fb2dd3cbc --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/serialize.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 +""" +Description: marshmallow serialize +""" +from marshmallow import Schema +from packageship.application.models.package import PackagesIssue, Packages + + +class IssueDownloadSchema(Schema): + """ + Field serialization for issue file download + """ + + class Meta: + """Model mapping serialized fields""" + model = PackagesIssue + fields = ('issue_id', 'issue_url', 'issue_content', + 'issue_title', 'issue_status', 'pkg_name', 'issue_type', 'related_release') + + +class PackagesDownloadSchema(Schema): + """ + Field serialization for package file download + """ + + class Meta: + """Model mapping serialized fields""" + model = Packages + fields = ('name', 'url', 'rpm_license', 'version', 'release', 'release_time', + 'used_time', 'latest_version', 'latest_version_time', + 'feature', 'cve', 'defect', 'maintainer', 'maintainlevel') diff --git a/packageship/packageship/application/apps/lifecycle/url.py b/packageship/packageship/application/apps/lifecycle/url.py new file mode 100644 index 0000000000000000000000000000000000000000..ea08d589483de3ab428582b56d5fe2368adfa670 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/url.py @@ -0,0 +1,14 @@ +#!/usr/bin/python3 +""" +Life cycle of url giant whale collection +""" +from . import view + +urls = [ # pylint: disable=invalid-name + # Download package data or iSsue data + (view.DownloadFile, '/lifeCycle/download/', {'query': ('GET')}), + # Get a collection of maintainers list + (view.MaintainerView, '/lifeCycle/maintainer', {'query': ('GET')}), + # Get the columns that need to be displayed by default in the package + (view.TableColView, '/packages/tablecol', {'query': ('GET')}), +] diff --git a/packageship/packageship/application/apps/lifecycle/view.py b/packageship/packageship/application/apps/lifecycle/view.py new file mode 100644 index 0000000000000000000000000000000000000000..eb171314c2c440c4067081333b253d4c7f108639 --- /dev/null +++ b/packageship/packageship/application/apps/lifecycle/view.py @@ -0,0 +1,223 @@ +#!/usr/bin/python3 +""" +Life cycle related api interface +""" +import io +import json +import math +import os +from concurrent.futures import ThreadPoolExecutor + +import pandas as pd +import yaml +from flask import request +from flask import jsonify, make_response +from flask import current_app +from flask_restful import Resource +from marshmallow import ValidationError +from sqlalchemy.exc import DisconnectionError, SQLAlchemyError + +from packageship import system_config +from packageship.application.apps.package.serialize import DataFormatVerfi, UpdatePackagesSchema +from packageship.libs.configutils.readconfig import ReadConfig +from packageship.libs.exception import Error +from packageship.application.apps.package.function.constants import ResponseCode +from packageship.libs.dbutils.sqlalchemy_helper import DBHelper +from packageship.application.models.package import PackagesIssue +from packageship.application.models.package import Packages +from packageship.application.models.package import PackagesMaintainer +from packageship.libs.log import Log +from .serialize import IssueDownloadSchema, PackagesDownloadSchema +from .function.gitee import Gitee as gitee + +LOGGER = Log(__name__) + + +# pylint: disable = no-self-use + +class DownloadFile(Resource): + """ + Download the content of the issue or the excel file of the package content + """ + + def _download_excel(self, file_type, table_name=None): + """ + Download excel file + """ + file_name = 'packages.xlsx' + if file_type == 'packages': + download_content = self.__get_packages_content(table_name) + else: + file_name = 'issues.xlsx' + download_content = self.__get_issues_content() + if download_content is None: + return jsonify( + ResponseCode.response_json( + ResponseCode.SERVICE_ERROR)) + pd_dataframe = self.__to_dataframe(download_content) + + _response = self.__bytes_save(pd_dataframe) + return self.__set_response_header(_response, file_name) + + def __bytes_save(self, data_frame): + """ + Save the file content in the form of a binary file stream + """ + try: + bytes_io = io.BytesIO() + writer = pd.ExcelWriter( # pylint: disable=abstract-class-instantiated + bytes_io, engine='xlsxwriter') + data_frame.to_excel(writer, sheet_name='Summary', index=False) + writer.save() + writer.close() + bytes_io.seek(0) + _response = make_response(bytes_io.getvalue()) + bytes_io.close() + return _response + except (IOError, Error) as io_error: + current_app.logger.error(io_error) + return make_response() + + def __set_response_header(self, response, file_name): + """ + Set http response header information + """ + response.headers['Content-Type'] = \ + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + response.headers["Cache-Control"] = "no-cache" + response.headers['Content-Disposition'] = 'attachment; filename={file_name}'.format( + file_name=file_name) + return response + + def __get_packages_content(self, table_name): + """ + Get package list information + """ + try: + with DBHelper(db_name='lifecycle') as database: + # Query all package data in the specified table + _model = Packages.package_meta(table_name) + _packageinfos = database.session.query(_model).all() + packages_dicts = PackagesDownloadSchema( + many=True).dump(_packageinfos) + return packages_dicts + + except (SQLAlchemyError, DisconnectionError) as error: + current_app.logger.error(error) + return None + + def __get_issues_content(self): + """ + Get the list of issues + """ + try: + with DBHelper(db_name='lifecycle') as database: + _issues = database.session.query(PackagesIssue).all() + issues_dicts = IssueDownloadSchema(many=True).dump(_issues) + return issues_dicts + except (SQLAlchemyError, DisconnectionError) as error: + current_app.logger.error(error) + return None + + def __to_dataframe(self, datas): + """ + Convert the obtained information into pandas content format + """ + + data_frame = pd.DataFrame(datas) + return data_frame + + def get(self, file_type): + """ + Download package collection information and isse list information + + """ + if file_type not in ['packages', 'issues']: + return jsonify( + ResponseCode.response_json( + ResponseCode.PARAM_ERROR)) + + table_name = request.args.get('table_name', None) + response = self._download_excel(file_type, table_name) + return response + + +class MaintainerView(Resource): + """ + Maintainer name collection + """ + + def __query_maintainers(self): + """ + Query the names of all maintainers in the specified table + """ + try: + with DBHelper(db_name='lifecycle') as database: + maintainers = database.session.query( + PackagesMaintainer.maintainer).group_by(PackagesMaintainer.maintainer).all() + return [maintainer_item[0] for maintainer_item in maintainers + if maintainer_item[0]] + except (SQLAlchemyError, DisconnectionError) as error: + current_app.logger.error(error) + return [] + + def get(self): + """ + Get the list of maintainers + """ + # Group query of the names of all maintainers in the current table + maintainers = self.__query_maintainers() + return jsonify(ResponseCode.response_json( + ResponseCode.SUCCESS, + maintainers)) + + +class TableColView(Resource): + """ + The default column of the package shows the interface + """ + + def __columns_names(self): + """ + Mapping of column name and title + """ + columns = [ + ('name', 'Name', True), + ('version', 'Version', True), + ('release', 'Release', True), + ('url', 'Url', True), + ('linense', 'License', False), + ('feature', 'Feature', False), + ('maintainer', 'Maintainer', True), + ('maintainlevel', 'Maintenance Level', True), + ('release_time', 'Release Time', False), + ('end_of_lifecycle', 'End of Life Cycle', True), + ('maintainer_status', 'Maintain Status', True), + ('latest_version', 'Latest Version', False), + ('latest_version_time', 'Latest Version Release Time', False), + ('issue', 'Issue', True)] + return columns + + def __columns_mapping(self): + """ + + """ + columns = list() + for column in self.__columns_names(): + columns.append({ + 'column_name': column[0], + 'label': column[1], + 'default_selected': column[2] + }) + return columns + + def get(self): + """ + Get the default display column of the package + + """ + table_mapping_columns = self.__columns_mapping() + return jsonify( + ResponseCode.response_json( + ResponseCode.SUCCESS, + table_mapping_columns)) diff --git a/packageship/packageship/application/apps/package/function/be_depend.py b/packageship/packageship/application/apps/package/function/be_depend.py index da50f0d4c6db7af261aec6302d0c4e2d6e7c8140..a9f9769c24875c8a0459b460b32e98635a9f801b 100644 --- a/packageship/packageship/application/apps/package/function/be_depend.py +++ b/packageship/packageship/application/apps/package/function/be_depend.py @@ -10,7 +10,7 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.sql import literal_column from flask import current_app from packageship.libs.dbutils import DBHelper -from packageship.application.models.package import src_pack +from packageship.application.models.package import SrcPack from packageship.application.apps.package.function.constants import ResponseCode @@ -59,7 +59,7 @@ class BeDepend(): """ with DBHelper(db_name=self.db_name) as data_base: src_obj = data_base.session.query( - src_pack).filter_by(name=self.source_name).first() + SrcPack).filter_by(name=self.source_name).first() if src_obj: # spell dictionary self.result_dict[self.source_name + "_src"] = [ diff --git a/packageship/packageship/application/apps/package/function/packages.py b/packageship/packageship/application/apps/package/function/packages.py index 5bf347d6ed186d9db1f9dc37d8e0c802d9e8679a..b5b5139b6310c1064e5ba63701c1201cf893ad52 100644 --- a/packageship/packageship/application/apps/package/function/packages.py +++ b/packageship/packageship/application/apps/package/function/packages.py @@ -12,12 +12,12 @@ from packageship.application.apps.package.function.constants import ResponseCode from packageship.application.apps.package.function.searchdb import db_priority from packageship.libs.dbutils import DBHelper -from packageship.application.models.package import src_pack -from packageship.application.models.package import src_requires -from packageship.application.models.package import bin_pack +from packageship.application.models.package import SrcPack +from packageship.application.models.package import SrcRequires +from packageship.application.models.package import BinPack from packageship.application.models.package import maintenance_info -from packageship.application.models.package import bin_requires -from packageship.application.models.package import bin_provides +from packageship.application.models.package import BinRequires +from packageship.application.models.package import BinProvides from packageship.libs.exception import Error @@ -34,7 +34,7 @@ def get_packages(dbname): Error: Abnormal error """ with DBHelper(db_name=dbname) as db_name: - src_pack_queryset = db_name.session.query(src_pack).all() + src_pack_queryset = db_name.session.query(SrcPack).all() if src_pack_queryset is None: return None resp_list = list() @@ -108,20 +108,20 @@ def buildep_packages(dbname, src_pack_pkgkey): with DBHelper(db_name=dbname) as db_name: # srcpack's pkgkey to src_requires find pkgkey s_pack_requires_set = db_name.session.query( - src_requires).filter_by(pkgKey=src_pack_pkgkey).all() + SrcRequires).filter_by(pkgKey=src_pack_pkgkey).all() # src_requires pkykey to find the name of the dependent component s_pack_requires_names = [ s_pack_requires_obj.name for s_pack_requires_obj in s_pack_requires_set] # Find pkgkey in bin_provides by the name of the dependent component - b_pack_provides_set = db_name.session.query(bin_provides).filter( - bin_provides.name.in_(s_pack_requires_names)).all() + b_pack_provides_set = db_name.session.query(BinProvides).filter( + BinProvides.name.in_(s_pack_requires_names)).all() b_pack_provides_pkg_list = [ b_pack_provides_obj.pkgKey for b_pack_provides_obj in b_pack_provides_set] # Go to bin_pack to find the name by pkgkey of bin_provides - b_bin_pack_set = db_name.session.query(bin_pack).filter( - bin_pack.pkgKey.in_(b_pack_provides_pkg_list)).all() + b_bin_pack_set = db_name.session.query(BinPack).filter( + BinPack.pkgKey.in_(b_pack_provides_pkg_list)).all() builddep = [b_bin_pack_obj.name for b_bin_pack_obj in b_bin_pack_set] return builddep @@ -143,7 +143,7 @@ def sub_packages(dbname, sourcename): subpack = dict() # The name of src_pack finds the sub-package bin_pack query set i_bin_pack_set = db_name.session.query( - bin_pack).filter_by(src_name=sourcename).all() + BinPack).filter_by(src_name=sourcename).all() if i_bin_pack_set is None: return subpack # Find the objects of each sub-package @@ -153,18 +153,18 @@ def sub_packages(dbname, sourcename): # Find the names of the components required to install bin_requires # dependencies i_bin_requires_set = db_name.session.query( - bin_requires).filter_by(pkgKey=i_bin_pack_pkgkey).all() + BinRequires).filter_by(pkgKey=i_bin_pack_pkgkey).all() i_bin_requires_names = [ b_bin_requires_obj.name for b_bin_requires_obj in i_bin_requires_set] # Find pkykey in bin_provides by the name of the dependent # component - i_bin_provides_set = db_name.session.query(bin_provides).filter( - bin_provides.name.in_(i_bin_requires_names)) + i_bin_provides_set = db_name.session.query(BinProvides).filter( + BinProvides.name.in_(i_bin_requires_names)) i_bin_provides_pkg_list = [ i_bin_provides_obj.pkgKey for i_bin_provides_obj in i_bin_provides_set] # Find the name in bin_pack by pkgkey - i_bin_pack_set = db_name.session.query(bin_pack).filter( - bin_pack.pkgKey.in_(i_bin_provides_pkg_list)) + i_bin_pack_set = db_name.session.query(BinPack).filter( + BinPack.pkgKey.in_(i_bin_provides_pkg_list)) i_bin_pack_names = [ in_bin_pack_obj.name for in_bin_pack_obj in i_bin_pack_set] subpack[i_bin_pack_name] = i_bin_pack_names @@ -185,7 +185,7 @@ def get_single_package(dbname, sourcename): """ with DBHelper(db_name=dbname) as db_name: package = dict() - src_pack_obj = db_name.session.query(src_pack).filter_by( + src_pack_obj = db_name.session.query(SrcPack).filter_by( name=sourcename).first() if src_pack_obj is None: return None @@ -273,7 +273,7 @@ def _update_package_info( result_data = True with DBHelper(db_name=dbname) as data_name: update_obj = data_name.session.query( - src_pack).filter_by(name=package_name).first() + SrcPack).filter_by(name=package_name).first() if update_obj is None: return False update_obj.maintaniner = maintainer diff --git a/packageship/packageship/application/apps/package/function/searchdb.py b/packageship/packageship/application/apps/package/function/searchdb.py index b8a91112aac12c4c2161f79454e5c6cbcf37bf60..fed2b753d6d17fe876cd505176097066c9d0b7bf 100644 --- a/packageship/packageship/application/apps/package/function/searchdb.py +++ b/packageship/packageship/application/apps/package/function/searchdb.py @@ -15,7 +15,7 @@ from sqlalchemy import exists from packageship.libs.dbutils import DBHelper from packageship.libs.log import Log -from packageship.application.models.package import bin_pack +from packageship.application.models.package import BinPack from packageship.libs.exception import ContentNoneException, Error from packageship.system_config import DATABASE_FILE_INFO from .constants import ResponseCode @@ -106,9 +106,11 @@ class SearchDB(): get_list.append(result.search_name) if not result.depend_name and result.req_name: if result.req_name in provides_not_found: - provides_not_found[result.req_name].append([result.search_name, result.search_src_name, result.search_version, db_name]) + provides_not_found[result.req_name].append( + [result.search_name, result.search_src_name, result.search_version, db_name]) else: - provides_not_found[result.req_name] = [[result.search_name, result.search_src_name, result.search_version, db_name]] + provides_not_found[result.req_name] = [ + [result.search_name, result.search_src_name, result.search_version, db_name]] else: obj = return_tuple( result.depend_name, @@ -123,7 +125,8 @@ class SearchDB(): get_list.clear() search_set.symmetric_difference_update(get_set) if not search_set: - install_result = self._get_install_pro_in_other_database(provides_not_found) + install_result = self._get_install_pro_in_other_database( + provides_not_found) result_list.extend(install_result) return result_list else: @@ -132,7 +135,8 @@ class SearchDB(): LOGGER.logger.error(error_msg) except SQLAlchemyError as error_msg: LOGGER.logger.error(error_msg) - install_result = self._get_install_pro_in_other_database(provides_not_found) + install_result = self._get_install_pro_in_other_database( + provides_not_found) result_list.extend(install_result) for binary_name in search_set: result_list.append((return_tuple(None, None, None, @@ -155,14 +159,14 @@ class SearchDB(): """ for db_name, data_base in self.db_object_dict.items(): try: - bin_obj = data_base.session.query(bin_pack).filter_by( + bin_obj = data_base.session.query(BinPack).filter_by( name=binary_name ).first() source_name = bin_obj.src_name source_version = bin_obj.version if source_name is not None: return ResponseCode.SUCCESS, db_name, \ - source_name, source_version + source_name, source_version except AttributeError as error_msg: LOGGER.logger.error(error_msg) except SQLAlchemyError as error_msg: @@ -283,7 +287,8 @@ class SearchDB(): if bin_set: for result in bin_set: if result.req_name not in not_found_binary: - LOGGER.logger.warning(result.req_name + " contains in two rpm packages!!!") + LOGGER.logger.warning( + result.req_name + " contains in two rpm packages!!!") else: for source_info in not_found_binary[result.req_name]: obj = return_tuple( @@ -305,7 +310,8 @@ class SearchDB(): if not_found_binary: for key, values in not_found_binary.items(): - LOGGER.logger.warning("CANNOT FOUND THE component" + key + " in all database") + LOGGER.logger.warning( + "CANNOT FOUND THE component" + key + " in all database") return result_list def _get_install_pro_in_other_database(self, not_found_binary): @@ -317,7 +323,7 @@ class SearchDB(): search_list = [] result_list = [] for db_name, data_base in self.db_object_dict.items(): - for key,values in not_found_binary.items(): + for key, values in not_found_binary.items(): search_list.append(key) search_set = set(search_list) search_list.clear() @@ -336,11 +342,12 @@ class SearchDB(): """.format(literal_column('name').in_(search_set))) bin_set = data_base.session. \ execute(sql_string, {'name_{}'.format(i): v - for i, v in enumerate(search_set, 1)}).fetchall() + for i, v in enumerate(search_set, 1)}).fetchall() if bin_set: for result in bin_set: if result.req_name not in not_found_binary: - LOGGER.logger.warning(result.req_name + " contains in two rpm packages!!!") + LOGGER.logger.warning( + result.req_name + " contains in two rpm packages!!!") else: for binary_info in not_found_binary[result.req_name]: obj = return_tuple( @@ -455,15 +462,18 @@ class SearchDB(): get_list.clear() s_name_set.symmetric_difference_update(get_set) if not s_name_set: - build_result = self._get_binary_in_other_database(provides_not_found) + build_result = self._get_binary_in_other_database( + provides_not_found) build_list.extend(build_result) return ResponseCode.SUCCESS, build_list if s_name_set: - build_result = self._get_binary_in_other_database(provides_not_found) + build_result = self._get_binary_in_other_database( + provides_not_found) build_list.extend(build_result) for source in s_name_set: - LOGGER.logger.warning("CANNOT FOUND THE source " + source + " in all database") + LOGGER.logger.warning( + "CANNOT FOUND THE source " + source + " in all database") return ResponseCode.SUCCESS, build_list def binary_search_database_for_first_time(self, binary_name): @@ -480,7 +490,7 @@ class SearchDB(): try: for db_name, data_base in self.db_object_dict.items(): if data_base.session.query( - exists().where(bin_pack.name == binary_name) + exists().where(BinPack.name == binary_name) ).scalar(): return db_name except AttributeError as attr_err: diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index a34cf9be8c0c17bf665faf89ce1bee195caed520..4dbcb9e51ba9837599dcc0ac5ee55a2e6d6ac1a7 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -14,12 +14,12 @@ from packageship.libs.exception import DatabaseRepeatException from packageship.libs.exception import Error from packageship.libs.configutils.readconfig import ReadConfig from packageship.libs.log import Log -from packageship.application.models.package import src_pack -from packageship.application.models.package import bin_pack -from packageship.application.models.package import bin_requires -from packageship.application.models.package import src_requires -from packageship.application.models.package import bin_provides -from packageship.application.models.package import packages +from packageship.application.models.package import SrcPack +from packageship.application.models.package import BinPack +from packageship.application.models.package import BinRequires +from packageship.application.models.package import SrcRequires +from packageship.application.models.package import BinProvides +from packageship.application.models.package import Packages from packageship import system_config LOGGER = Log(__name__) @@ -58,11 +58,11 @@ class InitDataBase(): 'mysql': MysqlDatabaseOperations } self.database_name = None - self._tables = ['src_pack', 'bin_pack', - 'bin_requires', 'src_requires', 'bin_provides'] + self._tables = ['SrcPack', 'BinPack', + 'BinRequires', 'SrcRequires', 'BinProvides'] # Create life cycle related databases and tables if not self.create_database(db_name='lifecycle', - tables=['packages_issue'], + tables=['PackagesIssue'], storage=True): raise SQLAlchemyError( 'Failed to create the specified database and table：lifecycle') @@ -309,7 +309,7 @@ class InitDataBase(): '{db_name}:There is no relevant data in the source \ package provided '.format(db_name=db_name)) with DBHelper(db_name=db_name) as database: - database.batch_add(packages_datas, src_pack) + database.batch_add(packages_datas, SrcPack) self._storage_packages(table_name, packages_datas) @@ -318,7 +318,7 @@ class InitDataBase(): """ The mapping relationship of the orm model """ - model = type("packages", (packages, DBHelper.BASE), { + model = type("packages", (Packages, DBHelper.BASE), { '__tablename__': table_name}) return model @@ -372,7 +372,7 @@ class InitDataBase(): raise ContentNoneException('{db_name}: The package data that the source package \ depends on is empty'.format(db_name=db_name)) with DBHelper(db_name=db_name) as database: - database.batch_add(requires_datas, src_requires) + database.batch_add(requires_datas, SrcRequires) def _save_bin_packages(self, db_name): """ @@ -402,7 +402,7 @@ class InitDataBase(): bin_packaegs[index]['src_name'] = src_package_name with DBHelper(db_name=db_name) as database: - database.batch_add(bin_packaegs, bin_pack) + database.batch_add(bin_packaegs, BinPack) def _save_bin_requires(self, db_name): """ @@ -423,7 +423,7 @@ class InitDataBase(): dependency package'.format(db_name=db_name)) with DBHelper(db_name=db_name) as database: - database.batch_add(requires_datas, bin_requires) + database.batch_add(requires_datas, BinRequires) def _save_bin_provides(self, db_name): """ @@ -444,7 +444,7 @@ class InitDataBase(): binary component '.format(db_name=db_name)) with DBHelper(db_name=db_name) as database: - database.batch_add(provides_datas, bin_provides) + database.batch_add(provides_datas, BinProvides) def __exists_repeat_database(self): """ diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index 725f4af7db3d68766eeb58da58f50b145be28cee..509bbd57b1a5866b6303550f83d5b0b027a2c915 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -2,11 +2,12 @@ """ Description: Database entity model mapping """ -from sqlalchemy import Column, Integer, String +import uuid +from sqlalchemy import Column, Integer, String, Text from packageship.libs.dbutils.sqlalchemy_helper import DBHelper -class src_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 +class SrcPack(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Source package model """ @@ -43,7 +44,7 @@ class src_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 maintainlevel = Column(String(100), nullable=True) -class bin_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 +class BinPack(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Description: functional description:Binary package data """ @@ -78,7 +79,7 @@ class bin_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 src_name = Column(String(500), nullable=True) -class bin_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 +class BinRequires(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Binary package dependent package entity model """ @@ -95,7 +96,7 @@ class bin_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 pre = Column(String(20), nullable=True) -class src_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 +class SrcRequires(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Source entity package dependent package entity model """ @@ -111,7 +112,7 @@ class src_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 pre = Column(String(20), nullable=True) -class bin_provides(DBHelper.BASE): # pylint: disable=C0103,R0903 +class BinProvides(DBHelper.BASE): # pylint: disable=C0103,R0903 """ Component entity model provided by binary package """ @@ -126,18 +127,66 @@ class bin_provides(DBHelper.BASE): # pylint: disable=C0103,R0903 pkgKey = Column(Integer, nullable=True) -class maintenance_info(DBHelper.BASE): # pylint: disable=C0103,R0903 +class Packages(): # pylint: disable=C0103,R0903 """ - Maintain data related to person information + Source code package version, issuer and other information """ - __tablename__ = 'maintenance_info' - + __table_args__ = {'extend_existing': True} id = Column(Integer, primary_key=True) - name = Column(String(500), nullable=True) + url = Column(String(500), nullable=True) + rpm_license = Column(String(500), nullable=True) + version = Column(String(200), nullable=True) + release = Column(String(200), nullable=True) + release_time = Column(String(50), nullable=True) + used_time = Column(Integer, default=0) + latest_version = Column(String(200), nullable=True) + latest_version_time = Column(String(50), nullable=True) + feature = Column(Integer, default=0) + cve = Column(Integer, default=0) + defect = Column(Integer, default=0) + maintainer = Column(String(200), nullable=True) + maintainlevel = Column(Integer, nullable=True) + version_control = Column(String(50), nullable=True) + src_repo = Column(String(500), nullable=True) + tag_prefix = Column(String(20), nullable=True) + summary = Column(String(500), nullable=True) + description = Column(String(500), nullable=True) - version = Column(String(500), nullable=True) + @classmethod + def package_meta(cls, table_name): + """ + Dynamically generate different classes through metaclasses + """ + _uuid = str(uuid.uuid1()) + model = type(_uuid, (cls, DBHelper.BASE), { + '__tablename__': table_name}) + return model - maintaniner = Column(String(100), nullable=True) - maintainlevel = Column(String(100), nullable=True) +class PackagesIssue(DBHelper.BASE): # pylint: disable=C0103,R0903 + """ + Source package issue + """ + __tablename__ = "packages_issue" + id = Column(Integer, primary_key=True) + issue_id = Column(String(50), nullable=True) + issue_url = Column(String(500), nullable=True) + issue_content = Column(Text, nullable=True) + issue_title = Column(String(1000), nullable=True) + issue_status = Column(String(20), nullable=True) + pkg_name = Column(String(500), nullable=False) + issue_download = Column(String(500), nullable=False) + issue_type = Column(String(50), nullable=True) + related_release = Column(String(500), nullable=True) + + +class PackagesMaintainer(DBHelper.BASE): # pylint: disable=C0103,R0903 + """ + Correspondence between source code package and maintainer + """ + __tablename__ = 'packages_maintainer' + id = Column(Integer, primary_key=True) + name = Column(String(200), nullable=True) + maintainer = Column(String(200), nullable=True) + maintainlevel = Column(Integer, nullable=True) diff --git a/packageship/packageship/application/settings.py b/packageship/packageship/application/settings.py index bc090439dbcae9b160247c04d171e6e0046e4e19..12d561cbf097d750018acc3f08fa5adf94676d68 100644 --- a/packageship/packageship/application/settings.py +++ b/packageship/packageship/application/settings.py @@ -3,6 +3,7 @@ Description: Basic configuration of flask framework """ import random +from packageship import system_config from packageship.libs.configutils.readconfig import ReadConfig @@ -19,9 +20,11 @@ class Config(): LOG_LEVEL = 'INFO' + SCHEDULER_API_ENABLED = True + def __init__(self): - self._read_config = ReadConfig() + self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH) self.set_config_val() @@ -33,14 +36,6 @@ class Config(): cls.SECRET_KEY = ''.join( [random.choice('abcdefghijklmnopqrstuvwxyz!@#$%^&*()') for index in range(random_len)]) - @classmethod - def _set_debug(cls, debug): - """ - Description: Set the debugging mode - """ - if debug == 'true': - cls.DEBUG = True - @classmethod def _set_log_level(cls, log_level): """ @@ -57,11 +52,6 @@ class Config(): """ Config._random_secret_key() - debug = self._read_config.get_system('debug') - - if debug: - Config._set_debug(debug) - log_level = self._read_config.get_config('LOG', 'log_level') if log_level: diff --git a/packageship/packageship/manage.py b/packageship/packageship/manage.py index dc79873eb00f8de9e87c358bb272deca8e0f99f5..25d8aa7762fd20e1d488a72a0d5943ba62fc2c99 100644 --- a/packageship/packageship/manage.py +++ b/packageship/packageship/manage.py @@ -5,8 +5,8 @@ Description: Entry for project initialization and service startupc import os from packageship.libs.exception import Error try: - from packageship.system_config import SYS_CONFIG_PATH - if not os.path.exists(SYS_CONFIG_PATH): + from packageship import system_config + if not os.path.exists(system_config.SYS_CONFIG_PATH): raise FileNotFoundError( 'the system configuration file does not exist and the log cannot be started') except FileNotFoundError as file_not_found: @@ -36,7 +36,7 @@ def before_request(): if __name__ == "__main__": - _readconfig = ReadConfig() + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) port = _readconfig.get_system('write_port') addr = _readconfig.get_system('write_ip_addr') app.run(port=port, host=addr) diff --git a/packageship/packageship/package.ini b/packageship/packageship/package.ini index c562d9ef1f4d60e2b0277c42cb7d6878f148ce90..e0f4f47291711a305ea6dccaf5db579b604ec498 100644 --- a/packageship/packageship/package.ini +++ b/packageship/packageship/package.ini @@ -55,4 +55,42 @@ buffer-size=65536 http-timeout=600 ; Server response time harakiri=600 +; Allow multiple threads to run in the system +enable-threads=true + + +[TIMEDTASK] +; Execution frequency and switch of timing tasks +; Whether to execute the switch for batch update of information such +; as the maintainer during initialization. When set to True, the maintainer +; and other information will be updated when the scheduled task starts +; to execute. When it is set to False, it will be updated when the scheduled +; task is executed. , Does not update information such as maintainers and maintenance levels + +open=True + +; The time point at which the timing task is executed in a cycle. +; Every day is a cycle, and the time value can only be any integer period between 0 and 23 +hour=3 + +; Recurring timing task starts to execute the task at the current time point. +; The value range of this configuration item is an integer between 0 and 59 +minute=0 + +[LIFECYCLE] +; Configuration during the life cycle for tag information, issue and other information acquisition + +; The yaml address of each package is stored in the remote address, which can be +; a remote warehouse address or the address of a static resource service +warehouse_remote=https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/ + +; When performing timing tasks, you can open multi-threaded execution, and you can set +; the number of threads in the thread pool according to the configuration of the server + +pool_workers=10 + + +; The main body of the warehouse, the owner of the warehouse +; When this value is not set, the system will default to src-openeuler +warehouse=src-openeuler diff --git a/packageship/packageship/selfpkg.py b/packageship/packageship/selfpkg.py index f748eb3fb615d2a59a812873bf837329d62a5729..b889553111f93bf484fc05f97ecd58832ac339f6 100644 --- a/packageship/packageship/selfpkg.py +++ b/packageship/packageship/selfpkg.py @@ -7,8 +7,8 @@ from packageship.libs.exception import Error from packageship.libs.configutils.readconfig import ReadConfig try: - from packageship.system_config import SYS_CONFIG_PATH - if not os.path.exists(SYS_CONFIG_PATH): + from packageship import system_config + if not os.path.exists(system_config.SYS_CONFIG_PATH): raise FileNotFoundError( 'the system configuration file does not exist and the log cannot be started') except FileNotFoundError as file_not_found: @@ -36,7 +36,7 @@ def before_request(): if __name__ == "__main__": - _readconfig = ReadConfig() + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) port = _readconfig.get_system('query_port') addr = _readconfig.get_system('query_ip_addr') app.run(port=port, host=addr) diff --git a/packageship/setup.py b/packageship/setup.py index b10dc708b045d27abf4e409b9bdda0824b2c2003..08488d7d3a161ce0035034d74ffdffac6ce0f40e 100644 --- a/packageship/setup.py +++ b/packageship/setup.py @@ -4,13 +4,9 @@ Package management program installation configuration file for software packaging """ from distutils.core import setup -import os -BASE_PATH = os.path.dirname(__file__) -path = os.path.join(BASE_PATH, 'Lib', 'site-packages', 'package') - -configpath = "/etc/pkgship/" +config_path = "/etc/pkgship/" # pylint: disable=invalid-name setup( name='packageship', @@ -29,6 +25,12 @@ setup( 'packageship.application.apps.package.function.packages', 'packageship.application.apps.package.function.searchdb', 'packageship.application.apps.package.function.self_depend', + 'packageship.application.apps.lifecycle.function.base', + 'packageship.application.apps.lifecycle.function.download_yaml', + 'packageship.application.apps.lifecycle.function.gitee', + 'packageship.application.apps.lifecycle.serialize', + 'packageship.application.apps.lifecycle.url', + 'packageship.application.apps.lifecycle.view', 'packageship.application.initsystem.data_import', 'packageship.application.models.package', 'packageship.application.settings', @@ -56,6 +58,6 @@ setup( long_description=open('README.md', encoding='utf-8').read(), author='gongzt', data_files=[ - (configpath, ['packageship/package.ini']), + (config_path, ['packageship/package.ini']), ('/usr/bin', ['packageship/pkgshipd'])] ) diff --git a/packageship/test/test_module/dependent_query_tests/test_be_depend.py b/packageship/test/test_module/dependent_query_tests/test_be_depend.py index fec663f76b73e23bacfdbf0a7cb2b9c7869bd8b9..8fcf171acfb82032fa68842c7955988b2a4d6b7b 100644 --- a/packageship/test/test_module/dependent_query_tests/test_be_depend.py +++ b/packageship/test/test_module/dependent_query_tests/test_be_depend.py @@ -10,11 +10,13 @@ from test.base_code.common_test_code import compare_two_values, get_correct_json from packageship.application.apps.package.function.constants import ResponseCode from packageship.application.apps.package.function.searchdb import db_priority + class TestBeDepend(ReadTestBase): """ The dependencies of the package are tested """ db_name = db_priority()[0] + def test_lack_parameter(self): """ Less transmission is always parameter transmission