diff --git a/packageship/packageship/application/apps/lifecycle/url.py b/packageship/packageship/application/apps/lifecycle/url.py index ea08d589483de3ab428582b56d5fe2368adfa670..7209a0f393149f42dd63647f697084e667f14f24 100644 --- a/packageship/packageship/application/apps/lifecycle/url.py +++ b/packageship/packageship/application/apps/lifecycle/url.py @@ -11,4 +11,8 @@ urls = [ # pylint: disable=invalid-name (view.MaintainerView, '/lifeCycle/maintainer', {'query': ('GET')}), # Get the columns that need to be displayed by default in the package (view.TableColView, '/packages/tablecol', {'query': ('GET')}), + # View all table names in the package-info database + (view.LifeTables, '/lifeCycle/tables', {'query': ('GET')}), +# update a package info + (view.UpdatePackages, '/lifeCycle/updatePkgInfo', {'write': ('PUT')}) ] diff --git a/packageship/packageship/application/apps/lifecycle/view.py b/packageship/packageship/application/apps/lifecycle/view.py index eb171314c2c440c4067081333b253d4c7f108639..ac5b6896595ba17bc2212d4c25659091ad6724a3 100644 --- a/packageship/packageship/application/apps/lifecycle/view.py +++ b/packageship/packageship/application/apps/lifecycle/view.py @@ -3,22 +3,21 @@ Life cycle related api interface """ import io -import json -import math import os from concurrent.futures import ThreadPoolExecutor import pandas as pd import yaml + from flask import request from flask import jsonify, make_response from flask import current_app from flask_restful import Resource from marshmallow import ValidationError + from sqlalchemy.exc import DisconnectionError, SQLAlchemyError from packageship import system_config -from packageship.application.apps.package.serialize import DataFormatVerfi, UpdatePackagesSchema from packageship.libs.configutils.readconfig import ReadConfig from packageship.libs.exception import Error from packageship.application.apps.package.function.constants import ResponseCode @@ -28,7 +27,8 @@ from packageship.application.models.package import Packages from packageship.application.models.package import PackagesMaintainer from packageship.libs.log import Log from .serialize import IssueDownloadSchema, PackagesDownloadSchema -from .function.gitee import Gitee as gitee +from ..package.serialize import DataFormatVerfi, UpdatePackagesSchema + LOGGER = Log(__name__) @@ -221,3 +221,322 @@ class TableColView(Resource): ResponseCode.response_json( ResponseCode.SUCCESS, table_mapping_columns)) + + +class LifeTables(Resource): + """ + description: LifeTables + Restful API: get + ChangeLog: + """ + + def get(self): + """ + return all table names in the database + + Returns: + Return the table names in the database as a list + """ + try: + with DBHelper(db_name="lifecycle") as database_name: + # View all table names in the package-info database + all_table_names = database_name.engine.table_names() + all_table_names.remove("packages_issue") + all_table_names.remove("packages_maintainer") + return jsonify( + ResponseCode.response_json( + ResponseCode.SUCCESS, data=all_table_names) + ) + except (SQLAlchemyError, DisconnectionError, Error, ValueError) as sql_error: + LOGGER.logger.error(sql_error) + return jsonify( + ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND) + ) + + +class UpdatePackages(Resource): + """ + description:Life cycle update information of a single package + Restful API: post + ChangeLog: + """ + + def _get_all_yaml_name(self, filepath): + """ + List of all yaml file names in the folder + + Args: + filepath: file path + + Returns: + yaml_file_list：List of all yaml file names in the folder + + Attributes： + Error：Error + NotADirectoryError：Invalid directory name + FileNotFoundError：File not found error + + """ + try: + yaml_file_list = os.listdir(filepath) + return yaml_file_list + except (Error, NotADirectoryError, FileNotFoundError) as error: + current_app.logger.error(error) + return None + + def _get_yaml_content(self, yaml_file, filepath): + """ + Read the content of the yaml file + + Args: + yaml_file: yaml file + filepath: file path + + Returns: + Return a dictionary containing name, maintainer and maintainlevel + """ + yaml_data_dict = dict() + if not yaml_file.endswith(".yaml"): + return None + pkg_name = yaml_file.rsplit('.yaml')[0] + single_yaml_path = os.path.join(filepath, yaml_file) + with open(single_yaml_path, 'r', encoding='utf-8') as file_context: + yaml_flie_data = yaml.load( + file_context.read(), Loader=yaml.FullLoader) + if yaml_flie_data is None or not isinstance(yaml_flie_data, dict): + return None + maintainer = yaml_flie_data.get("maintainer") + maintainlevel = yaml_flie_data.get("maintainlevel") + yaml_data_dict['name'] = pkg_name + if maintainer: + yaml_data_dict['maintainer'] = maintainer + if maintainlevel: + yaml_data_dict['maintainlevel'] = maintainlevel + return yaml_data_dict + + def _read_yaml_file(self, filepath): + """ + Read the yaml file and combine the data of the nested dictionary of the list + + Args: + filepath: file path + + Returns: + yaml.YAMLError：yaml file error + SQLAlchemyError：SQLAlchemy Error + DisconnectionError：Connect to database error + Error：Error + """ + yaml_file_list = self._get_all_yaml_name(filepath) + if not yaml_file_list: + return None + try: + yaml_data_list = list() + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') + if not isinstance(pool_workers, int): + pool_workers = 10 + with ThreadPoolExecutor(max_workers=pool_workers) as pool: + for yaml_file in yaml_file_list: + pool_result = pool.submit( + self._get_yaml_content, yaml_file, filepath) + yaml_data_dict = pool_result.result() + yaml_data_list.append(yaml_data_dict) + return yaml_data_list + except (yaml.YAMLError, SQLAlchemyError, DisconnectionError, Error) as error: + current_app.logger.error(error) + return None + + def _verification_yaml_data_list(self, yaml_data_list): + """ + Verify the data obtained in the yaml file + + Args: + yaml_data_list: yaml data list + + Returns: + yaml_data_list: After verification yaml data list + + Attributes: + ValidationError: Validation error + + """ + try: + DataFormatVerfi(many=True).load(yaml_data_list) + return yaml_data_list + except ValidationError as error: + current_app.logger.error(error.messages) + return None + + def _save_in_database(self, yaml_data_list): + """ + Save the data to the database + + Args: + tbname: Table Name + name_separate_list: Split name list + _update_pack_data: Split new list of combined data + + Returns: + SUCCESS or UPDATA_DATA_FAILED + + Attributes + DisconnectionError: Connect to database error + SQLAlchemyError: SQLAlchemy Error + Error: Error + + """ + try: + with DBHelper(db_name="lifecycle") as database_name: + if 'packages_maintainer' not in database_name.engine.table_names(): + return jsonify(ResponseCode.response_json( + ResponseCode.TABLE_NAME_NOT_EXIST)) + database_name.session.begin(subtransactions=True) + for yaml_data in yaml_data_list: + name = yaml_data.get("name") + maintainer = yaml_data.get("maintainer") + maintainlevel = yaml_data.get("maintainlevel") + packages_maintainer_obj = database_name.session.query( + PackagesMaintainer).filter_by(name=name).first() + if packages_maintainer_obj: + if maintainer: + packages_maintainer_obj.maintainer = maintainer + if maintainlevel: + packages_maintainer_obj.maintainlevel = maintainlevel + else: + database_name.add(PackagesMaintainer( + name=name, maintainer=maintainer, maintainlevel=maintainlevel + )) + database_name.session.commit() + return jsonify(ResponseCode.response_json( + ResponseCode.SUCCESS)) + except (DisconnectionError, SQLAlchemyError, Error, AttributeError) as error: + current_app.logger.error(error) + return jsonify(ResponseCode.response_json( + ResponseCode.UPDATA_DATA_FAILED)) + + def _overall_process( + self, + filepath): + """ + Call each method to complete the entire function + + Args: + filepath: file path + tbname: table name + + Returns: + SUCCESS or UPDATA_DATA_FAILED + + Attributes + DisconnectionError: Connect to database error + SQLAlchemyError: SQLAlchemy Error + Error: Error + """ + try: + if filepath is None or not os.path.exists(filepath): + return jsonify(ResponseCode.response_json( + ResponseCode.SPECIFIED_FILE_NOT_EXIST)) + yaml_file_list = self._get_all_yaml_name(filepath) + if not yaml_file_list: + return jsonify(ResponseCode.response_json( + ResponseCode.EMPTY_FOLDER)) + yaml_data_list_result = self._read_yaml_file(filepath) + yaml_data_list = self._verification_yaml_data_list( + yaml_data_list_result) + if yaml_data_list is None: + return jsonify(ResponseCode.response_json( + ResponseCode.YAML_FILE_ERROR)) + result = self._save_in_database( + yaml_data_list) + return result + except (DisconnectionError, SQLAlchemyError, Error) as error: + current_app.logger.error(error) + return jsonify(ResponseCode.response_json( + ResponseCode.UPDATA_DATA_FAILED)) + + def _update_single_package_info( + self, srcname, maintainer, maintainlevel): + """ + Update the maintainer field and maintainlevel + field of a single package + + Args: + srcname: The name of the source package + maintainer: Package maintainer + maintainlevel: Package maintenance level + + Returns: + success or failed + + Attributes + SQLAlchemyError: sqlalchemy error + DisconnectionError: Cannot connect to database error + Error: Error + """ + if not srcname: + return jsonify( + ResponseCode.response_json(ResponseCode.PACK_NAME_NOT_FOUND) + ) + if not maintainer and not maintainlevel: + return jsonify( + ResponseCode.response_json(ResponseCode.PARAM_ERROR) + ) + try: + with DBHelper(db_name='lifecycle') as database_name: + if 'packages_maintainer' not in database_name.engine.table_names(): + return jsonify(ResponseCode.response_json( + ResponseCode.TABLE_NAME_NOT_EXIST)) + update_obj = database_name.session.query( + PackagesMaintainer).filter_by(name=srcname).first() + if update_obj: + if maintainer: + update_obj.maintainer = maintainer + if maintainlevel: + update_obj.maintainlevel = maintainlevel + else: + database_name.add(PackagesMaintainer( + name=srcname, maintainer=maintainer, maintainlevel=maintainlevel + )) + database_name.session.commit() + return jsonify( + ResponseCode.response_json( + ResponseCode.SUCCESS)) + except (SQLAlchemyError, DisconnectionError, Error) as sql_error: + current_app.logger.error(sql_error) + database_name.session.rollback() + return jsonify(ResponseCode.response_json( + ResponseCode.UPDATA_DATA_FAILED + )) + + def put(self): + """ + Life cycle update information of a single package or + All packages + + Returns: + for example:: + { + "code": "", + "data": "", + "msg": "" + } + """ + schema = UpdatePackagesSchema() + data = request.get_json() + if schema.validate(data): + return jsonify( + ResponseCode.response_json(ResponseCode.PARAM_ERROR) + ) + srcname = data.get('pkg_name', None) + maintainer = data.get('maintainer', None) + maintainlevel = data.get('maintainlevel', None) + batch = data.get('batch') + filepath = data.get('filepath', None) + + if batch: + result = self._overall_process(filepath) + else: + result = self._update_single_package_info( + srcname, maintainer, maintainlevel) + return result diff --git a/packageship/packageship/application/apps/package/function/constants.py b/packageship/packageship/application/apps/package/function/constants.py index d201bcd2e4cbed16192a7666fb8a7c869d8c3d2e..a387e1a80b57fb62b4d7d3bc2ff9d93d83f7f684 100644 --- a/packageship/packageship/application/apps/package/function/constants.py +++ b/packageship/packageship/application/apps/package/function/constants.py @@ -40,6 +40,10 @@ class ResponseCode(): CONNECT_DB_ERROR = "4004" INPUT_NONE = "4005" FILE_NOT_FOUND = "4006" + SPECIFIED_FILE_NOT_EXIST = "4007" + UPDATA_OR_ADD_DATA_FAILED = "4008" + TABLE_NAME_NOT_EXIST = "4009" + UPDATA_DATA_FAILED = "4010" # Database operation module error status code DELETE_DB_ERROR = "40051" SERVICE_ERROR = "50000" @@ -49,6 +53,11 @@ class ResponseCode(): DATA_MERGE_ERROR = "50004" FILE_NOT_FIND_ERROR = "50005" DIS_CONNECTION_DB = "50006" + NO_PACKAGES_TABLE = "60001" + DATABASE_NOT_FOUND = "60002" + TABLE_NAME_NOT_EXIST_IN_DATABASE = "60003" + YAML_FILE_ERROR = " 70001" + EMPTY_FOLDER = "70002" CODE_MSG_MAP = { SUCCESS: "Successful Operation!", @@ -66,7 +75,17 @@ class ResponseCode(): DATA_MERGE_ERROR: "abnormal multi-file database integration", FILE_NOT_FIND_ERROR: "system initialization configuration file does not exist", DIS_CONNECTION_DB: "Unable to connect to the database, check the database configuration", - SERVICE_ERROR: "An exception occurred in the system, please try again later"} + SERVICE_ERROR: "An exception occurred in the system, please try again later", + SPECIFIED_FILE_NOT_EXIST: "The specified file does not exist", + NO_PACKAGES_TABLE: "There is no packages table in the database", + UPDATA_OR_ADD_DATA_FAILED: "Failed to update or add data", + DATABASE_NOT_FOUND: "There is no such database in the system", + TABLE_NAME_NOT_EXIST: "There is no such table in the database", + UPDATA_DATA_FAILED: "Failed to update data", + TABLE_NAME_NOT_EXIST_IN_DATABASE: "the table name dose not match the existed database", + YAML_FILE_ERROR: "Data error in yaml file", + EMPTY_FOLDER: "This is an empty folder, no yaml file" + } @classmethod def response_json(cls, code, data=None): diff --git a/packageship/packageship/application/apps/package/function/packages.py b/packageship/packageship/application/apps/package/function/packages.py index b5b5139b6310c1064e5ba63701c1201cf893ad52..ba1a74689f311684a9c615dabfddb8a186042eef 100644 --- a/packageship/packageship/application/apps/package/function/packages.py +++ b/packageship/packageship/application/apps/package/function/packages.py @@ -4,299 +4,317 @@ Description: Get package information and modify package information functions: get_packages, buildep_packages, sub_packages, get_single_package, update_single_package, update_maintaniner_info """ -from flask import current_app -from flask import jsonify -from sqlalchemy.exc import SQLAlchemyError +import math -from packageship.application.apps.package.function.constants import ResponseCode -from packageship.application.apps.package.function.searchdb import db_priority +from flask import current_app, jsonify +from sqlalchemy.exc import SQLAlchemyError, DisconnectionError + + +from packageship.application.apps.package.function.constants import ResponseCode +from packageship.application.apps.package.serialize import AllPackInfoSchema +from packageship.application.apps.package.serialize import SinglePackInfoSchema from packageship.libs.dbutils import DBHelper from packageship.application.models.package import SrcPack +from packageship.application.models.package import PackagesMaintainer +from packageship.application.models.package import PackagesIssue from packageship.application.models.package import SrcRequires from packageship.application.models.package import BinPack -from packageship.application.models.package import maintenance_info from packageship.application.models.package import BinRequires from packageship.application.models.package import BinProvides from packageship.libs.exception import Error +from packageship.application.models.package import Packages -def get_packages(dbname): +def get_all_package_info(tablename, pagenum, pagesize, + srcname, maintainner, maintainlevel): + """ + Args: + tablename: Table Name + pagenum: Page number + pagesize: Current page display quantity + + Returns: + package info + + Attributes: + SQLAlchemyError: sqlalchemy error + DisconnectionError: Cannot connect to database error + Error: Error + """ + try: + with DBHelper(db_name="lifecycle") as database_name: + if tablename not in database_name.engine.table_names(): + response = ResponseCode.response_json( + ResponseCode.TABLE_NAME_NOT_EXIST) + response['total_count'] = None + response['total_page'] = None + return jsonify(response) + cls_model = Packages.package_meta(tablename) + # If srcname is empty, it will query all the information in the + # library + package_info_set_one = database_name.session.query(cls_model).outerjoin( + PackagesMaintainer, cls_model.name == PackagesMaintainer.name) + if srcname: + package_info_set_one = package_info_set_one.filter( + cls_model.name.like('%{srcname}%'.format(srcname=srcname))) + if maintainner: + package_info_set_one = package_info_set_one.filter( + PackagesMaintainer.maintainer == maintainner) + if maintainlevel: + package_info_set_one = package_info_set_one.filter( + PackagesMaintainer.maintainlevel == maintainlevel) + package_info_set = package_info_set_one.limit( + int(pagesize)).offset((int(pagenum) - 1) * int(pagesize)).all() + packageinfo_dicts = AllPackInfoSchema( + many=True).dump(package_info_set) + total_count = package_info_set_one.count() + total_page = math.ceil(total_count / int(pagesize)) + packageinfo_dicts = parsing_dictionary_issuse(packageinfo_dicts) + packageinfo_dicts = parsing_dictionary_maintainner( + packageinfo_dicts) + response = ResponseCode.response_json( + ResponseCode.SUCCESS, packageinfo_dicts) + response["total_count"] = total_count + response["total_page"] = total_page + return response + except (SQLAlchemyError, DisconnectionError, Error) as error: + current_app.logger.error(error) + response = ResponseCode.response_json( + ResponseCode.TABLE_NAME_NOT_EXIST) + response["total_count"] = None + response["total_page"] = None + return jsonify(response) + + +def parsing_dictionary_issuse(packageinfo_dicts): """ - Get all packages info in search databases Args: - dbname: Database name + packageinfo_dicts: package info dict + Returns: - Package information is returned as a list - Raises: - AttributeError: Object does not have this property - Error: Abnormal error + packageinfo_dicts """ - with DBHelper(db_name=dbname) as db_name: - src_pack_queryset = db_name.session.query(SrcPack).all() - if src_pack_queryset is None: - return None - resp_list = list() - for src_pack_obj in src_pack_queryset: - package = dict() - package["sourceName"] = src_pack_obj.name - package["version"] = src_pack_obj.version - package["license"] = src_pack_obj.rpm_license - package["sourceURL"] = src_pack_obj.url - package["rpm_packager"] = src_pack_obj.rpm_packager - package["maintainer"] = src_pack_obj.maintaniner - package["maintainlevel"] = src_pack_obj.maintainlevel - package["dbname"] = dbname - resp_list.append(package) - return resp_list - - -def get_all_packages(db_name): + with DBHelper(db_name="lifecycle") as database_name: + for packageinfo_dict in packageinfo_dicts: + issue_count = database_name.session.query(PackagesIssue).filter_by( + pkg_name=packageinfo_dict.get("name")).count() + packageinfo_dict["issue"] = issue_count + return packageinfo_dicts + + +def parsing_dictionary_maintainner(packageinfo_dicts): """ - all packages info + parsing dictionary maintainner Args: - db_name: database name + packageinfo_dicts: + Returns: - response code: response status code + packageinfo_dicts """ - dbpreority = db_priority() - if dbpreority is None: - return jsonify( - ResponseCode.response_json(ResponseCode.FILE_NOT_FOUND) - ) - if not db_name: - response = [] - for dbname in dbpreority: - query_result = get_packages(dbname) - if query_result is None: - return None - for item in query_result: - if item is None: - query_result.remove(item) - response.append(item) - return jsonify( - ResponseCode.response_json(ResponseCode.SUCCESS, response) - ) - if db_name not in dbpreority: - return jsonify( - ResponseCode.response_json(ResponseCode.DB_NAME_ERROR) - ) - response = get_packages(db_name) - if not response: - return jsonify( - ResponseCode.response_json(ResponseCode.PACK_NAME_NOT_FOUND) - ) - return jsonify( - ResponseCode.response_json(ResponseCode.SUCCESS, response) - ) + with DBHelper(db_name="lifecycle") as database_name: + for packageinfo_dict in packageinfo_dicts: + maintainer_obj = database_name.session.query(PackagesMaintainer).filter_by( + name=packageinfo_dict.get("name")).first() + if maintainer_obj is None: + packageinfo_dict["maintainer"] = None + packageinfo_dict["maintainlevel"] = None + else: + packageinfo_dict["maintainer"] = maintainer_obj.maintainer + packageinfo_dict["maintainlevel"] = maintainer_obj.maintainlevel + return packageinfo_dicts -def buildep_packages(dbname, src_pack_pkgkey): +def sing_pack(srcname, tablename): """ - Query package layer 1 compilation dependency + Query information about a single source package, including a layer + of installation dependencies and compilation dependencies + Args: + srcname: The name of the source package + tablename: The name of the table in the database + + Returns: + single pack package info + Attributes: + SQLAlchemyError: sqlalchemy error + DisconnectionError: Cannot connect to database error + Error: Error + """ + try: + with DBHelper(db_name="lifecycle") as database_name: + if tablename not in database_name.engine.table_names(): + return jsonify( + ResponseCode.response_json( + ResponseCode.TABLE_NAME_NOT_EXIST)) + cls_model = Packages.package_meta(tablename) + package_info_obj = database_name.session.query( + cls_model).filter_by(name=srcname).first() + if package_info_obj is None: + return jsonify( + ResponseCode.response_json( + ResponseCode.PACK_NAME_NOT_FOUND)) + pack_info_dict = SinglePackInfoSchema( + many=False).dump(package_info_obj) + issue_count = database_name.session.query( + PackagesIssue).filter_by(pkg_name=package_info_obj.name).count() + pack_info_dict["issue"] = issue_count + buildrequired = buildrequired_search(srcname, tablename) + pack_info_dict["buildrequired"] = buildrequired + subpack = _sub_pack(srcname, tablename) + pack_info_dict["gitee_url"] = "www.gitee.com/src-openeuler/" + \ + str(srcname) + pack_info_dict["subpack"] = subpack + pack_info_dict.update( + {"license": pack_info_dict.pop("rpm_license")}) + pack_info_dict.update({"pkg_name": pack_info_dict.pop("name")}) + return jsonify( + ResponseCode.response_json( + ResponseCode.SUCCESS, pack_info_dict)) + except (SQLAlchemyError, DisconnectionError, Error, AttributeError) as error: + current_app.logger.error(error) + return jsonify( + ResponseCode.response_json( + ResponseCode.DIS_CONNECTION_DB)) + + +def buildrequired_search(srcname, tablename): + """ + Source code package one-level compilation dependency Args: - dbname: databases name - src_pack_pkgkey: The ID of the source package + srcname: The name of the source package + tablename: The name of the table in the database + Returns: - buildDep Compile dependencies of source packages - Raises: - AttributeError: Object does not have this property + Source code package one-level compilation dependency """ - with DBHelper(db_name=dbname) as db_name: - # srcpack's pkgkey to src_requires find pkgkey - s_pack_requires_set = db_name.session.query( + with DBHelper(db_name=tablename) as data_name: + + src_pack_obj = data_name.session.query( + SrcPack).filter_by(name=srcname).first() + if src_pack_obj is None: + return None + + src_pack_pkgkey = src_pack_obj.pkgKey + s_pack_requires_set = data_name.session.query( SrcRequires).filter_by(pkgKey=src_pack_pkgkey).all() # src_requires pkykey to find the name of the dependent component s_pack_requires_names = [ s_pack_requires_obj.name for s_pack_requires_obj in s_pack_requires_set] - # Find pkgkey in bin_provides by the name of the dependent component - b_pack_provides_set = db_name.session.query(BinProvides).filter( + # Find pkgkey in BinProvides by the name of the dependent component + b_pack_provides_set = data_name.session.query(BinProvides).filter( BinProvides.name.in_(s_pack_requires_names)).all() b_pack_provides_pkg_list = [ b_pack_provides_obj.pkgKey for b_pack_provides_obj in b_pack_provides_set] - # Go to bin_pack to find the name by pkgkey of bin_provides - b_bin_pack_set = db_name.session.query(BinPack).filter( + # Go to bin_pack to find the name by pkgkey of BinProvides + b_bin_pack_set = data_name.session.query(BinPack).filter( BinPack.pkgKey.in_(b_pack_provides_pkg_list)).all() builddep = [b_bin_pack_obj.name for b_bin_pack_obj in b_bin_pack_set] return builddep -def sub_packages(dbname, sourcename): +def _sub_pack(srcname, tablename): """ - Query package layer 1 installation dependency - + One-level installation dependency of the source package + to generate the binary package Args: - dbname: databases name - src_pack_pkgkey: srcpackage id - Returns: - subpack Source package to binary package, then find the installation dependencies - of the binary package - Raises: - AttributeError: Object does not have this property - """ - with DBHelper(db_name=dbname) as db_name: - subpack = dict() - # The name of src_pack finds the sub-package bin_pack query set - i_bin_pack_set = db_name.session.query( - BinPack).filter_by(src_name=sourcename).all() - if i_bin_pack_set is None: - return subpack - # Find the objects of each sub-package - for b_bin_pack_obj in i_bin_pack_set: - i_bin_pack_name = b_bin_pack_obj.name - i_bin_pack_pkgkey = b_bin_pack_obj.pkgKey - # Find the names of the components required to install bin_requires - # dependencies - i_bin_requires_set = db_name.session.query( - BinRequires).filter_by(pkgKey=i_bin_pack_pkgkey).all() - i_bin_requires_names = [ - b_bin_requires_obj.name for b_bin_requires_obj in i_bin_requires_set] - # Find pkykey in bin_provides by the name of the dependent - # component - i_bin_provides_set = db_name.session.query(BinProvides).filter( - BinProvides.name.in_(i_bin_requires_names)) - i_bin_provides_pkg_list = [ - i_bin_provides_obj.pkgKey for i_bin_provides_obj in i_bin_provides_set] - # Find the name in bin_pack by pkgkey - i_bin_pack_set = db_name.session.query(BinPack).filter( - BinPack.pkgKey.in_(i_bin_provides_pkg_list)) - i_bin_pack_names = [ - in_bin_pack_obj.name for in_bin_pack_obj in i_bin_pack_set] - subpack[i_bin_pack_name] = i_bin_pack_names - return subpack - - -def get_single_package(dbname, sourcename): - """ - Get single packages info + srcname: The name of the source package + tablename: The name of the table in the database - Args： - dbname: Database name - sourcename: Source package name - Returns： - package info - Raises: - AttributeError: Object does not have this property + Returns: + One-level installation dependency of the source package to + generate the binary package """ - with DBHelper(db_name=dbname) as db_name: - package = dict() - src_pack_obj = db_name.session.query(SrcPack).filter_by( - name=sourcename).first() + with DBHelper(db_name=tablename) as data_name: + src_pack_obj = data_name.session.query( + SrcPack).filter_by(name=srcname).first() if src_pack_obj is None: - return None - package["sourceName"] = src_pack_obj.name - package["version"] = src_pack_obj.version - package["license"] = src_pack_obj.rpm_license - package["sourceURL"] = src_pack_obj.url - package["rpm_packager"] = src_pack_obj.rpm_packager - package["maintainer"] = src_pack_obj.maintaniner - package["maintainlevel"] = src_pack_obj.maintainlevel - package["dbname"] = dbname - src_pack_pkgkey = src_pack_obj.pkgKey - builddep = buildep_packages(dbname, src_pack_pkgkey) - subpack = sub_packages(dbname, sourcename) - package['buildDep'] = builddep - package['subpack'] = subpack - return package + return [] + # Sub-packages generated by the source package + bin_pack_set = data_name.session.query( + BinPack).filter_by(src_name=src_pack_obj.name).all() + pack_list = list() + for bin_pack_obj in bin_pack_set: + bin_pack_dict = dict() + bin_pack_dict['id'] = bin_pack_obj.pkgKey + bin_pack_dict['name'] = bin_pack_obj.name + # Sub-package lookup provided components + + bin_provides_set = data_name.session.query( + BinProvides).filter_by(pkgKey=bin_pack_obj.pkgKey).all() + provide_list = provide_(tablename, bin_provides_set) + bin_pack_dict['provides'] = provide_list + bin_require_set = data_name.session.query( + BinRequires).filter_by(pkgKey=bin_pack_obj.pkgKey).all() + require_list = require_(tablename, bin_require_set) + bin_pack_dict['requires'] = require_list + pack_list.append(bin_pack_dict) + return pack_list -def get_single(dbnames, sourcename): +def provide_(tablename, bin_provides_set): """ - get single package + pkgKey goes to the bin_pack table to obtain + bin_name corresponding to requiredby Args: - dbname: database name - sourcename: source name + tablename: table name + bin_provides_set: Query set provided by the binary package + + Returns: + provide_list: Provided list """ - response_data = None - dbpreority = db_priority() - if db_priority is None: - response_data = ResponseCode.FILE_NOT_FOUND - - if not dbnames: - response = [] - for db_names in dbpreority: - query_result = get_single_package(db_names, sourcename) - response.append(query_result) - for key in response: - if key is None: - response.remove(key) - if not response: - return jsonify( - ResponseCode.response_json(ResponseCode.PACK_NAME_NOT_FOUND) - ) - return jsonify( - ResponseCode.response_json(ResponseCode.SUCCESS, response) - ) + with DBHelper(db_name=tablename) as data_name: + provide_list = [] + for bin_provides_obj in bin_provides_set: + bin_provides_dict = dict() + bin_provides_dict['id'] = bin_provides_obj.id + bin_provides_dict['name'] = bin_provides_obj.name + reqired_set = data_name.session.query( + BinProvides).filter_by(name=bin_provides_obj.name).all() + required_pkgkey_list = [ + reqired_obj.pkgKey for reqired_obj in reqired_set] + required_bin_set = data_name.session.query(BinPack).filter( + BinPack.pkgKey.in_(required_pkgkey_list)).all() + requiredby = [ + required_bin_obj.name for required_bin_obj in required_bin_set] + bin_provides_dict['requiredby'] = requiredby + provide_list.append(bin_provides_dict) + return provide_list - # Database queries data and catches exceptions - if dbnames not in dbpreority: - return jsonify( - ResponseCode.response_json(ResponseCode.DB_NAME_ERROR) - ) - response = get_single_package(dbnames, sourcename) - if response is None: - response_data = ResponseCode.PACK_NAME_NOT_FOUND - if response_data is not None: - return jsonify(ResponseCode.response_json(response_data)) - return jsonify( - ResponseCode.response_json(ResponseCode.SUCCESS, [response]) - ) - - -def _update_package_info( - package_name, - dbname, - maintainer, - maintain_level): + +def require_(tablename, bin_require_set): """ - change single package management + pkgKey goes to the bin_pack table to obtain + bin_name corresponding to provideBy Args: - package_name: package name - dbname: Database name - maintainer: maintainer info - maintain_level: maintain_level info + tablename: table name + bin_provides_set: Query set provided by the binary package + Returns: - message success or failed - Raises: - AttributeError: Object does not have this property - SQLAlchemyError: Exception of type - Error: Abnormal error + require_list: require list """ - try: - result_data = True - with DBHelper(db_name=dbname) as data_name: - update_obj = data_name.session.query( - SrcPack).filter_by(name=package_name).first() - if update_obj is None: - return False - update_obj.maintaniner = maintainer - update_obj.maintainlevel = maintain_level - data_name.session.commit() - name = update_obj.name - version = update_obj.version - with DBHelper(db_name='maintenance.information') as dbs_name: - information_obj = dbs_name.session.query(maintenance_info).filter_by( - name=package_name, version=version).first() - if information_obj is None: - information = maintenance_info( - name=name, - version=version, - maintaniner=maintainer, - maintainlevel=maintain_level) - dbs_name.session.add(information) - dbs_name.session.commit() - else: - information_obj.maintaniner = maintainer - information_obj.maintainlevel = maintain_level - dbs_name.session.commit() - return result_data - except (AttributeError, SQLAlchemyError, Error) as attri_error: - current_app.logger.error(attri_error) - raise attri_error + with DBHelper(db_name=tablename) as data_name: + # Sub-package to find the required components + require_list = [] + + for bin_require_obj in bin_require_set: + bin_require_dict = dict() + bin_require_dict['id'] = bin_require_obj.id + bin_require_dict['name'] = bin_require_obj.name + provide_set = data_name.session.query( + BinProvides).filter_by(name=bin_require_obj.name).all() + provide_pkg_list = [ + provide_obj.pkgKey for provide_obj in provide_set] + required_bin_set = data_name.session.query(BinPack).filter( + BinPack.pkgKey.in_(provide_pkg_list)).all() + providedby = [ + required_bin_obj.name for required_bin_obj in required_bin_set] + bin_require_dict['providedby'] = providedby + require_list.append(bin_require_dict) + return require_list diff --git a/packageship/packageship/application/apps/package/serialize.py b/packageship/packageship/application/apps/package/serialize.py index 8e271e3187dbab59a903c8b9f2306a560bb39c52..3aa63b21c32bb4654fffa0772da8fa1b5547141d 100644 --- a/packageship/packageship/application/apps/package/serialize.py +++ b/packageship/packageship/application/apps/package/serialize.py @@ -7,28 +7,35 @@ from marshmallow import fields from marshmallow import ValidationError from marshmallow import validate +from packageship.application.models.package import Packages -class PackagesSchema(Schema): + +def validate_pagenum(pagenum): """ - Description: PackagesSchema serialize + Description: Method test + Args： + pagenum: pagenum + Returns: + True or failure + Raises: + ValidationError: Test failed """ - dbName = fields.Str(validate=validate.Length( - max=50), required=False, allow_none=True) + if pagenum <= 0 or pagenum >= 65535: + raise ValidationError("pagenum is illegal data ") -class GetpackSchema(Schema): +def validate_pagesize(pagesize): """ - Description: GetpackSchema serialize + Description: Method test + Args： + pagesize: pagesize + Returns: + True or failure + Raises: + ValidationError: Test failed """ - sourceName = fields.Str( - required=True, - validate=validate.Length(min=1, - max=200)) - - dbName = fields.Str(validate=validate.Length( - max=30), required=False, allow_none=True) - version = fields.Str(validate=validate.Length( - max=200), required=False, allow_none=True) + if pagesize <= 0 or pagesize >= 65535: + raise ValidationError("pagesize is illegal data ") def validate_maintainlevel(maintainlevel): @@ -41,30 +48,82 @@ def validate_maintainlevel(maintainlevel): Raises: ValidationError: Test failed """ - if maintainlevel not in ['1', '2', '3', '4']: + if maintainlevel not in [1, 2, 3, 4, '']: raise ValidationError("maintainLevel is illegal data ") -class PutpackSchema(Schema): +def validate_maintainlevels(maintainlevel): """ - Description: PutpackSchema serialize + Description: Method test + Args： + maintainlevel: maintainlevel + Returns: + True or failure + Raises: + ValidationError: Test failed """ - sourceName = fields.Str( + if maintainlevel not in ['1', '2', '3', '4', '']: + raise ValidationError("maintainLevel is illegal data ") + + +class AllPackagesSchema(Schema): + """ + Description: AllPackagesSchema serialize + """ + table_name = fields.Str( required=True, - validate=validate.Length( - min=1, - max=200)) - dbName = fields.Str( + validate=validate.Length(min=1, + max=200)) + page_num = fields.Integer( + required=True, + validate=validate_pagenum + ) + page_size = fields.Integer( + required=True, + validate=validate_pagesize + ) + query_pkg_name = fields.Str(validate=validate.Length( + max=200), required=False, allow_none=True) + maintainner = fields.Str(validate=validate.Length( + max=200), required=False, allow_none=True) + maintainlevel = fields.Str( + validate=validate_maintainlevels, + required=False, + allow_none=True) + + +class SinglepackSchema(Schema): + """ + Description: GetpackSchema serialize + """ + pkg_name = fields.Str( required=True, + validate=validate.Length(min=1, + max=200)) + + table_name = fields.Str(required=True, + validate=validate.Length(min=1, + max=200)) + + +class UpdatePackagesSchema(Schema): + """ + Description: UpdatePackagesSchema serialize + """ + pkg_name = fields.Str( + required=False, validate=validate.Length( - min=1, - max=50)) + max=200)) maintainer = fields.Str(validate=validate.Length( max=50), required=False, allow_none=True) - maintainlevel = fields.Str( + maintainlevel = fields.Integer( validate=validate_maintainlevel, required=False, allow_none=True) + batch = fields.Boolean( + required=True) + filepath = fields.Str(validate=validate.Length( + max=200), required=False, allow_none=True) class InstallDependSchema(Schema): @@ -187,3 +246,64 @@ class InitSystemSchema(Schema): configfile = fields.Str( validate=validate.Length( max=500), required=False, allow_none=True) + + +class AllPackInfoSchema(Schema): # pylint: disable= too-few-public-methods + """ + Field serialization for package file download + """ + class Meta: + """Model mapping serialized fields""" + model = Packages + fields = ( + 'id', + 'name', + 'url', + 'version', + 'release', + 'release_time', + 'rpm_license', + 'maintainer', + 'maintainlevel', + 'feature', + 'release_time', + 'used_time', + 'maintainer_status', + 'latest_version', + 'latest_version_time') + + +class SinglePackInfoSchema(Schema): + """ + Field serialization for package file download + """ + + class Meta: + """Model mapping serialized fields""" + model = Packages + fields = ( + 'name', + 'version', + 'release', + 'url', + 'maintainer', + 'feature', + 'rpm_license', + 'maintainlevel', + 'summary', + 'description') + + +class DataFormatVerfi(Schema): + """ + Verify the data in yaml + """ + + maintainer = fields.Str(validate=validate.Length( + max=50), required=False, allow_none=True) + maintainlevel = fields.Int( + validate=validate_maintainlevel, + required=False, + allow_none=True) + name = fields.Str(validate=validate.Length(min=1, + max=50), required=True) diff --git a/packageship/packageship/application/apps/package/view.py b/packageship/packageship/application/apps/package/view.py index cdf32e4d37a21d090d58fba1a169f7d65154d6f7..807b45151efa0f8bff773b3729c9aeccc269c6e7 100644 --- a/packageship/packageship/application/apps/package/view.py +++ b/packageship/packageship/application/apps/package/view.py @@ -11,6 +11,7 @@ from flask import current_app from flask_restful import Resource from sqlalchemy.exc import DisconnectionError +from packageship import system_config from packageship.application.initsystem.data_import import InitDataBase from packageship.libs.configutils.readconfig import ReadConfig from packageship.libs.exception import Error @@ -19,13 +20,11 @@ from packageship.libs.exception import DataMergeException from packageship.libs.log import Log from packageship.system_config import DATABASE_FILE_INFO from .function.constants import ResponseCode -from .function.packages import get_all_packages -from .function.packages import _update_package_info -from .function.packages import get_single +from .function.packages import get_all_package_info +from .function.packages import sing_pack from .function.searchdb import db_priority -from .serialize import PackagesSchema -from .serialize import GetpackSchema -from .serialize import PutpackSchema +from .serialize import AllPackagesSchema, SinglepackSchema + from .serialize import DeletedbSchema from .serialize import InitSystemSchema from .serialize import BeDependSchema @@ -78,34 +77,33 @@ class Packages(Resource): Error: Abnormal error """ # Get verification parameters - schema = PackagesSchema() - data = schema.dump(request.args) + schema = AllPackagesSchema() + data = request.args if schema.validate(data): - return jsonify( - ResponseCode.response_json(ResponseCode.PARAM_ERROR) - ) - dbname = data.get("dbName", None) - # Call method to query - try: - response = get_all_packages(dbname) - return response - # Database queries data and catches exceptions - except DisconnectionError as dis_connection_error: - current_app.logger.error(dis_connection_error) - return jsonify( - ResponseCode.response_json( - ResponseCode.DIS_CONNECTION_DB)) - except (AttributeError, TypeError, Error) as attribute_error: - current_app.logger.error(attribute_error) - return jsonify( - ResponseCode.response_json( - ResponseCode.PACK_NAME_NOT_FOUND)) + response = ResponseCode.response_json(ResponseCode.PARAM_ERROR) + response['total_count'] = None + response['total_page'] = None + return jsonify(response) + table_name = data.get("table_name") + page_num = data.get("page_num") + page_size = data.get("page_size") + src_name = data.get("query_pkg_name", None) + maintainner = data.get("maintainner", None) + maintainlevel = data.get("maintainlevel", None) + result = get_all_package_info( + table_name, + page_num, + page_size, + src_name, + maintainner, + maintainlevel) + return result class SinglePack(Resource): """ description: single package management - Restful API: get, put + Restful API: get ChangeLog: """ @@ -141,93 +139,16 @@ class SinglePack(Resource): Error: Abnormal error """ # Get verification parameters - schema = GetpackSchema() - data = schema.dump(request.args) - if schema.validate(data): - return jsonify( - ResponseCode.response_json(ResponseCode.PARAM_ERROR) - ) - dbname = data.get("dbName", None) - sourcename = data.get("sourceName") - - # Call method to query - try: - response = get_single(dbname, sourcename) - return response - except DisconnectionError as dis_connection_error: - current_app.logger.error(dis_connection_error) - abnormal = ResponseCode.DIS_CONNECTION_DB - - except (AttributeError, TypeError, Error) as attribute_error: - current_app.logger.error(attribute_error) - abnormal = ResponseCode.PACK_NAME_NOT_FOUND - if abnormal is not None: - return jsonify(ResponseCode.response_json(abnormal)) - - def put(self): - """ - update a package info, - - Args: - dbName: Database name,Parameters are required - sourceName: The name of the source code package. Parameters are required - maintainer: Maintainer, parameter not required - maintainlevel: Maintenance level, parameter not required - Returns: - for - example:: - { - "code": "", - "data": "", - "msg": "" - } - Raises: - DisconnectionError: Unable to connect to database exception - AttributeError: Object does not have this property - TypeError: Exception of type - Error: Abnormal error - """ - # Get verification parameters - schema = PutpackSchema() - data = schema.dump(request.get_json()) + schema = SinglepackSchema() + data = request.args if schema.validate(data): return jsonify( ResponseCode.response_json(ResponseCode.PARAM_ERROR) ) - dbname = data.get('dbName') - sourcename = data.get('sourceName') - maintainer = data.get('maintainer', None) - maintain_level = data.get('maintainlevel', None) - - # Call method to query - if not maintainer and not maintain_level: - return jsonify( - ResponseCode.response_json(ResponseCode.PARAM_ERROR) - ) - - if dbname not in db_priority(): - return jsonify( - ResponseCode.response_json(ResponseCode.DB_NAME_ERROR) - ) - # Database queries data and catches exceptions - - try: - result_data = _update_package_info( - sourcename, dbname, maintainer, maintain_level) - if result_data is False: - return jsonify( - ResponseCode.response_json( - ResponseCode.PACK_NAME_NOT_FOUND)) - return jsonify( - ResponseCode.response_json(ResponseCode.SUCCESS)) - except DisconnectionError as dis_connection_error: - current_app.logger.error(dis_connection_error) - abnormal = ResponseCode.DIS_CONNECTION_DB - except (AttributeError, TypeError, Error) as attri_error: - current_app.logger.error(attri_error) - abnormal = ResponseCode.CONNECT_DB_ERROR - if abnormal is not None: - return jsonify(ResponseCode.response_json(abnormal)) + srcname = data.get("pkg_name") + tablename = data.get("table_name") + result = sing_pack(srcname, tablename) + return result class InstallDepend(Resource): @@ -576,7 +497,7 @@ class Repodatas(Resource): Error: Abnormal error """ schema = DeletedbSchema() - data = schema.dump(request.args) + data = request.args if schema.validate(data): return jsonify( ResponseCode.response_json(ResponseCode.PARAM_ERROR) @@ -646,7 +567,8 @@ class InitSystem(Resource): try: abnormal = None if not configfile: - _config_path = ReadConfig().get_system('init_conf_path') + _config_path = ReadConfig( + system_config.SYS_CONFIG_PATH).get_system('init_conf_path') InitDataBase(config_file_path=_config_path).init_data() else: InitDataBase(config_file_path=configfile).init_data() diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 4dbcb9e51ba9837599dcc0ac5ee55a2e6d6ac1a7..ffd9dc0ca2c1901d2e879a97ae146866c81155fb 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -58,11 +58,12 @@ class InitDataBase(): 'mysql': MysqlDatabaseOperations } self.database_name = None - self._tables = ['SrcPack', 'BinPack', - 'BinRequires', 'SrcRequires', 'BinProvides'] + self._tables = ['src_pack', 'bin_pack', + 'bin_requires', 'src_requires', 'bin_provides'] # Create life cycle related databases and tables if not self.create_database(db_name='lifecycle', - tables=['PackagesIssue'], + tables=['packages_issue', + 'packages_maintainer'], storage=True): raise SQLAlchemyError( 'Failed to create the specified database and table：lifecycle')