diff --git a/packageship/packageship/application/apps/package/function/searchdb.py b/packageship/packageship/application/apps/package/function/searchdb.py index e977f7a20e825fb15986ad99881153a85235fe80..400d422f5a9c7f79a40b34f2fbb0003fff589e25 100644 --- a/packageship/packageship/application/apps/package/function/searchdb.py +++ b/packageship/packageship/application/apps/package/function/searchdb.py @@ -15,7 +15,7 @@ from sqlalchemy import exists from packageship.libs.dbutils import DBHelper from packageship.libs.log import Log -from packageship.application.models.package import BinPack +from packageship.application.models.package import BinPack,SrcPack from packageship.libs.exception import ContentNoneException, Error from packageship.system_config import DATABASE_FILE_INFO from .constants import ResponseCode @@ -86,16 +86,18 @@ class SearchDB(): SELECT DISTINCT bin_pack.NAME AS depend_name, bin_pack.version AS depend_version, - bin_pack.src_name AS depend_src_name, + s2.name AS depend_src_name, bin_requires.NAME AS req_name, bin.NAME AS search_name, - bin.src_name AS search_src_name, + s1.name AS search_src_name, bin.version AS search_version FROM - ( SELECT pkgKey,NAME,version,src_name FROM bin_pack WHERE {} ) bin + ( SELECT pkgKey, NAME, version, rpm_sourcerpm FROM bin_pack WHERE {} ) bin + LEFT JOIN src_pack s1 ON bin.rpm_sourcerpm = s1.src_name LEFT JOIN bin_requires ON bin.pkgKey = bin_requires.pkgKey LEFT JOIN bin_provides ON bin_provides.name = bin_requires.name - LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey; + LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey + LEFT JOIN src_pack s2 ON bin_pack.rpm_sourcerpm = s2.src_name; """.format(name_in)) install_set = data_base.session. \ execute(sql_com, {'name_{}'.format(i): v @@ -159,20 +161,29 @@ class SearchDB(): SQLAlchemyError: sqlalchemy error """ for db_name, data_base in self.db_object_dict.items(): + sql_str = """ + SELECT DISTINCT + src_pack.name AS source_name, + src_pack.version AS source_version + FROM + bin_pack, + src_pack + WHERE + src_pack.src_name = bin_pack.rpm_sourcerpm + AND bin_pack.name = :binary_name; + """ try: - bin_obj = data_base.session.query(BinPack).filter_by( - name=binary_name - ).first() - source_name = bin_obj.src_name - source_version = bin_obj.version + bin_obj = data_base.session.execute(text(sql_str), {"binary_name": binary_name}).fetchone() + source_name = bin_obj.source_name + source_version = bin_obj.source_version if source_name is not None: return ResponseCode.SUCCESS, db_name, \ - source_name, source_version + source_name, source_version except AttributeError as error_msg: LOGGER.logger.error(error_msg) except SQLAlchemyError as error_msg: LOGGER.logger.error(error_msg) - return ResponseCode.DIS_CONNECTION_DB, None + return ResponseCode.DIS_CONNECTION_DB, None, None, None return ResponseCode.PACK_NAME_NOT_FOUND, None, None, None def get_sub_pack(self, source_name_list): @@ -202,11 +213,12 @@ class SearchDB(): sql_com = text(''' SELECT bin_pack.name AS subpack_name, + bin_pack.version AS sub_pack_version, src.name AS search_name, src.version AS search_version FROM - (SELECT name,version FROM src_pack WHERE {}) src - LEFT JOIN bin_pack on src.name = bin_pack.src_name'''.format(name_in)) + (SELECT name,version,src_name FROM src_pack WHERE {}) src + LEFT JOIN bin_pack on src.src_name = bin_pack.rpm_sourcerpm'''.format(name_in)) subpack_tuple = data_base.session. \ execute(sql_com, {'name_{}'.format(i): v for i, v in enumerate(search_set, 1)}).fetchall() @@ -225,10 +237,10 @@ class SearchDB(): except SQLAlchemyError as sql_error: current_app.logger.error(sql_error) return_tuple = namedtuple( - 'return_tuple', 'subpack_name search_version search_name') + 'return_tuple', 'subpack_name sub_pack_version search_version search_name') for search_name in search_set: result_list.append( - (return_tuple(None, None, search_name), 'NOT_FOUND')) + (return_tuple(None, None, None, search_name), 'NOT_FOUND')) return ResponseCode.SUCCESS, result_list def _get_binary_in_other_database(self, not_found_binary): @@ -268,17 +280,19 @@ class SearchDB(): search_list.clear() try: sql_string = text(""" - SELECT DISTINCT - t1.src_name AS source_name, - t1.NAME AS bin_name, - t1.version, - t2.NAME AS req_name - FROM - bin_pack t1, - bin_provides t2 - WHERE - t2.{} - AND t1.pkgKey = t2.pkgKey; + SELECT DISTINCT + s1.name AS source_name, + t1.NAME AS bin_name, + t1.version, + t2.NAME AS req_name + FROM + src_pack s1, + bin_pack t1, + bin_provides t2 + WHERE + t2.{} + AND t1.pkgKey = t2.pkgKey + AND t1.rpm_sourcerpm = s1.src_name; """.format(literal_column('name').in_(search_set))) bin_set = data_base.session. \ execute(sql_string, {'name_{}'.format(i): v @@ -335,17 +349,19 @@ class SearchDB(): search_set = set(search_list) search_list.clear() sql_string = text(""" - SELECT DISTINCT - t1.src_name AS source_name, + SELECT DISTINCT + s1.name AS source_name, t1.NAME AS bin_name, t1.version, t2.NAME AS req_name FROM + src_pack s1, bin_pack t1, - bin_provides t2 + bin_provides t2 WHERE t2.{} - AND t1.pkgKey = t2.pkgKey; + AND t1.pkgKey = t2.pkgKey + AND t1.rpm_sourcerpm = s1.src_name; """.format(literal_column('name').in_(search_set))) bin_set = data_base.session. \ execute(sql_string, {'name_{}'.format(i): v @@ -421,19 +437,20 @@ class SearchDB(): temp_list = list(s_name_set) for input_name_li in [temp_list[i:i + 900] for i in range(0, len(temp_list), 900)]: sql_com = text(""" - SELECT DISTINCT + SELECT DISTINCT src.NAME AS search_name, src.version AS search_version, - bin_pack.src_name AS source_name, + s1.name AS source_name, bin_provides.pkgKey AS bin_id, src_requires.NAME AS req_name, bin_pack.version AS version, - bin_pack.NAME AS bin_name + bin_pack.NAME AS bin_name FROM - ( SELECT pkgKey, NAME, version FROM src_pack WHERE {} ) src + ( SELECT pkgKey, NAME, version FROM src_pack WHERE {}) src LEFT JOIN src_requires ON src.pkgKey = src_requires.pkgKey LEFT JOIN bin_provides ON bin_provides.NAME = src_requires.NAME - LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey; + LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey + LEFT JOIN src_pack s1 on bin_pack.rpm_sourcerpm=s1.src_name; """.format(literal_column("name").in_(input_name_li))) res = data_base.session.execute( sql_com, @@ -516,6 +533,26 @@ class SearchDB(): return 'NOT FOUND' + def get_version_and_db(self, src_name): + """ + + Args: + src_name:the source package name + Returns: + this source package version and db_name + """ + try: + for db_name, data_base in self.db_object_dict.items(): + res = data_base.session.query(SrcPack.version).filter_by(name=src_name).first() + if res: + return db_name, res.version + except AttributeError as attr_err: + current_app.logger.error(attr_err) + except SQLAlchemyError as sql_err: + current_app.logger.error(sql_err) + + return None, None + def db_priority(): """ diff --git a/packageship/packageship/application/apps/package/function/self_depend.py b/packageship/packageship/application/apps/package/function/self_depend.py index 3f51e2a72c64a82191336b23a07f7126f2ce18bb..dd72bed74565b08bc8f6751d8c6cc91c7b3300c8 100644 --- a/packageship/packageship/application/apps/package/function/self_depend.py +++ b/packageship/packageship/application/apps/package/function/self_depend.py @@ -152,9 +152,10 @@ class SelfDepend(): if not source_name: LOGGER.logger.warning("source name is None") if source_name and source_name not in self.source_dicts.dictionary: + db_, src_version_ = self.search_db.get_version_and_db(source_name) self.source_dicts.append_src(key=source_name, - dbname=values[ListNode.DBNAME], - version=values[ListNode.VERSION]) + dbname=db_ if db_ else values[ListNode.DBNAME], + version=src_version_ if src_version_ else values[ListNode.VERSION]) self.search_build_list.append(source_name) if self.withsubpack == 1: self.search_subpack_list.append(source_name) @@ -173,10 +174,10 @@ class SelfDepend(): _, result_list = self.search_db.get_sub_pack(self.search_subpack_list) for subpack_tuple, dbname in result_list: if dbname != 'NOT_FOUND': - if subpack_tuple.subpack_name not in self.binary_dict.dictionary: + if subpack_tuple.subpack_name and subpack_tuple.subpack_name not in self.binary_dict.dictionary: self.binary_dict.append_bin(key=subpack_tuple.subpack_name, src=subpack_tuple.search_name, - version=subpack_tuple.search_version, + version=subpack_tuple.sub_pack_version, dbname=dbname, parent_node=[subpack_tuple.search_name, 'Subpack']) self.search_install_list.append(subpack_tuple.subpack_name) @@ -221,13 +222,14 @@ class SelfDepend(): if not source_name: LOGGER.logger.warning("source name is None") if source_name and source_name not in self.source_dicts.dictionary: + db_, src_version_ = self.search_db.get_version_and_db(source_name) self.source_dicts.append_src(key=source_name, - dbname=values[ListNode.DBNAME], - version=values[ListNode.VERSION]) + dbname=db_ if db_ else values[ListNode.DBNAME], + version=src_version_ if src_version_ else values[ListNode.VERSION]) if self.withsubpack == 1: self.search_subpack_list.append(source_name) - elif key in self.binary_dict.dictionary: - self.binary_dict.update_value(key=key, parent_list=values[ListNode.PARENT_LIST]) + elif key in self.binary_dict.dictionary: + self.binary_dict.update_value(key=key, parent_list=values[ListNode.PARENT_LIST]) def query_selfbuild(self): """ @@ -312,6 +314,7 @@ class DictionaryOperations(): Returns: Raises: """ - if parent_list: - self.dictionary[key][ListNode.PARENT_LIST].extend(parent_list) + for parent in parent_list: + if parent not in self.dictionary[key][ListNode.PARENT_LIST]: + self.dictionary[key][ListNode.PARENT_LIST].append(parent) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 1c84ca5735a32c984913eb4af58f5ba6f6e69439..c2169c153d6cc70d2ed2f6cd13e00f954173e2a6 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -326,6 +326,17 @@ class InitDataBase(): raise ContentNoneException( '{db_name}:There is no relevant data in the source \ package provided '.format(db_name=db_name)) + for index, src_package_item in enumerate(packages_datas): + try: + src_package_name = '-'.join([src_package_item.get('name'), + src_package_item.get('version'), + src_package_item.get('release') + '.src.rpm' + ]) + except AttributeError as exception_msg: + src_package_name = None + LOGGER.logger.warning(exception_msg) + finally: + packages_datas[index]['src_name'] = src_package_name with DBHelper(db_name=db_name) as database: database.batch_add(packages_datas, SrcPack) if lifecycle_status_val == 'enable': diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index a8b9140f0a7d6bbc8adf63e3f41d90bb9163bdad..f95bfcc242c5a2b939a5c2ec123c617bdf9ea24d 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -42,6 +42,7 @@ class SrcPack(DBHelper.BASE): checksum_type = Column(String(500), nullable=True) maintaniner = Column(String(100), nullable=True) maintainlevel = Column(String(100), nullable=True) + src_name = Column(String(100), nullable=True) class BinPack(DBHelper.BASE): diff --git a/packageship/test/common_files/correct_test_result_json/self_depend.json b/packageship/test/common_files/correct_test_result_json/self_depend.json index 6fee2dc84119c21efe60c92bb08e46994ea7d8c7..260aedb47209ab4216c6d563d20f985ffe45ced5 100644 --- a/packageship/test/common_files/correct_test_result_json/self_depend.json +++ b/packageship/test/common_files/correct_test_result_json/self_depend.json @@ -19,14 +19,6 @@ [ "D1", "install" - ], - [ - "B1", - "install" - ], - [ - "D", - "build" ] ] ], @@ -42,10 +34,6 @@ [ "C1", "install" - ], - [ - "C", - "build" ] ] ], @@ -80,17 +68,9 @@ "0.1", "mainline", [ - [ - "A", - "build" - ], [ "A2", "install" - ], - [ - "B2", - "install" ] ] ], @@ -146,14 +126,6 @@ [ "D1", "install" - ], - [ - "B1", - "install" - ], - [ - "D", - "build" ] ] ], @@ -169,10 +141,6 @@ [ "A1", "install" - ], - [ - "C", - "build" ] ] ], @@ -211,17 +179,9 @@ "root", null ], - [ - "A", - "build" - ], [ "A2", "install" - ], - [ - "B2", - "install" ] ] ], @@ -289,14 +249,6 @@ [ "D1", "install" - ], - [ - "B1", - "install" - ], - [ - "D", - "build" ] ] ], @@ -316,10 +268,6 @@ [ "A1", "install" - ], - [ - "C", - "build" ] ] ], @@ -357,14 +305,6 @@ [ "A2", "install" - ], - [ - "B2", - "install" - ], - [ - "A", - "build" ] ] ], @@ -452,14 +392,6 @@ [ "D1", "install" - ], - [ - "B1", - "install" - ], - [ - "D", - "build" ] ] ], @@ -479,178 +411,6 @@ [ "A1", "install" - ], - [ - "C", - "build" - ] - ] - ], - "B1": [ - "B", - "0.0.2", - "mainline", - [ - [ - "A", - "build" - ], - [ - "D", - "build" - ] - ] - ], - "B2": [ - "B", - "0.0.2", - "mainline", - [ - [ - "C", - "build" - ] - ] - ], - "C1": [ - "C", - "0.1", - "mainline", - [ - [ - "A2", - "install" - ], - [ - "B2", - "install" - ], - [ - "A", - "build" - ] - ] - ], - "D1": [ - "D", - "0.11", - "mainline", - [ - [ - "A2", - "install" - ], - [ - "D2", - "install" - ] - ] - ], - "C2": [ - "C", - "0.1", - "mainline", - [ - [ - "C", - "Subpack" - ] - ] - ], - "D2": [ - "D", - "0.11", - "mainline", - [ - [ - "D", - "Subpack" - ] - ] - ] - }, - "source_dicts": { - "A": [ - "mainline", - "0.0.23b" - ], - "B": [ - "mainline", - "0.0.2" - ], - "C": [ - "mainline", - "0.1" - ], - "D": [ - "mainline", - "0.11" - ] - }, - - "not_found_components": [] - }, - "msg": "Successful Operation!" - } - }, - { - "input": { - "packagename": "A", - "selfbuild": "1", - "withsubpack": "1", - "packtype": "source" - }, - "output": { - "code": "2001", - "data": { - "binary_dicts": { - "A1": [ - "A", - "0.0.23b", - "mainline", - [ - [ - "root", - null - ], - [ - "D1", - "install" - ], - [ - "B", - "build" - ], - [ - "D", - "build" - ], - [ - "B1", - "install" - ] - ] - ], - "A2": [ - "A", - "0.0.23b", - "mainline", - [ - [ - "root", - null - ], - [ - "A1", - "install" - ], - [ - "C1", - "install" - ], - [ - "C", - "build" ] ] ], @@ -688,18 +448,6 @@ [ "A2", "install" - ], - [ - "B", - "build" - ], - [ - "A", - "build" - ], - [ - "B2", - "install" ] ] ], @@ -759,294 +507,6 @@ "0.11" ] }, - - "not_found_components": [] - }, - "msg": "Successful Operation!" - } - }, - { - "input": { - "packagename": "A2", - "selfbuild": "1", - "withsubpack": "0" - }, - "output": { - "code": "2001", - "data": { - "binary_dicts": { - "A1": [ - "A", - "0.0.23b", - "mainline", - [ - [ - "D1", - "install" - ], - [ - "B", - "build" - ], - [ - "D", - "build" - ], - [ - "B1", - "install" - ] - ] - ], - "A2": [ - "A", - "0.0.23b", - "mainline", - [ - [ - "root", - null - ], - [ - "C1", - "install" - ], - [ - "A1", - "install" - ], - [ - "C", - "build" - ] - ] - ], - "B1": [ - "B", - "0.0.2", - "mainline", - [ - [ - "A", - "build" - ], - [ - "D", - "build" - ] - ] - ], - "B2": [ - "B", - "0.0.2", - "mainline", - [ - [ - "C", - "build" - ] - ] - ], - "C1": [ - "C", - "0.1", - "mainline", - [ - [ - "A2", - "install" - ], - [ - "B", - "build" - ], - [ - "A", - "build" - ], - [ - "B2", - "install" - ] - ] - ], - "D1": [ - "D", - "0.11", - "mainline", - [ - [ - "A2", - "install" - ] - ] - ] - }, - "source_dicts": { - "A": [ - "mainline", - "0.0.23b" - ], - "B": [ - "mainline", - "0.0.2" - ], - "C": [ - "mainline", - "0.1" - ], - "D": [ - "mainline", - "0.11" - ] - }, - - "not_found_components": [] - }, - "msg": "Successful Operation!" - } - }, - { - "input": { - "packagename": "A", - "selfbuild": "1", - "withsubpack": "0", - "packtype": "source" - }, - "output": { - "code": "2001", - "data": { - "binary_dicts": { - "A1": [ - "A", - "0.0.23b", - "mainline", - [ - [ - "root", - null - ], - [ - "D1", - "install" - ], - [ - "B", - "build" - ], - [ - "D", - "build" - ], - [ - "B1", - "install" - ] - ] - ], - "A2": [ - "A", - "0.0.23b", - "mainline", - [ - [ - "root", - null - ], - [ - "A1", - "install" - ], - [ - "C1", - "install" - ], - [ - "C", - "build" - ] - ] - ], - "B1": [ - "B", - "0.0.2", - "mainline", - [ - [ - "A", - "build" - ], - [ - "D", - "build" - ] - ] - ], - "B2": [ - "B", - "0.0.2", - "mainline", - [ - [ - "C", - "build" - ] - ] - ], - "C1": [ - "C", - "0.1", - "mainline", - [ - [ - "A2", - "install" - ], - [ - "B", - "build" - ], - [ - "A", - "build" - ], - [ - "B2", - "install" - ] - ] - ], - "D1": [ - "D", - "0.11", - "mainline", - [ - [ - "A2", - "install" - ] - ] - ] - }, - "source_dicts": { - "A": [ - "mainline", - "0.0.23b" - ], - "B": [ - "mainline", - "0.0.2" - ], - "C": [ - "mainline", - "0.1" - ], - "D": [ - "mainline", - "0.11" - ] - }, - "not_found_components": [] }, "msg": "Successful Operation!" diff --git a/packageship/test/common_files/db_origin/data_1_bin.sqlite b/packageship/test/common_files/db_origin/data_1_bin.sqlite index 8d5051866a938530ba81ffb7a1b971e4ef3c8f78..5a7d5a748d93f43596bd4c29aa3f38c94b80506a 100644 Binary files a/packageship/test/common_files/db_origin/data_1_bin.sqlite and b/packageship/test/common_files/db_origin/data_1_bin.sqlite differ diff --git a/packageship/test/common_files/db_origin/data_2_bin.sqlite b/packageship/test/common_files/db_origin/data_2_bin.sqlite index 76533ab871181f22041d56ed879eb4f841e4fd35..aa3d11c5a2a741750fa6ed08279514f9c87532de 100644 Binary files a/packageship/test/common_files/db_origin/data_2_bin.sqlite and b/packageship/test/common_files/db_origin/data_2_bin.sqlite differ diff --git a/packageship/test/common_files/dbs/fedora30.db b/packageship/test/common_files/dbs/fedora30.db index 2d6c77f9da54228843d3026dd79df0a5ad219588..cb66065433352c32ea7d4d71c9cdbce45582c38a 100644 Binary files a/packageship/test/common_files/dbs/fedora30.db and b/packageship/test/common_files/dbs/fedora30.db differ diff --git a/packageship/test/common_files/dbs/mainline.db b/packageship/test/common_files/dbs/mainline.db index a96d22a6301a92698fa34d0e3fa5dfca78d11245..c78effe5542c9170811ee769d564a45a5ddf498f 100644 Binary files a/packageship/test/common_files/dbs/mainline.db and b/packageship/test/common_files/dbs/mainline.db differ diff --git a/packageship/test/common_files/operate_dbs/fedora30.db b/packageship/test/common_files/operate_dbs/fedora30.db index 2d6c77f9da54228843d3026dd79df0a5ad219588..cb66065433352c32ea7d4d71c9cdbce45582c38a 100644 Binary files a/packageship/test/common_files/operate_dbs/fedora30.db and b/packageship/test/common_files/operate_dbs/fedora30.db differ diff --git a/packageship/test/common_files/operate_dbs/mainline.db b/packageship/test/common_files/operate_dbs/mainline.db index a96d22a6301a92698fa34d0e3fa5dfca78d11245..c78effe5542c9170811ee769d564a45a5ddf498f 100644 Binary files a/packageship/test/common_files/operate_dbs/mainline.db and b/packageship/test/common_files/operate_dbs/mainline.db differ diff --git a/packageship/test/common_files/package.ini b/packageship/test/common_files/package.ini index b312707c18d932406a6043f8f56fa6688691b69b..11eabd6d049d3fba9ac1e0a3527a1a4eb99ca186 100644 --- a/packageship/test/common_files/package.ini +++ b/packageship/test/common_files/package.ini @@ -1,22 +1,16 @@ [SYSTEM] -init_conf_path = -debug = false +init_conf_path = write_port = 8080 query_port = 8090 write_ip_addr = 127.0.0.1 query_ip_addr = 127.0.0.1 - -[DATABASE] -user_name = -password = -host = -port = -database = -dbtype = sqlite +remote_host = https://api.openeuler.org/pkgmanage [LOG] log_level = INFO log_name = log_info.log +backup_count = 10 +max_bytes = 314572800 [UWSGI] daemonize = /var/log/uwsgi.log @@ -24,3 +18,13 @@ buffer-size = 65536 http-timeout = 600 harakiri = 600 +[TIMEDTASK] +open = True +hour = 3 +minute = 0 + +[LIFECYCLE] +warehouse_remote = https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/ +pool_workers = 10 +warehouse = src-openeuler + diff --git a/packageship/test/test_module/init_system_tests/test_importdata.py b/packageship/test/test_module/init_system_tests/test_importdata.py index 59beb9603eaf3725ae70faf706791e4d6fb60d57..681be9c8c95e79176ab9bf35017b5e92288baf75 100644 --- a/packageship/test/test_module/init_system_tests/test_importdata.py +++ b/packageship/test/test_module/init_system_tests/test_importdata.py @@ -10,7 +10,7 @@ import warnings from configparser import ConfigParser import yaml from packageship import system_config -from packageship.libs.exception import Error +from packageship.libs.exception import Error,ConfigurationException try: @@ -71,10 +71,10 @@ class ImportData(unittest.TestCase): w_f.write("") InitDataBase(config_file_path=_config_path).init_data() - except ContentNoneException as error: + except ConfigurationException as error: self.assertEqual( error.__class__, - ContentNoneException, + ConfigurationException, msg="Yaml file exists, but the content is empty. The exception type is wrong") finally: # Restore files