From b7191d5f1cb1fca2854b2288fc750e91c3ea68ec Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Mon, 6 Mar 2023 15:40:21 +0800 Subject: [PATCH 01/14] change folder name of rom_ram_analzyer Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/README.md | 47 ++ tools/rom_ram_analyzer/lite_small/__init__.py | 0 .../lite_small/pkgs/__init__.py | 0 .../lite_small/pkgs/basic_tool.py | 142 ++++++ .../lite_small/pkgs/gn_common_tool.py | 289 +++++++++++ .../lite_small/pkgs/simple_excel_writer.py | 119 +++++ .../lite_small/pkgs/simple_yaml_tool.py | 15 + .../lite_small/src/__init__.py | 1 + .../rom_ram_analyzer/lite_small/src/config.py | 213 +++++++++ .../lite_small/src/config.yaml | 144 ++++++ .../lite_small/src/get_subsystem_component.py | 131 +++++ tools/rom_ram_analyzer/lite_small/src/misc.py | 228 +++++++++ .../lite_small/src/preprocess.py | 4 + .../lite_small/src/rom_analysis.py | 295 ++++++++++++ .../lite_small/src/template_processor.py | 244 ++++++++++ tools/rom_ram_analyzer/standard/README.md | 143 ++++++ tools/rom_ram_analyzer/standard/__init__.py | 0 .../standard/pkgs/__init__.py | 0 .../standard/pkgs/basic_tool.py | 33 ++ .../standard/pkgs/gn_common_tool.py | 157 ++++++ .../standard/pkgs/simple_excel_writer.py | 118 +++++ .../rom_ram_analyzer/standard/ram_analyzer.py | 452 ++++++++++++++++++ .../rom_ram_analyzer/standard/rom_analyzer.py | 200 ++++++++ 23 files changed, 2975 insertions(+) create mode 100644 tools/rom_ram_analyzer/lite_small/README.md create mode 100644 tools/rom_ram_analyzer/lite_small/__init__.py create mode 100644 tools/rom_ram_analyzer/lite_small/pkgs/__init__.py create mode 100644 tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py create mode 100644 tools/rom_ram_analyzer/lite_small/pkgs/gn_common_tool.py create mode 100644 tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py create mode 100644 tools/rom_ram_analyzer/lite_small/pkgs/simple_yaml_tool.py create mode 100644 tools/rom_ram_analyzer/lite_small/src/__init__.py create mode 100644 tools/rom_ram_analyzer/lite_small/src/config.py create mode 100644 tools/rom_ram_analyzer/lite_small/src/config.yaml create mode 100644 tools/rom_ram_analyzer/lite_small/src/get_subsystem_component.py create mode 100644 tools/rom_ram_analyzer/lite_small/src/misc.py create mode 100644 tools/rom_ram_analyzer/lite_small/src/preprocess.py create mode 100644 tools/rom_ram_analyzer/lite_small/src/rom_analysis.py create mode 100644 tools/rom_ram_analyzer/lite_small/src/template_processor.py create mode 100644 tools/rom_ram_analyzer/standard/README.md create mode 100644 tools/rom_ram_analyzer/standard/__init__.py create mode 100644 tools/rom_ram_analyzer/standard/pkgs/__init__.py create mode 100644 tools/rom_ram_analyzer/standard/pkgs/basic_tool.py create mode 100644 tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py create mode 100644 tools/rom_ram_analyzer/standard/pkgs/simple_excel_writer.py create mode 100644 tools/rom_ram_analyzer/standard/ram_analyzer.py create mode 100644 tools/rom_ram_analyzer/standard/rom_analyzer.py diff --git a/tools/rom_ram_analyzer/lite_small/README.md b/tools/rom_ram_analyzer/lite_small/README.md new file mode 100644 index 0000000..ce3da51 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/README.md @@ -0,0 +1,47 @@ +# rom_ram_analyzer + +## 目的 + +分析各部件的rom占用,结果以xls和json格式进行保存 + +## 支持产品 + +目标是支持所有的产品,但是目前由于配置文件没设计好,只支持:ipcamera_hispark_taurus ipcamera_hispark_taurus_linux wifiiot_hispark_pegasus + +## 代码思路 + +1. 扫描BUILD.gn文件,收集各个target的编译产物及其对应的component_name, subsystem_name信息,并存储到config.yaml中的gn_info_file字段指定的json文件中 +2. 根据配置文件config.yaml扫描产品的编译产物目录,得到真实的编译产物信息(主要是大小) +3. 用真实的编译产物与从BUILD.gn中收集的信息进行匹配,从而得到编译产物-大小-所属部件的对应信息 +4. 如果匹配失败,会直接利用grep到项目路径下进行搜索 +5. 如果还搜索失败,则将其归属到others + + +## 使用 + +前置条件： + +1. 获取整个本文件所在的整个目录 +1. 对系统进行编译 +1. linux平台 +1. python3.8及以后 +1. 安装requirements + ```txt + xlwt==1.3.0 + ``` + +1. `python3 rom_analysis.py --product_name {your_product_name} --oh_path {root_path_of_oh} [--recollect_gn bool]`运行代码,其中recollect_gn表示是需要重新扫描BUILD.gn还是直接使用已有结果.eg: `python3 rom_analysis.py --product_name ipcamera_hispark_taurus` +3. 运行完毕会产生4个json文件及一个xls文件,如果是默认配置,各文件描述如下: + - gn_info.json:BUILD.gn的分析结果 + - sub_com_info.json:从bundle.json中进行分析获得的各部件及其对应根目录的信息 + - {product_name}_product.json:该产品实际的编译产物信息,根据config.yaml进行收集 + - {product_name}_result.json:各部件的rom大小分析结果 + - {product_name}_result.xls:各部件的rom大小分析结果 + +## 新增template + +主要是在config.py中配置Processor,并在config.yaml中添加相应内容 + +## 后续工作 + +1. 部分log的输出有待优化 \ No newline at end of file diff --git a/tools/rom_ram_analyzer/lite_small/__init__.py b/tools/rom_ram_analyzer/lite_small/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/__init__.py b/tools/rom_ram_analyzer/lite_small/pkgs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py b/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py new file mode 100644 index 0000000..febe482 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2022 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import itertools +import os +import re +import glob +from typing import * + + +def do_nothing(x: Any) -> Any: + return x + + +class BasicTool: + @classmethod + def find_files_with_pattern(cls, folder: str, pattern: str = "/**", recursive: bool = True, apply_abs: bool = True, + real_path: bool = True, de_duplicate: bool = True, is_sort: bool = True, + post_handler: Callable[[List[str]], List[str]] = None) -> list: + """ + 根据指定paatern匹配folder下的所有文件，默认递归地查找所有文件 + folder：要查找的目录,会先经过cls.abspath处理(结尾不带/) + pattern：要查找的模式,需要以/开头,因为会和folder拼接 + recursive：是否递归查找 + apply_abs：是否将路径转换为绝对路径 + real_path：如果是软链接，是否转换为真正的路径 + de_duplicate：是否去重 + is_sort：是否排序 + post_handler: 对文件进行额外处理的方法，参数为文件名的List，返回值为字符串列表 + FIXME 有可能会卡住,可能原因为符号链接 + """ + file_list = glob.glob(cls.abspath(folder)+pattern, recursive=recursive) + if apply_abs: + file_list = list(map(lambda x: cls.abspath(x), file_list)) + if real_path: + file_list = list(map(lambda x: os.path.realpath(x), file_list)) + if de_duplicate: + file_list = list(set(file_list)) + if is_sort: + file_list = sorted(file_list, key=str.lower) + if post_handler: + file_list = post_handler(file_list) + if folder in file_list: + file_list.remove(folder) + return file_list + + @classmethod + def match_paragraph(cls, content: str, start_pattern: str = r"\w+\(\".*?\"\) *{", end_pattern: str = "\}") -> \ + Iterator[re.Match]: + """ + 匹配代码段，支持单行 + 注意：ptrn中已经包含前面的空格，所以start_pattern中可以省略 + :param content: 被匹配的字符串 + :param start_pattern: 模式的开头 + :param end_pattern: 模式的结尾 + :return: 匹配到的段落的迭代器 + """ + ptrn = r'^( *){s}(?#匹配开头).*?(?#中间非贪婪)\1(?#如果开头前面有空格,则结尾的前面应该有相同数量的空格)?{e}$(?#匹配结尾)'.format( + s=start_pattern, e=end_pattern) + ptrn = re.compile(ptrn, re.M | re.S) + result = re.finditer(ptrn, content) + return result + + @classmethod + def re_group_1(cls, content: str, pattern: str, **kwargs) -> str: + """ + 匹配正则表达式，如果有匹配到内容，返回group(1)的内容 + :param content: 要被匹配的内容 + :param pattern: 进行匹配的模式 + :return: 匹配到的结果（group(1)） + TODO 对（）的检查应该更严格 + """ + if not (r'(' in pattern and r')' in pattern): + raise ValueError("parentheses'()' must in the pattern") + result = re.search(pattern, content, **kwargs) + if result: + return result.group(1) + return str() + + @classmethod + def abspath(cls, path: str) -> str: + """ + 将路径转换为绝对路径，如果有~，展开 + :param path: 要被转换的路径 + :return: 绝对路径 + """ + return os.path.abspath(os.path.expanduser(path)) + + @classmethod + def grep_ern(cls, pattern: str, path: str, include: str = str(), exclude: tuple = tuple(), + post_handler: Callable[[Text], Any] = do_nothing) -> Any: + """ + 执行grep命令来查找内容 + :param exclude: 不搜索path下的的exclude目录 + :param include: 指定要搜索的文件 + :param pattern: 使用pattern进行grep + :param path: 搜索路径 + :param post_handler: 对grep的结果进行后处理 + :return: post_handler对grep结果进行处理后的结果 + TODO 将cls.execute用subprocess代替 + """ + cmd = f"grep -Ern '{pattern}' '{cls.abspath(path)}'" + if include: + cmd += f" --include='{include}'" + for e in exclude: + cmd += f" --exclude-dir={e}" + o = cls.execute(cmd) + if post_handler: + o = post_handler(o) + return o + + @classmethod + def execute(cls, cmd: str, post_processor: Callable[[Text], Text] = do_nothing) -> Any: + """ + 封装popen，返回标准输出的列表 + :param post_processor: 对执行结果进行处理 + :param cmd: 待执行的命令 + :return: 经处理过后的字符串列表 + + """ + output = os.popen(cmd).read() + output = post_processor(output) + return output + + +if __name__ == '__main__': + for i in BasicTool.grep_ern("^( *)ohos_shared_library", "/home/aodongbiao/oh", include="BUILD.gn", exclude=("out", "doc", ".ccache"), post_handler=lambda x: x.split('\n')): + if "oh/out" in i: + print(i) diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/gn_common_tool.py b/tools/rom_ram_analyzer/lite_small/pkgs/gn_common_tool.py new file mode 100644 index 0000000..66ebb21 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/pkgs/gn_common_tool.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2022 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os +import logging +import re +import ast +import json +import collections +from typing import * +if __name__ == '__main__': + from basic_tool import BasicTool +else: + from pkgs.basic_tool import BasicTool + + +class GnCommonTool: + """ + 处理BUILD.gn文件的通用方法 + """ + + @classmethod + def _find_gn_variable_list(cls, content: str) -> List: + """ + 获取s中${xxx}或$xxx形式的gn变量 + :param content: 待查找的字符串 + :param sep: 分隔符，使用本分隔符将内容进行分隔然后逐一查找 + :return: 变量名及其符号，eg：${abc}、$abc + :FIXME 没有对a = 'a' b = a中的b这种形式进行处理 + """ + result = list() + splited = content.split(os.sep) + patern = re.compile(r"\${.*?}") + for item in splited: + m = re.findall(patern, item) + result.extend(m) + if len(m) == 0 and "$" in item: + item = item.strip('"') + result.append(item[item.index("$"):]) + return result + + @classmethod + def is_gn_variable(cls, target: str, quote_processed: bool = False): + """ + 判断target是否是gn中的变量: + 规则：如果是quote_processed is False，则没有引号均认为是变量，有引号的情况下，如果是"$xxx"或${xx}的模式，则认为xxx是变量； + 如果quote_processed is True，则只要$开头就认为是变量 + b = "xxx" + c = b + c = "${b}" + "$p" + :param target: 要进行判断的字符串对象 + :param quote_processed: 引号是否已经去除 + :return: target是否为gn中的变量 + """ + target = target.strip() + if quote_processed: + return target.startswith("$") + if target.startswith('"') and target.endswith('"'): + target = target.strip('"') + if target.startswith("${") and target.endswith("}"): + return True + elif target.startswith("$"): + return True + return False + else: + return True + + @classmethod + def contains_gn_variable(cls, s: str, quote_processed: bool = False): + """ + 判断字符串s中是否包含gn变量 + """ + return cls.is_gn_variable(s, quote_processed) or ("$" in s) + + # 给__find_variables_in_gn用的，减少io + __var_val_mem_dict = collections.defaultdict(str) + + @classmethod + def find_variables_in_gn(cls, var_name_tuple: tuple, path: str, stop_tail: str = "home", use_cache: bool = False) -> \ + List[str]: + """ + 同时查找多个gn变量的值 + var_name_tuple：变量名的tuple，变量名应是未经过处理后的，如： + xxx + "${xxx}" + "$xxx" + :param var_name_tuple: 待查找的变量名的列表 + :param path: 变量名所在文件的路径 + :param stop_tail: 当path以stop_tail结尾时，停止查找 + :param use_cache: 是否使用缓存 + :return: 变量值的列表 + """ + if os.path.isfile(path): + path, _ = os.path.split(path) + var_val_dict = collections.defaultdict(str) + not_found_count = len(var_name_tuple) + if use_cache: + for var in var_name_tuple: + val = GnCommonTool.__var_val_mem_dict[var] + if val: + not_found_count -= 1 + var_val_dict[var] = val + while (stop_tail in path) and not_found_count: + for v in var_name_tuple: + pv = v.strip('"').lstrip("${").rstrip('}') + # 先直接grep出pv *= *\".*?\"的 + # 然后排除含有$符的 + # 再取第一个 + # 最后只取引号内的 + cmd = fr"grep -Ern '{pv} *= *\".*?\"' --include=*.gn* {path} | grep -Ev '\$' " \ + r"| head -n 1 | grep -E '\".*\"' -wo" + output = BasicTool.execute(cmd, lambda x: x.strip().strip('"')) + # backup:end + if not output: + continue + not_found_count -= 1 + var_val_dict[v] = output + GnCommonTool.__var_val_mem_dict[v] = output + path, _ = os.path.split(path) + return list(var_val_dict.values()) + + @classmethod + def replace_gn_variables(cls, s: str, gn_path: str, stop_tail: str) -> str: + """ + 替换字符串中的gn变量名为其值,注意,没有对s是否真的包含gn变量进行验证 + :param s: 待替换的字符串 + :param gn_path: 字符串所在的gn文件 + :param stop_tail: 当变量查找到stop_tail目录时停止 + :return: 将变量替换为其值的字符串 + """ + variable_list = GnCommonTool._find_gn_variable_list(s) + if len(variable_list) == 0: + return s + value_list = GnCommonTool.find_variables_in_gn( + tuple(variable_list), path=gn_path, stop_tail=stop_tail) + for k, v in dict(zip(variable_list, value_list)).items(): + s = s.replace(k, v) + return s + + @classmethod + def find_values_of_variable(cls, var_name: str, path: str, stop_tail: str = "home") -> list: + """ + 查找变量的值，如果有多个可能值，全部返回 + :param var_name: 变量名 + :param path: 变量名所在的文件 + :param stop_tail: 当变量查找到stop_tail目录时停止 + :return: 该变量的可能值 + """ + if os.path.isfile(path): + path, _ = os.path.split(path) + result = list() + v = var_name.strip('"').lstrip("${").rstrip('}') + while stop_tail in path: + cmd = fr"grep -Ern '^( *){v} *= *\".*?\"' --include=*.gn* {path}" + output = os.popen(cmd).readlines() + path = os.path.split(path)[0] + if not output: + continue + for line in output: + line = line.split('=')[-1].strip().strip('"') + if len(line) == 0: + continue + result.append(line) + break + return result + + +class GnVariableParser: + @classmethod + def string_parser(cls, var: str, content: str) -> str: + """ + 解析值为字符串的变量,没有对引号进行去除,如果是a = b这种b为变量的,则无法匹配 + :param content: 要进行解析的内容 + :param var: 变量名 + :return: 变量值[str] + """ + result = BasicTool.re_group_1( + content, r"{} *= *[\n]?(\".*?\")".format(var), flags=re.S | re.M) + return result + + @classmethod + def list_parser(cls, var: str, content: str) -> List[str]: + """ + 解析值为列表的变量，list的元素必须全为数字或字符串,且没有对引号进行去除,如果是a = b这种b为变量的,则无法匹配 + :param var: 变量名 + :param content: 要进行 + :return: 变量值[List] + """ + result = BasicTool.re_group_1( + content, r"{} *= *(\[.*?\])".format(var), flags=re.S | re.M) + result = ast.literal_eval(result.strip()) + return result + + +if __name__ == '__main__': + cc = \ + """ + target("shared_library", "mmp"){ + xxx + } + + ohos_shared_library("pinauthservice") { + sources = [ + "//base/useriam/pin_auth/services/modules/driver/src/pin_auth_driver_hdi.cpp", + "//base/useriam/pin_auth/services/modules/driver/src/pin_auth_interface_adapter.cpp", + "//base/useriam/pin_auth/services/modules/executors/src/pin_auth_executor_callback_hdi.cpp", + "//base/useriam/pin_auth/services/modules/executors/src/pin_auth_executor_hdi.cpp", + "//base/useriam/pin_auth/services/modules/inputters/src/i_inputer_data_impl.cpp", + "//base/useriam/pin_auth/services/modules/inputters/src/pin_auth_manager.cpp", + "//base/useriam/pin_auth/services/sa/src/pin_auth_service.cpp", + ] + + configs = [ + ":pin_auth_services_config", + "//base/useriam/user_auth_framework/common:iam_log_config", + "//base/useriam/user_auth_framework/common:iam_utils_config", + ] + + deps = [ + "//base/useriam/pin_auth/frameworks:pinauth_ipc", + "//third_party/openssl:libcrypto_shared", + ] + + external_deps = [ + "access_token:libaccesstoken_sdk", + "c_utils:utils", + "drivers_interface_pin_auth:libpin_auth_proxy_1.0", + "hisysevent_native:libhisysevent", + "hiviewdfx_hilog_native:libhilog", + "ipc:ipc_core", + "safwk:system_ability_fwk", + "user_auth_framework:userauth_executors", + ] + t = [ + 1, + 2, + 3 + ] + tt = [ + aaa, + bbb, + ccc + ] + remove_configs = [ "//build/config/compiler:no_exceptions" ] + + subsystem_name = "useriam" + part_name = "pin_auth" + }""" + s = """ +updater_usb_init_cfg_path = "//base/startup/init/services/etc/init.usb.cfg" +updater_init_usb_configfs_path_cfg = + "//drivers/peripheral/usb/cfg/init.usb.configfs.cfg" +updater_faultloggerd_cfg = +"//base/hiviewdfx/faultloggerd/services/config/faultloggerd.cfg" +updater_hilog_cfg = "//base/hiviewdfx/hilog/services/hilogd/etc/hilogd.cfg" + +ohos_prebuilt_etc("updater_hilog.cfg") { +source = "${updater_hilog_cfg}" +install_images = [ "updater" ] +part_name = "updater" +} +""" + s = "\"${updater_faultloggerd_cfg}\"" + print(GnCommonTool.contains_gn_variable(s)) + # print(GnVariableParser.string_parser("updater_faultloggerd_cfg",s)) + # print(re.search( + # "updater_faultloggerd_cfg *= *[\n]?(\".*?\")", s, flags=re.S | re.M).group()) + # print(GnVariableParser.list_parser("t", cc)) + # print(len(GnVariableParser.list_parscer("t", cc))) + # print(TargetNameParser.second_parser(cc)) + # print(GnCommonTool._find_gn_variable_list("a/${b}${e}/$c")) + # print(GnCommonTool._find_gn_variable_list("abc_$abc")) + # print(GnCommonTool.find_values_of_variable( + # "\"${OHOS_PROFILER_SUBSYS_NAME}\"", path="/home/aodongbiao/oh/third_party/abseil-cpp/absl/strings/BUILD.gn", stop_tail="/home/aodongbiao/oh")) + ... diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py b/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py new file mode 100644 index 0000000..ebe37e1 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py @@ -0,0 +1,119 @@ +import xlwt +from xlwt import Worksheet +import typing +import logging +from typing import Optional +from collections.abc import Iterable + + +class SimpleExcelWriter: + def __init__(self, default_sheet_name: str = "sheet1"): + self.__book = xlwt.Workbook(encoding='utf-8', style_compression=0) + self.__sheet_dict = { + default_sheet_name: self.__book.add_sheet( + sheetname=default_sheet_name, cell_overwrite_ok=True) + } + self.__sheet_pos = { + default_sheet_name: (0, 0) # 记录各个sheet页已经写到什么位置了，当前值为还没有写的 + } + self.__default_sheet_name = default_sheet_name + # 表头样式 + self.__head_style = xlwt.XFStyle() + # 内容样式 + self.__content_style = xlwt.XFStyle() + # 字体 + font = xlwt.Font() + font.bold = True + + # 设置背景颜色 + pattern = xlwt.Pattern() + pattern.pattern = xlwt.Pattern.SOLID_PATTERN + pattern.pattern_fore_colour = 22 # 背景颜色 + + # 居中对齐 + alignment = xlwt.Alignment() + alignment.horz = xlwt.Alignment.HORZ_CENTER # 水平方向 + alignment.vert = xlwt.Alignment.VERT_CENTER # 垂直方向 + + self.__head_style.font = font + self.__head_style.alignment = alignment + self.__head_style.pattern = pattern + self.__content_style.alignment = alignment + + def __increment_y(self, sheet_name: str, value: int = 1) -> int: + if sheet_name in self.__sheet_pos.keys(): + x, y = self.__sheet_pos.get(sheet_name) + y = y + value + self.__sheet_pos[sheet_name] = (x, y) + return y + + def __increment_x(self, sheet_name: str, value: int = 1) -> int: + if sheet_name in self.__sheet_pos.keys(): + x, y = self.__sheet_pos.get(sheet_name) + x = x + value + self.__sheet_pos[sheet_name] = (x, 0) + return x + + def append_line(self, content: list, sheet_name: str = None): + sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name + if sheet_name not in self.__sheet_dict.keys(): + logging.error("sheet name '{}' not exist".format(sheet_name)) + return + sheet: Worksheet = self.__sheet_dict.get(sheet_name) + x, y = self.__sheet_pos.get(sheet_name) + for ele in content: + sheet.write(x, y, ele, style=self.__content_style) + y = self.__increment_y(sheet_name) + self.__increment_x(sheet_name) + + def write_merge(self, x0: int, y0: int, x1: int, y1: int, content: typing.Any, + sheet_name: str = None): + sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name + if sheet_name not in self.__sheet_dict.keys(): + logging.error("sheet name '{}' not exist".format(sheet_name)) + return + sheet: Worksheet = self.__sheet_dict.get(sheet_name) + sheet.write_merge(x0, x1, y0, y1, content, style=self.__content_style) + + def set_sheet_header(self, headers: Iterable, sheet_name: str = None): + """ + 给sheet页设置表头 + """ + sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name + if sheet_name not in self.__sheet_dict.keys(): + logging.error("sheet name '{}' not exist".format(sheet_name)) + return + x, y = self.__sheet_pos.get(sheet_name) + if x != 0 or y != 0: + logging.error( + "pos of sheet '{}' is not (0,0). set_sheet_header must before write".format(sheet_name)) + return + sheet: Worksheet = self.__sheet_dict.get(sheet_name) + for h in headers: + sheet.write(x, y, h, self.__head_style) + y = self.__increment_y(sheet_name) + self.__increment_x(sheet_name) + + def add_sheet(self, sheet_name: str, cell_overwrite_ok=True) -> Optional[xlwt.Worksheet]: + if sheet_name in self.__sheet_dict.keys(): + logging.error("sheet name '{}' has exist".format(sheet_name)) + return + self.__sheet_dict[sheet_name] = self.__book.add_sheet( + sheetname=sheet_name, cell_overwrite_ok=cell_overwrite_ok) + self.__sheet_pos[sheet_name] = (0, 0) + return self.__sheet_dict.get(sheet_name) + + def save(self, file_name: str): + self.__book.save(file_name) + + +if __name__ == '__main__': + writer = SimpleExcelWriter(default_sheet_name="first") + writer.add_sheet("second") + writer.add_sheet("third") + writer.set_sheet_header(["h", "m", "n"]) + writer.append_line([1, 2, 3]) + writer.append_line([2, 3, 4], "second") + writer.append_line([3, 4, 5], "third") + writer.append_line([3, 2, 1]) + writer.save("demo.xls") diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/simple_yaml_tool.py b/tools/rom_ram_analyzer/lite_small/pkgs/simple_yaml_tool.py new file mode 100644 index 0000000..5edf612 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/pkgs/simple_yaml_tool.py @@ -0,0 +1,15 @@ +import yaml +from typing import * +from yaml.loader import SafeLoader + + +class SimpleYamlTool: + @classmethod + def read_yaml(cls, file_name: str, mode: str = "r", encoding: str = "utf-8") -> Dict: + with open(file_name, mode, encoding=encoding) as f: + return yaml.load(f, Loader=SafeLoader) + + +if __name__ == '__main__': + config = SimpleYamlTool.read_yaml("/home/aodongbiao/build_static_check/tools/component_tools/rom_ram_analyzer/src/config.yaml") + print(config["black_grep_dir"]) \ No newline at end of file diff --git a/tools/rom_ram_analyzer/lite_small/src/__init__.py b/tools/rom_ram_analyzer/lite_small/src/__init__.py new file mode 100644 index 0000000..341988c --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/src/__init__.py @@ -0,0 +1 @@ +VERSION = 1.0 diff --git a/tools/rom_ram_analyzer/lite_small/src/config.py b/tools/rom_ram_analyzer/lite_small/src/config.py new file mode 100644 index 0000000..05f1a26 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/src/config.py @@ -0,0 +1,213 @@ +import os +import sys +import argparse +import json +from typing import * + +import preprocess +from pkgs.simple_yaml_tool import SimpleYamlTool +from pkgs.basic_tool import do_nothing, BasicTool +from get_subsystem_component import SC +from misc import * +from template_processor import * +""" +只给rom_analysis.py使用 +""" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="analysis rom size of L0 and L1 product") + parser.add_argument("-p", "--product_name", type=str, + help="product name. eg: -p ipcamera_hispark_taurus") + parser.add_argument("-o", "--oh_path", type=str, + default=".", help="root path of openharmony") + parser.add_argument("-g", "--recollect_gn", action="store_false", help="recollect gn info or not") + parser.add_argument("-s", "--recollect_sc", action="store_false", help="recollect subsystem_component info or not") + args = parser.parse_args() + return args + + +_args = parse_args() + +# # global variables +configs = SimpleYamlTool.read_yaml("config.yaml") +result_dict: Dict[str, Any] = dict() + +project_path = BasicTool.abspath(_args.oh_path) +product_name = _args.product_name +recollect_gn = _args.recollect_gn +_recollect_sc = _args.recollect_sc +_sc_json: Dict[Text, Text] = configs.get("subsystem_component") +_sc_save = _sc_json.get("save") +_target_type = configs["target_type"] +_sc_output_path = _sc_json.get("filename") +if _recollect_sc: + sub_com_dict: Dict = SC.run(project_path, _sc_output_path, _sc_save) +else: + with open(_sc_output_path, 'r', encoding='utf-8') as f: + sub_com_dict = json.load(f) + +collector_config: Tuple[BaseProcessor] = ( + DefaultProcessor(project_path=project_path, # 项目根路径 + result_dict=result_dict, # 保存结果的字典 + # targte的类型名称,即xxx("yyy")中的xxx + target_type=_target_type[0], + # 用以进行匹配的模式串,包括匹配段落时作为前缀 + match_pattern=fr"^( *){_target_type[0]}\(.*?\)", + sub_com_dict=sub_com_dict, # 从bundle.json中收集的subsystem_name和component_name信息 + target_name_parser=TargetNameParser.single_parser, # 进行target_name解析的parser + other_info_handlers={ + "extension": extension_handler, + }, # 解析其他信息的parser,{"字段名":该字段的parser} + unit_post_handler=SOPostHandler() # 对即将进行存储的unit字典的handler,会返回一个str作为存储时的key + ), + DefaultProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[1], + match_pattern=fr"^( *){_target_type[1]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "extension": extension_handler, + }, + unit_post_handler=SOPostHandler(), + ), + DefaultProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[2], + match_pattern=fr"^( *){_target_type[2]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "extension": extension_handler, + }, + unit_post_handler=APostHandler(), + ), + DefaultProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[3], + match_pattern=fr"^( *){_target_type[3]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "extension": extension_handler, + }, + unit_post_handler=APostHandler(), + ), + DefaultProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[4], + match_pattern=fr"^( *){_target_type[4]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "extension": extension_handler, + }, + unit_post_handler=DefaultPostHandler(), + ), + DefaultProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[5], + match_pattern=fr"^( *){_target_type[5]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "extension": extension_handler, + }, + unit_post_handler=DefaultPostHandler(), + ), + DefaultProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[6], + match_pattern=fr"^( *){_target_type[6]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "real_target_type": target_type_handler, + "extension": extension_handler, + }, + unit_post_handler=LiteLibPostHandler(), + ud_post_handler=LiteLibS2MPostHandler, + ), + DefaultProcessor(project_path=project_path, # hap有个hap_name + result_dict=result_dict, + target_type=_target_type[7], + match_pattern=fr"^( *){_target_type[7]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "hap_name": hap_name_handler, + "extension": extension_handler, + }, + unit_post_handler=HAPPostHandler(), + ), + StrResourceProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[8], + match_pattern=fr"^( *){_target_type[8]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "extension": extension_handler, + }, + unit_post_handler=DefaultPostHandler(), + resource_field="source" + ), + StrResourceProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[9], + match_pattern=fr"^( *){_target_type[9]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "extension": extension_handler, + }, + unit_post_handler=DefaultPostHandler(), + resource_field="source" + ), + ListResourceProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[10], + match_pattern=fr"^( *){_target_type[10]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "extension": extension_handler, + }, + unit_post_handler=DefaultPostHandler(), + resource_field="sources" + ), + StrResourceProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[11], + match_pattern=fr"^( *){_target_type[11]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + # "extension": extension_handler, + }, + unit_post_handler=DefaultPostHandler(), + resource_field="source" + ), + DefaultProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[12], + match_pattern=fr"^( *){_target_type[12]}\(.*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.single_parser, + other_info_handlers={ + "real_target_type": target_type_handler, + # "extension": extension_handler, + }, + unit_post_handler=LiteComponentPostHandler(), + ), +) + +__all__ = ["configs", "result_dict", "collector_config", "sub_com_dict"] + +if __name__ == '__main__': + for c in collector_config: + c.run() + with open("demo.json", 'w', encoding='utf-8') as f: + json.dump(result_dict, f) diff --git a/tools/rom_ram_analyzer/lite_small/src/config.yaml b/tools/rom_ram_analyzer/lite_small/src/config.yaml new file mode 100644 index 0000000..f6e0f3e --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/src/config.yaml @@ -0,0 +1,144 @@ +# Copyright (c) 2022 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# 注意:如果target_type有了更改,要相应改变config.py中collector_config +target_type: + - shared_library + - ohos_shared_library + - static_library + - ohos_static_library + - executable + - ohos_executable + - lite_library + - ohos_hap + - ohos_prebuilt_etc + - ohos_prebuilt_para + - ohos_sa_profile + - ohos_prebuilt_shared_library + - lite_component + + +subsystem_component: + save: true + filename: sub_com_info.json +gn_info_file: gn_info.json + +ipcamera_hispark_taurus: + product_infofile: ipcamera_hispark_taurus_product.json # 编译产物的信息 + output_name: ipcamera_hispark_taurus_result.json # 输出文件的名字 + product_dir: # [required] + root: out/hispark_taurus/ipcamera_hispark_taurus/rootfs + relative: + bin: bin + so: usr/lib + etc: etc + rest: True # 是否分析其他目录下的并归到etc(copy的文件) + query_order: + so: + - shared_library + - ohos_shared_library + - ohos_prebuilt_shared_library + - lite_library + - lite_component + + bin: + - executable + - ohos_executable + - lite_component + + +ipcamera_hispark_taurus_linux: + product_infofile: ipcamera_hispark_taurus_linux_product.json + output_name: ipcamera_hispark_taurus_linux_result.json + product_dir: + root: out/hispark_taurus/ipcamera_hispark_taurus_linux/rootfs + relative: + bin: bin + so: usr/lib + etc: etc + rest: True + query_order: + so: + - shared_library + - ohos_shared_library + - ohos_prebuilt_shared_library + - lite_library + - lite_component + + bin: + - executable + - ohos_executable + - lite_component + + +wifiiot_hispark_pegasus: + product_infofile: wifiiot_hispark_pegasus_product.json + output_name: wifiiot_hispark_pegasus_result.json + product_dir: + root: out/hispark_pegasus/wifiiot_hispark_pegasus + relative: + a: libs + etc: etc + rest: False + query_order: + a: + - static_library + - ohos_static_library + - lite_library + + +rk3568: # rk的目前从packages/phone/system_module_info.json中分析准确度更高,因为rk基本都使用的是ohos_xxx,而L0和L1的更多的是使用的gn原生target template + product_infofile: rk3568_product.json + output_name: rk3568_result.json + product_dir: + root: out/rk3568/packages/phone/system + relative: + so: lib + bin: bin + hap: app + etc: etc + rest: True + query_order: + so: + - ohos_shared_library + - shared_library + - ohos_prebuilt_shared_library + - lite_library + - lite_component + + bin: + - ohos_executable + - executable + - lite_component + + hap: + - ohos_hap + +# extension and prefix of products +default_extension: + shared_library: .so + static_library: .a + app: .hap + +default_prefix: + shared_library: lib + static_library: lib +# black list for command 'grep' +black_list: + - .repo + - .ccache + - doc + - test + - build + - out diff --git a/tools/rom_ram_analyzer/lite_small/src/get_subsystem_component.py b/tools/rom_ram_analyzer/lite_small/src/get_subsystem_component.py new file mode 100644 index 0000000..3ed3847 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/src/get_subsystem_component.py @@ -0,0 +1,131 @@ +#! /usr/bin/python + +import argparse +import os +import json +import logging + +g_subsystem_path_error = list() # subsystem path exist in subsystem_config.json +# bundle.json path which cant get component path. +g_component_path_empty = list() +g_component_abs_path = list() # destPath can't be absolute path. + + +def get_subsystem_components(ohos_path: str): + subsystem_json_path = os.path.join( + ohos_path, r"build/subsystem_config.json") + subsystem_item = {} + + with open(subsystem_json_path, 'rb') as f: + subsystem_json = json.load(f) + # get sunsystems + for i in subsystem_json: + subsystem_name = subsystem_json[i]["name"] + subsystem_path = os.path.join(ohos_path, subsystem_json[i]["path"]) + if not os.path.exists(subsystem_path): + g_subsystem_path_error.append(subsystem_path) + continue + cmd = 'find ' + subsystem_path + ' -name bundle.json' + bundle_json_list = os.popen(cmd).readlines() + # get components + component_list = [] + for j in bundle_json_list: + bundle_path = j.strip() + with open(bundle_path, 'rb') as bundle_file: + bundle_json = json.load(bundle_file) + component_item = {} + if 'segment' in bundle_json and 'destPath' in bundle_json["segment"]: + destpath = bundle_json["segment"]["destPath"] + component_item[bundle_json["component"]["name"]] = destpath + if os.path.isabs(destpath): + g_component_abs_path.append(destpath) + else: + component_item[bundle_json["component"]["name"] + ] = "Unknow. Please check " + bundle_path + g_component_path_empty.append(bundle_path) + component_list.append(component_item) + subsystem_item[subsystem_name] = component_list + return subsystem_item + + +def get_subsystem_components_modified(ohos_root) -> dict: + ret = dict() + subsystem_info = get_subsystem_components(ohos_root) + if subsystem_info is None: + return None + for subsystem_k, subsystem_v in subsystem_info.items(): + for component in subsystem_v: + for k, v in component.items(): + ret.update({v: {'subsystem': subsystem_k, 'component': k}}) + return ret + + +def export_to_json(subsystem_item: dict, output_filename: str): + subsystem_item_json = json.dumps( + subsystem_item, indent=4, separators=(', ', ': ')) + with open(output_filename, 'w') as f: + f.write(subsystem_item_json) + logging.info("output path: " + output_filename) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("project", help="project root path.", type=str) + parser.add_argument("-o", "--outpath", + help="specify an output path.", type=str) + args = parser.parse_args() + + ohos_path = os.path.abspath(args.project) + if not is_project(ohos_path): + logging.error("'" + ohos_path + "' is not a valid project path.") + exit(1) + + output_path = r'.' + if args.outpath: + output_path = args.outpath + + return ohos_path, output_path + + +def is_project(path: str) -> bool: + ''' + @func: 判断是否源码工程。 + @note: 通过是否含有 .repo/manifests 目录粗略判断。 + ''' + p = os.path.normpath(path) + return os.path.exists(p + '/.repo/manifests') + + +def print_warning_info(): + '''@func: 打印一些异常信息。 + ''' + if g_component_path_empty or g_component_abs_path: + logging.warning("------------ warning info ------------------") + + if g_component_path_empty: + logging.warning("can't find destPath in:") + logging.warning(g_component_path_empty) + + if g_component_abs_path: + logging.warning("destPath can't be absolute path:") + logging.warning(g_component_abs_path) + + +class SC: + @classmethod + def run(cls, project_path: str, output_path: str = None, save_result: bool = True): + info = get_subsystem_components_modified( + os.path.abspath(os.path.expanduser(project_path))) + if save_result and output_path: + export_to_json(info, output_path) + print_warning_info() + return info + + +__all__ = ["SC"] + +if __name__ == '__main__': + ohos_path, output_path = parse_args() + info = get_subsystem_components_modified(ohos_path) + export_to_json(info, output_path) + print_warning_info() diff --git a/tools/rom_ram_analyzer/lite_small/src/misc.py b/tools/rom_ram_analyzer/lite_small/src/misc.py new file mode 100644 index 0000000..d000507 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/src/misc.py @@ -0,0 +1,228 @@ +import logging +import copy +import os +import logging +from abc import ABC, abstractmethod +from collections import defaultdict +from typing import * +import preprocess +from pkgs.gn_common_tool import GnVariableParser +from pkgs.simple_yaml_tool import SimpleYamlTool +from pkgs.basic_tool import BasicTool + + +_config = SimpleYamlTool.read_yaml("config.yaml") +""" +===============info handlers=============== +""" + + +def extension_handler(paragraph: Text): + return GnVariableParser.string_parser("output_extension", paragraph).strip('"') + + +def hap_name_handler(paragraph: Text): + return GnVariableParser.string_parser("hap_name", paragraph).strip('"') + + +def target_type_handler(paragraph: Text): + tt = GnVariableParser.string_parser("target_type", paragraph).strip('"') + if not tt: + logging.info("parse 'target_type' failed, maybe it's a variable") + return tt + + +""" +===============gn lineno collector=============== +""" + + +def gn_lineno_collect(match_pattern: str, project_path: str) -> DefaultDict[str, List[int]]: + """ + 在整个项目路径下搜索有特定target类型的BUILD.gn + :param match_pattern: 进行grep的pattern，支持扩展的正则 + :param project_path: 项目路径（搜索路径） + :return: {gn_file: [line_no_1, line_no_2, ..]} + """ + black_list = map(lambda x: os.path.join( + project_path, x), _config.get("black_list")) + + def handler(content: Text) -> List[str]: + return list(filter(lambda y: len(y) > 0, list(map(lambda x: x.strip(), content.split("\n"))))) + + grep_list = BasicTool.grep_ern(match_pattern, path=project_path, include="BUILD.gn", exclude=tuple(black_list), + post_handler=handler) + gn_line_dict: DefaultDict[str, List[int]] = defaultdict(list) + for gl in grep_list: + gn_file, line_no, _ = gl.split(":") + gn_line_dict[gn_file].append(line_no) + return gn_line_dict + + +""" +===============target name parser=============== +""" + + +class TargetNameParser: + @classmethod + def single_parser(cls, paragraph: Text) -> str: + """ + 查找类似shared_library("xxx")这种括号内只有一个参数的target的名称 + :param paragraph: 要解析的段落 + :return: target名称，如果是变量，不会对其进行解析 + """ + return BasicTool.re_group_1(paragraph, r"\w+\((.*)\)") + + @classmethod + def second_parser(cls, paragraph: Text) -> str: + """ + 查找类似target("shared_library","xxx")这种的target名称（括号内第二个参数） + :param paragraph: 要解析的段落 + :return: target名称，如果是变量，不会的其进行解析 + """ + return BasicTool.re_group_1(paragraph, r"\w+\(.*?, *(.*?)\)") + + +""" +===============post handlers=============== +""" + + +class BasePostHandler(ABC): + @abstractmethod + def run(self, unit: Dict[str, AnyStr]) -> str: + ... + + def __call__(self, unit: Dict[str, AnyStr]) -> str: + return self.run(unit) + + +class DefaultPostHandler(BasePostHandler): + def run(self, unit: Dict[str, AnyStr]): + return unit["output_name"] + + +class HAPPostHandler(BasePostHandler): + """ + for ohos_hap""" + + def run(self, unit: Dict[str, AnyStr]): + extension = _config.get("default_extension").get("app") + gn_hap_name = unit.get("hap_name") + if gn_hap_name: + return gn_hap_name+extension + return unit["output_name"]+extension + + +class SOPostHandler(BasePostHandler): + """ + for shared_library""" + + def run(self, unit: Dict[str, AnyStr]): + output_name = unit["output_name"] + prefix = _config.get("default_prefix").get("shared_library") + if unit.get("extension"): + extension = unit.get("extension") + else: + extension = _config.get("default_extension").get("shared_library") + if not extension.startswith('.'): + extension = '.'+extension + if output_name.startswith(prefix): + return output_name+extension + return prefix+output_name+extension + + +class APostHandler(BasePostHandler): + """ + for static library""" + + def run(self, unit: Dict[str, AnyStr]): + output_name = unit["output_name"] + prefix = _config.get("default_prefix").get("static_library") + extension: str = _config.get("default_extension").get("static_library") + if not extension.startswith('.'): + extension = '.'+extension + if output_name.startswith(prefix): + return output_name+extension + return prefix+output_name+extension + + +class LiteLibPostHandler(BasePostHandler): + """ + for lite_library""" + + def run(self, unit: Dict[str, AnyStr]): + tp = unit["real_target_type"] + output_name = unit["output_name"] + if tp == "static_library": + prefix = _config.get("default_prefix").get("static_library") + extension = _config.get("default_extension").get("static_library") + elif tp == "shared_library": + prefix = _config.get("default_prefix").get("shared_library") + extension = _config.get("default_extension").get("shared_library") + else: + prefix = str() + extension = str() + if not extension.startswith('.'): + extension = '.'+extension + if output_name.startswith(prefix): + return output_name+extension + return prefix+output_name+extension + + +class LiteComponentPostHandler(BasePostHandler): + """ + for lite_component""" + + def run(self, unit: Dict[str, AnyStr]): + tp = unit["real_target_type"] + output_name = unit["output_name"] + extension = unit.get("output_extension") + if tp == "shared_library": + prefix = _config.get("default_prefix").get("shared_library") + extension = _config.get("default_extension").get("shared_library") + else: + if tp != "executable": + unit["description"] = "virtual node" + prefix = str() + extension = str() + if not extension.startswith('.'): + extension = '.'+extension + return prefix+output_name+extension + + +class TargetPostHandler(BasePostHandler): + """ + for target(a,b){}""" + + def run(self, unit: Dict[str, AnyStr]): + ... + + +def LiteLibS2MPostHandler(unit: Dict, result_dict: Dict) -> None: + rt = unit.get("real_target_type") + new_unit = copy.deepcopy(unit) + if rt == "shared_library": + new_unit["real_target_type"] = "static_library" + k = LiteLibPostHandler()(new_unit) + new_unit["description"] = "may not exist" + result_dict["lite_library"][k] = new_unit + elif rt == "static_library": + new_unit["real_target_type"] = "shared_library" + k = LiteLibPostHandler()(new_unit) + new_unit["description"] = "may not exist" + result_dict["lite_library"][k] = new_unit + else: + logging.warning( + f"target type should be 'shared_library' or 'static_library', but got '{rt}'") + new_unit["real_target_type"] = "shared_library" + k = LiteLibPostHandler()(new_unit) + new_unit["description"] = "may not exist" + result_dict["lite_library"][k] = new_unit + + new_new_unit = copy.deepcopy(unit) + new_new_unit["real_target_type"] = "static_library" + k = LiteLibPostHandler()(new_new_unit) + new_new_unit["description"] = "may not exist" + result_dict["lite_library"][k] = new_new_unit diff --git a/tools/rom_ram_analyzer/lite_small/src/preprocess.py b/tools/rom_ram_analyzer/lite_small/src/preprocess.py new file mode 100644 index 0000000..19b9ae9 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/src/preprocess.py @@ -0,0 +1,4 @@ +import sys +import os +# 将上级目录加入到库的搜索路径 +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) \ No newline at end of file diff --git a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py new file mode 100644 index 0000000..ba1c6e1 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py @@ -0,0 +1,295 @@ +import sys +import argparse +import json +import logging +import os +from typing import * +import copy +import preprocess +from time import time +from concurrent.futures import ThreadPoolExecutor, Future +from threading import RLock +import collections + +from config import result_dict, collector_config, configs, \ + project_path, sub_com_dict, product_name, recollect_gn +# from gn_info_collect import GnInfoCollector +from pkgs.basic_tool import BasicTool +from pkgs.gn_common_tool import GnCommonTool +from pkgs.simple_excel_writer import SimpleExcelWriter +from misc import gn_lineno_collect + + +""" +1. 先收集BUILD.gn中的target信息 +2. 然后根据编译产物到1中进行搜索,匹配其所属的部件 + +对于ohos开头的template,主要根据其component字段和subsystem_name字段来归数其部件；同时也要考虑install_dir字段 +对于gn原生的template,主要根据bundle.json中的字段来归属其部件 + +对于找不到的,可以模糊匹配,如,有产物libxxx,则可以在所有的BUILD.gn中搜索xxx,并设置一个阀值予以过滤 +""" + + +class RomAnalysisTool: + @classmethod + def collect_gn_info(cls): + with ThreadPoolExecutor(max_workers=len(collector_config) + 1) as pool: + future_list: List[Future] = list() + for c in collector_config: + future_list.append(pool.submit(c)) + for f in future_list: + f.result() + gn_info_file = configs["gn_info_file"] + with open(gn_info_file, 'w', encoding='utf-8') as f: + json.dump(result_dict, f, indent=4) + + @classmethod + def _add_rest_dir(cls, top_dir: str, rela_path: str, sub_path: str, dir_list: List[str]) -> None: + """ + dir_list:相对于原始top目录的所有子目录的全路径 + """ + if not sub_path: + return + # 将其他目录添加到dir_list + all_subdir = os.listdir(os.path.join(top_dir, rela_path)) + for d in all_subdir: + t = os.path.join(rela_path, d) + if os.path.isdir(os.path.join(top_dir, t)) and t not in dir_list: + dir_list.append(t) + # 移除sub_path的当前层级的目录 + t = sub_path.split(os.sep) + if os.path.join(rela_path, t[0]) in dir_list: + dir_list.remove(os.path.join(rela_path, t[0])) + else: + logging.error( + f"'{os.path.join(rela_path,t[0])}' not in '{top_dir}'") + sp = str() + if len(t) == 1: + return + elif len(t) == 2: + sp = t[1] + else: + sp = os.path.join(*t[1:]) + cls._add_rest_dir(top_dir, os.path.join(rela_path, t[0]), sp, dir_list) + + @classmethod + def _find_files(cls, product_name: str) -> Dict[str, List[str]]: + product_dir: Dict[str, Dict] = configs[product_name]["product_dir"] + if not product_name: + logging.error( + f"product_name '{product_name}' not found in the config.yaml") + exit(1) + product_path_dit: Dict[str, str] = dict() # 存储编译产物的类型及目录 + root_dir = product_dir.get("root") + root_dir = os.path.join(project_path, root_dir) + relative_dir: Dict[str, str] = product_dir.get("relative") + if not relative_dir: + logging.warning( + f"'{relative_dir}' of {product_name} not found in the config.yaml") + exit(1) + # 除了so a hap bin外的全部归到etc里面 + for k, v in relative_dir.items(): + product_path_dit[k] = os.path.join(root_dir, v) + # 查找编译产物信息 + # product_dict格式: {"so": ["a.so", "b.so"]} + product_dict: Dict[str, List[str]] = dict() # 存储编译产物的名称 + for k, v in product_path_dit.items(): + if not os.path.exists(v): + logging.warning(f"dir '{v}' not exist") + product_dict[k] = BasicTool.find_files_with_pattern(v) # v是全路径 + if product_dir.get("rest"): + rest_dir_list: List[str] = os.listdir( + root_dir) # 除了配置在relative下之外的所有剩余目录,全部归到etc下 + for v in relative_dir.values(): + cls._add_rest_dir(root_dir, str(), v, rest_dir_list) + if "etc" not in product_dict.keys(): + product_dict["etc"] = list() + for r in rest_dir_list: + product_dict["etc"].extend( + BasicTool.find_files_with_pattern(os.path.join(root_dir, r))) + return product_dict + + @classmethod + def collect_product_info(cls, product_name: str): + product_dict: Dict[str, List[str]] = cls._find_files(product_name) + with open(configs[product_name]["product_infofile"], 'w', encoding='utf-8') as f: + json.dump(product_dict, f, indent=4) + return product_dict + + @classmethod + def _put(cls, sub: str, com: str, unit: Dict, rom_size_dict: Dict): + size = unit.get("size") + if not rom_size_dict.get("size"): # 总大小 + rom_size_dict["size"] = 0 + if not rom_size_dict.get(sub): # 子系统大小 + rom_size_dict[sub]: Dict[str, Dict] = dict() + rom_size_dict[sub]["size"] = 0 + rom_size_dict[sub]["count"] = 0 + + if not rom_size_dict.get(sub).get(com): # 部件 + rom_size_dict.get(sub)[com] = dict() + rom_size_dict[sub][com]["filelist"] = list() + rom_size_dict[sub][com]["size"] = 0 + rom_size_dict[sub][com]["count"] = 0 + + rom_size_dict[sub][com]["filelist"].append(unit) + rom_size_dict[sub][com]["size"] += size + rom_size_dict[sub][com]["count"] += 1 + rom_size_dict[sub]["size"] += size + rom_size_dict[sub]["count"] += 1 + rom_size_dict["size"] += size + + @classmethod + def _fuzzy_match(cls, file_name: str, extra_black_list: Tuple[str] = ("test",)) -> Tuple[str, str, str]: + """ + 直接grep,利用出现次数最多的BUILD.gn去定位subsystem_name和component_name""" + _, base_name = os.path.split(file_name) + if base_name.startswith("lib"): + base_name = base_name[3:] + if base_name.endswith(".a"): + base_name = base_name[:base_name.index(".a")] + elif base_name.endswith(".z.so"): + base_name = base_name[:base_name.index(".z.so")] + elif base_name.endswith(".so"): + base_name = base_name[:base_name.index(".so")] + exclude_dir = [os.path.join(project_path, x) + for x in configs["black_list"]] + exclude_dir.extend(list(extra_black_list)) + grep_result: List[str] = BasicTool.grep_ern( + base_name, + project_path, + include="BUILD.gn", + exclude=tuple(exclude_dir), + post_handler=lambda x: list(filter(lambda x: len(x) > 0, x.split('\n')))) + if not grep_result: + return str(), str(), str() + gn_dict: Dict[str, int] = collections.defaultdict(int) + for g in grep_result: + gn = g.split(':')[0].replace(project_path, "").lstrip(os.sep) + gn_dict[gn] += 1 + gn_file, _ = collections.Counter(gn_dict).most_common(1)[0] + for k, v in sub_com_dict.items(): + if gn_file.startswith(k): + return gn_file, v.get("subsystem"), v.get("component") + return str(), str(), str() + + @classmethod + def _save_as_xls(cls, result_dict: Dict, product_name: str) -> None: + header = ["subsystem_name", "component_name", + "output_file", "size(Byte)"] + tmp_dict = copy.deepcopy(result_dict) + excel_writer = SimpleExcelWriter("rom") + excel_writer.set_sheet_header(headers=header) + subsystem_start_row = 1 + subsystem_end_row = 0 + subsystem_col = 0 + component_start_row = 1 + component_end_row = 0 + component_col = 1 + del tmp_dict["size"] + for subsystem_name in tmp_dict.keys(): + subsystem_dict = tmp_dict.get(subsystem_name) + subsystem_size = subsystem_dict.get("size") + subsystem_file_count = subsystem_dict.get("count") + del subsystem_dict["count"] + del subsystem_dict["size"] + subsystem_end_row += subsystem_file_count + + for component_name in subsystem_dict.keys(): + component_dict: Dict[str, int] = subsystem_dict.get( + component_name) + component_size = component_dict.get("size") + component_file_count = component_dict.get("count") + del component_dict["count"] + del component_dict["size"] + component_end_row += component_file_count + + for fileinfo in component_dict.get("filelist"): + file_name = fileinfo.get("file_name") + file_size = fileinfo.get("size") + excel_writer.append_line( + [subsystem_name, component_name, file_name, file_size]) + excel_writer.write_merge(component_start_row, component_col, component_end_row, component_col, + component_name) + component_start_row = component_end_row + 1 + excel_writer.write_merge(subsystem_start_row, subsystem_col, subsystem_end_row, subsystem_col, + subsystem_name) + subsystem_start_row = subsystem_end_row + 1 + output_name: str = configs[product_name]["output_name"] + output_name = output_name.replace(".json", ".xls") + excel_writer.save(output_name) + + @ classmethod + def analysis(cls, product_name: str, product_dict: Dict[str, List[str]]): + gn_info_file = configs["gn_info_file"] + with open(gn_info_file, 'r', encoding='utf-8') as f: + gn_info = json.load(f) + query_order: Dict[str, List[str] + ] = configs[product_name]["query_order"] + query_order["etc"] = configs["target_type"] + rom_size_dict: Dict = dict() + # prodcut_dict: {"a":["a.txt", ...]} + for t, l in product_dict.items(): + for f in l: # 遍历所有文件 + # query_order: {"a":[static_library", ...]} + find_flag = False + type_list = query_order.get(t) + _, base_name = os.path.split(f) + size = os.path.getsize(f) + if not type_list: + logging.warning( + f"'{t}' not found in query_order of the config.yaml") + continue + for tn in type_list: # tn example: ohos_shared_library + output_dict: Dict[str, Dict] = gn_info.get(tn) + if not output_dict: + logging.warning( + f"'{tn}' not found in the {gn_info_file}") + continue + d = output_dict.get(base_name) + if not d: + continue + d["size"] = size + d["file_name"] = f.replace(project_path, "") + cls._put(d["subsystem_name"], + d["component_name"], d, rom_size_dict) + find_flag = True + if not find_flag: + # fuzzy match + psesudo_gn, sub, com = cls._fuzzy_match(f) + if sub and com: + cls._put(sub, com, { + "subsystem_name": sub, + "component_name": com, + "psesudo_gn_path": psesudo_gn, + "description": "fuzzy match", + "file_name": f.replace(project_path, ""), + "size": size, + }, rom_size_dict) + find_flag = True + if not find_flag: + cls._put("others", "others", { + "file_name": f.replace(project_path, ""), + "size": size, + }, rom_size_dict) + with open(configs[product_name]["output_name"], 'w', encoding='utf-8') as f: + json.dump(rom_size_dict, f, indent=4) + cls._save_as_xls(rom_size_dict, product_name) + + +def main(): + if recollect_gn: + RomAnalysisTool.collect_gn_info() + product_dict: Dict[str, List[str] + ] = RomAnalysisTool.collect_product_info(product_name) + RomAnalysisTool.analysis(product_name, product_dict) + + +if __name__ == "__main__": + main() + # t = os.listdir( + # "/home/aodongbiao/developtools_integration_verification/tools") + # RomAnalysisTool._add_rest_dir( + # "/home/aodongbiao/developtools_integration_verification/tools", "", "rom_ram_analyzer/L2/pkgs", t) + # print(t) diff --git a/tools/rom_ram_analyzer/lite_small/src/template_processor.py b/tools/rom_ram_analyzer/lite_small/src/template_processor.py new file mode 100644 index 0000000..e4124c8 --- /dev/null +++ b/tools/rom_ram_analyzer/lite_small/src/template_processor.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2022 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import * +from abc import ABC, abstractmethod +import os +import logging + +from pkgs.basic_tool import do_nothing, BasicTool +from pkgs.gn_common_tool import GnCommonTool, GnVariableParser +from misc import * + +TYPE = Literal["str", "list"] + + +class BaseProcessor(ABC): + """ + extend and usage: + DerivedClass(BaseProcessor): + def call(): + # your implementation + + DerivedClass(project_path,result_dict,...) + DerivedClass() + """ + + def __init__(self, + project_path: str, + result_dict: Dict[str, Dict[str, Dict]], + target_type: str, + match_pattern: str, + sub_com_dict: Dict[str, Dict[str, str]], + target_name_parser: Callable[[Text], Text] = do_nothing, + other_info_handlers: Dict[str, Callable[[ + Text], Union[str, list]]] = dict(), + unit_post_handler: BasePostHandler = do_nothing, + resource_field: str = None, + ud_post_handler: Callable[[Dict, Dict], None] = None + ): + """ + :param project_path: 项目根路径 + :param result_dict: 存储结果的字典 + :param target_type: target类型，eg："shared_library" + :param match_pattern: 用于进行的模式，eg：r"^( *)shared_library\(.*?\)" + :param sub_com_dict: 从get_subsystem_component.py运行结果加载进来的dict，包含oh整体上的子系统、部件及其路径信息 + :param target_name_parser: 解析target名字的Callable + :param other_info_handlers: 对其他信息进行收集处理，eg：{"sources": SourcesParser}——表示要处理target段落中的sources属性， + SourceParser是对target段落进行分析处理的Callable，接受一个字符串作为参数 + :param unit_post_handler: 对最终要存储的结果字典进行后处理，应当返回一个字符串作为存储时的key，且该key应为预期产物去除前后缀后的名字 + :resource_field: 针对资源类target,资源字段,如files = ["a.txt","b.txt"],则field为files + :ud_post_handler: 参数为unit和result_dict的handler + """ + if target_type not in result_dict.keys(): + result_dict[target_type] = dict() + self.project_path = project_path + self.result_dict = result_dict + self.target_type = target_type + self.match_pattern = match_pattern + self.gn_file_line_no_dict = gn_lineno_collect( + self.match_pattern, self.project_path) + self.sc_dict = sub_com_dict + self.target_name_parser = target_name_parser + self.other_info_handlers = other_info_handlers + self.unit_post_handler = unit_post_handler + self.resource_field = resource_field + self.ud_post_handler = ud_post_handler + + def _append(self, key: str, unit: Dict) -> None: + """ + 将target的结果存储到最终的结果字典中 + :param key：进行存储的key，应为预期的文件名 + :param unit: 要存储的target + :return: None + """ + self.result_dict.get(self.target_type)[key] = unit + + def _find_sc(self, gn_path: str): + # gn_path与project_path都应当是绝对路径 + if not gn_path.startswith(self.project_path): + logging.error("gn_path and project_path is not consistent: gn_path={}, project_path={}".format( + gn_path, self.project_path)) + return "", "" + k = gn_path.replace(self.project_path, "").lstrip(os.sep) + for k, v in self.sc_dict.items(): + if k.startswith(k): + return v.get("subsystem"), v.get("component") + return "", "" + + @abstractmethod + def run(self): + ... + + def __call__(self, *args, **kwargs): + self.run() + + +def _gn_var_process(project_path: str, gn_v: str, alt_v: str, gn_path: str, ifrom: str, efrom: str, strip_quote: bool = False) -> Tuple[str, str]: + if strip_quote: + gn_v = gn_v.strip('"') + if gn_v: + if GnCommonTool.contains_gn_variable(gn_v): + gn_v = GnCommonTool.replace_gn_variables( + gn_v, gn_path, project_path).strip('"') + else: + gn_v = gn_v.strip('"') + gn_f = ifrom + else: + gn_v = alt_v + gn_f = efrom + return gn_v, gn_f + + +class DefaultProcessor(BaseProcessor): + + def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: + output_name = GnVariableParser.string_parser("output_name", paragraph) + output_name, out_from = _gn_var_process(self.project_path, + output_name, target_name, gn_path, "target_name", "target_name", True) + sub = GnVariableParser.string_parser("subsystem_name", paragraph) + com = GnVariableParser.string_parser("part_name", paragraph) + sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json", True) + com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json", True) + result = { + "gn_path": gn_path, + "target_type": self.target_type, + "line_no": line_no, + "subsystem_name": sub, + "component_name": com, + "subsystem_from": sub_from, + "component_from": com_from, + "target_name": target_name, + "output_name": output_name, + "output_from": out_from, + } + for k, h in self.other_info_handlers.items(): + result[k] = h(paragraph) + key = self.unit_post_handler(result) + self._append(key, result) + if self.ud_post_handler: + self.ud_post_handler(result, self.result_dict) + + def run(self): + for gn_path, line_no_list in self.gn_file_line_no_dict.items(): + # 该路径下的主要的subsystem_name与component_name，如果target中没有指定，则取此值，如果指定了，则以target中的为准 + _sub, _com = self._find_sc(gn_path) + with open(gn_path, 'r', encoding='utf-8') as f: + content = f.read() + itr = BasicTool.match_paragraph( + content, start_pattern=self.target_type) + for line_no, p in zip(line_no_list, itr): + paragraph = p.group() + target_name = self.target_name_parser(paragraph).strip('"') + if not target_name: + continue + if GnCommonTool.contains_gn_variable(target_name, quote_processed=True): + possible_name_list = GnCommonTool.find_values_of_variable(target_name, path=gn_path, + stop_tail=self.project_path) + for n in possible_name_list: + self.helper(n, paragraph, gn_path, + line_no, _sub, _com) + else: + self.helper(target_name, paragraph, + gn_path, line_no, _sub, _com) + + +class StrResourceProcessor(DefaultProcessor): + def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: + resources = GnVariableParser.string_parser( + self.resource_field, paragraph) + if not resources: + return + _, resources = os.path.split(resources.strip('"')) + + if GnCommonTool.contains_gn_variable(resources): + resources = GnCommonTool.replace_gn_variables( + resources, gn_path, self.project_path).strip('"') + sub = GnVariableParser.string_parser("subsystem_name", paragraph) + com = GnVariableParser.string_parser("part_name", paragraph) + sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json") + com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json") + _, file_name = os.path.split(resources) + result = { + "gn_path": gn_path, + "target_type": self.target_type, + "line_no": line_no, + "subsystem_name": sub, + "component_name": com, + "subsystem_from": sub_from, + "component_from": com_from, + "target_name": target_name, + "output_name": file_name, + "output_from": "file_name", + } + for k, h in self.other_info_handlers.items(): + result[k] = h(paragraph) + key = self.unit_post_handler(result) + self._append(key, result) + + +class ListResourceProcessor(DefaultProcessor): + + def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: + resources = GnVariableParser.list_parser( + self.resource_field, paragraph) + if not resources: + return + sub = GnVariableParser.string_parser("subsystem_name", paragraph) + com = GnVariableParser.string_parser("part_name", paragraph) + sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json") + com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json") + for ff in resources: + _, file_name = os.path.split(ff) + result = { + "gn_path": gn_path, + "target_type": self.target_type, + "line_no": line_no, + "subsystem_name": sub, + "component_name": com, + "subsystem_from": sub_from, + "component_from": com_from, + "target_name": target_name, + "output_name": file_name, + "output_from": "file_name", + } + for k, h in self.other_info_handlers.items(): + result[k] = h(paragraph) + key = self.unit_post_handler(result) + self._append(key, result) + + +if __name__ == '__main__': + ... diff --git a/tools/rom_ram_analyzer/standard/README.md b/tools/rom_ram_analyzer/standard/README.md new file mode 100644 index 0000000..52bc5a9 --- /dev/null +++ b/tools/rom_ram_analyzer/standard/README.md @@ -0,0 +1,143 @@ +[toc] + +# Rom_analyzer.py + +## 功能介绍 + +基于BUILD.gn、bundle.json、编译产物system_module_info.json、out/{product_name}/packages/phone目录下的编译产物，分析各子系统及部件的rom大小。 + +结果以json与xls格式进行存储，其中，json格式是必输出的，xls格式需要-e参数控制。 + +## 使用说明 + +前置条件： + +1. 获取整个rom_ram_analyzer目录 +1. 对系统进行编译 +1. rom分析在linux平台,ram分析在windows平台 +1. python3.8及以后 +1. 安装requirements + ```txt + xlwt==1.3.0 + ``` + +命令介绍： + +1. `-h`或`--help`命令查看帮助 + ```shell + > python3 rom_analyzer.py -h + usage: rom_analyzer.py [-h] [-v] -p PROJECT_PATH -j MODULE_INFO_JSON -n PRODUCT_NAME -d PRODUCT_DIR [-o OUTPUT_FILE] [-e EXCEL] + + analyze rom size of component. + + optional arguments: + -h, --help show this help message and exit + -v, -version show program\'s version number and exit + -p PROJECT_PATH, --project_path PROJECT_PATH + root path of oh. eg: -p ~/oh + -j MODULE_INFO_JSON, --module_info_json MODULE_INFO_JSON + path of out/{product_name}/packages/phone/system_module_info.json + -n PRODUCT_NAME, --product_name PRODUCT_NAME + product name. eg: -n rk3568 + -d PRODUCT_DIR, --product_dir PRODUCT_DIR + subdirectories of out/{product_name}/packages/phone to be counted.eg: -d system -d vendor + -o OUTPUT_FILE, --output_file OUTPUT_FILE + basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result + -e EXCEL, --excel EXCEL + if output result as excel, default: False. eg: -e True + ``` +1. 使用示例 + ```shell + python3 rom_analyzer.py -p ~/nomodify_oh/ -j ../system_module_info.json -n rk3568 -d system -d vendor -d updater -o demo/demo -e True + # oh：rootpath of oh + # rk3568: product_name, same as out/{product_name} + # demo/demo: path of output file, where the second 'demo' is the basename of output file + # -e True：output result in excel format additionally + ``` + +## 输出格式说明(json) + + +```json +{ + 子系统名: { + "size": 整个子系统输出文件的总大小, + "file_count": 整个子系统产生的文件数, + 输出文件名: 本文件的大小, + ... + }, + ... +} +``` + +# ram_analyzer.py + +## 功能介绍 + +基于out/{product_name}/packages/phone下所有cfg文件、out/{product_name}/packages/phone/system/profile下所有xml文件，分析各进程及对应部件的ram占用（默认取Pss） + +结果以json与xls格式存储，其中，json格式是必输出的，xls格式需要-e参数控制。 + +## 使用说明 + +前置条件： + +1. 获取整个rom_ram_analyzer目录 +2. hdc可用 +2. 设备已连接 +3. 系统已烧录 +3. python3.8及以后 +4. 安装requirements + ```txt + xlwt==1.3.0 + ``` +5. 准备好相关数据： + 1. out/{product_name}/packages/phone下所有cfg文件，并将其放置于同一个目录中（ps：同名文件仅保存一份即可） + 1. out/{product_name}/packages/phone/system/profile下所有xml文件 +6. 运行rom_analyzer.py产生的json结果一份（即-o参数对应的文件，默认rom_analysis_result.json） + +命令介绍： + +1. 使用`-h`或`--help`查看帮助 + ```shell + > python .\ram_analyzer.py -h + usage: ram_analyzer.py [-h] [-v] -x XML_PATH -c CFG_PATH [-j ROM_RESULT] -n DEVICE_NUM [-o OUTPUT_FILENAME] [-e EXCEL] + + analyze ram size of component + + optional arguments: + -h, --help show this help message and exit + -v, -version show program\'s version number and exit + -x XML_PATH, --xml_path XML_PATH + path of xml file. eg: -x ~/oh/out/rk3568/packages/phone/system/profile + -c CFG_PATH, --cfg_path CFG_PATH + path of cfg files. eg: -c ./cfgs/ + -j ROM_RESULT, --rom_result ROM_RESULT + json file produced by rom_analyzer_v1.0.py, default: ./rom_analysis_result.json.eg: -j ./demo/rom_analysis_result.json + -n DEVICE_NUM, --device_num DEVICE_NUM + device number to be collect hidumper info. eg: -n 7001005458323933328a01fce16d3800 + -o OUTPUT_FILENAME, --output_filename OUTPUT_FILENAME + base name of output file, default: ram_analysis_result. eg: -o ram_analysis_result + -e EXCEL, --excel EXCEL + if output result as excel, default: False. eg: -e True + ``` +2. 使用示例： + ```shell + python .\ram_analyzer.py -x .\profile\ -c .\init\ -n 7001005458323933328a01fce16d3800 -j .\rom_analysis_result.json -o /demo/demo -e True + # demo/demo: path of output file, where the second 'demo' is the basename of output file + # -e True：output result in excel format additionally + ``` +## 输出格式说明（json） +```json +{ + 进程名:{ + "size": 本进程占用内存的大小, + 部件名: { + elf文件名: elf文件大小 + ... + } + ... + }, + ... +} +``` \ No newline at end of file diff --git a/tools/rom_ram_analyzer/standard/__init__.py b/tools/rom_ram_analyzer/standard/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/rom_ram_analyzer/standard/pkgs/__init__.py b/tools/rom_ram_analyzer/standard/pkgs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/rom_ram_analyzer/standard/pkgs/basic_tool.py b/tools/rom_ram_analyzer/standard/pkgs/basic_tool.py new file mode 100644 index 0000000..b4d548e --- /dev/null +++ b/tools/rom_ram_analyzer/standard/pkgs/basic_tool.py @@ -0,0 +1,33 @@ +import sys +import typing +import os +import glob +from pathlib import Path +from typing import * + + +class BasicTool: + @classmethod + def find_all_files(cls, folder: str, real_path: bool = True, apply_abs: bool = True, de_duplicate: bool = True, + p_filter: typing.Callable = lambda x: True) -> list: + filepath_list = set() + for root, _, file_names in os.walk(folder): + filepath_list.update( + [os.path.abspath(os.path.realpath( + os.path.join(root, f) if real_path else os.path.join(root, f))) if apply_abs else os.path.relpath( + os.path.realpath(os.path.join(root, f) if real_path else os.path.join(root, f))) for f in file_names + if p_filter(os.path.join(root, f))]) + if de_duplicate: + filepath_list = set(filepath_list) + filepath_list = sorted(filepath_list, key=str.lower) + return filepath_list + + @classmethod + def get_abs_path(cls, path: str) -> str: + return os.path.abspath(os.path.expanduser(path)) + + +if __name__ == '__main__': + # print(BasicTool.get_abs_path("~/git/..")) + for i in BasicTool.find_all_files(".", apply_abs=False): + print(i) \ No newline at end of file diff --git a/tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py b/tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py new file mode 100644 index 0000000..11c2718 --- /dev/null +++ b/tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py @@ -0,0 +1,157 @@ +import os +import json + + +class GnCommonTool: + """ + 处理BUILD.gn文件的通用方法 + """ + + @classmethod + def is_gn_variable(cls, target: str, has_quote: bool = True): + """ + 判断target是否是gn中的变量: + 规则：如果是有引号的模式，则没有引号均认为是变量，有引号的情况下，如有是"$xxx"的模式，则认为xxx是变量；如果是无引号模式，则只要$开头就认为是变量 + b = "xxx" + c = b + c = "${b}" + "$p" + """ + target = target.strip() + if not has_quote: + return target.startswith("$") + if target.startswith('"') and target.endswith('"'): + target = target.strip('"') + if target.startswith("${") and target.endswith("}"): + return True + elif target.startswith("$"): + return True + return False + else: + return True + + # 给__find_variables_in_gn用的，减少io + __var_val_mem_dict = dict() + + @classmethod + def find_variables_in_gn(cls, var_name_tuple: tuple, path: str, stop_tail: str = "home") -> tuple: + """ + 同时查找多个gn变量的值 + var_name_tuple：变量名的tuple，变量名应是未经过处理后的，如： + xxx + "${xxx}" + "$xxx" + """ + + if os.path.isfile(path): + path = os.path.split(path)[0] + var_val_dict = dict() + not_found_count = len(var_name_tuple) + for var in var_name_tuple: + val = GnCommonTool.__var_val_mem_dict.get(var) + if val is not None: + not_found_count -= 1 + var_val_dict[var] = val + while not path.endswith(stop_tail) and not_found_count != 0: + for v in var_name_tuple: + cmd = r"grep -Ern '^( *){} *= *\".*?\"' --include=*.gn* {}| grep -Ev '\$' | head -n 1 | grep -E '\".*\"' -wo".format( + v.strip('"').lstrip("${").rstrip('}'), path) + output = os.popen(cmd).read().strip().strip('"') + if len(output) != 0: + not_found_count -= 1 + var_val_dict[v] = output + GnCommonTool.__var_val_mem_dict[v] = output + path = os.path.split(path)[0] + return tuple(var_val_dict.values()) + + @classmethod + def __find_part_subsystem_from_bundle(cls, gnpath: str, stop_tail: str = "home") -> tuple: + """ + 根据BUILD.gn的全路径，一层层往上面查找bundle.json文件， + 并从bundle.json中查找part_name和subsystem + """ + filename = "bundle.json" + part_name = None + subsystem_name = None + if stop_tail not in gnpath: + return part_name, subsystem_name + if os.path.isfile(gnpath): + gnpath = os.path.split(gnpath)[0] + while not gnpath.endswith(stop_tail): + bundle_path = os.path.join(gnpath, filename) + if not os.path.isfile(bundle_path): # 如果该文件不在该目录下 + gnpath = os.path.split(gnpath)[0] + continue + with open(bundle_path, 'r', encoding='utf-8') as f: + content = json.load(f) + try: + part_name = content["component"]["name"] + subsystem_name = content["component"]["subsystem"] + except KeyError: + ... + finally: + break + part_name = None if (part_name is not None and len( + part_name) == 0) else part_name + subsystem_name = None if (subsystem_name is not None and len( + subsystem_name) == 0) else subsystem_name + return part_name, subsystem_name + + @classmethod + def find_part_subsystem(cls, gn_file: str, project_path: str) -> tuple: + """ + 查找gn_file对应的part_name和subsystem + 如果在gn中找不到，就到bundle.json中去找 + """ + part_name = None + subsystem_name = None + part_var_flag = False # 标识这个变量从gn中取出的原始值是不是变量 + subsystem_var_flag = False + var_list = list() + part_name_pattern = r"part_name *=\s*\S*" + subsystem_pattern = r"subsystem_name *=\s*\S*" + meta_grep_pattern = "grep -E '{}' {} | head -n 1" + part_cmd = meta_grep_pattern.format(part_name_pattern, gn_file) + subsystem_cmd = meta_grep_pattern.format(subsystem_pattern, gn_file) + part = os.popen(part_cmd).read().strip() + if len(part) != 0: + part = part.split('=')[-1].strip() + if GnCommonTool.is_gn_variable(part): + part_var_flag = True + var_list.append(part) + else: + part_name = part.strip('"') + if len(part_name) == 0: + part_name = None + subsystem = os.popen(subsystem_cmd).read().strip() + if len(subsystem) != 0: # 这里是只是看有没有grep到关键字 + subsystem = subsystem.split('=')[-1].strip() + if GnCommonTool.is_gn_variable(subsystem): + subsystem_var_flag = True + var_list.append(subsystem) + else: + subsystem_name = subsystem.strip('"') + if len(subsystem_name) == 0: + subsystem_name = None + if part_var_flag and subsystem_var_flag: + part_name, subsystem_name = GnCommonTool.find_variables_in_gn( + tuple(var_list), gn_file, project_path) + elif part_var_flag: + t = GnCommonTool.find_variables_in_gn( + tuple(var_list), gn_file, project_path)[0] + part_name = t if t is not None and len(t) != 0 else part_name + elif subsystem_var_flag: + t = GnCommonTool.find_variables_in_gn( + tuple(var_list), gn_file, project_path)[0] + subsystem_name = t if t is not None and len( + t) != 0 else subsystem_name + if part_name is not None and subsystem_name is not None: + return part_name, subsystem_name + # 如果有一个没有找到，就要一层层去找bundle.json文件 + t_part_name, t_subsystem_name = cls.__find_part_subsystem_from_bundle( + gn_file, stop_tail=project_path) + if t_part_name is not None: + part_name = t_part_name + if t_subsystem_name is not None: + subsystem_name = t_subsystem_name + return part_name, subsystem_name diff --git a/tools/rom_ram_analyzer/standard/pkgs/simple_excel_writer.py b/tools/rom_ram_analyzer/standard/pkgs/simple_excel_writer.py new file mode 100644 index 0000000..a1f2aa6 --- /dev/null +++ b/tools/rom_ram_analyzer/standard/pkgs/simple_excel_writer.py @@ -0,0 +1,118 @@ +import xlwt +from xlwt import Worksheet +import typing +from typing import Optional +from collections.abc import Iterable + + +class SimpleExcelWriter: + def __init__(self, default_sheet_name: str = "sheet1"): + self.__book = xlwt.Workbook(encoding='utf-8', style_compression=0) + self.__sheet_dict = { + default_sheet_name: self.__book.add_sheet( + sheetname=default_sheet_name, cell_overwrite_ok=True) + } + self.__sheet_pos = { + default_sheet_name: (0, 0) # 记录各个sheet页已经写到什么位置了，当前值为还没有写的 + } + self.__default_sheet_name = default_sheet_name + # 表头样式 + self.__head_style = xlwt.XFStyle() + # 内容样式 + self.__content_style = xlwt.XFStyle() + # 字体 + font = xlwt.Font() + font.bold = True + + # 居中对齐 + alignment = xlwt.Alignment() + alignment.horz = xlwt.Alignment.HORZ_CENTER # 水平方向 + alignment.vert = xlwt.Alignment.VERT_CENTER # 垂直方向 + + # 设置背景颜色 + pattern = xlwt.Pattern() + pattern.pattern = xlwt.Pattern.SOLID_PATTERN + pattern.pattern_fore_colour = 22 # 背景颜色 + + self.__head_style.font = font + self.__head_style.alignment = alignment + self.__head_style.pattern = pattern + self.__content_style.alignment = alignment + + def __increment_y(self, sheet_name: str, value: int = 1) -> int: + if sheet_name in self.__sheet_pos.keys(): + x, y = self.__sheet_pos.get(sheet_name) + y = y + value + self.__sheet_pos[sheet_name] = (x, y) + return y + + def __increment_x(self, sheet_name: str, value: int = 1) -> int: + if sheet_name in self.__sheet_pos.keys(): + x, y = self.__sheet_pos.get(sheet_name) + x = x + value + self.__sheet_pos[sheet_name] = (x, 0) + return x + + def append_line(self, content: list, sheet_name: str = None): + sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name + if sheet_name not in self.__sheet_dict.keys(): + print("error: sheet name '{}' not exist".format(sheet_name)) + return + sheet: Worksheet = self.__sheet_dict.get(sheet_name) + x, y = self.__sheet_pos.get(sheet_name) + for ele in content: + sheet.write(x, y, ele, style=self.__content_style) + y = self.__increment_y(sheet_name) + self.__increment_x(sheet_name) + + def write_merge(self, x0: int, y0: int, x1: int, y1: int, content: typing.Any, + sheet_name: str = None): + sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name + if sheet_name not in self.__sheet_dict.keys(): + print("error: sheet name '{}' not exist".format(sheet_name)) + return + sheet: Worksheet = self.__sheet_dict.get(sheet_name) + sheet.write_merge(x0, x1, y0, y1, content, style=self.__content_style) + + def set_sheet_header(self, headers: Iterable, sheet_name: str = None): + """ + 给sheet页设置表头 + """ + sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name + if sheet_name not in self.__sheet_dict.keys(): + print("error: sheet name '{}' not exist".format(sheet_name)) + return + x, y = self.__sheet_pos.get(sheet_name) + if x != 0 or y != 0: + print( + "error: pos of sheet '{}' is not (0,0). set_sheet_header must before write".format(sheet_name)) + return + sheet: Worksheet = self.__sheet_dict.get(sheet_name) + for h in headers: + sheet.write(x, y, h, self.__head_style) + y = self.__increment_y(sheet_name) + self.__increment_x(sheet_name) + + def add_sheet(self, sheet_name: str, cell_overwrite_ok=True) -> Optional[xlwt.Worksheet]: + if sheet_name in self.__sheet_dict.keys(): + print("error: sheet name '{}' has exist".format(sheet_name)) + return + self.__sheet_dict[sheet_name] = self.__book.add_sheet( + sheetname=sheet_name, cell_overwrite_ok=cell_overwrite_ok) + self.__sheet_pos[sheet_name] = (0, 0) + return self.__sheet_dict.get(sheet_name) + + def save(self, file_name: str): + self.__book.save(file_name) + + +if __name__ == '__main__': + writer = SimpleExcelWriter(default_sheet_name="first") + writer.add_sheet("second") + writer.add_sheet("third") + writer.set_sheet_header(["h", "m", "n"]) + writer.append_line([1, 2, 3]) + writer.append_line([2, 3, 4], "second") + writer.append_line([3, 4, 5], "third") + writer.append_line([3, 2, 1]) + writer.save("demo.xls") diff --git a/tools/rom_ram_analyzer/standard/ram_analyzer.py b/tools/rom_ram_analyzer/standard/ram_analyzer.py new file mode 100644 index 0000000..c3863af --- /dev/null +++ b/tools/rom_ram_analyzer/standard/ram_analyzer.py @@ -0,0 +1,452 @@ +import argparse +import copy +import glob +import json +import os +import re +import sys +import subprocess +import typing +import xml.dom.minidom as dom +from pprint import pprint + +from pkgs.simple_excel_writer import SimpleExcelWriter + +debug = True if sys.gettrace() else False + + +class HDCTool: + @classmethod + def verify_hdc(cls, verify_str: str = "OpenHarmony") -> bool: + """ + 验证hdc是否可用 + True：可用 + False：不可用 + """ + cp = subprocess.run(["hdc"], capture_output=True) + stdout = str(cp.stdout) + stderr = str(cp.stderr) + return verify_str in stdout or verify_str in stderr + + @classmethod + def verify_device(cls, device_num: str) -> bool: + """ + 验证设备是否已经连接 + True：已连接 + False：未连接 + """ + cp = subprocess.run(["hdc", "list", "targets"], capture_output=True) + stdout = str(cp.stdout) + stderr = str(cp.stderr) + return device_num in stderr or device_num in stdout + + + @classmethod + def exec(cls, args: list, output_from: str = "stdout"): + cp = subprocess.run(args, capture_output=True) + if output_from == "stdout": + return cp.stdout.decode() + elif output_from == "stderr": + return cp.stderr.decode() + else: + print("error: 'output_from' must be stdout or stdin") + + +def delete_values_from_dict(target_dict: typing.Dict, key_list: typing.Iterable): + for k in key_list: + if k not in target_dict.keys(): + continue + del target_dict[k] + + +class RamAnalyzer: + @classmethod + def __hidumper_mem_line_process(cls, content: typing.Text) -> typing.List[typing.Text]: + """ + 将hidumper的拥有的数据行进行分割，得到 + [pid, name, pss, vss, rss, uss]格式的list + """ + trival_pattern = re.compile(r"kB|\(.*\)(?#去除单位kB以及小括号内的任意数据，包括小括号)") + content = re.sub(trival_pattern, "", content) + blank_pattern = re.compile(r"\s+(?#匹配一个或多个空格)") + return re.sub(blank_pattern, ' ', content.strip()).split() + + __ss_dict: typing.Dict[str, int] = { + "Pss": 2, + "Vss": 3, + "Rss": 4, + "Uss": 5 + } + + @classmethod + def __parse_hidumper_mem(cls, content: typing.Text, device_num: str, ss: str = "Pss") -> typing.Dict[ + typing.Text, int]: + """ + 解析：hidumper --meme的结果 + 返回{process_name: pss}形式的字典 + '248 samgr 1464(0 in SwapPss) kB 15064 kB 6928 kB 1072 kB\r' + """ + + def find_full_process_name(hname: str) -> str: + for lname in __process_name_list: + if lname.startswith(hname): + return lname + + def process_ps_ef(content: str) -> list: + line_list = content.strip().split("\n")[1:] + process_name_list = list() + for line in line_list: + process_name = line.split()[7] + if process_name.startswith('['): + # 内核进程 + continue + process_name_list.append(process_name) + return process_name_list + + if ss not in cls.__ss_dict.keys(): + print("error: {} is not a valid parameter".format(ss)) + return dict() + output = content.split('\n') + process_pss_dict = dict() + __process_name_list: typing.List[str] = process_ps_ef( + HDCTool.exec(["hdc", "-t", device_num, "shell", "ps", "-ef"])) + for line in output: + if "Total Memory Usage by Size" in line: + break + if line.isspace(): + continue + processed: typing.List[typing.Text] = cls.__hidumper_mem_line_process(line) + if not processed or not processed[0].isnumeric(): # 如果第一列不是数字（pid），就过 + continue + name = processed[1] # 否则的话就取名字，和对应的size + size = int(processed[cls.__ss_dict.get(ss)]) + process_pss_dict[find_full_process_name(name)] = size + return process_pss_dict + + @classmethod + def process_hidumper_info(cls, device_num: str, ss:str) -> typing.Dict[str, int]: + """ + 处理进程名与对应进程大小 + """ + + def exec_once() -> typing.Dict[str, int]: + stdout = HDCTool.exec(["hdc", "-t", device_num, "shell", "hidumper", "--mem"]) + name_size_dict = cls.__parse_hidumper_mem(stdout, device_num, ss) + return name_size_dict + + if not HDCTool.verify_hdc(): + print("error: Command 'hdc' not found") + return dict() + if not HDCTool.verify_device(device_num): + print("error: {} is inaccessible or not found".format(device_num)) + return dict() + + return exec_once() + + @classmethod + def __parse_process_xml(cls, file_path: str, result_dict: typing.Dict[str, typing.List[str]]): + """ + 解析xml文件，结存存入 result_dict中，格式：{process_name: os_list} + 其中，so_list中是so的base_name + """ + if not (os.path.isfile(file_path) and file_path.endswith(".xml")): + print("warning: {} not exist or not a xml file".format(file_path)) + return + doc = dom.parse(file_path) + info = doc.getElementsByTagName("info")[0] + process = info.getElementsByTagName("process")[0] + process_name = process.childNodes[0].data + result_dict[process_name] = list() + libs = info.getElementsByTagName("loadlibs")[0].getElementsByTagName("libpath") + for lib in libs: + so = lib.childNodes[0].data + result_dict.get(process_name).append(os.path.split(so)[-1]) + if debug: + print(process_name, " ", so) + + @classmethod + def get_elf_info_from_rom_result(cls, rom_result_json: str) -> typing.Dict[str, typing.Dict[str, str]]: + """ + 利用rom_analyzer.py的分析结果，重组成 + {file_base_name: {"subsystem_name":subsystem_name, "component_name":component_name}} + 的形式 + """ + with open(rom_result_json, 'r', encoding='utf-8') as f: + rom_info_dict = json.load(f) + elf_info_dict: typing.Dict[str, typing.Dict[str, str]] = dict() + for subsystem_name in rom_info_dict.keys(): + sub_val_dict: typing.Dict[str, typing.Any] = rom_info_dict.get(subsystem_name) + delete_values_from_dict(sub_val_dict, ["size", "file_count"]) + for component_name in sub_val_dict.keys(): + component_val_dict: typing.Dict[str, str] = sub_val_dict.get(component_name) + delete_values_from_dict(component_val_dict, ["size", "file_count"]) + for file_name, size in component_val_dict.items(): + file_basename: str = os.path.split(file_name)[-1] + elf_info_dict[file_basename] = { + "subsystem_name": subsystem_name, + "component_name": component_name, + "size": size + } + return elf_info_dict + + @classmethod + def __parse_process_cfg(cls, cfg_path: str, profile_path: str, result_dict: dict): + """ + 解析cfg，因为有的cfg会拉起xml中的进程，所以也可能会去解析xml + """ + with open(cfg_path, 'r', encoding='utf-8') as f: + cfg_dict = json.loads(f.read()) + services = cfg_dict.get("services") + if services is None: + print("warning: 'services' not in {}".format(cfg_path)) + return + for service in services: + process_name = service.get("name") + first, *path_list = service.get("path") + if first.endswith("sa_main"): + # 由sa_main去来起进程 + xml_base_name = os.path.split(path_list[0])[-1] + cls.__parse_process_xml(os.path.join(profile_path, xml_base_name), result_dict) + else: + # 直接执行 + # process_name = os.path.split(first)[-1] + if result_dict.get(process_name) is None: + result_dict[process_name] = list() + result_dict.get(process_name).append(os.path.split(first)[-1]) + + @classmethod + def get_process_so_relationship(cls, xml_path: str, cfg_path: str, profile_path: str) -> typing.Dict[ + str, typing.List[str]]: + """ + 从out/{product_name}/packages/phone/sa_profile/merged_sa查找xml文件并处理得到进程与so的对应关系 + """ + # xml_path = os.path.join(product_path, "packages", "phone", "sa_profile", "merged_sa") + # 从merged_sa里面收集 + xml_list = glob.glob(xml_path + os.sep + "*[.]xml", recursive=True) + process_elf_dict: typing.Dict[str, typing.List[str]] = dict() + for xml in xml_list: + if debug: + print("parsing: ", xml) + try: + cls.__parse_process_xml(xml, process_elf_dict) + except: + print("parse '{}' failed".format(xml)) + finally: + ... + # 从system/etc/init/*.cfg中收集，如果是sa_main拉起的，则从system/profile/*.xml中进行解析 + cfg_list = glob.glob(cfg_path + os.sep + "*[.]cfg", recursive=True) + for cfg in cfg_list: + if debug: + print("parsing: ", cfg) + try: + cls.__parse_process_cfg(cfg, profile_path, process_elf_dict) + except: + print("parse '{}' failed".format(cfg)) + finally: + ... + return process_elf_dict + + @classmethod + def __save_result_as_excel(cls, data_dict: dict, filename: str, ss: str): + """ + 保存结果到excel中 + 进程名:{ + "size": xxx, + 子系统名:{ + 部件名:{ + 二进制文件: xxx, + ... + } + } + } + """ + tmp_dict = copy.deepcopy(data_dict) + writer = SimpleExcelWriter("ram_info") + writer.set_sheet_header( + ["process_name", "process_size({}, KB)".format(ss), "subsystem_name","component_name", "elf_name", "elf_size(KB)"]) + process_start_r = 1 + process_end_r = 0 + process_c = 0 + subsystem_c = 2 + subsystem_start_r = 1 + subsystem_end_r = 0 + process_size_c = 1 + component_start_r = 1 + component_end_r = 0 + component_c = 3 + for process_name in tmp_dict.keys(): + process_val_dict: typing.Dict[str, typing.Dict[str, int]] = tmp_dict.get(process_name) + process_size = process_val_dict.get("size") + delete_values_from_dict(process_val_dict, ["size"]) + for subsystem_name, subsystem_val_dict in process_val_dict.items(): # 遍历subsystem + for component_name, component_val_dict in subsystem_val_dict.items(): # 遍历component + elf_count_of_component = len(component_val_dict) + for elf_name, size in component_val_dict.items(): # 遍里elf + writer.append_line([process_name, process_size, subsystem_name, component_name, elf_name, "%.2f" % (size / 1024)]) + component_end_r += elf_count_of_component + subsystem_end_r += elf_count_of_component + # 重写component + writer.write_merge(component_start_r, component_c, component_end_r, + component_c, component_name) + component_start_r = component_end_r + 1 + process_end_r += elf_count_of_component + writer.write_merge(subsystem_start_r, subsystem_c, subsystem_end_r, subsystem_c, subsystem_name) + subsystem_start_r = subsystem_end_r+1 + writer.write_merge(process_start_r, process_c, process_end_r, process_c, process_name) + writer.write_merge(process_start_r, process_size_c, process_end_r, process_size_c, process_size) + process_start_r = process_end_r + 1 + writer.save(filename) + + @classmethod + def find_elf_size_from_rom_result(cls, service_name: str, subsystem_name: str, component_name: str, + evaluator: typing.Callable, rom_result_dict: typing.Dict[str, typing.Dict]) -> \ + typing.Tuple[ + bool, str, str, int]: + """ + 全局查找进程的相关elf文件 + subsystem_name与component_name可明确指定，或为*以遍历整个dict + evaluator：评估elf文件的从phone下面开始的路径与service_name的关系，评判如何才是找到了 + returns: 是否查找到，elf文件名，部件名，size + """ + subsystem_name_list = [subsystem_name] if subsystem_name != "*" else rom_result_dict.keys() + for sn in subsystem_name_list: + sub_val_dict = rom_result_dict.get(sn) + component_name_list = [component_name] if component_name != '*' else sub_val_dict.keys() + for cn in component_name_list: + if cn == "size" or cn == "file_count": + continue + component_val_dict: typing.Dict[str, int] = sub_val_dict.get(cn) + for k, v in component_val_dict.items(): + if k == "size" or k == "file_count": + continue + if not evaluator(service_name, k): + continue + return True, os.path.split(k)[-1],sn, cn, v + return False, str(), str(), str(), int() + + @classmethod + def analysis(cls, cfg_path: str, xml_path: str, rom_result_json: str, device_num: str, + output_file: str, ss: str, output_excel: bool): + """ + process size subsystem/component so so_size + """ + if not HDCTool.verify_hdc(): + print("error: Command 'hdc' not found") + return + if not HDCTool.verify_device(device_num): + print("error: {} is inaccessible or not found".format(device_num)) + return + with open(rom_result_json, 'r', encoding='utf-8') as f: + rom_result_dict: typing.Dict = json.loads(f.read()) + # 从rom的分析结果中将需要的elf信息重组 + so_info_dict: typing.Dict[ + str, typing.Dict[str["component_name|subsystem_name|size"], str]] = cls.get_elf_info_from_rom_result( + rom_result_json) + process_elf_dict: typing.Dict[str, typing.List[str]] = cls.get_process_so_relationship(xml_path, cfg_path, + profile_path) + process_size_dict: typing.Dict[str, int] = cls.process_hidumper_info(device_num, ss) + result_dict: typing.Dict[str, typing.Dict[str, typing.Any]] = dict() + + def get(key: typing.Any, dt: typing.Dict[str, typing.Any]): + for k, v in dt.items(): + if k.startswith(key) or (len(v) > 0 and key == v[0]): + # 要么uinput_inject的对应key为mmi_uinput_inject。对于此类特殊处理，即：如果service_name找不到，但是直接执行的bin等于这个名字，也认为找到 + return v + + for process_name, process_size in process_size_dict.items(): # 从进程出发 + # 如果部件是init,特殊处理 + if process_name == "init": + _, elf,_, _, size = cls.find_elf_size_from_rom_result(process_name, "startup", "init", + lambda x, y: os.path.split(y)[ + -1].lower() == x.lower(), + rom_result_dict) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name]["startup"] = dict() + result_dict[process_name]["startup"]["init"] = dict() + result_dict[process_name]["startup"]["init"][elf if len(elf) != 0 else "UNKNOWN"] = size + continue + # 如果是hap，特殊处理 + if (process_name.startswith("com.") or process_name.startswith("ohos.")): + _, hap_name, subsystem_name, component_name, size = cls.find_elf_size_from_rom_result(process_name, "*", "*", + lambda x, y: len( + y.split( + '/')) >= 3 and x.lower().startswith( + y.split('/')[2].lower()), + rom_result_dict) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name][subsystem_name] = dict() + result_dict[process_name][subsystem_name][component_name] = dict() + result_dict[process_name][subsystem_name][component_name][hap_name if len(hap_name) != 0 else "UNKNOWN"] = size + continue + so_list: list = get(process_name, process_elf_dict) # 得到进程相关的elf文件list + if so_list is None: + print("warning: process '{}' not found in .xml or .cfg".format(process_name)) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"]["UNKNOWN"] = int() + continue + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + for so in so_list: + unit = so_info_dict.get(so) + if unit is None: + result_dict[process_name]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"][so] = int() + print("warning: '{}' in {} not found in json from rom analysis result".format(so, process_name)) + continue + component_name = unit.get("component_name") + subsystem_name = unit.get("subsystem_name") + so_size = unit.get("size") + if result_dict.get(process_name).get(subsystem_name) is None: + result_dict[process_name][subsystem_name] = dict() + if result_dict.get(process_name).get(subsystem_name).get(component_name) is None: + result_dict[process_name][subsystem_name][component_name] = dict() + result_dict[process_name][subsystem_name][component_name][so] = so_size + base_dir, _ = os.path.split(output_file) + if len(base_dir) != 0 and not os.path.isdir(base_dir): + os.makedirs(base_dir, exist_ok=True) + with open(output_file + ".json", 'w', encoding='utf-8') as f: + f.write(json.dumps(result_dict, indent=4)) + if output_excel: + cls.__save_result_as_excel(result_dict, output_file + ".xls", ss) + + +def get_args(): + VERSION = 1.0 + parser = argparse.ArgumentParser( + description="analyze ram size of component" + ) + parser.add_argument("-v", "-version", action="version", + version=f"version {VERSION}") + parser.add_argument("-x", "--xml_path", type=str, required=True, + help="path of xml file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile") + parser.add_argument("-c", "--cfg_path", type=str, required=True, + help="path of cfg files. eg: -c ./cfgs/") + parser.add_argument("-j", "--rom_result", type=str, default="./rom_analysis_result.json", + help="json file produced by rom_analyzer_v1.0.py, default: ./rom_analysis_result.json." + "eg: -j ./demo/rom_analysis_result.json") + parser.add_argument("-n", "--device_num", type=str, required=True, + help="device number to be collect hidumper info. eg: -n 7001005458323933328a01fce16d3800") + parser.add_argument("-o", "--output_filename", default="ram_analysis_result", type=str, + help="base name of output file, default: ram_analysis_result. eg: -o ram_analysis_result") + parser.add_argument("-e", "--excel", type=bool, default=False, + help="if output result as excel, default: False. eg: -e True") + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = get_args() + cfg_path = args.cfg_path + profile_path = args.xml_path + rom_result = args.rom_result + device_num = args.device_num + output_filename = args.output_filename + output_excel = args.excel + RamAnalyzer.analysis(cfg_path, profile_path, rom_result, + device_num=device_num, output_file=output_filename, ss="Pss", output_excel=output_excel) diff --git a/tools/rom_ram_analyzer/standard/rom_analyzer.py b/tools/rom_ram_analyzer/standard/rom_analyzer.py new file mode 100644 index 0000000..b4f337d --- /dev/null +++ b/tools/rom_ram_analyzer/standard/rom_analyzer.py @@ -0,0 +1,200 @@ +import argparse +import json +import os +import sys +import typing +from copy import deepcopy +from typing import * + +from pkgs.basic_tool import BasicTool +from pkgs.gn_common_tool import GnCommonTool +from pkgs.simple_excel_writer import SimpleExcelWriter + +debug = bool(sys.gettrace()) + + +class RomAnalyzer: + @classmethod + def __collect_product_info(cls, system_module_info_json: Text, + project_path: Text) -> Dict[Text, Dict[Text, Text]]: + """ + 根据system_module_info.json生成target字典 + """ + with open(system_module_info_json, 'r', encoding='utf-8') as f: + product_list = json.loads(f.read()) + project_path = BasicTool.get_abs_path(project_path) + product_info_dict: Dict[Text, Dict[Text, Text]] = dict() + for unit in product_list: + dest: List = unit.get("dest") + if dest is None: + print("warning: keyword 'dest' not found in {}".format( + system_module_info_json)) + continue + label: Text = unit.get("label") + gn_path = component_name = subsystem_name = None + if label: + gn_path = os.path.join(project_path, label.split(':')[ + 0].lstrip('/'), "BUILD.gn") + component_name = unit.get("part_name") + subsystem_name = unit.get("subsystem_name") + if (not component_name) or (not subsystem_name): + cn, sn = GnCommonTool.find_part_subsystem( + gn_path, project_path) + component_name = cn if not component_name else component_name + subsystem_name = sn if not subsystem_name else subsystem_name + else: + print("warning: keyword 'label' not found in {}".format(unit)) + for target in dest: + product_info_dict[target] = { + "component_name": component_name, + "subsystem_name": subsystem_name, + "gn_path": gn_path, + } + return product_info_dict + + @classmethod + def __save_result_as_excel(cls, result_dict: dict, output_name: str): + header = ["subsystem_name", "component_name", + "output_file", "size(Byte)"] + tmp_dict = deepcopy(result_dict) + excel_writer = SimpleExcelWriter("rom") + excel_writer.set_sheet_header(headers=header) + subsystem_start_row = 1 + subsystem_end_row = 0 + subsystem_col = 0 + component_start_row = 1 + component_end_row = 0 + component_col = 1 + + for subsystem_name in tmp_dict.keys(): + subsystem_dict = tmp_dict.get(subsystem_name) + subsystem_size = subsystem_dict.get("size") + subsystem_file_count = subsystem_dict.get("file_count") + del subsystem_dict["file_count"] + del subsystem_dict["size"] + subsystem_end_row += subsystem_file_count + + for component_name in subsystem_dict.keys(): + component_dict: Dict[str, int] = subsystem_dict.get( + component_name) + component_size = component_dict.get("size") + component_file_count = component_dict.get("file_count") + del component_dict["file_count"] + del component_dict["size"] + component_end_row += component_file_count + + for file_name, size in component_dict.items(): + excel_writer.append_line( + [subsystem_name, component_name, file_name, size]) + excel_writer.write_merge(component_start_row, component_col, component_end_row, component_col, + component_name) + component_start_row = component_end_row + 1 + excel_writer.write_merge(subsystem_start_row, subsystem_col, subsystem_end_row, subsystem_col, + subsystem_name) + subsystem_start_row = subsystem_end_row + 1 + excel_writer.save(output_name + ".xls") + + @classmethod + def __put(cls, unit: typing.Dict[Text, Any], result_dict: typing.Dict[Text, Dict]): + """ + { + subsystem_name:{ + component_name: { + file_name: file_size + } + } + } + """ + component_name = "others" if unit.get( + "component_name") is None else unit.get("component_name") + subsystem_name = "others" if unit.get( + "subsystem_name") is None else unit.get("subsystem_name") + size = unit.get("size") + relative_filepath = unit.get("relative_filepath") + if result_dict.get(subsystem_name) is None: # 子系统 + result_dict[subsystem_name] = dict() + result_dict[subsystem_name]["size"] = 0 + result_dict[subsystem_name]["file_count"] = 0 + + if result_dict.get(subsystem_name).get(component_name) is None: # 部件 + result_dict[subsystem_name][component_name] = dict() + result_dict[subsystem_name][component_name]["size"] = 0 + result_dict[subsystem_name][component_name]["file_count"] = 0 + result_dict[subsystem_name]["size"] += size + result_dict[subsystem_name]["file_count"] += 1 + result_dict[subsystem_name][component_name]["size"] += size + result_dict[subsystem_name][component_name]["file_count"] += 1 + result_dict[subsystem_name][component_name][relative_filepath] = size + + + @classmethod + def analysis(cls, system_module_info_json: Text, product_dirs: List[str], + project_path: Text, product_name: Text, output_file: Text, output_execel: bool): + """ + system_module_info_json: json文件 + product_dirs：要处理的产物的路径列表如["vendor", "system/"] + project_path: 项目根路径 + product_name: eg，rk3568 + output_file: basename of output file + """ + project_path = BasicTool.get_abs_path(project_path) + phone_dir = os.path.join( + project_path, "out", product_name, "packages", "phone") + product_dirs = [os.path.join(phone_dir, d) for d in product_dirs] + product_info_dict = cls.__collect_product_info( + system_module_info_json, project_path) # 所有产物信息 + result_dict: Dict[Text:Dict] = dict() + for d in product_dirs: + file_list: List[Text] = BasicTool.find_all_files(d) + for f in file_list: + size = os.path.getsize(f) + relative_filepath = f.replace(phone_dir, "").lstrip(os.sep) + unit: Dict[Text, Any] = product_info_dict.get( + relative_filepath) + if unit is None: + unit = dict() + unit["size"] = size + unit["relative_filepath"] = relative_filepath + cls.__put(unit, result_dict) + output_dir, _ = os.path.split(output_file) + if len(output_dir) != 0: + os.makedirs(output_dir, exist_ok=True) + with open(output_file + ".json", 'w', encoding='utf-8') as f: + f.write(json.dumps(result_dict, indent=4)) + if output_execel: + cls.__save_result_as_excel(result_dict, output_file) + + +def get_args(): + VERSION = 2.0 + parser = argparse.ArgumentParser( + description=f"analyze rom size of component.\n") + parser.add_argument("-v", "-version", action="version", + version=f"version {VERSION}") + parser.add_argument("-p", "--project_path", type=str, required=True, + help="root path of openharmony. eg: -p ~/openharmony") + parser.add_argument("-j", "--module_info_json", required=True, type=str, + help="path of out/{product_name}/packages/phone/system_module_info.json") + parser.add_argument("-n", "--product_name", required=True, + type=str, help="product name. eg: -n rk3568") + parser.add_argument("-d", "--product_dir", required=True, action="append", + help="subdirectories of out/{product_name}/packages/phone to be counted." + "eg: -d system -d vendor") + parser.add_argument("-o", "--output_file", type=str, default="rom_analysis_result", + help="basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result") + parser.add_argument("-e", "--excel", type=bool, default=False, + help="if output result as excel, default: False. eg: -e True") + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = get_args() + module_info_json = args.module_info_json + project_path = args.project_path + product_name = args.product_name + product_dirs = args.product_dir + output_file = args.output_file + output_excel = args.excel + RomAnalyzer.analysis(module_info_json, product_dirs, + project_path, product_name, output_file, output_excel) -- Gitee From 2aff8154c0cf9ad840fc51650954fa400a021741 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Mon, 6 Mar 2023 15:41:30 +0800 Subject: [PATCH 02/14] change folder name of rom_ram_analyzer Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/L0L1/README.md | 47 -- tools/rom_ram_analyzer/L0L1/__init__.py | 0 tools/rom_ram_analyzer/L0L1/pkgs/__init__.py | 0 .../rom_ram_analyzer/L0L1/pkgs/basic_tool.py | 142 ------ .../L0L1/pkgs/gn_common_tool.py | 289 ----------- .../L0L1/pkgs/simple_excel_writer.py | 119 ----- .../L0L1/pkgs/simple_yaml_tool.py | 15 - tools/rom_ram_analyzer/L0L1/src/__init__.py | 1 - tools/rom_ram_analyzer/L0L1/src/config.py | 213 --------- tools/rom_ram_analyzer/L0L1/src/config.yaml | 144 ------ .../L0L1/src/get_subsystem_component.py | 131 ----- tools/rom_ram_analyzer/L0L1/src/misc.py | 228 --------- tools/rom_ram_analyzer/L0L1/src/preprocess.py | 4 - .../rom_ram_analyzer/L0L1/src/rom_analysis.py | 295 ------------ .../L0L1/src/template_processor.py | 244 ---------- tools/rom_ram_analyzer/L2/README.md | 143 ------ tools/rom_ram_analyzer/L2/__init__.py | 0 tools/rom_ram_analyzer/L2/pkgs/__init__.py | 0 tools/rom_ram_analyzer/L2/pkgs/basic_tool.py | 33 -- .../L2/pkgs/gn_common_tool.py | 157 ------ .../L2/pkgs/simple_excel_writer.py | 118 ----- tools/rom_ram_analyzer/L2/ram_analyzer.py | 452 ------------------ tools/rom_ram_analyzer/L2/rom_analyzer.py | 200 -------- 23 files changed, 2975 deletions(-) delete mode 100644 tools/rom_ram_analyzer/L0L1/README.md delete mode 100644 tools/rom_ram_analyzer/L0L1/__init__.py delete mode 100644 tools/rom_ram_analyzer/L0L1/pkgs/__init__.py delete mode 100644 tools/rom_ram_analyzer/L0L1/pkgs/basic_tool.py delete mode 100644 tools/rom_ram_analyzer/L0L1/pkgs/gn_common_tool.py delete mode 100644 tools/rom_ram_analyzer/L0L1/pkgs/simple_excel_writer.py delete mode 100644 tools/rom_ram_analyzer/L0L1/pkgs/simple_yaml_tool.py delete mode 100644 tools/rom_ram_analyzer/L0L1/src/__init__.py delete mode 100644 tools/rom_ram_analyzer/L0L1/src/config.py delete mode 100644 tools/rom_ram_analyzer/L0L1/src/config.yaml delete mode 100644 tools/rom_ram_analyzer/L0L1/src/get_subsystem_component.py delete mode 100644 tools/rom_ram_analyzer/L0L1/src/misc.py delete mode 100644 tools/rom_ram_analyzer/L0L1/src/preprocess.py delete mode 100644 tools/rom_ram_analyzer/L0L1/src/rom_analysis.py delete mode 100644 tools/rom_ram_analyzer/L0L1/src/template_processor.py delete mode 100644 tools/rom_ram_analyzer/L2/README.md delete mode 100644 tools/rom_ram_analyzer/L2/__init__.py delete mode 100644 tools/rom_ram_analyzer/L2/pkgs/__init__.py delete mode 100644 tools/rom_ram_analyzer/L2/pkgs/basic_tool.py delete mode 100644 tools/rom_ram_analyzer/L2/pkgs/gn_common_tool.py delete mode 100644 tools/rom_ram_analyzer/L2/pkgs/simple_excel_writer.py delete mode 100644 tools/rom_ram_analyzer/L2/ram_analyzer.py delete mode 100644 tools/rom_ram_analyzer/L2/rom_analyzer.py diff --git a/tools/rom_ram_analyzer/L0L1/README.md b/tools/rom_ram_analyzer/L0L1/README.md deleted file mode 100644 index ce3da51..0000000 --- a/tools/rom_ram_analyzer/L0L1/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# rom_ram_analyzer - -## 目的 - -分析各部件的rom占用,结果以xls和json格式进行保存 - -## 支持产品 - -目标是支持所有的产品,但是目前由于配置文件没设计好,只支持:ipcamera_hispark_taurus ipcamera_hispark_taurus_linux wifiiot_hispark_pegasus - -## 代码思路 - -1. 扫描BUILD.gn文件,收集各个target的编译产物及其对应的component_name, subsystem_name信息,并存储到config.yaml中的gn_info_file字段指定的json文件中 -2. 根据配置文件config.yaml扫描产品的编译产物目录,得到真实的编译产物信息(主要是大小) -3. 用真实的编译产物与从BUILD.gn中收集的信息进行匹配,从而得到编译产物-大小-所属部件的对应信息 -4. 如果匹配失败,会直接利用grep到项目路径下进行搜索 -5. 如果还搜索失败,则将其归属到others - - -## 使用 - -前置条件： - -1. 获取整个本文件所在的整个目录 -1. 对系统进行编译 -1. linux平台 -1. python3.8及以后 -1. 安装requirements - ```txt - xlwt==1.3.0 - ``` - -1. `python3 rom_analysis.py --product_name {your_product_name} --oh_path {root_path_of_oh} [--recollect_gn bool]`运行代码,其中recollect_gn表示是需要重新扫描BUILD.gn还是直接使用已有结果.eg: `python3 rom_analysis.py --product_name ipcamera_hispark_taurus` -3. 运行完毕会产生4个json文件及一个xls文件,如果是默认配置,各文件描述如下: - - gn_info.json:BUILD.gn的分析结果 - - sub_com_info.json:从bundle.json中进行分析获得的各部件及其对应根目录的信息 - - {product_name}_product.json:该产品实际的编译产物信息,根据config.yaml进行收集 - - {product_name}_result.json:各部件的rom大小分析结果 - - {product_name}_result.xls:各部件的rom大小分析结果 - -## 新增template - -主要是在config.py中配置Processor,并在config.yaml中添加相应内容 - -## 后续工作 - -1. 部分log的输出有待优化 \ No newline at end of file diff --git a/tools/rom_ram_analyzer/L0L1/__init__.py b/tools/rom_ram_analyzer/L0L1/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tools/rom_ram_analyzer/L0L1/pkgs/__init__.py b/tools/rom_ram_analyzer/L0L1/pkgs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tools/rom_ram_analyzer/L0L1/pkgs/basic_tool.py b/tools/rom_ram_analyzer/L0L1/pkgs/basic_tool.py deleted file mode 100644 index febe482..0000000 --- a/tools/rom_ram_analyzer/L0L1/pkgs/basic_tool.py +++ /dev/null @@ -1,142 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2022 Huawei Device Co., Ltd. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import itertools -import os -import re -import glob -from typing import * - - -def do_nothing(x: Any) -> Any: - return x - - -class BasicTool: - @classmethod - def find_files_with_pattern(cls, folder: str, pattern: str = "/**", recursive: bool = True, apply_abs: bool = True, - real_path: bool = True, de_duplicate: bool = True, is_sort: bool = True, - post_handler: Callable[[List[str]], List[str]] = None) -> list: - """ - 根据指定paatern匹配folder下的所有文件，默认递归地查找所有文件 - folder：要查找的目录,会先经过cls.abspath处理(结尾不带/) - pattern：要查找的模式,需要以/开头,因为会和folder拼接 - recursive：是否递归查找 - apply_abs：是否将路径转换为绝对路径 - real_path：如果是软链接，是否转换为真正的路径 - de_duplicate：是否去重 - is_sort：是否排序 - post_handler: 对文件进行额外处理的方法，参数为文件名的List，返回值为字符串列表 - FIXME 有可能会卡住,可能原因为符号链接 - """ - file_list = glob.glob(cls.abspath(folder)+pattern, recursive=recursive) - if apply_abs: - file_list = list(map(lambda x: cls.abspath(x), file_list)) - if real_path: - file_list = list(map(lambda x: os.path.realpath(x), file_list)) - if de_duplicate: - file_list = list(set(file_list)) - if is_sort: - file_list = sorted(file_list, key=str.lower) - if post_handler: - file_list = post_handler(file_list) - if folder in file_list: - file_list.remove(folder) - return file_list - - @classmethod - def match_paragraph(cls, content: str, start_pattern: str = r"\w+\(\".*?\"\) *{", end_pattern: str = "\}") -> \ - Iterator[re.Match]: - """ - 匹配代码段，支持单行 - 注意：ptrn中已经包含前面的空格，所以start_pattern中可以省略 - :param content: 被匹配的字符串 - :param start_pattern: 模式的开头 - :param end_pattern: 模式的结尾 - :return: 匹配到的段落的迭代器 - """ - ptrn = r'^( *){s}(?#匹配开头).*?(?#中间非贪婪)\1(?#如果开头前面有空格,则结尾的前面应该有相同数量的空格)?{e}$(?#匹配结尾)'.format( - s=start_pattern, e=end_pattern) - ptrn = re.compile(ptrn, re.M | re.S) - result = re.finditer(ptrn, content) - return result - - @classmethod - def re_group_1(cls, content: str, pattern: str, **kwargs) -> str: - """ - 匹配正则表达式，如果有匹配到内容，返回group(1)的内容 - :param content: 要被匹配的内容 - :param pattern: 进行匹配的模式 - :return: 匹配到的结果（group(1)） - TODO 对（）的检查应该更严格 - """ - if not (r'(' in pattern and r')' in pattern): - raise ValueError("parentheses'()' must in the pattern") - result = re.search(pattern, content, **kwargs) - if result: - return result.group(1) - return str() - - @classmethod - def abspath(cls, path: str) -> str: - """ - 将路径转换为绝对路径，如果有~，展开 - :param path: 要被转换的路径 - :return: 绝对路径 - """ - return os.path.abspath(os.path.expanduser(path)) - - @classmethod - def grep_ern(cls, pattern: str, path: str, include: str = str(), exclude: tuple = tuple(), - post_handler: Callable[[Text], Any] = do_nothing) -> Any: - """ - 执行grep命令来查找内容 - :param exclude: 不搜索path下的的exclude目录 - :param include: 指定要搜索的文件 - :param pattern: 使用pattern进行grep - :param path: 搜索路径 - :param post_handler: 对grep的结果进行后处理 - :return: post_handler对grep结果进行处理后的结果 - TODO 将cls.execute用subprocess代替 - """ - cmd = f"grep -Ern '{pattern}' '{cls.abspath(path)}'" - if include: - cmd += f" --include='{include}'" - for e in exclude: - cmd += f" --exclude-dir={e}" - o = cls.execute(cmd) - if post_handler: - o = post_handler(o) - return o - - @classmethod - def execute(cls, cmd: str, post_processor: Callable[[Text], Text] = do_nothing) -> Any: - """ - 封装popen，返回标准输出的列表 - :param post_processor: 对执行结果进行处理 - :param cmd: 待执行的命令 - :return: 经处理过后的字符串列表 - - """ - output = os.popen(cmd).read() - output = post_processor(output) - return output - - -if __name__ == '__main__': - for i in BasicTool.grep_ern("^( *)ohos_shared_library", "/home/aodongbiao/oh", include="BUILD.gn", exclude=("out", "doc", ".ccache"), post_handler=lambda x: x.split('\n')): - if "oh/out" in i: - print(i) diff --git a/tools/rom_ram_analyzer/L0L1/pkgs/gn_common_tool.py b/tools/rom_ram_analyzer/L0L1/pkgs/gn_common_tool.py deleted file mode 100644 index 66ebb21..0000000 --- a/tools/rom_ram_analyzer/L0L1/pkgs/gn_common_tool.py +++ /dev/null @@ -1,289 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2022 Huawei Device Co., Ltd. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import os -import logging -import re -import ast -import json -import collections -from typing import * -if __name__ == '__main__': - from basic_tool import BasicTool -else: - from pkgs.basic_tool import BasicTool - - -class GnCommonTool: - """ - 处理BUILD.gn文件的通用方法 - """ - - @classmethod - def _find_gn_variable_list(cls, content: str) -> List: - """ - 获取s中${xxx}或$xxx形式的gn变量 - :param content: 待查找的字符串 - :param sep: 分隔符，使用本分隔符将内容进行分隔然后逐一查找 - :return: 变量名及其符号，eg：${abc}、$abc - :FIXME 没有对a = 'a' b = a中的b这种形式进行处理 - """ - result = list() - splited = content.split(os.sep) - patern = re.compile(r"\${.*?}") - for item in splited: - m = re.findall(patern, item) - result.extend(m) - if len(m) == 0 and "$" in item: - item = item.strip('"') - result.append(item[item.index("$"):]) - return result - - @classmethod - def is_gn_variable(cls, target: str, quote_processed: bool = False): - """ - 判断target是否是gn中的变量: - 规则：如果是quote_processed is False，则没有引号均认为是变量，有引号的情况下，如果是"$xxx"或${xx}的模式，则认为xxx是变量； - 如果quote_processed is True，则只要$开头就认为是变量 - b = "xxx" - c = b - c = "${b}" - "$p" - :param target: 要进行判断的字符串对象 - :param quote_processed: 引号是否已经去除 - :return: target是否为gn中的变量 - """ - target = target.strip() - if quote_processed: - return target.startswith("$") - if target.startswith('"') and target.endswith('"'): - target = target.strip('"') - if target.startswith("${") and target.endswith("}"): - return True - elif target.startswith("$"): - return True - return False - else: - return True - - @classmethod - def contains_gn_variable(cls, s: str, quote_processed: bool = False): - """ - 判断字符串s中是否包含gn变量 - """ - return cls.is_gn_variable(s, quote_processed) or ("$" in s) - - # 给__find_variables_in_gn用的，减少io - __var_val_mem_dict = collections.defaultdict(str) - - @classmethod - def find_variables_in_gn(cls, var_name_tuple: tuple, path: str, stop_tail: str = "home", use_cache: bool = False) -> \ - List[str]: - """ - 同时查找多个gn变量的值 - var_name_tuple：变量名的tuple，变量名应是未经过处理后的，如： - xxx - "${xxx}" - "$xxx" - :param var_name_tuple: 待查找的变量名的列表 - :param path: 变量名所在文件的路径 - :param stop_tail: 当path以stop_tail结尾时，停止查找 - :param use_cache: 是否使用缓存 - :return: 变量值的列表 - """ - if os.path.isfile(path): - path, _ = os.path.split(path) - var_val_dict = collections.defaultdict(str) - not_found_count = len(var_name_tuple) - if use_cache: - for var in var_name_tuple: - val = GnCommonTool.__var_val_mem_dict[var] - if val: - not_found_count -= 1 - var_val_dict[var] = val - while (stop_tail in path) and not_found_count: - for v in var_name_tuple: - pv = v.strip('"').lstrip("${").rstrip('}') - # 先直接grep出pv *= *\".*?\"的 - # 然后排除含有$符的 - # 再取第一个 - # 最后只取引号内的 - cmd = fr"grep -Ern '{pv} *= *\".*?\"' --include=*.gn* {path} | grep -Ev '\$' " \ - r"| head -n 1 | grep -E '\".*\"' -wo" - output = BasicTool.execute(cmd, lambda x: x.strip().strip('"')) - # backup:end - if not output: - continue - not_found_count -= 1 - var_val_dict[v] = output - GnCommonTool.__var_val_mem_dict[v] = output - path, _ = os.path.split(path) - return list(var_val_dict.values()) - - @classmethod - def replace_gn_variables(cls, s: str, gn_path: str, stop_tail: str) -> str: - """ - 替换字符串中的gn变量名为其值,注意,没有对s是否真的包含gn变量进行验证 - :param s: 待替换的字符串 - :param gn_path: 字符串所在的gn文件 - :param stop_tail: 当变量查找到stop_tail目录时停止 - :return: 将变量替换为其值的字符串 - """ - variable_list = GnCommonTool._find_gn_variable_list(s) - if len(variable_list) == 0: - return s - value_list = GnCommonTool.find_variables_in_gn( - tuple(variable_list), path=gn_path, stop_tail=stop_tail) - for k, v in dict(zip(variable_list, value_list)).items(): - s = s.replace(k, v) - return s - - @classmethod - def find_values_of_variable(cls, var_name: str, path: str, stop_tail: str = "home") -> list: - """ - 查找变量的值，如果有多个可能值，全部返回 - :param var_name: 变量名 - :param path: 变量名所在的文件 - :param stop_tail: 当变量查找到stop_tail目录时停止 - :return: 该变量的可能值 - """ - if os.path.isfile(path): - path, _ = os.path.split(path) - result = list() - v = var_name.strip('"').lstrip("${").rstrip('}') - while stop_tail in path: - cmd = fr"grep -Ern '^( *){v} *= *\".*?\"' --include=*.gn* {path}" - output = os.popen(cmd).readlines() - path = os.path.split(path)[0] - if not output: - continue - for line in output: - line = line.split('=')[-1].strip().strip('"') - if len(line) == 0: - continue - result.append(line) - break - return result - - -class GnVariableParser: - @classmethod - def string_parser(cls, var: str, content: str) -> str: - """ - 解析值为字符串的变量,没有对引号进行去除,如果是a = b这种b为变量的,则无法匹配 - :param content: 要进行解析的内容 - :param var: 变量名 - :return: 变量值[str] - """ - result = BasicTool.re_group_1( - content, r"{} *= *[\n]?(\".*?\")".format(var), flags=re.S | re.M) - return result - - @classmethod - def list_parser(cls, var: str, content: str) -> List[str]: - """ - 解析值为列表的变量，list的元素必须全为数字或字符串,且没有对引号进行去除,如果是a = b这种b为变量的,则无法匹配 - :param var: 变量名 - :param content: 要进行 - :return: 变量值[List] - """ - result = BasicTool.re_group_1( - content, r"{} *= *(\[.*?\])".format(var), flags=re.S | re.M) - result = ast.literal_eval(result.strip()) - return result - - -if __name__ == '__main__': - cc = \ - """ - target("shared_library", "mmp"){ - xxx - } - - ohos_shared_library("pinauthservice") { - sources = [ - "//base/useriam/pin_auth/services/modules/driver/src/pin_auth_driver_hdi.cpp", - "//base/useriam/pin_auth/services/modules/driver/src/pin_auth_interface_adapter.cpp", - "//base/useriam/pin_auth/services/modules/executors/src/pin_auth_executor_callback_hdi.cpp", - "//base/useriam/pin_auth/services/modules/executors/src/pin_auth_executor_hdi.cpp", - "//base/useriam/pin_auth/services/modules/inputters/src/i_inputer_data_impl.cpp", - "//base/useriam/pin_auth/services/modules/inputters/src/pin_auth_manager.cpp", - "//base/useriam/pin_auth/services/sa/src/pin_auth_service.cpp", - ] - - configs = [ - ":pin_auth_services_config", - "//base/useriam/user_auth_framework/common:iam_log_config", - "//base/useriam/user_auth_framework/common:iam_utils_config", - ] - - deps = [ - "//base/useriam/pin_auth/frameworks:pinauth_ipc", - "//third_party/openssl:libcrypto_shared", - ] - - external_deps = [ - "access_token:libaccesstoken_sdk", - "c_utils:utils", - "drivers_interface_pin_auth:libpin_auth_proxy_1.0", - "hisysevent_native:libhisysevent", - "hiviewdfx_hilog_native:libhilog", - "ipc:ipc_core", - "safwk:system_ability_fwk", - "user_auth_framework:userauth_executors", - ] - t = [ - 1, - 2, - 3 - ] - tt = [ - aaa, - bbb, - ccc - ] - remove_configs = [ "//build/config/compiler:no_exceptions" ] - - subsystem_name = "useriam" - part_name = "pin_auth" - }""" - s = """ -updater_usb_init_cfg_path = "//base/startup/init/services/etc/init.usb.cfg" -updater_init_usb_configfs_path_cfg = - "//drivers/peripheral/usb/cfg/init.usb.configfs.cfg" -updater_faultloggerd_cfg = -"//base/hiviewdfx/faultloggerd/services/config/faultloggerd.cfg" -updater_hilog_cfg = "//base/hiviewdfx/hilog/services/hilogd/etc/hilogd.cfg" - -ohos_prebuilt_etc("updater_hilog.cfg") { -source = "${updater_hilog_cfg}" -install_images = [ "updater" ] -part_name = "updater" -} -""" - s = "\"${updater_faultloggerd_cfg}\"" - print(GnCommonTool.contains_gn_variable(s)) - # print(GnVariableParser.string_parser("updater_faultloggerd_cfg",s)) - # print(re.search( - # "updater_faultloggerd_cfg *= *[\n]?(\".*?\")", s, flags=re.S | re.M).group()) - # print(GnVariableParser.list_parser("t", cc)) - # print(len(GnVariableParser.list_parscer("t", cc))) - # print(TargetNameParser.second_parser(cc)) - # print(GnCommonTool._find_gn_variable_list("a/${b}${e}/$c")) - # print(GnCommonTool._find_gn_variable_list("abc_$abc")) - # print(GnCommonTool.find_values_of_variable( - # "\"${OHOS_PROFILER_SUBSYS_NAME}\"", path="/home/aodongbiao/oh/third_party/abseil-cpp/absl/strings/BUILD.gn", stop_tail="/home/aodongbiao/oh")) - ... diff --git a/tools/rom_ram_analyzer/L0L1/pkgs/simple_excel_writer.py b/tools/rom_ram_analyzer/L0L1/pkgs/simple_excel_writer.py deleted file mode 100644 index ebe37e1..0000000 --- a/tools/rom_ram_analyzer/L0L1/pkgs/simple_excel_writer.py +++ /dev/null @@ -1,119 +0,0 @@ -import xlwt -from xlwt import Worksheet -import typing -import logging -from typing import Optional -from collections.abc import Iterable - - -class SimpleExcelWriter: - def __init__(self, default_sheet_name: str = "sheet1"): - self.__book = xlwt.Workbook(encoding='utf-8', style_compression=0) - self.__sheet_dict = { - default_sheet_name: self.__book.add_sheet( - sheetname=default_sheet_name, cell_overwrite_ok=True) - } - self.__sheet_pos = { - default_sheet_name: (0, 0) # 记录各个sheet页已经写到什么位置了，当前值为还没有写的 - } - self.__default_sheet_name = default_sheet_name - # 表头样式 - self.__head_style = xlwt.XFStyle() - # 内容样式 - self.__content_style = xlwt.XFStyle() - # 字体 - font = xlwt.Font() - font.bold = True - - # 设置背景颜色 - pattern = xlwt.Pattern() - pattern.pattern = xlwt.Pattern.SOLID_PATTERN - pattern.pattern_fore_colour = 22 # 背景颜色 - - # 居中对齐 - alignment = xlwt.Alignment() - alignment.horz = xlwt.Alignment.HORZ_CENTER # 水平方向 - alignment.vert = xlwt.Alignment.VERT_CENTER # 垂直方向 - - self.__head_style.font = font - self.__head_style.alignment = alignment - self.__head_style.pattern = pattern - self.__content_style.alignment = alignment - - def __increment_y(self, sheet_name: str, value: int = 1) -> int: - if sheet_name in self.__sheet_pos.keys(): - x, y = self.__sheet_pos.get(sheet_name) - y = y + value - self.__sheet_pos[sheet_name] = (x, y) - return y - - def __increment_x(self, sheet_name: str, value: int = 1) -> int: - if sheet_name in self.__sheet_pos.keys(): - x, y = self.__sheet_pos.get(sheet_name) - x = x + value - self.__sheet_pos[sheet_name] = (x, 0) - return x - - def append_line(self, content: list, sheet_name: str = None): - sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name - if sheet_name not in self.__sheet_dict.keys(): - logging.error("sheet name '{}' not exist".format(sheet_name)) - return - sheet: Worksheet = self.__sheet_dict.get(sheet_name) - x, y = self.__sheet_pos.get(sheet_name) - for ele in content: - sheet.write(x, y, ele, style=self.__content_style) - y = self.__increment_y(sheet_name) - self.__increment_x(sheet_name) - - def write_merge(self, x0: int, y0: int, x1: int, y1: int, content: typing.Any, - sheet_name: str = None): - sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name - if sheet_name not in self.__sheet_dict.keys(): - logging.error("sheet name '{}' not exist".format(sheet_name)) - return - sheet: Worksheet = self.__sheet_dict.get(sheet_name) - sheet.write_merge(x0, x1, y0, y1, content, style=self.__content_style) - - def set_sheet_header(self, headers: Iterable, sheet_name: str = None): - """ - 给sheet页设置表头 - """ - sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name - if sheet_name not in self.__sheet_dict.keys(): - logging.error("sheet name '{}' not exist".format(sheet_name)) - return - x, y = self.__sheet_pos.get(sheet_name) - if x != 0 or y != 0: - logging.error( - "pos of sheet '{}' is not (0,0). set_sheet_header must before write".format(sheet_name)) - return - sheet: Worksheet = self.__sheet_dict.get(sheet_name) - for h in headers: - sheet.write(x, y, h, self.__head_style) - y = self.__increment_y(sheet_name) - self.__increment_x(sheet_name) - - def add_sheet(self, sheet_name: str, cell_overwrite_ok=True) -> Optional[xlwt.Worksheet]: - if sheet_name in self.__sheet_dict.keys(): - logging.error("sheet name '{}' has exist".format(sheet_name)) - return - self.__sheet_dict[sheet_name] = self.__book.add_sheet( - sheetname=sheet_name, cell_overwrite_ok=cell_overwrite_ok) - self.__sheet_pos[sheet_name] = (0, 0) - return self.__sheet_dict.get(sheet_name) - - def save(self, file_name: str): - self.__book.save(file_name) - - -if __name__ == '__main__': - writer = SimpleExcelWriter(default_sheet_name="first") - writer.add_sheet("second") - writer.add_sheet("third") - writer.set_sheet_header(["h", "m", "n"]) - writer.append_line([1, 2, 3]) - writer.append_line([2, 3, 4], "second") - writer.append_line([3, 4, 5], "third") - writer.append_line([3, 2, 1]) - writer.save("demo.xls") diff --git a/tools/rom_ram_analyzer/L0L1/pkgs/simple_yaml_tool.py b/tools/rom_ram_analyzer/L0L1/pkgs/simple_yaml_tool.py deleted file mode 100644 index 5edf612..0000000 --- a/tools/rom_ram_analyzer/L0L1/pkgs/simple_yaml_tool.py +++ /dev/null @@ -1,15 +0,0 @@ -import yaml -from typing import * -from yaml.loader import SafeLoader - - -class SimpleYamlTool: - @classmethod - def read_yaml(cls, file_name: str, mode: str = "r", encoding: str = "utf-8") -> Dict: - with open(file_name, mode, encoding=encoding) as f: - return yaml.load(f, Loader=SafeLoader) - - -if __name__ == '__main__': - config = SimpleYamlTool.read_yaml("/home/aodongbiao/build_static_check/tools/component_tools/rom_ram_analyzer/src/config.yaml") - print(config["black_grep_dir"]) \ No newline at end of file diff --git a/tools/rom_ram_analyzer/L0L1/src/__init__.py b/tools/rom_ram_analyzer/L0L1/src/__init__.py deleted file mode 100644 index 341988c..0000000 --- a/tools/rom_ram_analyzer/L0L1/src/__init__.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = 1.0 diff --git a/tools/rom_ram_analyzer/L0L1/src/config.py b/tools/rom_ram_analyzer/L0L1/src/config.py deleted file mode 100644 index 05f1a26..0000000 --- a/tools/rom_ram_analyzer/L0L1/src/config.py +++ /dev/null @@ -1,213 +0,0 @@ -import os -import sys -import argparse -import json -from typing import * - -import preprocess -from pkgs.simple_yaml_tool import SimpleYamlTool -from pkgs.basic_tool import do_nothing, BasicTool -from get_subsystem_component import SC -from misc import * -from template_processor import * -""" -只给rom_analysis.py使用 -""" - - -def parse_args(): - parser = argparse.ArgumentParser( - description="analysis rom size of L0 and L1 product") - parser.add_argument("-p", "--product_name", type=str, - help="product name. eg: -p ipcamera_hispark_taurus") - parser.add_argument("-o", "--oh_path", type=str, - default=".", help="root path of openharmony") - parser.add_argument("-g", "--recollect_gn", action="store_false", help="recollect gn info or not") - parser.add_argument("-s", "--recollect_sc", action="store_false", help="recollect subsystem_component info or not") - args = parser.parse_args() - return args - - -_args = parse_args() - -# # global variables -configs = SimpleYamlTool.read_yaml("config.yaml") -result_dict: Dict[str, Any] = dict() - -project_path = BasicTool.abspath(_args.oh_path) -product_name = _args.product_name -recollect_gn = _args.recollect_gn -_recollect_sc = _args.recollect_sc -_sc_json: Dict[Text, Text] = configs.get("subsystem_component") -_sc_save = _sc_json.get("save") -_target_type = configs["target_type"] -_sc_output_path = _sc_json.get("filename") -if _recollect_sc: - sub_com_dict: Dict = SC.run(project_path, _sc_output_path, _sc_save) -else: - with open(_sc_output_path, 'r', encoding='utf-8') as f: - sub_com_dict = json.load(f) - -collector_config: Tuple[BaseProcessor] = ( - DefaultProcessor(project_path=project_path, # 项目根路径 - result_dict=result_dict, # 保存结果的字典 - # targte的类型名称,即xxx("yyy")中的xxx - target_type=_target_type[0], - # 用以进行匹配的模式串,包括匹配段落时作为前缀 - match_pattern=fr"^( *){_target_type[0]}\(.*?\)", - sub_com_dict=sub_com_dict, # 从bundle.json中收集的subsystem_name和component_name信息 - target_name_parser=TargetNameParser.single_parser, # 进行target_name解析的parser - other_info_handlers={ - "extension": extension_handler, - }, # 解析其他信息的parser,{"字段名":该字段的parser} - unit_post_handler=SOPostHandler() # 对即将进行存储的unit字典的handler,会返回一个str作为存储时的key - ), - DefaultProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[1], - match_pattern=fr"^( *){_target_type[1]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "extension": extension_handler, - }, - unit_post_handler=SOPostHandler(), - ), - DefaultProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[2], - match_pattern=fr"^( *){_target_type[2]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "extension": extension_handler, - }, - unit_post_handler=APostHandler(), - ), - DefaultProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[3], - match_pattern=fr"^( *){_target_type[3]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "extension": extension_handler, - }, - unit_post_handler=APostHandler(), - ), - DefaultProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[4], - match_pattern=fr"^( *){_target_type[4]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "extension": extension_handler, - }, - unit_post_handler=DefaultPostHandler(), - ), - DefaultProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[5], - match_pattern=fr"^( *){_target_type[5]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "extension": extension_handler, - }, - unit_post_handler=DefaultPostHandler(), - ), - DefaultProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[6], - match_pattern=fr"^( *){_target_type[6]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "real_target_type": target_type_handler, - "extension": extension_handler, - }, - unit_post_handler=LiteLibPostHandler(), - ud_post_handler=LiteLibS2MPostHandler, - ), - DefaultProcessor(project_path=project_path, # hap有个hap_name - result_dict=result_dict, - target_type=_target_type[7], - match_pattern=fr"^( *){_target_type[7]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "hap_name": hap_name_handler, - "extension": extension_handler, - }, - unit_post_handler=HAPPostHandler(), - ), - StrResourceProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[8], - match_pattern=fr"^( *){_target_type[8]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "extension": extension_handler, - }, - unit_post_handler=DefaultPostHandler(), - resource_field="source" - ), - StrResourceProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[9], - match_pattern=fr"^( *){_target_type[9]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "extension": extension_handler, - }, - unit_post_handler=DefaultPostHandler(), - resource_field="source" - ), - ListResourceProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[10], - match_pattern=fr"^( *){_target_type[10]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "extension": extension_handler, - }, - unit_post_handler=DefaultPostHandler(), - resource_field="sources" - ), - StrResourceProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[11], - match_pattern=fr"^( *){_target_type[11]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - # "extension": extension_handler, - }, - unit_post_handler=DefaultPostHandler(), - resource_field="source" - ), - DefaultProcessor(project_path=project_path, - result_dict=result_dict, - target_type=_target_type[12], - match_pattern=fr"^( *){_target_type[12]}\(.*?\)", - sub_com_dict=sub_com_dict, - target_name_parser=TargetNameParser.single_parser, - other_info_handlers={ - "real_target_type": target_type_handler, - # "extension": extension_handler, - }, - unit_post_handler=LiteComponentPostHandler(), - ), -) - -__all__ = ["configs", "result_dict", "collector_config", "sub_com_dict"] - -if __name__ == '__main__': - for c in collector_config: - c.run() - with open("demo.json", 'w', encoding='utf-8') as f: - json.dump(result_dict, f) diff --git a/tools/rom_ram_analyzer/L0L1/src/config.yaml b/tools/rom_ram_analyzer/L0L1/src/config.yaml deleted file mode 100644 index f6e0f3e..0000000 --- a/tools/rom_ram_analyzer/L0L1/src/config.yaml +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2022 Huawei Device Co., Ltd. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# 注意:如果target_type有了更改,要相应改变config.py中collector_config -target_type: - - shared_library - - ohos_shared_library - - static_library - - ohos_static_library - - executable - - ohos_executable - - lite_library - - ohos_hap - - ohos_prebuilt_etc - - ohos_prebuilt_para - - ohos_sa_profile - - ohos_prebuilt_shared_library - - lite_component - - -subsystem_component: - save: true - filename: sub_com_info.json -gn_info_file: gn_info.json - -ipcamera_hispark_taurus: - product_infofile: ipcamera_hispark_taurus_product.json # 编译产物的信息 - output_name: ipcamera_hispark_taurus_result.json # 输出文件的名字 - product_dir: # [required] - root: out/hispark_taurus/ipcamera_hispark_taurus/rootfs - relative: - bin: bin - so: usr/lib - etc: etc - rest: True # 是否分析其他目录下的并归到etc(copy的文件) - query_order: - so: - - shared_library - - ohos_shared_library - - ohos_prebuilt_shared_library - - lite_library - - lite_component - - bin: - - executable - - ohos_executable - - lite_component - - -ipcamera_hispark_taurus_linux: - product_infofile: ipcamera_hispark_taurus_linux_product.json - output_name: ipcamera_hispark_taurus_linux_result.json - product_dir: - root: out/hispark_taurus/ipcamera_hispark_taurus_linux/rootfs - relative: - bin: bin - so: usr/lib - etc: etc - rest: True - query_order: - so: - - shared_library - - ohos_shared_library - - ohos_prebuilt_shared_library - - lite_library - - lite_component - - bin: - - executable - - ohos_executable - - lite_component - - -wifiiot_hispark_pegasus: - product_infofile: wifiiot_hispark_pegasus_product.json - output_name: wifiiot_hispark_pegasus_result.json - product_dir: - root: out/hispark_pegasus/wifiiot_hispark_pegasus - relative: - a: libs - etc: etc - rest: False - query_order: - a: - - static_library - - ohos_static_library - - lite_library - - -rk3568: # rk的目前从packages/phone/system_module_info.json中分析准确度更高,因为rk基本都使用的是ohos_xxx,而L0和L1的更多的是使用的gn原生target template - product_infofile: rk3568_product.json - output_name: rk3568_result.json - product_dir: - root: out/rk3568/packages/phone/system - relative: - so: lib - bin: bin - hap: app - etc: etc - rest: True - query_order: - so: - - ohos_shared_library - - shared_library - - ohos_prebuilt_shared_library - - lite_library - - lite_component - - bin: - - ohos_executable - - executable - - lite_component - - hap: - - ohos_hap - -# extension and prefix of products -default_extension: - shared_library: .so - static_library: .a - app: .hap - -default_prefix: - shared_library: lib - static_library: lib -# black list for command 'grep' -black_list: - - .repo - - .ccache - - doc - - test - - build - - out diff --git a/tools/rom_ram_analyzer/L0L1/src/get_subsystem_component.py b/tools/rom_ram_analyzer/L0L1/src/get_subsystem_component.py deleted file mode 100644 index 3ed3847..0000000 --- a/tools/rom_ram_analyzer/L0L1/src/get_subsystem_component.py +++ /dev/null @@ -1,131 +0,0 @@ -#! /usr/bin/python - -import argparse -import os -import json -import logging - -g_subsystem_path_error = list() # subsystem path exist in subsystem_config.json -# bundle.json path which cant get component path. -g_component_path_empty = list() -g_component_abs_path = list() # destPath can't be absolute path. - - -def get_subsystem_components(ohos_path: str): - subsystem_json_path = os.path.join( - ohos_path, r"build/subsystem_config.json") - subsystem_item = {} - - with open(subsystem_json_path, 'rb') as f: - subsystem_json = json.load(f) - # get sunsystems - for i in subsystem_json: - subsystem_name = subsystem_json[i]["name"] - subsystem_path = os.path.join(ohos_path, subsystem_json[i]["path"]) - if not os.path.exists(subsystem_path): - g_subsystem_path_error.append(subsystem_path) - continue - cmd = 'find ' + subsystem_path + ' -name bundle.json' - bundle_json_list = os.popen(cmd).readlines() - # get components - component_list = [] - for j in bundle_json_list: - bundle_path = j.strip() - with open(bundle_path, 'rb') as bundle_file: - bundle_json = json.load(bundle_file) - component_item = {} - if 'segment' in bundle_json and 'destPath' in bundle_json["segment"]: - destpath = bundle_json["segment"]["destPath"] - component_item[bundle_json["component"]["name"]] = destpath - if os.path.isabs(destpath): - g_component_abs_path.append(destpath) - else: - component_item[bundle_json["component"]["name"] - ] = "Unknow. Please check " + bundle_path - g_component_path_empty.append(bundle_path) - component_list.append(component_item) - subsystem_item[subsystem_name] = component_list - return subsystem_item - - -def get_subsystem_components_modified(ohos_root) -> dict: - ret = dict() - subsystem_info = get_subsystem_components(ohos_root) - if subsystem_info is None: - return None - for subsystem_k, subsystem_v in subsystem_info.items(): - for component in subsystem_v: - for k, v in component.items(): - ret.update({v: {'subsystem': subsystem_k, 'component': k}}) - return ret - - -def export_to_json(subsystem_item: dict, output_filename: str): - subsystem_item_json = json.dumps( - subsystem_item, indent=4, separators=(', ', ': ')) - with open(output_filename, 'w') as f: - f.write(subsystem_item_json) - logging.info("output path: " + output_filename) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("project", help="project root path.", type=str) - parser.add_argument("-o", "--outpath", - help="specify an output path.", type=str) - args = parser.parse_args() - - ohos_path = os.path.abspath(args.project) - if not is_project(ohos_path): - logging.error("'" + ohos_path + "' is not a valid project path.") - exit(1) - - output_path = r'.' - if args.outpath: - output_path = args.outpath - - return ohos_path, output_path - - -def is_project(path: str) -> bool: - ''' - @func: 判断是否源码工程。 - @note: 通过是否含有 .repo/manifests 目录粗略判断。 - ''' - p = os.path.normpath(path) - return os.path.exists(p + '/.repo/manifests') - - -def print_warning_info(): - '''@func: 打印一些异常信息。 - ''' - if g_component_path_empty or g_component_abs_path: - logging.warning("------------ warning info ------------------") - - if g_component_path_empty: - logging.warning("can't find destPath in:") - logging.warning(g_component_path_empty) - - if g_component_abs_path: - logging.warning("destPath can't be absolute path:") - logging.warning(g_component_abs_path) - - -class SC: - @classmethod - def run(cls, project_path: str, output_path: str = None, save_result: bool = True): - info = get_subsystem_components_modified( - os.path.abspath(os.path.expanduser(project_path))) - if save_result and output_path: - export_to_json(info, output_path) - print_warning_info() - return info - - -__all__ = ["SC"] - -if __name__ == '__main__': - ohos_path, output_path = parse_args() - info = get_subsystem_components_modified(ohos_path) - export_to_json(info, output_path) - print_warning_info() diff --git a/tools/rom_ram_analyzer/L0L1/src/misc.py b/tools/rom_ram_analyzer/L0L1/src/misc.py deleted file mode 100644 index d000507..0000000 --- a/tools/rom_ram_analyzer/L0L1/src/misc.py +++ /dev/null @@ -1,228 +0,0 @@ -import logging -import copy -import os -import logging -from abc import ABC, abstractmethod -from collections import defaultdict -from typing import * -import preprocess -from pkgs.gn_common_tool import GnVariableParser -from pkgs.simple_yaml_tool import SimpleYamlTool -from pkgs.basic_tool import BasicTool - - -_config = SimpleYamlTool.read_yaml("config.yaml") -""" -===============info handlers=============== -""" - - -def extension_handler(paragraph: Text): - return GnVariableParser.string_parser("output_extension", paragraph).strip('"') - - -def hap_name_handler(paragraph: Text): - return GnVariableParser.string_parser("hap_name", paragraph).strip('"') - - -def target_type_handler(paragraph: Text): - tt = GnVariableParser.string_parser("target_type", paragraph).strip('"') - if not tt: - logging.info("parse 'target_type' failed, maybe it's a variable") - return tt - - -""" -===============gn lineno collector=============== -""" - - -def gn_lineno_collect(match_pattern: str, project_path: str) -> DefaultDict[str, List[int]]: - """ - 在整个项目路径下搜索有特定target类型的BUILD.gn - :param match_pattern: 进行grep的pattern，支持扩展的正则 - :param project_path: 项目路径（搜索路径） - :return: {gn_file: [line_no_1, line_no_2, ..]} - """ - black_list = map(lambda x: os.path.join( - project_path, x), _config.get("black_list")) - - def handler(content: Text) -> List[str]: - return list(filter(lambda y: len(y) > 0, list(map(lambda x: x.strip(), content.split("\n"))))) - - grep_list = BasicTool.grep_ern(match_pattern, path=project_path, include="BUILD.gn", exclude=tuple(black_list), - post_handler=handler) - gn_line_dict: DefaultDict[str, List[int]] = defaultdict(list) - for gl in grep_list: - gn_file, line_no, _ = gl.split(":") - gn_line_dict[gn_file].append(line_no) - return gn_line_dict - - -""" -===============target name parser=============== -""" - - -class TargetNameParser: - @classmethod - def single_parser(cls, paragraph: Text) -> str: - """ - 查找类似shared_library("xxx")这种括号内只有一个参数的target的名称 - :param paragraph: 要解析的段落 - :return: target名称，如果是变量，不会对其进行解析 - """ - return BasicTool.re_group_1(paragraph, r"\w+\((.*)\)") - - @classmethod - def second_parser(cls, paragraph: Text) -> str: - """ - 查找类似target("shared_library","xxx")这种的target名称（括号内第二个参数） - :param paragraph: 要解析的段落 - :return: target名称，如果是变量，不会的其进行解析 - """ - return BasicTool.re_group_1(paragraph, r"\w+\(.*?, *(.*?)\)") - - -""" -===============post handlers=============== -""" - - -class BasePostHandler(ABC): - @abstractmethod - def run(self, unit: Dict[str, AnyStr]) -> str: - ... - - def __call__(self, unit: Dict[str, AnyStr]) -> str: - return self.run(unit) - - -class DefaultPostHandler(BasePostHandler): - def run(self, unit: Dict[str, AnyStr]): - return unit["output_name"] - - -class HAPPostHandler(BasePostHandler): - """ - for ohos_hap""" - - def run(self, unit: Dict[str, AnyStr]): - extension = _config.get("default_extension").get("app") - gn_hap_name = unit.get("hap_name") - if gn_hap_name: - return gn_hap_name+extension - return unit["output_name"]+extension - - -class SOPostHandler(BasePostHandler): - """ - for shared_library""" - - def run(self, unit: Dict[str, AnyStr]): - output_name = unit["output_name"] - prefix = _config.get("default_prefix").get("shared_library") - if unit.get("extension"): - extension = unit.get("extension") - else: - extension = _config.get("default_extension").get("shared_library") - if not extension.startswith('.'): - extension = '.'+extension - if output_name.startswith(prefix): - return output_name+extension - return prefix+output_name+extension - - -class APostHandler(BasePostHandler): - """ - for static library""" - - def run(self, unit: Dict[str, AnyStr]): - output_name = unit["output_name"] - prefix = _config.get("default_prefix").get("static_library") - extension: str = _config.get("default_extension").get("static_library") - if not extension.startswith('.'): - extension = '.'+extension - if output_name.startswith(prefix): - return output_name+extension - return prefix+output_name+extension - - -class LiteLibPostHandler(BasePostHandler): - """ - for lite_library""" - - def run(self, unit: Dict[str, AnyStr]): - tp = unit["real_target_type"] - output_name = unit["output_name"] - if tp == "static_library": - prefix = _config.get("default_prefix").get("static_library") - extension = _config.get("default_extension").get("static_library") - elif tp == "shared_library": - prefix = _config.get("default_prefix").get("shared_library") - extension = _config.get("default_extension").get("shared_library") - else: - prefix = str() - extension = str() - if not extension.startswith('.'): - extension = '.'+extension - if output_name.startswith(prefix): - return output_name+extension - return prefix+output_name+extension - - -class LiteComponentPostHandler(BasePostHandler): - """ - for lite_component""" - - def run(self, unit: Dict[str, AnyStr]): - tp = unit["real_target_type"] - output_name = unit["output_name"] - extension = unit.get("output_extension") - if tp == "shared_library": - prefix = _config.get("default_prefix").get("shared_library") - extension = _config.get("default_extension").get("shared_library") - else: - if tp != "executable": - unit["description"] = "virtual node" - prefix = str() - extension = str() - if not extension.startswith('.'): - extension = '.'+extension - return prefix+output_name+extension - - -class TargetPostHandler(BasePostHandler): - """ - for target(a,b){}""" - - def run(self, unit: Dict[str, AnyStr]): - ... - - -def LiteLibS2MPostHandler(unit: Dict, result_dict: Dict) -> None: - rt = unit.get("real_target_type") - new_unit = copy.deepcopy(unit) - if rt == "shared_library": - new_unit["real_target_type"] = "static_library" - k = LiteLibPostHandler()(new_unit) - new_unit["description"] = "may not exist" - result_dict["lite_library"][k] = new_unit - elif rt == "static_library": - new_unit["real_target_type"] = "shared_library" - k = LiteLibPostHandler()(new_unit) - new_unit["description"] = "may not exist" - result_dict["lite_library"][k] = new_unit - else: - logging.warning( - f"target type should be 'shared_library' or 'static_library', but got '{rt}'") - new_unit["real_target_type"] = "shared_library" - k = LiteLibPostHandler()(new_unit) - new_unit["description"] = "may not exist" - result_dict["lite_library"][k] = new_unit - - new_new_unit = copy.deepcopy(unit) - new_new_unit["real_target_type"] = "static_library" - k = LiteLibPostHandler()(new_new_unit) - new_new_unit["description"] = "may not exist" - result_dict["lite_library"][k] = new_new_unit diff --git a/tools/rom_ram_analyzer/L0L1/src/preprocess.py b/tools/rom_ram_analyzer/L0L1/src/preprocess.py deleted file mode 100644 index 19b9ae9..0000000 --- a/tools/rom_ram_analyzer/L0L1/src/preprocess.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys -import os -# 将上级目录加入到库的搜索路径 -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) \ No newline at end of file diff --git a/tools/rom_ram_analyzer/L0L1/src/rom_analysis.py b/tools/rom_ram_analyzer/L0L1/src/rom_analysis.py deleted file mode 100644 index ba1c6e1..0000000 --- a/tools/rom_ram_analyzer/L0L1/src/rom_analysis.py +++ /dev/null @@ -1,295 +0,0 @@ -import sys -import argparse -import json -import logging -import os -from typing import * -import copy -import preprocess -from time import time -from concurrent.futures import ThreadPoolExecutor, Future -from threading import RLock -import collections - -from config import result_dict, collector_config, configs, \ - project_path, sub_com_dict, product_name, recollect_gn -# from gn_info_collect import GnInfoCollector -from pkgs.basic_tool import BasicTool -from pkgs.gn_common_tool import GnCommonTool -from pkgs.simple_excel_writer import SimpleExcelWriter -from misc import gn_lineno_collect - - -""" -1. 先收集BUILD.gn中的target信息 -2. 然后根据编译产物到1中进行搜索,匹配其所属的部件 - -对于ohos开头的template,主要根据其component字段和subsystem_name字段来归数其部件；同时也要考虑install_dir字段 -对于gn原生的template,主要根据bundle.json中的字段来归属其部件 - -对于找不到的,可以模糊匹配,如,有产物libxxx,则可以在所有的BUILD.gn中搜索xxx,并设置一个阀值予以过滤 -""" - - -class RomAnalysisTool: - @classmethod - def collect_gn_info(cls): - with ThreadPoolExecutor(max_workers=len(collector_config) + 1) as pool: - future_list: List[Future] = list() - for c in collector_config: - future_list.append(pool.submit(c)) - for f in future_list: - f.result() - gn_info_file = configs["gn_info_file"] - with open(gn_info_file, 'w', encoding='utf-8') as f: - json.dump(result_dict, f, indent=4) - - @classmethod - def _add_rest_dir(cls, top_dir: str, rela_path: str, sub_path: str, dir_list: List[str]) -> None: - """ - dir_list:相对于原始top目录的所有子目录的全路径 - """ - if not sub_path: - return - # 将其他目录添加到dir_list - all_subdir = os.listdir(os.path.join(top_dir, rela_path)) - for d in all_subdir: - t = os.path.join(rela_path, d) - if os.path.isdir(os.path.join(top_dir, t)) and t not in dir_list: - dir_list.append(t) - # 移除sub_path的当前层级的目录 - t = sub_path.split(os.sep) - if os.path.join(rela_path, t[0]) in dir_list: - dir_list.remove(os.path.join(rela_path, t[0])) - else: - logging.error( - f"'{os.path.join(rela_path,t[0])}' not in '{top_dir}'") - sp = str() - if len(t) == 1: - return - elif len(t) == 2: - sp = t[1] - else: - sp = os.path.join(*t[1:]) - cls._add_rest_dir(top_dir, os.path.join(rela_path, t[0]), sp, dir_list) - - @classmethod - def _find_files(cls, product_name: str) -> Dict[str, List[str]]: - product_dir: Dict[str, Dict] = configs[product_name]["product_dir"] - if not product_name: - logging.error( - f"product_name '{product_name}' not found in the config.yaml") - exit(1) - product_path_dit: Dict[str, str] = dict() # 存储编译产物的类型及目录 - root_dir = product_dir.get("root") - root_dir = os.path.join(project_path, root_dir) - relative_dir: Dict[str, str] = product_dir.get("relative") - if not relative_dir: - logging.warning( - f"'{relative_dir}' of {product_name} not found in the config.yaml") - exit(1) - # 除了so a hap bin外的全部归到etc里面 - for k, v in relative_dir.items(): - product_path_dit[k] = os.path.join(root_dir, v) - # 查找编译产物信息 - # product_dict格式: {"so": ["a.so", "b.so"]} - product_dict: Dict[str, List[str]] = dict() # 存储编译产物的名称 - for k, v in product_path_dit.items(): - if not os.path.exists(v): - logging.warning(f"dir '{v}' not exist") - product_dict[k] = BasicTool.find_files_with_pattern(v) # v是全路径 - if product_dir.get("rest"): - rest_dir_list: List[str] = os.listdir( - root_dir) # 除了配置在relative下之外的所有剩余目录,全部归到etc下 - for v in relative_dir.values(): - cls._add_rest_dir(root_dir, str(), v, rest_dir_list) - if "etc" not in product_dict.keys(): - product_dict["etc"] = list() - for r in rest_dir_list: - product_dict["etc"].extend( - BasicTool.find_files_with_pattern(os.path.join(root_dir, r))) - return product_dict - - @classmethod - def collect_product_info(cls, product_name: str): - product_dict: Dict[str, List[str]] = cls._find_files(product_name) - with open(configs[product_name]["product_infofile"], 'w', encoding='utf-8') as f: - json.dump(product_dict, f, indent=4) - return product_dict - - @classmethod - def _put(cls, sub: str, com: str, unit: Dict, rom_size_dict: Dict): - size = unit.get("size") - if not rom_size_dict.get("size"): # 总大小 - rom_size_dict["size"] = 0 - if not rom_size_dict.get(sub): # 子系统大小 - rom_size_dict[sub]: Dict[str, Dict] = dict() - rom_size_dict[sub]["size"] = 0 - rom_size_dict[sub]["count"] = 0 - - if not rom_size_dict.get(sub).get(com): # 部件 - rom_size_dict.get(sub)[com] = dict() - rom_size_dict[sub][com]["filelist"] = list() - rom_size_dict[sub][com]["size"] = 0 - rom_size_dict[sub][com]["count"] = 0 - - rom_size_dict[sub][com]["filelist"].append(unit) - rom_size_dict[sub][com]["size"] += size - rom_size_dict[sub][com]["count"] += 1 - rom_size_dict[sub]["size"] += size - rom_size_dict[sub]["count"] += 1 - rom_size_dict["size"] += size - - @classmethod - def _fuzzy_match(cls, file_name: str, extra_black_list: Tuple[str] = ("test",)) -> Tuple[str, str, str]: - """ - 直接grep,利用出现次数最多的BUILD.gn去定位subsystem_name和component_name""" - _, base_name = os.path.split(file_name) - if base_name.startswith("lib"): - base_name = base_name[3:] - if base_name.endswith(".a"): - base_name = base_name[:base_name.index(".a")] - elif base_name.endswith(".z.so"): - base_name = base_name[:base_name.index(".z.so")] - elif base_name.endswith(".so"): - base_name = base_name[:base_name.index(".so")] - exclude_dir = [os.path.join(project_path, x) - for x in configs["black_list"]] - exclude_dir.extend(list(extra_black_list)) - grep_result: List[str] = BasicTool.grep_ern( - base_name, - project_path, - include="BUILD.gn", - exclude=tuple(exclude_dir), - post_handler=lambda x: list(filter(lambda x: len(x) > 0, x.split('\n')))) - if not grep_result: - return str(), str(), str() - gn_dict: Dict[str, int] = collections.defaultdict(int) - for g in grep_result: - gn = g.split(':')[0].replace(project_path, "").lstrip(os.sep) - gn_dict[gn] += 1 - gn_file, _ = collections.Counter(gn_dict).most_common(1)[0] - for k, v in sub_com_dict.items(): - if gn_file.startswith(k): - return gn_file, v.get("subsystem"), v.get("component") - return str(), str(), str() - - @classmethod - def _save_as_xls(cls, result_dict: Dict, product_name: str) -> None: - header = ["subsystem_name", "component_name", - "output_file", "size(Byte)"] - tmp_dict = copy.deepcopy(result_dict) - excel_writer = SimpleExcelWriter("rom") - excel_writer.set_sheet_header(headers=header) - subsystem_start_row = 1 - subsystem_end_row = 0 - subsystem_col = 0 - component_start_row = 1 - component_end_row = 0 - component_col = 1 - del tmp_dict["size"] - for subsystem_name in tmp_dict.keys(): - subsystem_dict = tmp_dict.get(subsystem_name) - subsystem_size = subsystem_dict.get("size") - subsystem_file_count = subsystem_dict.get("count") - del subsystem_dict["count"] - del subsystem_dict["size"] - subsystem_end_row += subsystem_file_count - - for component_name in subsystem_dict.keys(): - component_dict: Dict[str, int] = subsystem_dict.get( - component_name) - component_size = component_dict.get("size") - component_file_count = component_dict.get("count") - del component_dict["count"] - del component_dict["size"] - component_end_row += component_file_count - - for fileinfo in component_dict.get("filelist"): - file_name = fileinfo.get("file_name") - file_size = fileinfo.get("size") - excel_writer.append_line( - [subsystem_name, component_name, file_name, file_size]) - excel_writer.write_merge(component_start_row, component_col, component_end_row, component_col, - component_name) - component_start_row = component_end_row + 1 - excel_writer.write_merge(subsystem_start_row, subsystem_col, subsystem_end_row, subsystem_col, - subsystem_name) - subsystem_start_row = subsystem_end_row + 1 - output_name: str = configs[product_name]["output_name"] - output_name = output_name.replace(".json", ".xls") - excel_writer.save(output_name) - - @ classmethod - def analysis(cls, product_name: str, product_dict: Dict[str, List[str]]): - gn_info_file = configs["gn_info_file"] - with open(gn_info_file, 'r', encoding='utf-8') as f: - gn_info = json.load(f) - query_order: Dict[str, List[str] - ] = configs[product_name]["query_order"] - query_order["etc"] = configs["target_type"] - rom_size_dict: Dict = dict() - # prodcut_dict: {"a":["a.txt", ...]} - for t, l in product_dict.items(): - for f in l: # 遍历所有文件 - # query_order: {"a":[static_library", ...]} - find_flag = False - type_list = query_order.get(t) - _, base_name = os.path.split(f) - size = os.path.getsize(f) - if not type_list: - logging.warning( - f"'{t}' not found in query_order of the config.yaml") - continue - for tn in type_list: # tn example: ohos_shared_library - output_dict: Dict[str, Dict] = gn_info.get(tn) - if not output_dict: - logging.warning( - f"'{tn}' not found in the {gn_info_file}") - continue - d = output_dict.get(base_name) - if not d: - continue - d["size"] = size - d["file_name"] = f.replace(project_path, "") - cls._put(d["subsystem_name"], - d["component_name"], d, rom_size_dict) - find_flag = True - if not find_flag: - # fuzzy match - psesudo_gn, sub, com = cls._fuzzy_match(f) - if sub and com: - cls._put(sub, com, { - "subsystem_name": sub, - "component_name": com, - "psesudo_gn_path": psesudo_gn, - "description": "fuzzy match", - "file_name": f.replace(project_path, ""), - "size": size, - }, rom_size_dict) - find_flag = True - if not find_flag: - cls._put("others", "others", { - "file_name": f.replace(project_path, ""), - "size": size, - }, rom_size_dict) - with open(configs[product_name]["output_name"], 'w', encoding='utf-8') as f: - json.dump(rom_size_dict, f, indent=4) - cls._save_as_xls(rom_size_dict, product_name) - - -def main(): - if recollect_gn: - RomAnalysisTool.collect_gn_info() - product_dict: Dict[str, List[str] - ] = RomAnalysisTool.collect_product_info(product_name) - RomAnalysisTool.analysis(product_name, product_dict) - - -if __name__ == "__main__": - main() - # t = os.listdir( - # "/home/aodongbiao/developtools_integration_verification/tools") - # RomAnalysisTool._add_rest_dir( - # "/home/aodongbiao/developtools_integration_verification/tools", "", "rom_ram_analyzer/L2/pkgs", t) - # print(t) diff --git a/tools/rom_ram_analyzer/L0L1/src/template_processor.py b/tools/rom_ram_analyzer/L0L1/src/template_processor.py deleted file mode 100644 index e4124c8..0000000 --- a/tools/rom_ram_analyzer/L0L1/src/template_processor.py +++ /dev/null @@ -1,244 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2022 Huawei Device Co., Ltd. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from typing import * -from abc import ABC, abstractmethod -import os -import logging - -from pkgs.basic_tool import do_nothing, BasicTool -from pkgs.gn_common_tool import GnCommonTool, GnVariableParser -from misc import * - -TYPE = Literal["str", "list"] - - -class BaseProcessor(ABC): - """ - extend and usage: - DerivedClass(BaseProcessor): - def call(): - # your implementation - - DerivedClass(project_path,result_dict,...) - DerivedClass() - """ - - def __init__(self, - project_path: str, - result_dict: Dict[str, Dict[str, Dict]], - target_type: str, - match_pattern: str, - sub_com_dict: Dict[str, Dict[str, str]], - target_name_parser: Callable[[Text], Text] = do_nothing, - other_info_handlers: Dict[str, Callable[[ - Text], Union[str, list]]] = dict(), - unit_post_handler: BasePostHandler = do_nothing, - resource_field: str = None, - ud_post_handler: Callable[[Dict, Dict], None] = None - ): - """ - :param project_path: 项目根路径 - :param result_dict: 存储结果的字典 - :param target_type: target类型，eg："shared_library" - :param match_pattern: 用于进行的模式，eg：r"^( *)shared_library\(.*?\)" - :param sub_com_dict: 从get_subsystem_component.py运行结果加载进来的dict，包含oh整体上的子系统、部件及其路径信息 - :param target_name_parser: 解析target名字的Callable - :param other_info_handlers: 对其他信息进行收集处理，eg：{"sources": SourcesParser}——表示要处理target段落中的sources属性， - SourceParser是对target段落进行分析处理的Callable，接受一个字符串作为参数 - :param unit_post_handler: 对最终要存储的结果字典进行后处理，应当返回一个字符串作为存储时的key，且该key应为预期产物去除前后缀后的名字 - :resource_field: 针对资源类target,资源字段,如files = ["a.txt","b.txt"],则field为files - :ud_post_handler: 参数为unit和result_dict的handler - """ - if target_type not in result_dict.keys(): - result_dict[target_type] = dict() - self.project_path = project_path - self.result_dict = result_dict - self.target_type = target_type - self.match_pattern = match_pattern - self.gn_file_line_no_dict = gn_lineno_collect( - self.match_pattern, self.project_path) - self.sc_dict = sub_com_dict - self.target_name_parser = target_name_parser - self.other_info_handlers = other_info_handlers - self.unit_post_handler = unit_post_handler - self.resource_field = resource_field - self.ud_post_handler = ud_post_handler - - def _append(self, key: str, unit: Dict) -> None: - """ - 将target的结果存储到最终的结果字典中 - :param key：进行存储的key，应为预期的文件名 - :param unit: 要存储的target - :return: None - """ - self.result_dict.get(self.target_type)[key] = unit - - def _find_sc(self, gn_path: str): - # gn_path与project_path都应当是绝对路径 - if not gn_path.startswith(self.project_path): - logging.error("gn_path and project_path is not consistent: gn_path={}, project_path={}".format( - gn_path, self.project_path)) - return "", "" - k = gn_path.replace(self.project_path, "").lstrip(os.sep) - for k, v in self.sc_dict.items(): - if k.startswith(k): - return v.get("subsystem"), v.get("component") - return "", "" - - @abstractmethod - def run(self): - ... - - def __call__(self, *args, **kwargs): - self.run() - - -def _gn_var_process(project_path: str, gn_v: str, alt_v: str, gn_path: str, ifrom: str, efrom: str, strip_quote: bool = False) -> Tuple[str, str]: - if strip_quote: - gn_v = gn_v.strip('"') - if gn_v: - if GnCommonTool.contains_gn_variable(gn_v): - gn_v = GnCommonTool.replace_gn_variables( - gn_v, gn_path, project_path).strip('"') - else: - gn_v = gn_v.strip('"') - gn_f = ifrom - else: - gn_v = alt_v - gn_f = efrom - return gn_v, gn_f - - -class DefaultProcessor(BaseProcessor): - - def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: - output_name = GnVariableParser.string_parser("output_name", paragraph) - output_name, out_from = _gn_var_process(self.project_path, - output_name, target_name, gn_path, "target_name", "target_name", True) - sub = GnVariableParser.string_parser("subsystem_name", paragraph) - com = GnVariableParser.string_parser("part_name", paragraph) - sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json", True) - com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json", True) - result = { - "gn_path": gn_path, - "target_type": self.target_type, - "line_no": line_no, - "subsystem_name": sub, - "component_name": com, - "subsystem_from": sub_from, - "component_from": com_from, - "target_name": target_name, - "output_name": output_name, - "output_from": out_from, - } - for k, h in self.other_info_handlers.items(): - result[k] = h(paragraph) - key = self.unit_post_handler(result) - self._append(key, result) - if self.ud_post_handler: - self.ud_post_handler(result, self.result_dict) - - def run(self): - for gn_path, line_no_list in self.gn_file_line_no_dict.items(): - # 该路径下的主要的subsystem_name与component_name，如果target中没有指定，则取此值，如果指定了，则以target中的为准 - _sub, _com = self._find_sc(gn_path) - with open(gn_path, 'r', encoding='utf-8') as f: - content = f.read() - itr = BasicTool.match_paragraph( - content, start_pattern=self.target_type) - for line_no, p in zip(line_no_list, itr): - paragraph = p.group() - target_name = self.target_name_parser(paragraph).strip('"') - if not target_name: - continue - if GnCommonTool.contains_gn_variable(target_name, quote_processed=True): - possible_name_list = GnCommonTool.find_values_of_variable(target_name, path=gn_path, - stop_tail=self.project_path) - for n in possible_name_list: - self.helper(n, paragraph, gn_path, - line_no, _sub, _com) - else: - self.helper(target_name, paragraph, - gn_path, line_no, _sub, _com) - - -class StrResourceProcessor(DefaultProcessor): - def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: - resources = GnVariableParser.string_parser( - self.resource_field, paragraph) - if not resources: - return - _, resources = os.path.split(resources.strip('"')) - - if GnCommonTool.contains_gn_variable(resources): - resources = GnCommonTool.replace_gn_variables( - resources, gn_path, self.project_path).strip('"') - sub = GnVariableParser.string_parser("subsystem_name", paragraph) - com = GnVariableParser.string_parser("part_name", paragraph) - sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json") - com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json") - _, file_name = os.path.split(resources) - result = { - "gn_path": gn_path, - "target_type": self.target_type, - "line_no": line_no, - "subsystem_name": sub, - "component_name": com, - "subsystem_from": sub_from, - "component_from": com_from, - "target_name": target_name, - "output_name": file_name, - "output_from": "file_name", - } - for k, h in self.other_info_handlers.items(): - result[k] = h(paragraph) - key = self.unit_post_handler(result) - self._append(key, result) - - -class ListResourceProcessor(DefaultProcessor): - - def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: - resources = GnVariableParser.list_parser( - self.resource_field, paragraph) - if not resources: - return - sub = GnVariableParser.string_parser("subsystem_name", paragraph) - com = GnVariableParser.string_parser("part_name", paragraph) - sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json") - com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json") - for ff in resources: - _, file_name = os.path.split(ff) - result = { - "gn_path": gn_path, - "target_type": self.target_type, - "line_no": line_no, - "subsystem_name": sub, - "component_name": com, - "subsystem_from": sub_from, - "component_from": com_from, - "target_name": target_name, - "output_name": file_name, - "output_from": "file_name", - } - for k, h in self.other_info_handlers.items(): - result[k] = h(paragraph) - key = self.unit_post_handler(result) - self._append(key, result) - - -if __name__ == '__main__': - ... diff --git a/tools/rom_ram_analyzer/L2/README.md b/tools/rom_ram_analyzer/L2/README.md deleted file mode 100644 index 52bc5a9..0000000 --- a/tools/rom_ram_analyzer/L2/README.md +++ /dev/null @@ -1,143 +0,0 @@ -[toc] - -# Rom_analyzer.py - -## 功能介绍 - -基于BUILD.gn、bundle.json、编译产物system_module_info.json、out/{product_name}/packages/phone目录下的编译产物，分析各子系统及部件的rom大小。 - -结果以json与xls格式进行存储，其中，json格式是必输出的，xls格式需要-e参数控制。 - -## 使用说明 - -前置条件： - -1. 获取整个rom_ram_analyzer目录 -1. 对系统进行编译 -1. rom分析在linux平台,ram分析在windows平台 -1. python3.8及以后 -1. 安装requirements - ```txt - xlwt==1.3.0 - ``` - -命令介绍： - -1. `-h`或`--help`命令查看帮助 - ```shell - > python3 rom_analyzer.py -h - usage: rom_analyzer.py [-h] [-v] -p PROJECT_PATH -j MODULE_INFO_JSON -n PRODUCT_NAME -d PRODUCT_DIR [-o OUTPUT_FILE] [-e EXCEL] - - analyze rom size of component. - - optional arguments: - -h, --help show this help message and exit - -v, -version show program\'s version number and exit - -p PROJECT_PATH, --project_path PROJECT_PATH - root path of oh. eg: -p ~/oh - -j MODULE_INFO_JSON, --module_info_json MODULE_INFO_JSON - path of out/{product_name}/packages/phone/system_module_info.json - -n PRODUCT_NAME, --product_name PRODUCT_NAME - product name. eg: -n rk3568 - -d PRODUCT_DIR, --product_dir PRODUCT_DIR - subdirectories of out/{product_name}/packages/phone to be counted.eg: -d system -d vendor - -o OUTPUT_FILE, --output_file OUTPUT_FILE - basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result - -e EXCEL, --excel EXCEL - if output result as excel, default: False. eg: -e True - ``` -1. 使用示例 - ```shell - python3 rom_analyzer.py -p ~/nomodify_oh/ -j ../system_module_info.json -n rk3568 -d system -d vendor -d updater -o demo/demo -e True - # oh：rootpath of oh - # rk3568: product_name, same as out/{product_name} - # demo/demo: path of output file, where the second 'demo' is the basename of output file - # -e True：output result in excel format additionally - ``` - -## 输出格式说明(json) - - -```json -{ - 子系统名: { - "size": 整个子系统输出文件的总大小, - "file_count": 整个子系统产生的文件数, - 输出文件名: 本文件的大小, - ... - }, - ... -} -``` - -# ram_analyzer.py - -## 功能介绍 - -基于out/{product_name}/packages/phone下所有cfg文件、out/{product_name}/packages/phone/system/profile下所有xml文件，分析各进程及对应部件的ram占用（默认取Pss） - -结果以json与xls格式存储，其中，json格式是必输出的，xls格式需要-e参数控制。 - -## 使用说明 - -前置条件： - -1. 获取整个rom_ram_analyzer目录 -2. hdc可用 -2. 设备已连接 -3. 系统已烧录 -3. python3.8及以后 -4. 安装requirements - ```txt - xlwt==1.3.0 - ``` -5. 准备好相关数据： - 1. out/{product_name}/packages/phone下所有cfg文件，并将其放置于同一个目录中（ps：同名文件仅保存一份即可） - 1. out/{product_name}/packages/phone/system/profile下所有xml文件 -6. 运行rom_analyzer.py产生的json结果一份（即-o参数对应的文件，默认rom_analysis_result.json） - -命令介绍： - -1. 使用`-h`或`--help`查看帮助 - ```shell - > python .\ram_analyzer.py -h - usage: ram_analyzer.py [-h] [-v] -x XML_PATH -c CFG_PATH [-j ROM_RESULT] -n DEVICE_NUM [-o OUTPUT_FILENAME] [-e EXCEL] - - analyze ram size of component - - optional arguments: - -h, --help show this help message and exit - -v, -version show program\'s version number and exit - -x XML_PATH, --xml_path XML_PATH - path of xml file. eg: -x ~/oh/out/rk3568/packages/phone/system/profile - -c CFG_PATH, --cfg_path CFG_PATH - path of cfg files. eg: -c ./cfgs/ - -j ROM_RESULT, --rom_result ROM_RESULT - json file produced by rom_analyzer_v1.0.py, default: ./rom_analysis_result.json.eg: -j ./demo/rom_analysis_result.json - -n DEVICE_NUM, --device_num DEVICE_NUM - device number to be collect hidumper info. eg: -n 7001005458323933328a01fce16d3800 - -o OUTPUT_FILENAME, --output_filename OUTPUT_FILENAME - base name of output file, default: ram_analysis_result. eg: -o ram_analysis_result - -e EXCEL, --excel EXCEL - if output result as excel, default: False. eg: -e True - ``` -2. 使用示例： - ```shell - python .\ram_analyzer.py -x .\profile\ -c .\init\ -n 7001005458323933328a01fce16d3800 -j .\rom_analysis_result.json -o /demo/demo -e True - # demo/demo: path of output file, where the second 'demo' is the basename of output file - # -e True：output result in excel format additionally - ``` -## 输出格式说明（json） -```json -{ - 进程名:{ - "size": 本进程占用内存的大小, - 部件名: { - elf文件名: elf文件大小 - ... - } - ... - }, - ... -} -``` \ No newline at end of file diff --git a/tools/rom_ram_analyzer/L2/__init__.py b/tools/rom_ram_analyzer/L2/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tools/rom_ram_analyzer/L2/pkgs/__init__.py b/tools/rom_ram_analyzer/L2/pkgs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tools/rom_ram_analyzer/L2/pkgs/basic_tool.py b/tools/rom_ram_analyzer/L2/pkgs/basic_tool.py deleted file mode 100644 index b4d548e..0000000 --- a/tools/rom_ram_analyzer/L2/pkgs/basic_tool.py +++ /dev/null @@ -1,33 +0,0 @@ -import sys -import typing -import os -import glob -from pathlib import Path -from typing import * - - -class BasicTool: - @classmethod - def find_all_files(cls, folder: str, real_path: bool = True, apply_abs: bool = True, de_duplicate: bool = True, - p_filter: typing.Callable = lambda x: True) -> list: - filepath_list = set() - for root, _, file_names in os.walk(folder): - filepath_list.update( - [os.path.abspath(os.path.realpath( - os.path.join(root, f) if real_path else os.path.join(root, f))) if apply_abs else os.path.relpath( - os.path.realpath(os.path.join(root, f) if real_path else os.path.join(root, f))) for f in file_names - if p_filter(os.path.join(root, f))]) - if de_duplicate: - filepath_list = set(filepath_list) - filepath_list = sorted(filepath_list, key=str.lower) - return filepath_list - - @classmethod - def get_abs_path(cls, path: str) -> str: - return os.path.abspath(os.path.expanduser(path)) - - -if __name__ == '__main__': - # print(BasicTool.get_abs_path("~/git/..")) - for i in BasicTool.find_all_files(".", apply_abs=False): - print(i) \ No newline at end of file diff --git a/tools/rom_ram_analyzer/L2/pkgs/gn_common_tool.py b/tools/rom_ram_analyzer/L2/pkgs/gn_common_tool.py deleted file mode 100644 index 11c2718..0000000 --- a/tools/rom_ram_analyzer/L2/pkgs/gn_common_tool.py +++ /dev/null @@ -1,157 +0,0 @@ -import os -import json - - -class GnCommonTool: - """ - 处理BUILD.gn文件的通用方法 - """ - - @classmethod - def is_gn_variable(cls, target: str, has_quote: bool = True): - """ - 判断target是否是gn中的变量: - 规则：如果是有引号的模式，则没有引号均认为是变量，有引号的情况下，如有是"$xxx"的模式，则认为xxx是变量；如果是无引号模式，则只要$开头就认为是变量 - b = "xxx" - c = b - c = "${b}" - "$p" - """ - target = target.strip() - if not has_quote: - return target.startswith("$") - if target.startswith('"') and target.endswith('"'): - target = target.strip('"') - if target.startswith("${") and target.endswith("}"): - return True - elif target.startswith("$"): - return True - return False - else: - return True - - # 给__find_variables_in_gn用的，减少io - __var_val_mem_dict = dict() - - @classmethod - def find_variables_in_gn(cls, var_name_tuple: tuple, path: str, stop_tail: str = "home") -> tuple: - """ - 同时查找多个gn变量的值 - var_name_tuple：变量名的tuple，变量名应是未经过处理后的，如： - xxx - "${xxx}" - "$xxx" - """ - - if os.path.isfile(path): - path = os.path.split(path)[0] - var_val_dict = dict() - not_found_count = len(var_name_tuple) - for var in var_name_tuple: - val = GnCommonTool.__var_val_mem_dict.get(var) - if val is not None: - not_found_count -= 1 - var_val_dict[var] = val - while not path.endswith(stop_tail) and not_found_count != 0: - for v in var_name_tuple: - cmd = r"grep -Ern '^( *){} *= *\".*?\"' --include=*.gn* {}| grep -Ev '\$' | head -n 1 | grep -E '\".*\"' -wo".format( - v.strip('"').lstrip("${").rstrip('}'), path) - output = os.popen(cmd).read().strip().strip('"') - if len(output) != 0: - not_found_count -= 1 - var_val_dict[v] = output - GnCommonTool.__var_val_mem_dict[v] = output - path = os.path.split(path)[0] - return tuple(var_val_dict.values()) - - @classmethod - def __find_part_subsystem_from_bundle(cls, gnpath: str, stop_tail: str = "home") -> tuple: - """ - 根据BUILD.gn的全路径，一层层往上面查找bundle.json文件， - 并从bundle.json中查找part_name和subsystem - """ - filename = "bundle.json" - part_name = None - subsystem_name = None - if stop_tail not in gnpath: - return part_name, subsystem_name - if os.path.isfile(gnpath): - gnpath = os.path.split(gnpath)[0] - while not gnpath.endswith(stop_tail): - bundle_path = os.path.join(gnpath, filename) - if not os.path.isfile(bundle_path): # 如果该文件不在该目录下 - gnpath = os.path.split(gnpath)[0] - continue - with open(bundle_path, 'r', encoding='utf-8') as f: - content = json.load(f) - try: - part_name = content["component"]["name"] - subsystem_name = content["component"]["subsystem"] - except KeyError: - ... - finally: - break - part_name = None if (part_name is not None and len( - part_name) == 0) else part_name - subsystem_name = None if (subsystem_name is not None and len( - subsystem_name) == 0) else subsystem_name - return part_name, subsystem_name - - @classmethod - def find_part_subsystem(cls, gn_file: str, project_path: str) -> tuple: - """ - 查找gn_file对应的part_name和subsystem - 如果在gn中找不到，就到bundle.json中去找 - """ - part_name = None - subsystem_name = None - part_var_flag = False # 标识这个变量从gn中取出的原始值是不是变量 - subsystem_var_flag = False - var_list = list() - part_name_pattern = r"part_name *=\s*\S*" - subsystem_pattern = r"subsystem_name *=\s*\S*" - meta_grep_pattern = "grep -E '{}' {} | head -n 1" - part_cmd = meta_grep_pattern.format(part_name_pattern, gn_file) - subsystem_cmd = meta_grep_pattern.format(subsystem_pattern, gn_file) - part = os.popen(part_cmd).read().strip() - if len(part) != 0: - part = part.split('=')[-1].strip() - if GnCommonTool.is_gn_variable(part): - part_var_flag = True - var_list.append(part) - else: - part_name = part.strip('"') - if len(part_name) == 0: - part_name = None - subsystem = os.popen(subsystem_cmd).read().strip() - if len(subsystem) != 0: # 这里是只是看有没有grep到关键字 - subsystem = subsystem.split('=')[-1].strip() - if GnCommonTool.is_gn_variable(subsystem): - subsystem_var_flag = True - var_list.append(subsystem) - else: - subsystem_name = subsystem.strip('"') - if len(subsystem_name) == 0: - subsystem_name = None - if part_var_flag and subsystem_var_flag: - part_name, subsystem_name = GnCommonTool.find_variables_in_gn( - tuple(var_list), gn_file, project_path) - elif part_var_flag: - t = GnCommonTool.find_variables_in_gn( - tuple(var_list), gn_file, project_path)[0] - part_name = t if t is not None and len(t) != 0 else part_name - elif subsystem_var_flag: - t = GnCommonTool.find_variables_in_gn( - tuple(var_list), gn_file, project_path)[0] - subsystem_name = t if t is not None and len( - t) != 0 else subsystem_name - if part_name is not None and subsystem_name is not None: - return part_name, subsystem_name - # 如果有一个没有找到，就要一层层去找bundle.json文件 - t_part_name, t_subsystem_name = cls.__find_part_subsystem_from_bundle( - gn_file, stop_tail=project_path) - if t_part_name is not None: - part_name = t_part_name - if t_subsystem_name is not None: - subsystem_name = t_subsystem_name - return part_name, subsystem_name diff --git a/tools/rom_ram_analyzer/L2/pkgs/simple_excel_writer.py b/tools/rom_ram_analyzer/L2/pkgs/simple_excel_writer.py deleted file mode 100644 index a1f2aa6..0000000 --- a/tools/rom_ram_analyzer/L2/pkgs/simple_excel_writer.py +++ /dev/null @@ -1,118 +0,0 @@ -import xlwt -from xlwt import Worksheet -import typing -from typing import Optional -from collections.abc import Iterable - - -class SimpleExcelWriter: - def __init__(self, default_sheet_name: str = "sheet1"): - self.__book = xlwt.Workbook(encoding='utf-8', style_compression=0) - self.__sheet_dict = { - default_sheet_name: self.__book.add_sheet( - sheetname=default_sheet_name, cell_overwrite_ok=True) - } - self.__sheet_pos = { - default_sheet_name: (0, 0) # 记录各个sheet页已经写到什么位置了，当前值为还没有写的 - } - self.__default_sheet_name = default_sheet_name - # 表头样式 - self.__head_style = xlwt.XFStyle() - # 内容样式 - self.__content_style = xlwt.XFStyle() - # 字体 - font = xlwt.Font() - font.bold = True - - # 居中对齐 - alignment = xlwt.Alignment() - alignment.horz = xlwt.Alignment.HORZ_CENTER # 水平方向 - alignment.vert = xlwt.Alignment.VERT_CENTER # 垂直方向 - - # 设置背景颜色 - pattern = xlwt.Pattern() - pattern.pattern = xlwt.Pattern.SOLID_PATTERN - pattern.pattern_fore_colour = 22 # 背景颜色 - - self.__head_style.font = font - self.__head_style.alignment = alignment - self.__head_style.pattern = pattern - self.__content_style.alignment = alignment - - def __increment_y(self, sheet_name: str, value: int = 1) -> int: - if sheet_name in self.__sheet_pos.keys(): - x, y = self.__sheet_pos.get(sheet_name) - y = y + value - self.__sheet_pos[sheet_name] = (x, y) - return y - - def __increment_x(self, sheet_name: str, value: int = 1) -> int: - if sheet_name in self.__sheet_pos.keys(): - x, y = self.__sheet_pos.get(sheet_name) - x = x + value - self.__sheet_pos[sheet_name] = (x, 0) - return x - - def append_line(self, content: list, sheet_name: str = None): - sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name - if sheet_name not in self.__sheet_dict.keys(): - print("error: sheet name '{}' not exist".format(sheet_name)) - return - sheet: Worksheet = self.__sheet_dict.get(sheet_name) - x, y = self.__sheet_pos.get(sheet_name) - for ele in content: - sheet.write(x, y, ele, style=self.__content_style) - y = self.__increment_y(sheet_name) - self.__increment_x(sheet_name) - - def write_merge(self, x0: int, y0: int, x1: int, y1: int, content: typing.Any, - sheet_name: str = None): - sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name - if sheet_name not in self.__sheet_dict.keys(): - print("error: sheet name '{}' not exist".format(sheet_name)) - return - sheet: Worksheet = self.__sheet_dict.get(sheet_name) - sheet.write_merge(x0, x1, y0, y1, content, style=self.__content_style) - - def set_sheet_header(self, headers: Iterable, sheet_name: str = None): - """ - 给sheet页设置表头 - """ - sheet_name = self.__default_sheet_name if sheet_name is None else sheet_name - if sheet_name not in self.__sheet_dict.keys(): - print("error: sheet name '{}' not exist".format(sheet_name)) - return - x, y = self.__sheet_pos.get(sheet_name) - if x != 0 or y != 0: - print( - "error: pos of sheet '{}' is not (0,0). set_sheet_header must before write".format(sheet_name)) - return - sheet: Worksheet = self.__sheet_dict.get(sheet_name) - for h in headers: - sheet.write(x, y, h, self.__head_style) - y = self.__increment_y(sheet_name) - self.__increment_x(sheet_name) - - def add_sheet(self, sheet_name: str, cell_overwrite_ok=True) -> Optional[xlwt.Worksheet]: - if sheet_name in self.__sheet_dict.keys(): - print("error: sheet name '{}' has exist".format(sheet_name)) - return - self.__sheet_dict[sheet_name] = self.__book.add_sheet( - sheetname=sheet_name, cell_overwrite_ok=cell_overwrite_ok) - self.__sheet_pos[sheet_name] = (0, 0) - return self.__sheet_dict.get(sheet_name) - - def save(self, file_name: str): - self.__book.save(file_name) - - -if __name__ == '__main__': - writer = SimpleExcelWriter(default_sheet_name="first") - writer.add_sheet("second") - writer.add_sheet("third") - writer.set_sheet_header(["h", "m", "n"]) - writer.append_line([1, 2, 3]) - writer.append_line([2, 3, 4], "second") - writer.append_line([3, 4, 5], "third") - writer.append_line([3, 2, 1]) - writer.save("demo.xls") diff --git a/tools/rom_ram_analyzer/L2/ram_analyzer.py b/tools/rom_ram_analyzer/L2/ram_analyzer.py deleted file mode 100644 index c3863af..0000000 --- a/tools/rom_ram_analyzer/L2/ram_analyzer.py +++ /dev/null @@ -1,452 +0,0 @@ -import argparse -import copy -import glob -import json -import os -import re -import sys -import subprocess -import typing -import xml.dom.minidom as dom -from pprint import pprint - -from pkgs.simple_excel_writer import SimpleExcelWriter - -debug = True if sys.gettrace() else False - - -class HDCTool: - @classmethod - def verify_hdc(cls, verify_str: str = "OpenHarmony") -> bool: - """ - 验证hdc是否可用 - True：可用 - False：不可用 - """ - cp = subprocess.run(["hdc"], capture_output=True) - stdout = str(cp.stdout) - stderr = str(cp.stderr) - return verify_str in stdout or verify_str in stderr - - @classmethod - def verify_device(cls, device_num: str) -> bool: - """ - 验证设备是否已经连接 - True：已连接 - False：未连接 - """ - cp = subprocess.run(["hdc", "list", "targets"], capture_output=True) - stdout = str(cp.stdout) - stderr = str(cp.stderr) - return device_num in stderr or device_num in stdout - - - @classmethod - def exec(cls, args: list, output_from: str = "stdout"): - cp = subprocess.run(args, capture_output=True) - if output_from == "stdout": - return cp.stdout.decode() - elif output_from == "stderr": - return cp.stderr.decode() - else: - print("error: 'output_from' must be stdout or stdin") - - -def delete_values_from_dict(target_dict: typing.Dict, key_list: typing.Iterable): - for k in key_list: - if k not in target_dict.keys(): - continue - del target_dict[k] - - -class RamAnalyzer: - @classmethod - def __hidumper_mem_line_process(cls, content: typing.Text) -> typing.List[typing.Text]: - """ - 将hidumper的拥有的数据行进行分割，得到 - [pid, name, pss, vss, rss, uss]格式的list - """ - trival_pattern = re.compile(r"kB|\(.*\)(?#去除单位kB以及小括号内的任意数据，包括小括号)") - content = re.sub(trival_pattern, "", content) - blank_pattern = re.compile(r"\s+(?#匹配一个或多个空格)") - return re.sub(blank_pattern, ' ', content.strip()).split() - - __ss_dict: typing.Dict[str, int] = { - "Pss": 2, - "Vss": 3, - "Rss": 4, - "Uss": 5 - } - - @classmethod - def __parse_hidumper_mem(cls, content: typing.Text, device_num: str, ss: str = "Pss") -> typing.Dict[ - typing.Text, int]: - """ - 解析：hidumper --meme的结果 - 返回{process_name: pss}形式的字典 - '248 samgr 1464(0 in SwapPss) kB 15064 kB 6928 kB 1072 kB\r' - """ - - def find_full_process_name(hname: str) -> str: - for lname in __process_name_list: - if lname.startswith(hname): - return lname - - def process_ps_ef(content: str) -> list: - line_list = content.strip().split("\n")[1:] - process_name_list = list() - for line in line_list: - process_name = line.split()[7] - if process_name.startswith('['): - # 内核进程 - continue - process_name_list.append(process_name) - return process_name_list - - if ss not in cls.__ss_dict.keys(): - print("error: {} is not a valid parameter".format(ss)) - return dict() - output = content.split('\n') - process_pss_dict = dict() - __process_name_list: typing.List[str] = process_ps_ef( - HDCTool.exec(["hdc", "-t", device_num, "shell", "ps", "-ef"])) - for line in output: - if "Total Memory Usage by Size" in line: - break - if line.isspace(): - continue - processed: typing.List[typing.Text] = cls.__hidumper_mem_line_process(line) - if not processed or not processed[0].isnumeric(): # 如果第一列不是数字（pid），就过 - continue - name = processed[1] # 否则的话就取名字，和对应的size - size = int(processed[cls.__ss_dict.get(ss)]) - process_pss_dict[find_full_process_name(name)] = size - return process_pss_dict - - @classmethod - def process_hidumper_info(cls, device_num: str, ss:str) -> typing.Dict[str, int]: - """ - 处理进程名与对应进程大小 - """ - - def exec_once() -> typing.Dict[str, int]: - stdout = HDCTool.exec(["hdc", "-t", device_num, "shell", "hidumper", "--mem"]) - name_size_dict = cls.__parse_hidumper_mem(stdout, device_num, ss) - return name_size_dict - - if not HDCTool.verify_hdc(): - print("error: Command 'hdc' not found") - return dict() - if not HDCTool.verify_device(device_num): - print("error: {} is inaccessible or not found".format(device_num)) - return dict() - - return exec_once() - - @classmethod - def __parse_process_xml(cls, file_path: str, result_dict: typing.Dict[str, typing.List[str]]): - """ - 解析xml文件，结存存入 result_dict中，格式：{process_name: os_list} - 其中，so_list中是so的base_name - """ - if not (os.path.isfile(file_path) and file_path.endswith(".xml")): - print("warning: {} not exist or not a xml file".format(file_path)) - return - doc = dom.parse(file_path) - info = doc.getElementsByTagName("info")[0] - process = info.getElementsByTagName("process")[0] - process_name = process.childNodes[0].data - result_dict[process_name] = list() - libs = info.getElementsByTagName("loadlibs")[0].getElementsByTagName("libpath") - for lib in libs: - so = lib.childNodes[0].data - result_dict.get(process_name).append(os.path.split(so)[-1]) - if debug: - print(process_name, " ", so) - - @classmethod - def get_elf_info_from_rom_result(cls, rom_result_json: str) -> typing.Dict[str, typing.Dict[str, str]]: - """ - 利用rom_analyzer.py的分析结果，重组成 - {file_base_name: {"subsystem_name":subsystem_name, "component_name":component_name}} - 的形式 - """ - with open(rom_result_json, 'r', encoding='utf-8') as f: - rom_info_dict = json.load(f) - elf_info_dict: typing.Dict[str, typing.Dict[str, str]] = dict() - for subsystem_name in rom_info_dict.keys(): - sub_val_dict: typing.Dict[str, typing.Any] = rom_info_dict.get(subsystem_name) - delete_values_from_dict(sub_val_dict, ["size", "file_count"]) - for component_name in sub_val_dict.keys(): - component_val_dict: typing.Dict[str, str] = sub_val_dict.get(component_name) - delete_values_from_dict(component_val_dict, ["size", "file_count"]) - for file_name, size in component_val_dict.items(): - file_basename: str = os.path.split(file_name)[-1] - elf_info_dict[file_basename] = { - "subsystem_name": subsystem_name, - "component_name": component_name, - "size": size - } - return elf_info_dict - - @classmethod - def __parse_process_cfg(cls, cfg_path: str, profile_path: str, result_dict: dict): - """ - 解析cfg，因为有的cfg会拉起xml中的进程，所以也可能会去解析xml - """ - with open(cfg_path, 'r', encoding='utf-8') as f: - cfg_dict = json.loads(f.read()) - services = cfg_dict.get("services") - if services is None: - print("warning: 'services' not in {}".format(cfg_path)) - return - for service in services: - process_name = service.get("name") - first, *path_list = service.get("path") - if first.endswith("sa_main"): - # 由sa_main去来起进程 - xml_base_name = os.path.split(path_list[0])[-1] - cls.__parse_process_xml(os.path.join(profile_path, xml_base_name), result_dict) - else: - # 直接执行 - # process_name = os.path.split(first)[-1] - if result_dict.get(process_name) is None: - result_dict[process_name] = list() - result_dict.get(process_name).append(os.path.split(first)[-1]) - - @classmethod - def get_process_so_relationship(cls, xml_path: str, cfg_path: str, profile_path: str) -> typing.Dict[ - str, typing.List[str]]: - """ - 从out/{product_name}/packages/phone/sa_profile/merged_sa查找xml文件并处理得到进程与so的对应关系 - """ - # xml_path = os.path.join(product_path, "packages", "phone", "sa_profile", "merged_sa") - # 从merged_sa里面收集 - xml_list = glob.glob(xml_path + os.sep + "*[.]xml", recursive=True) - process_elf_dict: typing.Dict[str, typing.List[str]] = dict() - for xml in xml_list: - if debug: - print("parsing: ", xml) - try: - cls.__parse_process_xml(xml, process_elf_dict) - except: - print("parse '{}' failed".format(xml)) - finally: - ... - # 从system/etc/init/*.cfg中收集，如果是sa_main拉起的，则从system/profile/*.xml中进行解析 - cfg_list = glob.glob(cfg_path + os.sep + "*[.]cfg", recursive=True) - for cfg in cfg_list: - if debug: - print("parsing: ", cfg) - try: - cls.__parse_process_cfg(cfg, profile_path, process_elf_dict) - except: - print("parse '{}' failed".format(cfg)) - finally: - ... - return process_elf_dict - - @classmethod - def __save_result_as_excel(cls, data_dict: dict, filename: str, ss: str): - """ - 保存结果到excel中 - 进程名:{ - "size": xxx, - 子系统名:{ - 部件名:{ - 二进制文件: xxx, - ... - } - } - } - """ - tmp_dict = copy.deepcopy(data_dict) - writer = SimpleExcelWriter("ram_info") - writer.set_sheet_header( - ["process_name", "process_size({}, KB)".format(ss), "subsystem_name","component_name", "elf_name", "elf_size(KB)"]) - process_start_r = 1 - process_end_r = 0 - process_c = 0 - subsystem_c = 2 - subsystem_start_r = 1 - subsystem_end_r = 0 - process_size_c = 1 - component_start_r = 1 - component_end_r = 0 - component_c = 3 - for process_name in tmp_dict.keys(): - process_val_dict: typing.Dict[str, typing.Dict[str, int]] = tmp_dict.get(process_name) - process_size = process_val_dict.get("size") - delete_values_from_dict(process_val_dict, ["size"]) - for subsystem_name, subsystem_val_dict in process_val_dict.items(): # 遍历subsystem - for component_name, component_val_dict in subsystem_val_dict.items(): # 遍历component - elf_count_of_component = len(component_val_dict) - for elf_name, size in component_val_dict.items(): # 遍里elf - writer.append_line([process_name, process_size, subsystem_name, component_name, elf_name, "%.2f" % (size / 1024)]) - component_end_r += elf_count_of_component - subsystem_end_r += elf_count_of_component - # 重写component - writer.write_merge(component_start_r, component_c, component_end_r, - component_c, component_name) - component_start_r = component_end_r + 1 - process_end_r += elf_count_of_component - writer.write_merge(subsystem_start_r, subsystem_c, subsystem_end_r, subsystem_c, subsystem_name) - subsystem_start_r = subsystem_end_r+1 - writer.write_merge(process_start_r, process_c, process_end_r, process_c, process_name) - writer.write_merge(process_start_r, process_size_c, process_end_r, process_size_c, process_size) - process_start_r = process_end_r + 1 - writer.save(filename) - - @classmethod - def find_elf_size_from_rom_result(cls, service_name: str, subsystem_name: str, component_name: str, - evaluator: typing.Callable, rom_result_dict: typing.Dict[str, typing.Dict]) -> \ - typing.Tuple[ - bool, str, str, int]: - """ - 全局查找进程的相关elf文件 - subsystem_name与component_name可明确指定，或为*以遍历整个dict - evaluator：评估elf文件的从phone下面开始的路径与service_name的关系，评判如何才是找到了 - returns: 是否查找到，elf文件名，部件名，size - """ - subsystem_name_list = [subsystem_name] if subsystem_name != "*" else rom_result_dict.keys() - for sn in subsystem_name_list: - sub_val_dict = rom_result_dict.get(sn) - component_name_list = [component_name] if component_name != '*' else sub_val_dict.keys() - for cn in component_name_list: - if cn == "size" or cn == "file_count": - continue - component_val_dict: typing.Dict[str, int] = sub_val_dict.get(cn) - for k, v in component_val_dict.items(): - if k == "size" or k == "file_count": - continue - if not evaluator(service_name, k): - continue - return True, os.path.split(k)[-1],sn, cn, v - return False, str(), str(), str(), int() - - @classmethod - def analysis(cls, cfg_path: str, xml_path: str, rom_result_json: str, device_num: str, - output_file: str, ss: str, output_excel: bool): - """ - process size subsystem/component so so_size - """ - if not HDCTool.verify_hdc(): - print("error: Command 'hdc' not found") - return - if not HDCTool.verify_device(device_num): - print("error: {} is inaccessible or not found".format(device_num)) - return - with open(rom_result_json, 'r', encoding='utf-8') as f: - rom_result_dict: typing.Dict = json.loads(f.read()) - # 从rom的分析结果中将需要的elf信息重组 - so_info_dict: typing.Dict[ - str, typing.Dict[str["component_name|subsystem_name|size"], str]] = cls.get_elf_info_from_rom_result( - rom_result_json) - process_elf_dict: typing.Dict[str, typing.List[str]] = cls.get_process_so_relationship(xml_path, cfg_path, - profile_path) - process_size_dict: typing.Dict[str, int] = cls.process_hidumper_info(device_num, ss) - result_dict: typing.Dict[str, typing.Dict[str, typing.Any]] = dict() - - def get(key: typing.Any, dt: typing.Dict[str, typing.Any]): - for k, v in dt.items(): - if k.startswith(key) or (len(v) > 0 and key == v[0]): - # 要么uinput_inject的对应key为mmi_uinput_inject。对于此类特殊处理，即：如果service_name找不到，但是直接执行的bin等于这个名字，也认为找到 - return v - - for process_name, process_size in process_size_dict.items(): # 从进程出发 - # 如果部件是init,特殊处理 - if process_name == "init": - _, elf,_, _, size = cls.find_elf_size_from_rom_result(process_name, "startup", "init", - lambda x, y: os.path.split(y)[ - -1].lower() == x.lower(), - rom_result_dict) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name]["startup"] = dict() - result_dict[process_name]["startup"]["init"] = dict() - result_dict[process_name]["startup"]["init"][elf if len(elf) != 0 else "UNKNOWN"] = size - continue - # 如果是hap，特殊处理 - if (process_name.startswith("com.") or process_name.startswith("ohos.")): - _, hap_name, subsystem_name, component_name, size = cls.find_elf_size_from_rom_result(process_name, "*", "*", - lambda x, y: len( - y.split( - '/')) >= 3 and x.lower().startswith( - y.split('/')[2].lower()), - rom_result_dict) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name][subsystem_name] = dict() - result_dict[process_name][subsystem_name][component_name] = dict() - result_dict[process_name][subsystem_name][component_name][hap_name if len(hap_name) != 0 else "UNKNOWN"] = size - continue - so_list: list = get(process_name, process_elf_dict) # 得到进程相关的elf文件list - if so_list is None: - print("warning: process '{}' not found in .xml or .cfg".format(process_name)) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"]["UNKNOWN"] = int() - continue - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - for so in so_list: - unit = so_info_dict.get(so) - if unit is None: - result_dict[process_name]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"][so] = int() - print("warning: '{}' in {} not found in json from rom analysis result".format(so, process_name)) - continue - component_name = unit.get("component_name") - subsystem_name = unit.get("subsystem_name") - so_size = unit.get("size") - if result_dict.get(process_name).get(subsystem_name) is None: - result_dict[process_name][subsystem_name] = dict() - if result_dict.get(process_name).get(subsystem_name).get(component_name) is None: - result_dict[process_name][subsystem_name][component_name] = dict() - result_dict[process_name][subsystem_name][component_name][so] = so_size - base_dir, _ = os.path.split(output_file) - if len(base_dir) != 0 and not os.path.isdir(base_dir): - os.makedirs(base_dir, exist_ok=True) - with open(output_file + ".json", 'w', encoding='utf-8') as f: - f.write(json.dumps(result_dict, indent=4)) - if output_excel: - cls.__save_result_as_excel(result_dict, output_file + ".xls", ss) - - -def get_args(): - VERSION = 1.0 - parser = argparse.ArgumentParser( - description="analyze ram size of component" - ) - parser.add_argument("-v", "-version", action="version", - version=f"version {VERSION}") - parser.add_argument("-x", "--xml_path", type=str, required=True, - help="path of xml file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile") - parser.add_argument("-c", "--cfg_path", type=str, required=True, - help="path of cfg files. eg: -c ./cfgs/") - parser.add_argument("-j", "--rom_result", type=str, default="./rom_analysis_result.json", - help="json file produced by rom_analyzer_v1.0.py, default: ./rom_analysis_result.json." - "eg: -j ./demo/rom_analysis_result.json") - parser.add_argument("-n", "--device_num", type=str, required=True, - help="device number to be collect hidumper info. eg: -n 7001005458323933328a01fce16d3800") - parser.add_argument("-o", "--output_filename", default="ram_analysis_result", type=str, - help="base name of output file, default: ram_analysis_result. eg: -o ram_analysis_result") - parser.add_argument("-e", "--excel", type=bool, default=False, - help="if output result as excel, default: False. eg: -e True") - args = parser.parse_args() - return args - - -if __name__ == '__main__': - args = get_args() - cfg_path = args.cfg_path - profile_path = args.xml_path - rom_result = args.rom_result - device_num = args.device_num - output_filename = args.output_filename - output_excel = args.excel - RamAnalyzer.analysis(cfg_path, profile_path, rom_result, - device_num=device_num, output_file=output_filename, ss="Pss", output_excel=output_excel) diff --git a/tools/rom_ram_analyzer/L2/rom_analyzer.py b/tools/rom_ram_analyzer/L2/rom_analyzer.py deleted file mode 100644 index b4f337d..0000000 --- a/tools/rom_ram_analyzer/L2/rom_analyzer.py +++ /dev/null @@ -1,200 +0,0 @@ -import argparse -import json -import os -import sys -import typing -from copy import deepcopy -from typing import * - -from pkgs.basic_tool import BasicTool -from pkgs.gn_common_tool import GnCommonTool -from pkgs.simple_excel_writer import SimpleExcelWriter - -debug = bool(sys.gettrace()) - - -class RomAnalyzer: - @classmethod - def __collect_product_info(cls, system_module_info_json: Text, - project_path: Text) -> Dict[Text, Dict[Text, Text]]: - """ - 根据system_module_info.json生成target字典 - """ - with open(system_module_info_json, 'r', encoding='utf-8') as f: - product_list = json.loads(f.read()) - project_path = BasicTool.get_abs_path(project_path) - product_info_dict: Dict[Text, Dict[Text, Text]] = dict() - for unit in product_list: - dest: List = unit.get("dest") - if dest is None: - print("warning: keyword 'dest' not found in {}".format( - system_module_info_json)) - continue - label: Text = unit.get("label") - gn_path = component_name = subsystem_name = None - if label: - gn_path = os.path.join(project_path, label.split(':')[ - 0].lstrip('/'), "BUILD.gn") - component_name = unit.get("part_name") - subsystem_name = unit.get("subsystem_name") - if (not component_name) or (not subsystem_name): - cn, sn = GnCommonTool.find_part_subsystem( - gn_path, project_path) - component_name = cn if not component_name else component_name - subsystem_name = sn if not subsystem_name else subsystem_name - else: - print("warning: keyword 'label' not found in {}".format(unit)) - for target in dest: - product_info_dict[target] = { - "component_name": component_name, - "subsystem_name": subsystem_name, - "gn_path": gn_path, - } - return product_info_dict - - @classmethod - def __save_result_as_excel(cls, result_dict: dict, output_name: str): - header = ["subsystem_name", "component_name", - "output_file", "size(Byte)"] - tmp_dict = deepcopy(result_dict) - excel_writer = SimpleExcelWriter("rom") - excel_writer.set_sheet_header(headers=header) - subsystem_start_row = 1 - subsystem_end_row = 0 - subsystem_col = 0 - component_start_row = 1 - component_end_row = 0 - component_col = 1 - - for subsystem_name in tmp_dict.keys(): - subsystem_dict = tmp_dict.get(subsystem_name) - subsystem_size = subsystem_dict.get("size") - subsystem_file_count = subsystem_dict.get("file_count") - del subsystem_dict["file_count"] - del subsystem_dict["size"] - subsystem_end_row += subsystem_file_count - - for component_name in subsystem_dict.keys(): - component_dict: Dict[str, int] = subsystem_dict.get( - component_name) - component_size = component_dict.get("size") - component_file_count = component_dict.get("file_count") - del component_dict["file_count"] - del component_dict["size"] - component_end_row += component_file_count - - for file_name, size in component_dict.items(): - excel_writer.append_line( - [subsystem_name, component_name, file_name, size]) - excel_writer.write_merge(component_start_row, component_col, component_end_row, component_col, - component_name) - component_start_row = component_end_row + 1 - excel_writer.write_merge(subsystem_start_row, subsystem_col, subsystem_end_row, subsystem_col, - subsystem_name) - subsystem_start_row = subsystem_end_row + 1 - excel_writer.save(output_name + ".xls") - - @classmethod - def __put(cls, unit: typing.Dict[Text, Any], result_dict: typing.Dict[Text, Dict]): - """ - { - subsystem_name:{ - component_name: { - file_name: file_size - } - } - } - """ - component_name = "others" if unit.get( - "component_name") is None else unit.get("component_name") - subsystem_name = "others" if unit.get( - "subsystem_name") is None else unit.get("subsystem_name") - size = unit.get("size") - relative_filepath = unit.get("relative_filepath") - if result_dict.get(subsystem_name) is None: # 子系统 - result_dict[subsystem_name] = dict() - result_dict[subsystem_name]["size"] = 0 - result_dict[subsystem_name]["file_count"] = 0 - - if result_dict.get(subsystem_name).get(component_name) is None: # 部件 - result_dict[subsystem_name][component_name] = dict() - result_dict[subsystem_name][component_name]["size"] = 0 - result_dict[subsystem_name][component_name]["file_count"] = 0 - result_dict[subsystem_name]["size"] += size - result_dict[subsystem_name]["file_count"] += 1 - result_dict[subsystem_name][component_name]["size"] += size - result_dict[subsystem_name][component_name]["file_count"] += 1 - result_dict[subsystem_name][component_name][relative_filepath] = size - - - @classmethod - def analysis(cls, system_module_info_json: Text, product_dirs: List[str], - project_path: Text, product_name: Text, output_file: Text, output_execel: bool): - """ - system_module_info_json: json文件 - product_dirs：要处理的产物的路径列表如["vendor", "system/"] - project_path: 项目根路径 - product_name: eg，rk3568 - output_file: basename of output file - """ - project_path = BasicTool.get_abs_path(project_path) - phone_dir = os.path.join( - project_path, "out", product_name, "packages", "phone") - product_dirs = [os.path.join(phone_dir, d) for d in product_dirs] - product_info_dict = cls.__collect_product_info( - system_module_info_json, project_path) # 所有产物信息 - result_dict: Dict[Text:Dict] = dict() - for d in product_dirs: - file_list: List[Text] = BasicTool.find_all_files(d) - for f in file_list: - size = os.path.getsize(f) - relative_filepath = f.replace(phone_dir, "").lstrip(os.sep) - unit: Dict[Text, Any] = product_info_dict.get( - relative_filepath) - if unit is None: - unit = dict() - unit["size"] = size - unit["relative_filepath"] = relative_filepath - cls.__put(unit, result_dict) - output_dir, _ = os.path.split(output_file) - if len(output_dir) != 0: - os.makedirs(output_dir, exist_ok=True) - with open(output_file + ".json", 'w', encoding='utf-8') as f: - f.write(json.dumps(result_dict, indent=4)) - if output_execel: - cls.__save_result_as_excel(result_dict, output_file) - - -def get_args(): - VERSION = 2.0 - parser = argparse.ArgumentParser( - description=f"analyze rom size of component.\n") - parser.add_argument("-v", "-version", action="version", - version=f"version {VERSION}") - parser.add_argument("-p", "--project_path", type=str, required=True, - help="root path of openharmony. eg: -p ~/openharmony") - parser.add_argument("-j", "--module_info_json", required=True, type=str, - help="path of out/{product_name}/packages/phone/system_module_info.json") - parser.add_argument("-n", "--product_name", required=True, - type=str, help="product name. eg: -n rk3568") - parser.add_argument("-d", "--product_dir", required=True, action="append", - help="subdirectories of out/{product_name}/packages/phone to be counted." - "eg: -d system -d vendor") - parser.add_argument("-o", "--output_file", type=str, default="rom_analysis_result", - help="basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result") - parser.add_argument("-e", "--excel", type=bool, default=False, - help="if output result as excel, default: False. eg: -e True") - args = parser.parse_args() - return args - - -if __name__ == '__main__': - args = get_args() - module_info_json = args.module_info_json - project_path = args.project_path - product_name = args.product_name - product_dirs = args.product_dir - output_file = args.output_file - output_excel = args.excel - RomAnalyzer.analysis(module_info_json, product_dirs, - project_path, product_name, output_file, output_excel) -- Gitee From 9fdc2988906cfcf2f850a9a1a045a07e8de66271 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Mon, 6 Mar 2023 16:10:18 +0800 Subject: [PATCH 03/14] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=91=8A=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- .../lite_small/pkgs/simple_excel_writer.py | 31 +++++++++-------- .../standard/pkgs/simple_excel_writer.py | 33 ++++++++++--------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py b/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py index ebe37e1..9aa5058 100644 --- a/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py +++ b/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py @@ -1,5 +1,5 @@ import xlwt -from xlwt import Worksheet +from xlwt import Worksheet, Pattern, Alignment, Font, XFStyle import typing import logging from typing import Optional @@ -18,27 +18,26 @@ class SimpleExcelWriter: } self.__default_sheet_name = default_sheet_name # 表头样式 - self.__head_style = xlwt.XFStyle() + self.__head_style = XFStyle() # 内容样式 - self.__content_style = xlwt.XFStyle() + self.__content_style = XFStyle() # 字体 - font = xlwt.Font() - font.bold = True + ft = Font() + ft.bold = True # 设置背景颜色 - pattern = xlwt.Pattern() - pattern.pattern = xlwt.Pattern.SOLID_PATTERN - pattern.pattern_fore_colour = 22 # 背景颜色 + ptrn = Pattern() + ptrn.pattern = Pattern.SOLID_PATTERN + ptrn.pattern_fore_colour = 22 # 背景颜色 # 居中对齐 - alignment = xlwt.Alignment() - alignment.horz = xlwt.Alignment.HORZ_CENTER # 水平方向 - alignment.vert = xlwt.Alignment.VERT_CENTER # 垂直方向 - - self.__head_style.font = font - self.__head_style.alignment = alignment - self.__head_style.pattern = pattern - self.__content_style.alignment = alignment + algmt = Alignment() + algmt.horz = Alignment.HORZ_CENTER # 水平方向 + algmt.vert = Alignment.VERT_CENTER # 垂直方向 + self.__head_style.font = ft + self.__head_style.alignment = algmt + self.__head_style.pattern = ft + self.__content_style.alignment = algmt def __increment_y(self, sheet_name: str, value: int = 1) -> int: if sheet_name in self.__sheet_pos.keys(): diff --git a/tools/rom_ram_analyzer/standard/pkgs/simple_excel_writer.py b/tools/rom_ram_analyzer/standard/pkgs/simple_excel_writer.py index a1f2aa6..7ce9d2d 100644 --- a/tools/rom_ram_analyzer/standard/pkgs/simple_excel_writer.py +++ b/tools/rom_ram_analyzer/standard/pkgs/simple_excel_writer.py @@ -1,5 +1,6 @@ import xlwt from xlwt import Worksheet +from xlwt import Pattern as PTRN, Alignment as ALMT, Font as FT, XFStyle as XFSTL import typing from typing import Optional from collections.abc import Iterable @@ -17,27 +18,27 @@ class SimpleExcelWriter: } self.__default_sheet_name = default_sheet_name # 表头样式 - self.__head_style = xlwt.XFStyle() + self.__head_style = XFSTL() # 内容样式 - self.__content_style = xlwt.XFStyle() + self.__content_style = XFSTL() # 字体 - font = xlwt.Font() - font.bold = True - - # 居中对齐 - alignment = xlwt.Alignment() - alignment.horz = xlwt.Alignment.HORZ_CENTER # 水平方向 - alignment.vert = xlwt.Alignment.VERT_CENTER # 垂直方向 + ft = FT() + ft.bold = True # 设置背景颜色 - pattern = xlwt.Pattern() - pattern.pattern = xlwt.Pattern.SOLID_PATTERN - pattern.pattern_fore_colour = 22 # 背景颜色 + ptrn = PTRN() + ptrn.pattern = PTRN.SOLID_PATTERN + ptrn.pattern_fore_colour = 22 # 背景颜色 + + # 居中对齐 + algmt = ALMT() + algmt.horz = ALMT.HORZ_CENTER # 水平方向 + algmt.vert = ALMT.VERT_CENTER # 垂直方向 - self.__head_style.font = font - self.__head_style.alignment = alignment - self.__head_style.pattern = pattern - self.__content_style.alignment = alignment + self.__head_style.font = ft + self.__head_style.alignment = algmt + self.__head_style.pattern = ptrn + self.__content_style.alignment = algmt def __increment_y(self, sheet_name: str, value: int = 1) -> int: if sheet_name in self.__sheet_pos.keys(): -- Gitee From 2014d9b24db3c8fb2d6531d986e09330251eb3f2 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Mon, 6 Mar 2023 16:27:32 +0800 Subject: [PATCH 04/14] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=91=8A=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- .../lite_small/pkgs/simple_excel_writer.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py b/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py index 9aa5058..3c5c4f3 100644 --- a/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py +++ b/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py @@ -1,5 +1,5 @@ import xlwt -from xlwt import Worksheet, Pattern, Alignment, Font, XFStyle +from xlwt import Worksheet, Pattern as PTRN, Alignment as ALGT, Font as FT, XFStyle as XFSL import typing import logging from typing import Optional @@ -18,22 +18,22 @@ class SimpleExcelWriter: } self.__default_sheet_name = default_sheet_name # 表头样式 - self.__head_style = XFStyle() + self.__head_style = XFSL() # 内容样式 - self.__content_style = XFStyle() + self.__content_style = XFSL() # 字体 - ft = Font() + ft = FT() ft.bold = True # 设置背景颜色 - ptrn = Pattern() - ptrn.pattern = Pattern.SOLID_PATTERN + ptrn = PTRN() + ptrn.pattern = PTRN.SOLID_PATTERN ptrn.pattern_fore_colour = 22 # 背景颜色 # 居中对齐 - algmt = Alignment() - algmt.horz = Alignment.HORZ_CENTER # 水平方向 - algmt.vert = Alignment.VERT_CENTER # 垂直方向 + algmt = ALGT() + algmt.horz = ALGT.HORZ_CENTER # 水平方向 + algmt.vert = ALGT.VERT_CENTER # 垂直方向 self.__head_style.font = ft self.__head_style.alignment = algmt self.__head_style.pattern = ft -- Gitee From e1f463f4d9c88e592d4e78f49067026174d9dc46 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Tue, 7 Mar 2023 10:48:09 +0800 Subject: [PATCH 05/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]1.=20bugfix:?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E5=91=BD=E5=90=8D=E9=87=8D=E5=A4=8D=E5=BC=95?= =?UTF-8?q?=E5=8F=91=E7=9A=84=E6=9F=A5=E6=89=BE=E9=83=A8=E4=BB=B6=E5=92=8C?= =?UTF-8?q?=E5=AD=90=E7=B3=BB=E7=BB=9F=E5=87=BA=E9=94=99;=202.=20=E4=BF=AE?= =?UTF-8?q?=E6=94=B9readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/README.md | 11 ++++++----- .../rom_ram_analyzer/lite_small/src/rom_analysis.py | 6 +++--- .../lite_small/src/template_processor.py | 13 +++++++++++-- tools/rom_ram_analyzer/standard/README.md | 13 ++++++++----- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/README.md b/tools/rom_ram_analyzer/lite_small/README.md index ce3da51..dae93c2 100644 --- a/tools/rom_ram_analyzer/lite_small/README.md +++ b/tools/rom_ram_analyzer/lite_small/README.md @@ -6,7 +6,7 @@ ## 支持产品 -目标是支持所有的产品,但是目前由于配置文件没设计好,只支持:ipcamera_hispark_taurus ipcamera_hispark_taurus_linux wifiiot_hispark_pegasus +支持产品:ipcamera_hispark_taurus ipcamera_hispark_taurus_linux wifiiot_hispark_pegasus ## 代码思路 @@ -24,14 +24,14 @@ 1. 获取整个本文件所在的整个目录 1. 对系统进行编译 1. linux平台 -1. python3.8及以后 +1. python3 1. 安装requirements ```txt xlwt==1.3.0 ``` -1. `python3 rom_analysis.py --product_name {your_product_name} --oh_path {root_path_of_oh} [--recollect_gn bool]`运行代码,其中recollect_gn表示是需要重新扫描BUILD.gn还是直接使用已有结果.eg: `python3 rom_analysis.py --product_name ipcamera_hispark_taurus` -3. 运行完毕会产生4个json文件及一个xls文件,如果是默认配置,各文件描述如下: +1. `python3 rom_analysis.py --product_name {your_product_name} --oh_path {root_path_of_oh} [--recollect_gn bool] [--recollect_sc bool]`运行代码,其中recollect_gn表示是需要重新扫描BUILD.gn还是直接使用已有结果,recollect_sc表示是需要重新扫描子系统和部件信息还是直接使用已有的.eg: `python3 rom_analysis.py --product_name ipcamera_hispark_taurus`. +1. 运行完毕会产生4个json文件及一个xls文件,如果是默认配置,各文件描述如下: - gn_info.json:BUILD.gn的分析结果 - sub_com_info.json:从bundle.json中进行分析获得的各部件及其对应根目录的信息 - {product_name}_product.json:该产品实际的编译产物信息,根据config.yaml进行收集 @@ -44,4 +44,5 @@ ## 后续工作 -1. 部分log的输出有待优化 \ No newline at end of file +1. 部分log的输出有待优化 +1. 准确率有待进一步提升 \ No newline at end of file diff --git a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py index ba1c6e1..4d5c106 100644 --- a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py +++ b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py @@ -229,7 +229,7 @@ class RomAnalysisTool: ] = configs[product_name]["query_order"] query_order["etc"] = configs["target_type"] rom_size_dict: Dict = dict() - # prodcut_dict: {"a":["a.txt", ...]} + # prodcut_dict: {"so":["a.so", ...]} for t, l in product_dict.items(): for f in l: # 遍历所有文件 # query_order: {"a":[static_library", ...]} @@ -240,9 +240,9 @@ class RomAnalysisTool: if not type_list: logging.warning( f"'{t}' not found in query_order of the config.yaml") - continue + break for tn in type_list: # tn example: ohos_shared_library - output_dict: Dict[str, Dict] = gn_info.get(tn) + output_dict: Dict[str, Dict] = gn_info.get(tn) # 这个模板对应的所有可能编译产物 if not output_dict: logging.warning( f"'{tn}' not found in the {gn_info_file}") diff --git a/tools/rom_ram_analyzer/lite_small/src/template_processor.py b/tools/rom_ram_analyzer/lite_small/src/template_processor.py index e4124c8..9e7e818 100644 --- a/tools/rom_ram_analyzer/lite_small/src/template_processor.py +++ b/tools/rom_ram_analyzer/lite_small/src/template_processor.py @@ -17,6 +17,7 @@ from typing import * from abc import ABC, abstractmethod import os import logging +from pprint import pprint from pkgs.basic_tool import do_nothing, BasicTool from pkgs.gn_common_tool import GnCommonTool, GnVariableParser @@ -92,9 +93,9 @@ class BaseProcessor(ABC): logging.error("gn_path and project_path is not consistent: gn_path={}, project_path={}".format( gn_path, self.project_path)) return "", "" - k = gn_path.replace(self.project_path, "").lstrip(os.sep) + gp = gn_path.replace(self.project_path, "").lstrip(os.sep) for k, v in self.sc_dict.items(): - if k.startswith(k): + if gp.startswith(k): return v.get("subsystem"), v.get("component") return "", "" @@ -107,6 +108,14 @@ class BaseProcessor(ABC): def _gn_var_process(project_path: str, gn_v: str, alt_v: str, gn_path: str, ifrom: str, efrom: str, strip_quote: bool = False) -> Tuple[str, str]: + """ + :param project_path:项目根路径 + :gn_v:gn中的值(可能为变量或空) + :alt_v: 如果gn_v为空,则直接使用alt_v代替 + :gn_path: gn文件的路径 + :ifrom: 如果gn_v不为空,则其来自哪个字段 + :efrom: 如果gn_v为空,则其(准确来说是alt_v)来自哪个字段 + """ if strip_quote: gn_v = gn_v.strip('"') if gn_v: diff --git a/tools/rom_ram_analyzer/standard/README.md b/tools/rom_ram_analyzer/standard/README.md index 52bc5a9..5e774c0 100644 --- a/tools/rom_ram_analyzer/standard/README.md +++ b/tools/rom_ram_analyzer/standard/README.md @@ -15,7 +15,7 @@ 1. 获取整个rom_ram_analyzer目录 1. 对系统进行编译 1. rom分析在linux平台,ram分析在windows平台 -1. python3.8及以后 +1. python3及以后 1. 安装requirements ```txt xlwt==1.3.0 @@ -86,7 +86,7 @@ 2. hdc可用 2. 设备已连接 3. 系统已烧录 -3. python3.8及以后 +3. python3 4. 安装requirements ```txt xlwt==1.3.0 @@ -132,9 +132,12 @@ { 进程名:{ "size": 本进程占用内存的大小, - 部件名: { - elf文件名: elf文件大小 - ... + 子系统名: { + 部件名: { + elf文件名: elf文件大小 + ... + } + ... } ... }, -- Gitee From 06c959fa32cf934a0a6cd4a76f7f6d40a5f30a6e Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Tue, 7 Mar 2023 11:28:55 +0800 Subject: [PATCH 06/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]bugfix:=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E4=B8=80=E4=B8=AA=E9=83=A8=E4=BB=B6=E7=9A=84=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E4=B8=BA=E5=8F=A6=E4=B8=80=E4=B8=AA=E9=83=A8=E4=BB=B6?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E7=9A=84=E5=89=8D=E7=BC=80=E6=97=B6=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E9=83=A8=E4=BB=B6=E9=94=99=E8=AF=AF=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- .../lite_small/src/template_processor.py | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/src/template_processor.py b/tools/rom_ram_analyzer/lite_small/src/template_processor.py index 9e7e818..0457f5d 100644 --- a/tools/rom_ram_analyzer/lite_small/src/template_processor.py +++ b/tools/rom_ram_analyzer/lite_small/src/template_processor.py @@ -94,10 +94,20 @@ class BaseProcessor(ABC): gn_path, self.project_path)) return "", "" gp = gn_path.replace(self.project_path, "").lstrip(os.sep) + alter_list = list() for k, v in self.sc_dict.items(): if gp.startswith(k): - return v.get("subsystem"), v.get("component") - return "", "" + alter_list.append(k) + # return v.get("subsystem"), v.get("component") + if not alter_list: + return str(), str() + + def bylength(str1: str, str2: str): + return len(str1)-len(str2) + pprint(alter_list) + alter_list.sort(cmp=bylength) + pprint(alter_list) + return alter_list[0].get("subsystem"), alter_list[0].get("component") @abstractmethod def run(self): @@ -139,8 +149,10 @@ class DefaultProcessor(BaseProcessor): output_name, target_name, gn_path, "target_name", "target_name", True) sub = GnVariableParser.string_parser("subsystem_name", paragraph) com = GnVariableParser.string_parser("part_name", paragraph) - sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json", True) - com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json", True) + sub, sub_from = _gn_var_process( + self.project_path, sub, _sub, gn_path, "gn", "json", True) + com, com_from = _gn_var_process( + self.project_path, com, _com, gn_path, "gn", "json", True) result = { "gn_path": gn_path, "target_type": self.target_type, @@ -197,8 +209,10 @@ class StrResourceProcessor(DefaultProcessor): resources, gn_path, self.project_path).strip('"') sub = GnVariableParser.string_parser("subsystem_name", paragraph) com = GnVariableParser.string_parser("part_name", paragraph) - sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json") - com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json") + sub, sub_from = _gn_var_process( + self.project_path, sub, _sub, gn_path, "gn", "json") + com, com_from = _gn_var_process( + self.project_path, com, _com, gn_path, "gn", "json") _, file_name = os.path.split(resources) result = { "gn_path": gn_path, @@ -227,8 +241,10 @@ class ListResourceProcessor(DefaultProcessor): return sub = GnVariableParser.string_parser("subsystem_name", paragraph) com = GnVariableParser.string_parser("part_name", paragraph) - sub, sub_from = _gn_var_process(self.project_path, sub, _sub, gn_path, "gn", "json") - com, com_from = _gn_var_process(self.project_path, com, _com, gn_path, "gn", "json") + sub, sub_from = _gn_var_process( + self.project_path, sub, _sub, gn_path, "gn", "json") + com, com_from = _gn_var_process( + self.project_path, com, _com, gn_path, "gn", "json") for ff in resources: _, file_name = os.path.split(ff) result = { -- Gitee From 72136f11fa2657229fae8f79cb752a1852fbd2fc Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Tue, 7 Mar 2023 15:56:02 +0800 Subject: [PATCH 07/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]bugfix:=E8=A7=A3?= =?UTF-8?q?=E5=86=B3grep=E6=97=B6=E8=BF=87=E6=BB=A4=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E5=A4=B1=E6=95=88=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- .../lite_small/pkgs/basic_tool.py | 5 ++++- .../lite_small/pkgs/simple_excel_writer.py | 2 +- .../lite_small/src/config.yaml | 4 ++-- tools/rom_ram_analyzer/lite_small/src/misc.py | 5 +---- .../lite_small/src/rom_analysis.py | 22 ++++++++++++++----- .../lite_small/src/template_processor.py | 12 +++------- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py b/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py index febe482..16c362e 100644 --- a/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py +++ b/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py @@ -113,6 +113,7 @@ class BasicTool: TODO 将cls.execute用subprocess代替 """ cmd = f"grep -Ern '{pattern}' '{cls.abspath(path)}'" + # E:启用正则表达式 r:递归搜索 n:显示行号 if include: cmd += f" --include='{include}'" for e in exclude: @@ -137,6 +138,8 @@ class BasicTool: if __name__ == '__main__': - for i in BasicTool.grep_ern("^( *)ohos_shared_library", "/home/aodongbiao/oh", include="BUILD.gn", exclude=("out", "doc", ".ccache"), post_handler=lambda x: x.split('\n')): + res = BasicTool.grep_ern("^( *)ohos_prebuilt_shared_library", "/home/aodongbiao/oh", include="BUILD.gn", exclude=("/home/aodongbiao/oh/out","doc", ".ccache"), post_handler=lambda x: x.split('\n')) + # print(res) + for i in res: if "oh/out" in i: print(i) diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py b/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py index 3c5c4f3..94a9496 100644 --- a/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py +++ b/tools/rom_ram_analyzer/lite_small/pkgs/simple_excel_writer.py @@ -36,7 +36,7 @@ class SimpleExcelWriter: algmt.vert = ALGT.VERT_CENTER # 垂直方向 self.__head_style.font = ft self.__head_style.alignment = algmt - self.__head_style.pattern = ft + self.__head_style.pattern = ptrn self.__content_style.alignment = algmt def __increment_y(self, sheet_name: str, value: int = 1) -> int: diff --git a/tools/rom_ram_analyzer/lite_small/src/config.yaml b/tools/rom_ram_analyzer/lite_small/src/config.yaml index f6e0f3e..90cf9fb 100644 --- a/tools/rom_ram_analyzer/lite_small/src/config.yaml +++ b/tools/rom_ram_analyzer/lite_small/src/config.yaml @@ -134,8 +134,8 @@ default_extension: default_prefix: shared_library: lib static_library: lib -# black list for command 'grep' -black_list: +# black list for command 'grep', it should be the subdirectory of the root path of oh +black_list: - .repo - .ccache - doc diff --git a/tools/rom_ram_analyzer/lite_small/src/misc.py b/tools/rom_ram_analyzer/lite_small/src/misc.py index d000507..e9d6ae6 100644 --- a/tools/rom_ram_analyzer/lite_small/src/misc.py +++ b/tools/rom_ram_analyzer/lite_small/src/misc.py @@ -43,10 +43,7 @@ def gn_lineno_collect(match_pattern: str, project_path: str) -> DefaultDict[str, :param match_pattern: 进行grep的pattern，支持扩展的正则 :param project_path: 项目路径（搜索路径） :return: {gn_file: [line_no_1, line_no_2, ..]} - """ - black_list = map(lambda x: os.path.join( - project_path, x), _config.get("black_list")) - + black_list = _config.get("black_list") def handler(content: Text) -> List[str]: return list(filter(lambda y: len(y) > 0, list(map(lambda x: x.strip(), content.split("\n"))))) diff --git a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py index 4d5c106..5276274 100644 --- a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py +++ b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py @@ -47,7 +47,10 @@ class RomAnalysisTool: @classmethod def _add_rest_dir(cls, top_dir: str, rela_path: str, sub_path: str, dir_list: List[str]) -> None: """ - dir_list:相对于原始top目录的所有子目录的全路径 + :top_dir 顶层目录,不会变化 + :rela_path 最顶层的值为空 + :sub_path 相对于top_dir的路径,最外层是包含在dir_list当中的 + :dir_list 相对于原始top目录的所有子目录的全路径 """ if not sub_path: return @@ -86,7 +89,7 @@ class RomAnalysisTool: relative_dir: Dict[str, str] = product_dir.get("relative") if not relative_dir: logging.warning( - f"'{relative_dir}' of {product_name} not found in the config.yaml") + f"'relative_dir' of {product_name} not found in the config.yaml") exit(1) # 除了so a hap bin外的全部归到etc里面 for k, v in relative_dir.items(): @@ -141,7 +144,7 @@ class RomAnalysisTool: rom_size_dict["size"] += size @classmethod - def _fuzzy_match(cls, file_name: str, extra_black_list: Tuple[str] = ("test",)) -> Tuple[str, str, str]: + def _fuzzy_match(cls, file_name: str, filter_path_keyword: Tuple[str] = ("test",)) -> Tuple[str, str, str]: """ 直接grep,利用出现次数最多的BUILD.gn去定位subsystem_name和component_name""" _, base_name = os.path.split(file_name) @@ -153,15 +156,22 @@ class RomAnalysisTool: base_name = base_name[:base_name.index(".z.so")] elif base_name.endswith(".so"): base_name = base_name[:base_name.index(".so")] - exclude_dir = [os.path.join(project_path, x) - for x in configs["black_list"]] - exclude_dir.extend(list(extra_black_list)) + # exclude_dir = [os.path.join(project_path, x) + # for x in configs["black_list"]] + exclude_dir = configs["black_list"] grep_result: List[str] = BasicTool.grep_ern( base_name, project_path, include="BUILD.gn", exclude=tuple(exclude_dir), post_handler=lambda x: list(filter(lambda x: len(x) > 0, x.split('\n')))) + tmp = list() + for gr in grep_result: + for item in filter_path_keyword: + if item in gr: + continue + tmp.append(gr) + grep_result = tmp if not grep_result: return str(), str(), str() gn_dict: Dict[str, int] = collections.defaultdict(int) diff --git a/tools/rom_ram_analyzer/lite_small/src/template_processor.py b/tools/rom_ram_analyzer/lite_small/src/template_processor.py index 0457f5d..b5b58d4 100644 --- a/tools/rom_ram_analyzer/lite_small/src/template_processor.py +++ b/tools/rom_ram_analyzer/lite_small/src/template_processor.py @@ -92,22 +92,16 @@ class BaseProcessor(ABC): if not gn_path.startswith(self.project_path): logging.error("gn_path and project_path is not consistent: gn_path={}, project_path={}".format( gn_path, self.project_path)) - return "", "" + return str(), str() gp = gn_path.replace(self.project_path, "").lstrip(os.sep) alter_list = list() for k, v in self.sc_dict.items(): if gp.startswith(k): alter_list.append(k) - # return v.get("subsystem"), v.get("component") if not alter_list: return str(), str() - - def bylength(str1: str, str2: str): - return len(str1)-len(str2) - pprint(alter_list) - alter_list.sort(cmp=bylength) - pprint(alter_list) - return alter_list[0].get("subsystem"), alter_list[0].get("component") + alter_list.sort(key=lambda x: len(x), reverse=True) + return self.sc_dict[alter_list[0]].get("subsystem"), self.sc_dict[alter_list[0]].get("component") @abstractmethod def run(self): -- Gitee From b1aaf59efc374e905777e789cae3ecf6891e3f8d Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Tue, 7 Mar 2023 15:57:55 +0800 Subject: [PATCH 08/14] [rom_ram_analyzer]bugfix Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/src/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/rom_ram_analyzer/lite_small/src/misc.py b/tools/rom_ram_analyzer/lite_small/src/misc.py index e9d6ae6..ec2609c 100644 --- a/tools/rom_ram_analyzer/lite_small/src/misc.py +++ b/tools/rom_ram_analyzer/lite_small/src/misc.py @@ -43,6 +43,7 @@ def gn_lineno_collect(match_pattern: str, project_path: str) -> DefaultDict[str, :param match_pattern: 进行grep的pattern，支持扩展的正则 :param project_path: 项目路径（搜索路径） :return: {gn_file: [line_no_1, line_no_2, ..]} + """ black_list = _config.get("black_list") def handler(content: Text) -> List[str]: return list(filter(lambda y: len(y) > 0, list(map(lambda x: x.strip(), content.split("\n"))))) -- Gitee From 544925d2b982ce6554e608c73c9322b35a266261 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Tue, 7 Mar 2023 19:36:42 +0800 Subject: [PATCH 09/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]1.=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E4=BA=86=E4=B8=80=E4=BA=9Bbug.=202.=20=E5=A4=A7?= =?UTF-8?q?=E5=B9=85=E6=8F=90=E5=8D=87=E4=BA=86=E5=8C=B9=E9=85=8D=E5=87=86?= =?UTF-8?q?=E7=A1=AE=E7=8E=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- .../lite_small/pkgs/basic_tool.py | 2 +- .../lite_small/src/config.yaml | 9 ++- tools/rom_ram_analyzer/lite_small/src/misc.py | 1 + .../lite_small/src/rom_analysis.py | 72 +++++++++++-------- .../lite_small/src/template_processor.py | 4 ++ 5 files changed, 56 insertions(+), 32 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py b/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py index 16c362e..b04afab 100644 --- a/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py +++ b/tools/rom_ram_analyzer/lite_small/pkgs/basic_tool.py @@ -117,7 +117,7 @@ class BasicTool: if include: cmd += f" --include='{include}'" for e in exclude: - cmd += f" --exclude-dir={e}" + cmd += f" --exclude-dir='{e}'" o = cls.execute(cmd) if post_handler: o = post_handler(o) diff --git a/tools/rom_ram_analyzer/lite_small/src/config.yaml b/tools/rom_ram_analyzer/lite_small/src/config.yaml index 90cf9fb..9db3e40 100644 --- a/tools/rom_ram_analyzer/lite_small/src/config.yaml +++ b/tools/rom_ram_analyzer/lite_small/src/config.yaml @@ -141,4 +141,11 @@ black_list: - doc - test - build - - out + # 排除out目录,为了避免排除其他路径下的out目录,这里详细列出了out下的所有目录 + - out/gen + - out/hispark_pegasus + - out/hispark_taurus + - out/kernel + - out/preloader + - out/rk3568 + # 如果发现了跨形态的错误,如:检查A形态,但是发现将A形态的产品归属到B形态下了,则也可以将B形态的对应路径加在这里进行过滤 diff --git a/tools/rom_ram_analyzer/lite_small/src/misc.py b/tools/rom_ram_analyzer/lite_small/src/misc.py index ec2609c..a99314e 100644 --- a/tools/rom_ram_analyzer/lite_small/src/misc.py +++ b/tools/rom_ram_analyzer/lite_small/src/misc.py @@ -5,6 +5,7 @@ import logging from abc import ABC, abstractmethod from collections import defaultdict from typing import * +from pprint import pprint import preprocess from pkgs.gn_common_tool import GnVariableParser from pkgs.simple_yaml_tool import SimpleYamlTool diff --git a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py index 5276274..e60dd3b 100644 --- a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py +++ b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py @@ -49,32 +49,38 @@ class RomAnalysisTool: """ :top_dir 顶层目录,不会变化 :rela_path 最顶层的值为空 - :sub_path 相对于top_dir的路径,最外层是包含在dir_list当中的 - :dir_list 相对于原始top目录的所有子目录的全路径 + :sub_path 一般是a/b/c这种形式 + :dir_list 相对于原始top目录的子目录的全路径 + example: + / + |-a + |-b + |-c + |-|-d + |-|-e + |-|-f + |-|-|-g + |-|-|-h + top_dir: / + rela_path: "" + sub_path: c/e + dir_list: [c] + => [c/d, c/f], assume 'a' and 'b' has been removed from dir_list """ - if not sub_path: + if (not sub_path) or (os.sep not in sub_path): return # 将其他目录添加到dir_list - all_subdir = os.listdir(os.path.join(top_dir, rela_path)) - for d in all_subdir: - t = os.path.join(rela_path, d) - if os.path.isdir(os.path.join(top_dir, t)) and t not in dir_list: - dir_list.append(t) - # 移除sub_path的当前层级的目录 - t = sub_path.split(os.sep) - if os.path.join(rela_path, t[0]) in dir_list: - dir_list.remove(os.path.join(rela_path, t[0])) - else: - logging.error( - f"'{os.path.join(rela_path,t[0])}' not in '{top_dir}'") - sp = str() - if len(t) == 1: + t, sub_sub_path = sub_path.split(os.sep, 1) # 如果是c/e,分割成c,e + t = os.path.join(rela_path, t) + if t in dir_list: + dir_list.remove(t) + sub_sub_dir_list = os.listdir(os.path.join(top_dir, t)) + for ssdl in sub_sub_dir_list: + if os.path.join(rela_path,sub_path) != os.path.join(t,ssdl): + dir_list.append(os.path.join(t, ssdl)) + if not sub_sub_dir_list: return - elif len(t) == 2: - sp = t[1] - else: - sp = os.path.join(*t[1:]) - cls._add_rest_dir(top_dir, os.path.join(rela_path, t[0]), sp, dir_list) + cls._add_rest_dir(top_dir, t, sub_sub_path, dir_list) @classmethod def _find_files(cls, product_name: str) -> Dict[str, List[str]]: @@ -105,7 +111,11 @@ class RomAnalysisTool: rest_dir_list: List[str] = os.listdir( root_dir) # 除了配置在relative下之外的所有剩余目录,全部归到etc下 for v in relative_dir.values(): - cls._add_rest_dir(root_dir, str(), v, rest_dir_list) + if v in rest_dir_list: + rest_dir_list.remove(v) + for v in relative_dir.values(): + if os.sep in v: + cls._add_rest_dir(root_dir, str(), v, rest_dir_list) if "etc" not in product_dict.keys(): product_dict["etc"] = list() for r in rest_dir_list: @@ -242,6 +252,8 @@ class RomAnalysisTool: # prodcut_dict: {"so":["a.so", ...]} for t, l in product_dict.items(): for f in l: # 遍历所有文件 + if os.path.isdir(f): + continue # query_order: {"a":[static_library", ...]} find_flag = False type_list = query_order.get(t) @@ -252,7 +264,8 @@ class RomAnalysisTool: f"'{t}' not found in query_order of the config.yaml") break for tn in type_list: # tn example: ohos_shared_library - output_dict: Dict[str, Dict] = gn_info.get(tn) # 这个模板对应的所有可能编译产物 + output_dict: Dict[str, Dict] = gn_info.get( + tn) # 这个模板对应的所有可能编译产物 if not output_dict: logging.warning( f"'{tn}' not found in the {gn_info_file}") @@ -279,7 +292,7 @@ class RomAnalysisTool: }, rom_size_dict) find_flag = True if not find_flag: - cls._put("others", "others", { + cls._put("NOTFOUND", "NOTFOUND", { "file_name": f.replace(project_path, ""), "size": size, }, rom_size_dict) @@ -298,8 +311,7 @@ def main(): if __name__ == "__main__": main() - # t = os.listdir( - # "/home/aodongbiao/developtools_integration_verification/tools") - # RomAnalysisTool._add_rest_dir( - # "/home/aodongbiao/developtools_integration_verification/tools", "", "rom_ram_analyzer/L2/pkgs", t) - # print(t) + # relative_dir = ["bin", "usr/lib", "etc"] + # root_dir = "/home/aodongbiao/oh/out/hispark_taurus/ipcamera_hispark_taurus/rootfs" + # rest_dir_list = os.listdir(root_dir) + # RomAnalysisTool._find_files("ipcamera_hispark_taurus") diff --git a/tools/rom_ram_analyzer/lite_small/src/template_processor.py b/tools/rom_ram_analyzer/lite_small/src/template_processor.py index b5b58d4..2777a23 100644 --- a/tools/rom_ram_analyzer/lite_small/src/template_processor.py +++ b/tools/rom_ram_analyzer/lite_small/src/template_processor.py @@ -147,6 +147,10 @@ class DefaultProcessor(BaseProcessor): self.project_path, sub, _sub, gn_path, "gn", "json", True) com, com_from = _gn_var_process( self.project_path, com, _com, gn_path, "gn", "json", True) + if not sub: + sub = "UNDEFINED" + if not com: + com = "UNDEFINED" result = { "gn_path": gn_path, "target_type": self.target_type, -- Gitee From 862c1c8989711b7d1f7d2cb1eeae74e21b45f8d8 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Tue, 7 Mar 2023 20:18:32 +0800 Subject: [PATCH 10/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]=E5=AE=8C=E5=96=84u?= =?UTF-8?q?ndefined?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/README.md | 6 +++++- .../rom_ram_analyzer/lite_small/src/config.py | 18 +++++++++++++++-- .../lite_small/src/config.yaml | 1 + .../lite_small/src/template_processor.py | 20 +++++++++++++++++-- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/README.md b/tools/rom_ram_analyzer/lite_small/README.md index dae93c2..102cdcd 100644 --- a/tools/rom_ram_analyzer/lite_small/README.md +++ b/tools/rom_ram_analyzer/lite_small/README.md @@ -42,7 +42,11 @@ 主要是在config.py中配置Processor,并在config.yaml中添加相应内容 +## 如何提高准确率 + +1. 如果已知编译产物不可能从某些目录下的BUILD.gn产生,则可以将其加入到config.yaml的black_list,从而不对该目录下的BUILD.gn进行扫描,以减少出错概率 + ## 后续工作 1. 部分log的输出有待优化 -1. 准确率有待进一步提升 \ No newline at end of file +1. 新增对target(a,b)类型的检查 \ No newline at end of file diff --git a/tools/rom_ram_analyzer/lite_small/src/config.py b/tools/rom_ram_analyzer/lite_small/src/config.py index 05f1a26..76580da 100644 --- a/tools/rom_ram_analyzer/lite_small/src/config.py +++ b/tools/rom_ram_analyzer/lite_small/src/config.py @@ -22,8 +22,10 @@ def parse_args(): help="product name. eg: -p ipcamera_hispark_taurus") parser.add_argument("-o", "--oh_path", type=str, default=".", help="root path of openharmony") - parser.add_argument("-g", "--recollect_gn", action="store_false", help="recollect gn info or not") - parser.add_argument("-s", "--recollect_sc", action="store_false", help="recollect subsystem_component info or not") + parser.add_argument("-g", "--recollect_gn", + action="store_false", help="recollect gn info or not") + parser.add_argument("-s", "--recollect_sc", action="store_false", + help="recollect subsystem_component info or not") args = parser.parse_args() return args @@ -202,6 +204,18 @@ collector_config: Tuple[BaseProcessor] = ( }, unit_post_handler=LiteComponentPostHandler(), ), + # TargetProcessor(project_path=project_path, + # result_dict=result_dict, + # target_type=_target_type[13], + # match_pattern=fr"^( *){_target_type[13]}\(.*?\, .*?\)", + # sub_com_dict=sub_com_dict, + # target_name_parser=TargetNameParser.second_parser, + # other_info_handlers={ + # "real_target_type": target_type_handler, + # # "extension": extension_handler, + # }, + # unit_post_handler=LiteComponentPostHandler(), + # ) ) __all__ = ["configs", "result_dict", "collector_config", "sub_com_dict"] diff --git a/tools/rom_ram_analyzer/lite_small/src/config.yaml b/tools/rom_ram_analyzer/lite_small/src/config.yaml index 9db3e40..f0fd95f 100644 --- a/tools/rom_ram_analyzer/lite_small/src/config.yaml +++ b/tools/rom_ram_analyzer/lite_small/src/config.yaml @@ -27,6 +27,7 @@ target_type: - ohos_sa_profile - ohos_prebuilt_shared_library - lite_component + # - target subsystem_component: diff --git a/tools/rom_ram_analyzer/lite_small/src/template_processor.py b/tools/rom_ram_analyzer/lite_small/src/template_processor.py index 2777a23..b2ccf3d 100644 --- a/tools/rom_ram_analyzer/lite_small/src/template_processor.py +++ b/tools/rom_ram_analyzer/lite_small/src/template_processor.py @@ -136,6 +136,10 @@ def _gn_var_process(project_path: str, gn_v: str, alt_v: str, gn_path: str, ifro class DefaultProcessor(BaseProcessor): + + @property + def UNDEFINED(self): + return "UNDEFINED" def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: output_name = GnVariableParser.string_parser("output_name", paragraph) @@ -148,9 +152,9 @@ class DefaultProcessor(BaseProcessor): com, com_from = _gn_var_process( self.project_path, com, _com, gn_path, "gn", "json", True) if not sub: - sub = "UNDEFINED" + sub = self.UNDEFINED if not com: - com = "UNDEFINED" + com = self.UNDEFINED result = { "gn_path": gn_path, "target_type": self.target_type, @@ -211,6 +215,10 @@ class StrResourceProcessor(DefaultProcessor): self.project_path, sub, _sub, gn_path, "gn", "json") com, com_from = _gn_var_process( self.project_path, com, _com, gn_path, "gn", "json") + if not sub: + sub = self.UNDEFINED + if not com: + com = self.UNDEFINED _, file_name = os.path.split(resources) result = { "gn_path": gn_path, @@ -243,6 +251,10 @@ class ListResourceProcessor(DefaultProcessor): self.project_path, sub, _sub, gn_path, "gn", "json") com, com_from = _gn_var_process( self.project_path, com, _com, gn_path, "gn", "json") + if not sub: + sub = self.UNDEFINED + if not com: + com = self.UNDEFINED for ff in resources: _, file_name = os.path.split(ff) result = { @@ -262,6 +274,10 @@ class ListResourceProcessor(DefaultProcessor): key = self.unit_post_handler(result) self._append(key, result) +class TargetProcessor(DefaultProcessor): + def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: + + ... if __name__ == '__main__': ... -- Gitee From 3f4db827a44be080b7ea98aa170f22efd68dba16 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Wed, 8 Mar 2023 09:25:57 +0800 Subject: [PATCH 11/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]=E5=87=86=E7=A1=AE?= =?UTF-8?q?=E7=8E=87=E6=8F=90=E5=8D=87:=E5=88=9D=E6=AD=A5=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E4=BA=86=E5=AF=B9target(a,b)=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E7=9A=84=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/README.md | 3 +-- .../rom_ram_analyzer/lite_small/src/config.py | 23 +++++++++---------- .../lite_small/src/config.yaml | 6 ++++- tools/rom_ram_analyzer/lite_small/src/misc.py | 16 +++++++++++++ .../lite_small/src/template_processor.py | 4 ---- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/README.md b/tools/rom_ram_analyzer/lite_small/README.md index 102cdcd..aa8fa22 100644 --- a/tools/rom_ram_analyzer/lite_small/README.md +++ b/tools/rom_ram_analyzer/lite_small/README.md @@ -48,5 +48,4 @@ ## 后续工作 -1. 部分log的输出有待优化 -1. 新增对target(a,b)类型的检查 \ No newline at end of file +1. 部分log的输出有待优化 \ No newline at end of file diff --git a/tools/rom_ram_analyzer/lite_small/src/config.py b/tools/rom_ram_analyzer/lite_small/src/config.py index 76580da..aa9b1ed 100644 --- a/tools/rom_ram_analyzer/lite_small/src/config.py +++ b/tools/rom_ram_analyzer/lite_small/src/config.py @@ -204,18 +204,17 @@ collector_config: Tuple[BaseProcessor] = ( }, unit_post_handler=LiteComponentPostHandler(), ), - # TargetProcessor(project_path=project_path, - # result_dict=result_dict, - # target_type=_target_type[13], - # match_pattern=fr"^( *){_target_type[13]}\(.*?\, .*?\)", - # sub_com_dict=sub_com_dict, - # target_name_parser=TargetNameParser.second_parser, - # other_info_handlers={ - # "real_target_type": target_type_handler, - # # "extension": extension_handler, - # }, - # unit_post_handler=LiteComponentPostHandler(), - # ) + DefaultProcessor(project_path=project_path, + result_dict=result_dict, + target_type=_target_type[13], + match_pattern=fr"^( *){_target_type[13]}\(.*?\, .*?\)", + sub_com_dict=sub_com_dict, + target_name_parser=TargetNameParser.second_parser, + other_info_handlers={ + }, + unit_post_handler=DefaultPostHandler(), + ud_post_handler=TargetS2MPostHandler + ) ) __all__ = ["configs", "result_dict", "collector_config", "sub_com_dict"] diff --git a/tools/rom_ram_analyzer/lite_small/src/config.yaml b/tools/rom_ram_analyzer/lite_small/src/config.yaml index f0fd95f..254c4cd 100644 --- a/tools/rom_ram_analyzer/lite_small/src/config.yaml +++ b/tools/rom_ram_analyzer/lite_small/src/config.yaml @@ -27,7 +27,7 @@ target_type: - ohos_sa_profile - ohos_prebuilt_shared_library - lite_component - # - target + - target subsystem_component: @@ -52,6 +52,7 @@ ipcamera_hispark_taurus: - ohos_prebuilt_shared_library - lite_library - lite_component + - target bin: - executable @@ -76,6 +77,7 @@ ipcamera_hispark_taurus_linux: - ohos_prebuilt_shared_library - lite_library - lite_component + - target bin: - executable @@ -97,6 +99,7 @@ wifiiot_hispark_pegasus: - static_library - ohos_static_library - lite_library + - target rk3568: # rk的目前从packages/phone/system_module_info.json中分析准确度更高,因为rk基本都使用的是ohos_xxx,而L0和L1的更多的是使用的gn原生target template @@ -117,6 +120,7 @@ rk3568: # rk的目前从packages/phone/system_module_info.json中分析准确度 - ohos_prebuilt_shared_library - lite_library - lite_component + - target bin: - ohos_executable diff --git a/tools/rom_ram_analyzer/lite_small/src/misc.py b/tools/rom_ram_analyzer/lite_small/src/misc.py index a99314e..aecfc60 100644 --- a/tools/rom_ram_analyzer/lite_small/src/misc.py +++ b/tools/rom_ram_analyzer/lite_small/src/misc.py @@ -46,6 +46,7 @@ def gn_lineno_collect(match_pattern: str, project_path: str) -> DefaultDict[str, :return: {gn_file: [line_no_1, line_no_2, ..]} """ black_list = _config.get("black_list") + def handler(content: Text) -> List[str]: return list(filter(lambda y: len(y) > 0, list(map(lambda x: x.strip(), content.split("\n"))))) @@ -225,3 +226,18 @@ def LiteLibS2MPostHandler(unit: Dict, result_dict: Dict) -> None: k = LiteLibPostHandler()(new_new_unit) new_new_unit["description"] = "may not exist" result_dict["lite_library"][k] = new_new_unit + + +def TargetS2MPostHandler(unit: Dict, result_dict: Dict) -> None: + unit["description"] = "may not exist" + tmp_a = copy.deepcopy(unit) + tmp_a["real_target_type"] = "static_library" + k = LiteLibPostHandler()(tmp_a) + result_dict["target"][k] = tmp_a + + tmp_s = copy.deepcopy(unit) + tmp_s["real_target_type"] = "shared_library" + k = LiteLibPostHandler()(tmp_s) + result_dict["target"][k] = tmp_s + pprint(tmp_a) + pprint(tmp_s) \ No newline at end of file diff --git a/tools/rom_ram_analyzer/lite_small/src/template_processor.py b/tools/rom_ram_analyzer/lite_small/src/template_processor.py index b2ccf3d..8e3a9e4 100644 --- a/tools/rom_ram_analyzer/lite_small/src/template_processor.py +++ b/tools/rom_ram_analyzer/lite_small/src/template_processor.py @@ -274,10 +274,6 @@ class ListResourceProcessor(DefaultProcessor): key = self.unit_post_handler(result) self._append(key, result) -class TargetProcessor(DefaultProcessor): - def helper(self, target_name: str, paragraph: str, gn_path: str, line_no: int, _sub: str, _com: str) -> Tuple[str]: - - ... if __name__ == '__main__': ... -- Gitee From 2344ac888c2ce8d8a3cde9f37ce525fc71c359fb Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Wed, 8 Mar 2023 09:59:36 +0800 Subject: [PATCH 12/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]bugfix:=E8=A7=A3?= =?UTF-8?q?=E5=86=B3config.yaml=E4=B8=AD=E9=BB=91=E5=90=8D=E5=8D=95?= =?UTF-8?q?=E5=A4=B1=E6=95=88=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/src/misc.py | 18 +++++++++++------- .../lite_small/src/rom_analysis.py | 12 +++++++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/src/misc.py b/tools/rom_ram_analyzer/lite_small/src/misc.py index aecfc60..189a78f 100644 --- a/tools/rom_ram_analyzer/lite_small/src/misc.py +++ b/tools/rom_ram_analyzer/lite_small/src/misc.py @@ -46,12 +46,18 @@ def gn_lineno_collect(match_pattern: str, project_path: str) -> DefaultDict[str, :return: {gn_file: [line_no_1, line_no_2, ..]} """ black_list = _config.get("black_list") + tbl = [x for x in black_list if os.sep in x] def handler(content: Text) -> List[str]: - return list(filter(lambda y: len(y) > 0, list(map(lambda x: x.strip(), content.split("\n"))))) - - grep_list = BasicTool.grep_ern(match_pattern, path=project_path, include="BUILD.gn", exclude=tuple(black_list), - post_handler=handler) + t = list(filter(lambda y: len(y) > 0, list( + map(lambda x: x.strip(), content.split("\n"))))) + for item in tbl: + p = os.path.join(project_path, item) + t = list(filter(lambda x: p not in x, t)) + return t + + grep_list = BasicTool.grep_ern(match_pattern, path=project_path, + include="BUILD.gn", exclude=tuple(black_list), post_handler=handler) gn_line_dict: DefaultDict[str, List[int]] = defaultdict(list) for gl in grep_list: gn_file, line_no, _ = gl.split(":") @@ -234,10 +240,8 @@ def TargetS2MPostHandler(unit: Dict, result_dict: Dict) -> None: tmp_a["real_target_type"] = "static_library" k = LiteLibPostHandler()(tmp_a) result_dict["target"][k] = tmp_a - + tmp_s = copy.deepcopy(unit) tmp_s["real_target_type"] = "shared_library" k = LiteLibPostHandler()(tmp_s) result_dict["target"][k] = tmp_s - pprint(tmp_a) - pprint(tmp_s) \ No newline at end of file diff --git a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py index e60dd3b..26824e9 100644 --- a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py +++ b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py @@ -166,15 +166,21 @@ class RomAnalysisTool: base_name = base_name[:base_name.index(".z.so")] elif base_name.endswith(".so"): base_name = base_name[:base_name.index(".so")] - # exclude_dir = [os.path.join(project_path, x) - # for x in configs["black_list"]] exclude_dir = configs["black_list"] + tbl = [x for x in exclude_dir if os.sep in x] + def handler(content: Text) -> List[str]: + t = list(filter(lambda y: len(y) > 0, list( + map(lambda x: x.strip(), content.split("\n"))))) + for item in tbl: + p = os.path.join(project_path, item) + t = list(filter(lambda x: p not in x, t)) + return t grep_result: List[str] = BasicTool.grep_ern( base_name, project_path, include="BUILD.gn", exclude=tuple(exclude_dir), - post_handler=lambda x: list(filter(lambda x: len(x) > 0, x.split('\n')))) + post_handler=handler) tmp = list() for gr in grep_result: for item in filter_path_keyword: -- Gitee From ea2bf8ab628c4e000458e645bc1efa43b1b89544 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Wed, 8 Mar 2023 11:17:28 +0800 Subject: [PATCH 13/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]=E5=AE=8C=E5=96=84r?= =?UTF-8?q?eademe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/README.md | 16 +++++++++++++--- tools/rom_ram_analyzer/standard/README.md | 6 +++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/README.md b/tools/rom_ram_analyzer/lite_small/README.md index aa8fa22..9f2c0b1 100644 --- a/tools/rom_ram_analyzer/lite_small/README.md +++ b/tools/rom_ram_analyzer/lite_small/README.md @@ -13,9 +13,19 @@ 1. 扫描BUILD.gn文件,收集各个target的编译产物及其对应的component_name, subsystem_name信息,并存储到config.yaml中的gn_info_file字段指定的json文件中 2. 根据配置文件config.yaml扫描产品的编译产物目录,得到真实的编译产物信息(主要是大小) 3. 用真实的编译产物与从BUILD.gn中收集的信息进行匹配,从而得到编译产物-大小-所属部件的对应信息 -4. 如果匹配失败,会直接利用grep到项目路径下进行搜索 +4. 如果匹配失败,会直接利用grep到项目路径下进行模糊搜索,取出现次数最多的BUILD.gn,并根据该BUILD.gn文件去查找子系统和部件 5. 如果还搜索失败,则将其归属到others +## 说明 + +1. 关于UNDEFINED:如果target的声明中和get_subsystem_component.py(原作者:陈毓德)脚本的运行结果中都没有查找到子系统和部件信息,则为UNDEFINED +1. 关于NOTFOUND:表示对应的编译产物没有在BUILD.gn的扫描结果中匹配(包括模糊匹配)到 +1. 本工具是基于gn的template进行匹配,如果新增了自定义的template,则需要相应在代码中进行配置 +1. 由于本工具是进行的静态扫描,且部分gn文件中使用了较为复杂的gn语法,因此本工具的**准确率无法达到100%,结果仅供参考** + +**建议** + +因为是静态扫描,因此过于复杂的gn语法会导致扫描BUILD.gn文件时无法确定其编译产物,典型的:在for循环中嵌套target声明.因此就本工具来说,不建议使用过于复杂的gn语法. ## 使用 @@ -30,7 +40,7 @@ xlwt==1.3.0 ``` -1. `python3 rom_analysis.py --product_name {your_product_name} --oh_path {root_path_of_oh} [--recollect_gn bool] [--recollect_sc bool]`运行代码,其中recollect_gn表示是需要重新扫描BUILD.gn还是直接使用已有结果,recollect_sc表示是需要重新扫描子系统和部件信息还是直接使用已有的.eg: `python3 rom_analysis.py --product_name ipcamera_hispark_taurus`. +1. `python3 rom_analysis.py --product_name {your_product_name} --oh_path {root_path_of_oh} [-g] [-s]`运行代码,其中-g表示直接使用上次扫描的BUILD.gn的结果,-s表示直接使用已有的子系统和部件信息,默认都会重新扫描.eg: `python3 rom_analysis.py --product_name ipcamera_hispark_taurus`. 1. 运行完毕会产生4个json文件及一个xls文件,如果是默认配置,各文件描述如下: - gn_info.json:BUILD.gn的分析结果 - sub_com_info.json:从bundle.json中进行分析获得的各部件及其对应根目录的信息 @@ -44,7 +54,7 @@ ## 如何提高准确率 -1. 如果已知编译产物不可能从某些目录下的BUILD.gn产生,则可以将其加入到config.yaml的black_list,从而不对该目录下的BUILD.gn进行扫描,以减少出错概率 +1. 如果已知编译产物不可能从某些目录下的BUILD.gn产生,则可以将其对应目录加入到config.yaml的black_list,从而不对该目录下的BUILD.gn进行扫描,以减少出错概率 ## 后续工作 diff --git a/tools/rom_ram_analyzer/standard/README.md b/tools/rom_ram_analyzer/standard/README.md index 5e774c0..74e6933 100644 --- a/tools/rom_ram_analyzer/standard/README.md +++ b/tools/rom_ram_analyzer/standard/README.md @@ -55,7 +55,7 @@ # -e True：output result in excel format additionally ``` -## 输出格式说明(json) +## 输出格式介绍(json) ```json @@ -70,6 +70,10 @@ } ``` +## 其他说明 + +1. 由于目前standard产品使用的基本都是自定义的template,能够有效收集更多信息,因此相较于lite_small的分析脚本,本脚本能够具有更高的准确率,请放心使用 + # ram_analyzer.py ## 功能介绍 -- Gitee From 79104051b308701c4e7db78253917298c906c302 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Wed, 8 Mar 2023 11:23:18 +0800 Subject: [PATCH 14/14] =?UTF-8?q?[rom=5Fram=5Fanalyzer]=E8=A7=A3=E5=86=B3?= =?UTF-8?q?=E5=91=8A=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/README.md | 2 +- tools/rom_ram_analyzer/standard/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/README.md b/tools/rom_ram_analyzer/lite_small/README.md index 9f2c0b1..790e5db 100644 --- a/tools/rom_ram_analyzer/lite_small/README.md +++ b/tools/rom_ram_analyzer/lite_small/README.md @@ -13,7 +13,7 @@ 1. 扫描BUILD.gn文件,收集各个target的编译产物及其对应的component_name, subsystem_name信息,并存储到config.yaml中的gn_info_file字段指定的json文件中 2. 根据配置文件config.yaml扫描产品的编译产物目录,得到真实的编译产物信息(主要是大小) 3. 用真实的编译产物与从BUILD.gn中收集的信息进行匹配,从而得到编译产物-大小-所属部件的对应信息 -4. 如果匹配失败,会直接利用grep到项目路径下进行模糊搜索,取出现次数最多的BUILD.gn,并根据该BUILD.gn文件去查找子系统和部件 +4. 如果匹配失败,会直接利用grep到项目路径下进行模糊搜索,取出现次数top1的BUILD.gn,并根据该BUILD.gn文件去查找子系统和部件 5. 如果还搜索失败,则将其归属到others ## 说明 diff --git a/tools/rom_ram_analyzer/standard/README.md b/tools/rom_ram_analyzer/standard/README.md index 74e6933..69c6b52 100644 --- a/tools/rom_ram_analyzer/standard/README.md +++ b/tools/rom_ram_analyzer/standard/README.md @@ -70,7 +70,7 @@ } ``` -## 其他说明 +## 附加说明 1. 由于目前standard产品使用的基本都是自定义的template,能够有效收集更多信息,因此相较于lite_small的分析脚本,本脚本能够具有更高的准确率,请放心使用 -- Gitee