From 83be1560ca13bc19eab578eab824a11c44b5cd08 Mon Sep 17 00:00:00 2001 From: yangpan Date: Mon, 8 Jul 2024 09:58:02 +0800 Subject: [PATCH] =?UTF-8?q?bolt=E6=A8=A1=E5=BC=8F=E5=9C=A82203=E4=B8=8A?= =?UTF-8?q?=E8=83=BD=E8=B7=91=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/elfmerge/elf_link_elf.c | 694 ++++++++++++++--------------- src/elfmerge/meson.build | 5 - src/sysboost_loader/binfmt_rto.c | 433 +++++++++++++----- src/sysboost_loader/rto_populate.c | 248 ++++++----- src/sysboostd/bolt.rs | 5 +- 5 files changed, 807 insertions(+), 578 deletions(-) diff --git a/src/elfmerge/elf_link_elf.c b/src/elfmerge/elf_link_elf.c index 7d363dc..742a728 100644 --- a/src/elfmerge/elf_link_elf.c +++ b/src/elfmerge/elf_link_elf.c @@ -1327,353 +1327,353 @@ static void modify_elf_header(elf_link_t *elf_link) elf_set_hugepage(elf_link); } -/* debug modify start */ - -#include -#include -#include -#include -#include -#include -#include - -struct dwarf_unit_header -{ - uint32_t length; - uint16_t version; - uint8_t unit_type; - uint8_t pointer_size; - uint32_t abbrev_offset; -}; - -void check_unit_header(struct dwarf_unit_header *unit_header) -{ - /* - * 32-bit DWARF format's length must < 0xfffffff0, - * we only support 32-bit now. - */ - if (unit_header->length >= 0xfffffff0) - si_panic("64-bit DWARF format is not supported\n"); - - if (unit_header->version != 5) - si_panic("only support DWARF version 5\n"); +// /* debug modify start */ + +// #include +// #include +// #include +// #include +// #include +// #include +// #include + +// struct dwarf_unit_header +// { +// uint32_t length; +// uint16_t version; +// uint8_t unit_type; +// uint8_t pointer_size; +// uint32_t abbrev_offset; +// }; + +// void check_unit_header(struct dwarf_unit_header *unit_header) +// { +// /* +// * 32-bit DWARF format's length must < 0xfffffff0, +// * we only support 32-bit now. +// */ +// if (unit_header->length >= 0xfffffff0) +// si_panic("64-bit DWARF format is not supported\n"); + +// if (unit_header->version != 5) +// si_panic("only support DWARF version 5\n"); - if (unit_header->pointer_size != 8) - si_panic("only support 64-bit target machine\n"); -} - -void check_cu_header(struct dwarf_unit_header *cu_header) -{ - check_unit_header(cu_header); - - if (cu_header->unit_type != DW_UT_compile) - si_panic("current unit_header is not cu_header\n"); -} - -/* modify abbrev offset stored in .debug_info */ -void modify_debug_info_abbrev_offset(elf_link_t *elf_link) -{ - elf_file_t *ef; - uint32_t da_offset = 0; - void *di_base = elf_find_section_ptr_by_name(get_out_ef(elf_link), ".debug_info"); - uint32_t cu_offset = 0; - - foreach_infile(elf_link, ef) { - Elf64_Shdr *di_sec = elf_find_section_by_name(ef, ".debug_info"); - Elf64_Shdr *da_sec = elf_find_section_by_name(ef, ".debug_abbrev"); - uint32_t in_ef_cu_offset = 0; - - while (in_ef_cu_offset < di_sec->sh_size) { - struct dwarf_unit_header *cu_header = di_base + cu_offset; - check_cu_header(cu_header); - cu_header->abbrev_offset += da_offset; - /* - * each cu have additional 4 bytes, - * because length doesn't count itself's space. - */ - cu_offset += cu_header->length + 4; - in_ef_cu_offset += cu_header->length + 4; - } - - da_offset += da_sec->sh_size; - } -} - -Dwarf_Debug dwarf_init(const char *path) -{ - static char true_pathbuf[FILENAME_MAX]; - Dwarf_Debug dbg = 0; - int res; - - res = dwarf_init_path( - path, true_pathbuf, - FILENAME_MAX, DW_GROUPNUMBER_ANY, NULL, - NULL, &dbg, NULL - ); - - if (res != DW_DLV_OK) - return NULL; - - return dbg; -} - -int dwarf_get_first_die_of_next_cu(Dwarf_Debug dbg, Dwarf_Die* first_die) -{ - int ret; - // Dwarf_Error error; - ret = dwarf_next_cu_header_d(dbg, true, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL); - - /* no next cu */ - if (ret == DW_DLV_NO_ENTRY) - return ret; - - if (ret != DW_DLV_OK) { - si_panic("dwarf_next_cu_header_d ERROR, ret: %d\n", ret); - return ret; - } - // else printf("OK\n"); - - ret = dwarf_siblingof_b(dbg, NULL, true, first_die, NULL); - /* - * if there is no entry, dwarf_siblingof_b will return DW_DLV_NO_ENTRY, - * but now we just ignore this condition for quick dev. - */ - if (ret != DW_DLV_OK) { - si_panic("dwarf_siblingof_b ERROR %d\n", ret); - return ret; - } - return ret; -} - -struct dwarf_bias_info { - uint64_t text; - uint64_t debug_str; - uint64_t debug_line_str; -}; - -int dwarf_modify_di_abbrev(Dwarf_Die die, void *di_ptr, struct dwarf_bias_info *bias_info) -{ - Dwarf_Debug dbg = die->di_cu_context->cc_dbg; - Dwarf_Byte_Ptr die_info_end = - _dwarf_calculate_info_section_end_ptr(die->di_cu_context); - void *abbrev_ptr = die->di_debug_ptr; - Dwarf_Error error; - - Dwarf_Unsigned unused = 0; - Dwarf_Unsigned len; - int ret = dwarf_decode_leb128( - abbrev_ptr, &len, &unused, (char *)die_info_end - ); - if (ret != DW_DLV_OK) - return ret; - abbrev_ptr += len; - void *di_base = dbg->de_debug_info.dss_data; - - for (Dwarf_Unsigned i = 0; i < die->di_abbrev_list->abl_abbrev_count; i++) { - Dwarf_Unsigned attr_form = die->di_abbrev_list->abl_form[i]; - Dwarf_Unsigned sov = 0; - int ret; - - /* todo test if this is needed */ - if (attr_form == DW_FORM_implicit_const) { - continue; - } - - ret = _dwarf_get_size_of_val( - dbg, attr_form, - die->di_cu_context->cc_version_stamp, - die->di_cu_context->cc_address_size, - abbrev_ptr, - die->di_cu_context->cc_length_size, - &sov, - die_info_end, - &error - ); - if (ret != DW_DLV_OK) - si_panic("_dwarf_get_size_of_val fail, ret: %d\n", ret); - - uint32_t *dst_ptr = di_ptr + (abbrev_ptr - di_base); - switch (die->di_abbrev_list->abl_form[i]) { - case DW_FORM_addr: - *dst_ptr += bias_info->text; - break; - case DW_FORM_line_strp: - *dst_ptr += bias_info->debug_line_str; - break; - case DW_FORM_strp: - *dst_ptr += bias_info->debug_str; - // printf("offset: %lx, *abbrev_ptr: %x *dst_ptr: %x\n", - // (abbrev_ptr - di_base), - // *(uint32_t *)abbrev_ptr, *dst_ptr); - break; - case DW_FORM_data1: - case DW_FORM_data2: - case DW_FORM_data4: - case DW_FORM_data8: - /* no need to modify */ - break; - case DW_FORM_block2: - case DW_FORM_string: - case DW_FORM_sdata: - case DW_FORM_ref4: - case DW_FORM_implicit_const: - case DW_FORM_exprloc: - case DW_FORM_flag_present: - case DW_FORM_sec_offset: - /* TODO */ - break; - case DW_FORM_block4: - case DW_FORM_block: - case DW_FORM_block1: - case DW_FORM_flag: - case DW_FORM_udata: - case DW_FORM_ref_addr: - case DW_FORM_ref1: - case DW_FORM_ref2: - case DW_FORM_ref8: - case DW_FORM_ref_udata: - case DW_FORM_indirect: - case DW_FORM_strx: - case DW_FORM_addrx: - case DW_FORM_addrx1: - case DW_FORM_addrx2: - case DW_FORM_addrx3: - case DW_FORM_addrx4: - case DW_FORM_ref_sup4: - case DW_FORM_strp_sup: - case DW_FORM_data16: - case DW_FORM_loclistx: - case DW_FORM_rnglistx: - case DW_FORM_ref_sup8: - case DW_FORM_strx1: - case DW_FORM_strx2: - case DW_FORM_strx3: - case DW_FORM_strx4: - case DW_FORM_ref_sig8: - /* not present in bash */ - si_panic("unsupported die FORM 0x%x\n", - die->di_abbrev_list->abl_form[i]); - break; - default: - si_panic("unknown die FORM 0x%x\n", - die->di_abbrev_list->abl_form[i]); - break; - } - abbrev_ptr += sov; - } - - return DW_DLV_OK; -} - -int dwarf_traverse_die(Dwarf_Debug dbg, Dwarf_Die parent_die, - void *di_ptr, struct dwarf_bias_info *bias_info) -{ - Dwarf_Die son_die; - int res; - - dwarf_modify_di_abbrev(parent_die, di_ptr, bias_info); - - res = dwarf_child(parent_die, &son_die, NULL); - while (res == DW_DLV_OK) { - dwarf_traverse_die(dbg, son_die, di_ptr, bias_info); - res = dwarf_siblingof_b(dbg, son_die, true, &son_die, NULL); - } - if (res == DW_DLV_NO_ENTRY) { - // no more child - return DW_DLV_OK; - } else { - printf("dwarf_child or dwarf_siblingof_b ERROR\n"); - return res; - } -} - -void dwarf_traverse_cu(Dwarf_Debug dbg, void *di_ptr, struct dwarf_bias_info *bias_info) -{ - int res = 0; - - Dwarf_Die first_die; - for (;;) { - res = dwarf_get_first_die_of_next_cu(dbg, &first_die); - if (res == DW_DLV_NO_ENTRY) { - /* no entry */ - break; - } - dwarf_traverse_die(dbg, first_die, di_ptr, bias_info); - } -} - -/* delete it later */ -char *temp_get_file_name(char *name) -{ - char *result = malloc(strlen(name)); - memset(result, 0, strlen(name)); - memcpy(result, name, strlen(name) - 11); - return result; -} - -void prep_bias_info(elf_link_t *elf_link, elf_file_t *ef, struct dwarf_bias_info *bias_info) -{ - /* .text starts from .init */ - unsigned long text_base_addr = - elf_find_section_by_name(ef, ".init")->sh_addr; - unsigned long ds_base_offset = - elf_find_section_by_name(get_out_ef(elf_link), ".debug_str")->sh_offset; - unsigned long dls_base_offset = - elf_find_section_by_name(get_out_ef(elf_link), ".debug_line_str")->sh_offset; - - Elf64_Shdr *text_sec = elf_find_section_by_name(ef, ".init"); - unsigned long text_addr = get_new_addr_by_old_addr( - elf_link, ef, text_sec->sh_addr - ); - Elf64_Shdr *ds_sec = elf_find_section_by_name(ef, ".debug_str"); - unsigned long ds_offset = get_new_offset_by_old_offset( - elf_link, ef, ds_sec->sh_offset - ); - Elf64_Shdr *dls_sec = elf_find_section_by_name(ef, ".debug_line_str"); - unsigned long dls_offset = get_new_offset_by_old_offset( - elf_link, ef, dls_sec->sh_offset - ); - - bias_info->text = text_addr - text_base_addr; - bias_info->debug_str = ds_offset - ds_base_offset; - bias_info->debug_line_str = dls_offset - dls_base_offset; - SI_LOG_DEBUG("%s, text: %lx, debug_str: %lx, debug_line_str: %lx\n", - ef->file_name, - bias_info->text, bias_info->debug_str, bias_info->debug_line_str); - SI_LOG_DEBUG("text_addr: %lx, out_text_base_addr: %lx\n", - text_addr, text_base_addr); -} - -int modify_debug_info(elf_link_t *elf_link) -{ - elf_file_t *ef; - char *temp_path; - struct dwarf_bias_info bias_info; - - foreach_infile(elf_link, ef) { - Elf64_Shdr *di_sec = elf_find_section_by_name(ef, ".debug_info"); - unsigned long dst_offset = get_new_offset_by_old_offset( - elf_link, ef, di_sec->sh_offset - ); - void *di_ptr = get_out_ef(elf_link)->data + dst_offset; - prep_bias_info(elf_link, ef, &bias_info); - - temp_path = temp_get_file_name(ef->file_name); - Dwarf_Debug dbg = dwarf_init(temp_path); - if (!dbg) - si_panic("dwarf_init fail, file: %s\n", temp_path); - dwarf_traverse_cu(dbg, di_ptr, &bias_info); - dwarf_finish(dbg); - } - return 0; -} - -static void modify_debug(elf_link_t *elf_link) -{ - modify_debug_info_abbrev_offset(elf_link); - modify_debug_info(elf_link); -} +// if (unit_header->pointer_size != 8) +// si_panic("only support 64-bit target machine\n"); +// } + +// void check_cu_header(struct dwarf_unit_header *cu_header) +// { +// check_unit_header(cu_header); + +// if (cu_header->unit_type != DW_UT_compile) +// si_panic("current unit_header is not cu_header\n"); +// } + +// /* modify abbrev offset stored in .debug_info */ +// void modify_debug_info_abbrev_offset(elf_link_t *elf_link) +// { +// elf_file_t *ef; +// uint32_t da_offset = 0; +// void *di_base = elf_find_section_ptr_by_name(get_out_ef(elf_link), ".debug_info"); +// uint32_t cu_offset = 0; + +// foreach_infile(elf_link, ef) { +// Elf64_Shdr *di_sec = elf_find_section_by_name(ef, ".debug_info"); +// Elf64_Shdr *da_sec = elf_find_section_by_name(ef, ".debug_abbrev"); +// uint32_t in_ef_cu_offset = 0; + +// while (in_ef_cu_offset < di_sec->sh_size) { +// struct dwarf_unit_header *cu_header = di_base + cu_offset; +// check_cu_header(cu_header); +// cu_header->abbrev_offset += da_offset; +// /* +// * each cu have additional 4 bytes, +// * because length doesn't count itself's space. +// */ +// cu_offset += cu_header->length + 4; +// in_ef_cu_offset += cu_header->length + 4; +// } + +// da_offset += da_sec->sh_size; +// } +// } + +// Dwarf_Debug dwarf_init(const char *path) +// { +// static char true_pathbuf[FILENAME_MAX]; +// Dwarf_Debug dbg = 0; +// int res; + +// res = dwarf_init_path( +// path, true_pathbuf, +// FILENAME_MAX, DW_GROUPNUMBER_ANY, NULL, +// NULL, &dbg, NULL +// ); + +// if (res != DW_DLV_OK) +// return NULL; + +// return dbg; +// } + +// int dwarf_get_first_die_of_next_cu(Dwarf_Debug dbg, Dwarf_Die* first_die) +// { +// int ret; +// // Dwarf_Error error; +// ret = dwarf_next_cu_header_d(dbg, true, NULL, NULL, NULL, NULL, +// NULL, NULL, NULL, NULL, NULL, NULL, NULL); + +// /* no next cu */ +// if (ret == DW_DLV_NO_ENTRY) +// return ret; + +// if (ret != DW_DLV_OK) { +// si_panic("dwarf_next_cu_header_d ERROR, ret: %d\n", ret); +// return ret; +// } +// // else printf("OK\n"); + +// ret = dwarf_siblingof_b(dbg, NULL, true, first_die, NULL); +// /* +// * if there is no entry, dwarf_siblingof_b will return DW_DLV_NO_ENTRY, +// * but now we just ignore this condition for quick dev. +// */ +// if (ret != DW_DLV_OK) { +// si_panic("dwarf_siblingof_b ERROR %d\n", ret); +// return ret; +// } +// return ret; +// } + +// struct dwarf_bias_info { +// uint64_t text; +// uint64_t debug_str; +// uint64_t debug_line_str; +// }; + +// int dwarf_modify_di_abbrev(Dwarf_Die die, void *di_ptr, struct dwarf_bias_info *bias_info) +// { +// Dwarf_Debug dbg = die->di_cu_context->cc_dbg; +// Dwarf_Byte_Ptr die_info_end = +// _dwarf_calculate_info_section_end_ptr(die->di_cu_context); +// void *abbrev_ptr = die->di_debug_ptr; +// Dwarf_Error error; + +// Dwarf_Unsigned unused = 0; +// Dwarf_Unsigned len; +// int ret = dwarf_decode_leb128( +// abbrev_ptr, &len, &unused, (char *)die_info_end +// ); +// if (ret != DW_DLV_OK) +// return ret; +// abbrev_ptr += len; +// void *di_base = dbg->de_debug_info.dss_data; + +// for (Dwarf_Unsigned i = 0; i < die->di_abbrev_list->abl_abbrev_count; i++) { +// Dwarf_Unsigned attr_form = die->di_abbrev_list->abl_form[i]; +// Dwarf_Unsigned sov = 0; +// int ret; + +// /* todo test if this is needed */ +// if (attr_form == DW_FORM_implicit_const) { +// continue; +// } + +// ret = _dwarf_get_size_of_val( +// dbg, attr_form, +// die->di_cu_context->cc_version_stamp, +// die->di_cu_context->cc_address_size, +// abbrev_ptr, +// die->di_cu_context->cc_length_size, +// &sov, +// die_info_end, +// &error +// ); +// if (ret != DW_DLV_OK) +// si_panic("_dwarf_get_size_of_val fail, ret: %d\n", ret); + +// uint32_t *dst_ptr = di_ptr + (abbrev_ptr - di_base); +// switch (die->di_abbrev_list->abl_form[i]) { +// case DW_FORM_addr: +// *dst_ptr += bias_info->text; +// break; +// case DW_FORM_line_strp: +// *dst_ptr += bias_info->debug_line_str; +// break; +// case DW_FORM_strp: +// *dst_ptr += bias_info->debug_str; +// // printf("offset: %lx, *abbrev_ptr: %x *dst_ptr: %x\n", +// // (abbrev_ptr - di_base), +// // *(uint32_t *)abbrev_ptr, *dst_ptr); +// break; +// case DW_FORM_data1: +// case DW_FORM_data2: +// case DW_FORM_data4: +// case DW_FORM_data8: +// /* no need to modify */ +// break; +// case DW_FORM_block2: +// case DW_FORM_string: +// case DW_FORM_sdata: +// case DW_FORM_ref4: +// case DW_FORM_implicit_const: +// case DW_FORM_exprloc: +// case DW_FORM_flag_present: +// case DW_FORM_sec_offset: +// /* TODO */ +// break; +// case DW_FORM_block4: +// case DW_FORM_block: +// case DW_FORM_block1: +// case DW_FORM_flag: +// case DW_FORM_udata: +// case DW_FORM_ref_addr: +// case DW_FORM_ref1: +// case DW_FORM_ref2: +// case DW_FORM_ref8: +// case DW_FORM_ref_udata: +// case DW_FORM_indirect: +// case DW_FORM_strx: +// case DW_FORM_addrx: +// case DW_FORM_addrx1: +// case DW_FORM_addrx2: +// case DW_FORM_addrx3: +// case DW_FORM_addrx4: +// case DW_FORM_ref_sup4: +// case DW_FORM_strp_sup: +// case DW_FORM_data16: +// case DW_FORM_loclistx: +// case DW_FORM_rnglistx: +// case DW_FORM_ref_sup8: +// case DW_FORM_strx1: +// case DW_FORM_strx2: +// case DW_FORM_strx3: +// case DW_FORM_strx4: +// case DW_FORM_ref_sig8: +// /* not present in bash */ +// si_panic("unsupported die FORM 0x%x\n", +// die->di_abbrev_list->abl_form[i]); +// break; +// default: +// si_panic("unknown die FORM 0x%x\n", +// die->di_abbrev_list->abl_form[i]); +// break; +// } +// abbrev_ptr += sov; +// } + +// return DW_DLV_OK; +// } + +// int dwarf_traverse_die(Dwarf_Debug dbg, Dwarf_Die parent_die, +// void *di_ptr, struct dwarf_bias_info *bias_info) +// { +// Dwarf_Die son_die; +// int res; + +// dwarf_modify_di_abbrev(parent_die, di_ptr, bias_info); + +// res = dwarf_child(parent_die, &son_die, NULL); +// while (res == DW_DLV_OK) { +// dwarf_traverse_die(dbg, son_die, di_ptr, bias_info); +// res = dwarf_siblingof_b(dbg, son_die, true, &son_die, NULL); +// } +// if (res == DW_DLV_NO_ENTRY) { +// // no more child +// return DW_DLV_OK; +// } else { +// printf("dwarf_child or dwarf_siblingof_b ERROR\n"); +// return res; +// } +// } + +// void dwarf_traverse_cu(Dwarf_Debug dbg, void *di_ptr, struct dwarf_bias_info *bias_info) +// { +// int res = 0; + +// Dwarf_Die first_die; +// for (;;) { +// res = dwarf_get_first_die_of_next_cu(dbg, &first_die); +// if (res == DW_DLV_NO_ENTRY) { +// /* no entry */ +// break; +// } +// dwarf_traverse_die(dbg, first_die, di_ptr, bias_info); +// } +// } + +// /* delete it later */ +// char *temp_get_file_name(char *name) +// { +// char *result = malloc(strlen(name)); +// memset(result, 0, strlen(name)); +// memcpy(result, name, strlen(name) - 11); +// return result; +// } + +// void prep_bias_info(elf_link_t *elf_link, elf_file_t *ef, struct dwarf_bias_info *bias_info) +// { +// /* .text starts from .init */ +// unsigned long text_base_addr = +// elf_find_section_by_name(ef, ".init")->sh_addr; +// unsigned long ds_base_offset = +// elf_find_section_by_name(get_out_ef(elf_link), ".debug_str")->sh_offset; +// unsigned long dls_base_offset = +// elf_find_section_by_name(get_out_ef(elf_link), ".debug_line_str")->sh_offset; + +// Elf64_Shdr *text_sec = elf_find_section_by_name(ef, ".init"); +// unsigned long text_addr = get_new_addr_by_old_addr( +// elf_link, ef, text_sec->sh_addr +// ); +// Elf64_Shdr *ds_sec = elf_find_section_by_name(ef, ".debug_str"); +// unsigned long ds_offset = get_new_offset_by_old_offset( +// elf_link, ef, ds_sec->sh_offset +// ); +// Elf64_Shdr *dls_sec = elf_find_section_by_name(ef, ".debug_line_str"); +// unsigned long dls_offset = get_new_offset_by_old_offset( +// elf_link, ef, dls_sec->sh_offset +// ); + +// bias_info->text = text_addr - text_base_addr; +// bias_info->debug_str = ds_offset - ds_base_offset; +// bias_info->debug_line_str = dls_offset - dls_base_offset; +// SI_LOG_DEBUG("%s, text: %lx, debug_str: %lx, debug_line_str: %lx\n", +// ef->file_name, +// bias_info->text, bias_info->debug_str, bias_info->debug_line_str); +// SI_LOG_DEBUG("text_addr: %lx, out_text_base_addr: %lx\n", +// text_addr, text_base_addr); +// } + +// int modify_debug_info(elf_link_t *elf_link) +// { +// elf_file_t *ef; +// char *temp_path; +// struct dwarf_bias_info bias_info; + +// foreach_infile(elf_link, ef) { +// Elf64_Shdr *di_sec = elf_find_section_by_name(ef, ".debug_info"); +// unsigned long dst_offset = get_new_offset_by_old_offset( +// elf_link, ef, di_sec->sh_offset +// ); +// void *di_ptr = get_out_ef(elf_link)->data + dst_offset; +// prep_bias_info(elf_link, ef, &bias_info); + +// temp_path = temp_get_file_name(ef->file_name); +// Dwarf_Debug dbg = dwarf_init(temp_path); +// if (!dbg) +// si_panic("dwarf_init fail, file: %s\n", temp_path); +// dwarf_traverse_cu(dbg, di_ptr, &bias_info); +// dwarf_finish(dbg); +// } +// return 0; +// } + +// static void modify_debug(elf_link_t *elf_link) +// { +// modify_debug_info_abbrev_offset(elf_link); +// modify_debug_info(elf_link); +// } /* debug modify end */ @@ -1867,7 +1867,7 @@ int elf_link_write(elf_link_t *elf_link) // .rela.init .rela.text .rela.rodata .rela.tdata .rela.init_array .rela.data modify_local_call(elf_link); - modify_debug(elf_link); + //modify_debug(elf_link); // modify ELF header and write sections modify_elf_header(elf_link); diff --git a/src/elfmerge/meson.build b/src/elfmerge/meson.build index d425113..1e0d0a9 100644 --- a/src/elfmerge/meson.build +++ b/src/elfmerge/meson.build @@ -18,11 +18,6 @@ core_sources = files( cflags += ['-fpic', '-pie'] -default_ldflags += ['/usr/lib64/libdwarf.a'] - -includes += '/usr/src/debug/libdwarf-0.7.0-1.oe2309.aarch64/src/lib/libdwarf/' -includes += '/usr/include/libdwarf-0/' - executable( 'elfmerge', core_sources, install: true, diff --git a/src/sysboost_loader/binfmt_rto.c b/src/sysboost_loader/binfmt_rto.c index dd7abe3..b4ab845 100644 --- a/src/sysboost_loader/binfmt_rto.c +++ b/src/sysboost_loader/binfmt_rto.c @@ -10,6 +10,7 @@ * Copyright 1993, 1994: Eric Youngdale (ericy@cais.com). */ +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -51,8 +54,8 @@ #include #ifdef CONFIG_X86 #include -/* x86, 22.03 LTS map_vdso is undefine */ -extern int map_vdso(const struct vdso_image *image, unsigned long addr); +// /* x86, 22.03 LTS map_vdso is undefine */ +// extern int map_vdso(const struct vdso_image *image, unsigned long addr); #endif #include "main.h" #include "binfmt_rto.h" @@ -62,13 +65,14 @@ extern int map_vdso(const struct vdso_image *image, unsigned long addr); #endif #ifdef CONFIG_ELF_SYSBOOST -#include "../elf_ext.h" - -/* compat 22.03 LTS, 22.03 LTS SP2 */ -#ifndef MM_SAVED_AUXV -#define MM_SAVED_AUXV(mm) mm->saved_auxv +#include "../elfmerge/elf_ext.h" #endif +// /* compat 22.03 LTS, 22.03 LTS SP2 */ +// #ifndef MM_SAVED_AUXV +// #define MM_SAVED_AUXV(mm) mm->saved_auxv +// #endif + #define proc_symbol(SYM) typeof(SYM) *(SYM) static struct global_symbols { #ifdef CONFIG_ARM64 @@ -98,6 +102,11 @@ static struct global_symbols { proc_symbol(task_cputime); proc_symbol(thread_group_cputime); proc_symbol(do_mm_populate); + // proc_symbol(elf_core_extra_phdrs); + // proc_symbol(elf_core_extra_data_size); + // proc_symbol(elf_core_write_extra_phdrs); + // proc_symbol(elf_core_write_extra_data); + proc_symbol(get_mm_exe_file); } rto_sym; #define proc_symbol_char(x) #x @@ -128,6 +137,11 @@ static char *global_symbol_names[] = { proc_symbol_char(task_cputime), proc_symbol_char(thread_group_cputime), proc_symbol_char(do_mm_populate), + // proc_symbol_char(elf_core_extra_phdrs), + // proc_symbol_char(elf_core_extra_data_size), + // proc_symbol_char(elf_core_write_extra_phdrs), + // proc_symbol_char(elf_core_write_extra_data), + proc_symbol_char(get_mm_exe_file), }; static int init_symbols(void) @@ -142,20 +156,6 @@ static int init_symbols(void) return 0; } -#ifdef ELF_HWCAP -#undef ELF_HWCAP -#define ELF_HWCAP (__cpu_get_elf_hwcap()) -static inline unsigned long __cpu_get_elf_hwcap(void) -{ -#ifdef CONFIG_ARM64 - return rto_sym.cpu_get_elf_hwcap(); -#else - // x86 boot_cpu_data is export - return (boot_cpu_data.x86_capability[CPUID_1_EDX]); -#endif -} -#endif - #ifdef ELF_HWCAP2 #undef ELF_HWCAP2 #define ELF_HWCAP2 (__cpu_get_elf_hwcap2()) @@ -170,6 +170,38 @@ static inline unsigned long __cpu_get_elf_hwcap2(void) } #endif +#ifdef CONFIG_ARM64 +#ifdef start_thread +#undef start_thread +#endif + +#define start_thread ___start_thread + +// arm64 start_thread is inline function, so copy it +static inline void ___start_thread(struct pt_regs *regs, unsigned long pc, + unsigned long sp) +{ + start_thread_common(regs, pc); + regs->pstate = PSR_MODE_EL0t; + rto_sym.spectre_v4_enable_task_mitigation(current); + regs->sp = sp; +} +#endif /* CONFIG_ARM64 */ + +#ifdef ELF_HWCAP +#undef ELF_HWCAP +#define ELF_HWCAP (__cpu_get_elf_hwcap()) +static inline unsigned long __cpu_get_elf_hwcap(void) +{ +#ifdef CONFIG_ARM64 + return rto_sym.cpu_get_elf_hwcap(); +#else + // x86 boot_cpu_data is export + return (boot_cpu_data.x86_capability[CPUID_1_EDX]); +#endif +} +#endif + #ifdef ARCH_DLINFO #undef ARCH_DLINFO #endif @@ -194,11 +226,11 @@ do { \ // TODO: vdso layout for ARM64 #define __arch_setup_additional_pages(bprm, uses_interp, load_bias, is_rto_format) (rto_sym.arch_setup_additional_pages(bprm, uses_interp)) -#ifdef arch_elf_adjust_prot -#undef arch_elf_adjust_prot -#endif +// #ifdef arch_elf_adjust_prot +// #undef arch_elf_adjust_prot +// #endif -#define arch_elf_adjust_prot rto_sym.arch_elf_adjust_prot +// #define arch_elf_adjust_prot rto_sym.arch_elf_adjust_prot #else // x86 @@ -244,6 +276,21 @@ int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp, un #endif + +#ifdef CONFIG_ELF_SYSBOOST + +#define AT_RSEQ_FEATURE_SIZE 27 /* rseq supported feature size */ +#define AT_RSEQ_ALIGN 28 /* rseq allocation alignment */ + +/* compat 22.03 LTS, 22.03 LTS SP2 */ +#ifndef MM_SAVED_AUXV +#define MM_SAVED_AUXV(mm) mm->saved_auxv +#endif + +#ifdef CONFIG_X86 +extern int map_vdso(const struct vdso_image *image, unsigned long addr); +#endif + #endif /* CONFIG_ELF_SYSBOOST */ #ifndef ELF_COMPAT @@ -280,6 +327,12 @@ static int elf_core_dump(struct coredump_params *cprm); #define elf_core_dump NULL #endif +#if ELF_EXEC_PAGESIZE > PAGE_SIZE +#define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE +#else +#define ELF_MIN_ALIGN PAGE_SIZE +#endif + #ifndef ELF_CORE_EFLAGS #define ELF_CORE_EFLAGS 0 #endif @@ -288,8 +341,10 @@ static struct linux_binfmt elf_format = { .module = THIS_MODULE, .load_binary = load_rto_binary, .load_shlib = load_elf_library, +#ifdef CONFIG_COREDUMP .core_dump = elf_core_dump, .min_coredump = ELF_EXEC_PAGESIZE, +#endif }; #define BAD_ADDR(x) (unlikely((unsigned long)(x) >= TASK_SIZE)) @@ -389,6 +444,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, unsigned char k_rand_bytes[16]; int items; elf_addr_t *elf_info; + elf_addr_t flags = 0; int ei_index; const struct cred *cred = current_cred(); struct vm_area_struct *vma; @@ -560,15 +616,25 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, return 0; } +#ifdef CONFIG_ELF_SYSBOOST static unsigned long elf_map(struct file *filep, unsigned long addr, const struct elf_phdr *eppnt, int prot, int type, unsigned long total_size, bool use_pmd_mapping, bool is_exec_seg) +#else +static unsigned long elf_map(struct file *filep, unsigned long addr, + const struct elf_phdr *eppnt, int prot, int type, + unsigned long total_size) +#endif { - unsigned long map_addr, size, off; + unsigned long map_addr; + +#ifdef CONFIG_ELF_SYSBOOST + unsigned long size, off; if (use_pmd_mapping) { size = eppnt->p_filesz + ELF_HPAGEOFFSET(eppnt->p_vaddr); off = eppnt->p_offset - ELF_HPAGEOFFSET(eppnt->p_vaddr); + pr_info("vm_mmap, addr: %lx, size: %lx, off: %lx", addr, size, off); addr = ELF_HPAGESTART(addr); if (is_exec_seg) size = ELF_HPAGEALIGN(size); @@ -580,6 +646,12 @@ static unsigned long elf_map(struct file *filep, unsigned long addr, addr = ELF_PAGESTART(addr); size = ELF_PAGEALIGN(size); } +#else + unsigned long size = eppnt->p_filesz + ELF_PAGEOFFSET(eppnt->p_vaddr); + unsigned long off = eppnt->p_offset - ELF_PAGEOFFSET(eppnt->p_vaddr); + addr = ELF_PAGESTART(addr); + size = ELF_PAGEALIGN(size); +#endif /* mmap() will return -EINVAL if given a zero size, but a * segment with zero filesize is perfectly valid */ @@ -595,21 +667,24 @@ static unsigned long elf_map(struct file *filep, unsigned long addr, * the end. (which unmap is needed for ELF images with holes.) */ if (total_size) { - if (use_pmd_mapping) - total_size = ELF_HPAGEALIGN(total_size); - else + // if (use_pmd_mapping) + // total_size = ELF_HPAGEALIGN(total_size); + // else total_size = ELF_PAGEALIGN(total_size); if (debug) pr_info("vm_mmap, addr: %lx, total_size: %lx, off: %lx", addr, total_size, off); map_addr = vm_mmap(filep, addr, total_size, prot, type, off); - if (!BAD_ADDR(map_addr)) + if (!BAD_ADDR(map_addr)) { vm_munmap(map_addr+size, total_size-size); + pr_info("vm_mmap total_size, map_addr: %lx", map_addr); + } } else { if (debug) pr_info("vm_mmap, addr: %lx, size: %lx, off: %lx", addr, size, off); map_addr = vm_mmap(filep, addr, size, prot, type, off); + pr_info("vm_mmap size, map_addr: %lx", map_addr); } if ((type & MAP_FIXED_NOREPLACE) && @@ -805,7 +880,7 @@ static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, if (p_flags & PF_X) prot |= PROT_EXEC; - return arch_elf_adjust_prot(prot, arch_state, has_interp, is_interp); + return rto_sym.arch_elf_adjust_prot(prot, arch_state, has_interp, is_interp); } /* This is much more generalized than the library routine read function, @@ -813,10 +888,17 @@ static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, is only provided so that we can read a.out libraries that have an ELF header */ +#ifdef CONFIG_ELF_SYSBOOST static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex, struct file *interpreter, unsigned long no_base, struct elf_phdr *interp_elf_phdata, struct arch_elf_state *arch_state, bool is_rto_format) +#else +static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex, + struct file *interpreter, + unsigned long no_base, struct elf_phdr *interp_elf_phdata, + struct arch_elf_state *arch_state) +#endif { struct elf_phdr *eppnt; unsigned long load_addr = 0; @@ -872,9 +954,13 @@ static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex, else if (no_base && interp_elf_ex->e_type == ET_DYN) load_addr = -vaddr; +#ifdef CONFIG_ELF_SYSBOOST map_addr = elf_map(interpreter, load_addr + vaddr, eppnt, elf_prot, elf_type, total_size, false, false); - +#else + map_addr = elf_map(interpreter, load_addr + vaddr, + eppnt, elf_prot, elf_type, total_size); +#endif total_size = 0; error = map_addr; if (BAD_ADDR(map_addr)) @@ -1060,40 +1146,40 @@ static int parse_elf_properties(struct file *f, const struct elf_phdr *phdr, } #ifdef CONFIG_ELF_SYSBOOST -struct file *try_get_rto_file(struct file *file) -{ - char *buffer, *rto_path; - struct file *rto_file; - - buffer = kmalloc(PATH_MAX, GFP_KERNEL); - rto_path = file_path(file, buffer, PATH_MAX - 5); - strcat(rto_path, ".rto"); - rto_file = open_exec(rto_path); - - kfree(buffer); - return rto_file; -} - -void *load_bprm_buf(struct file *file) -{ - ssize_t ret; - char *buffer; - loff_t pos = 0; - - buffer = kmalloc(BINPRM_BUF_SIZE, GFP_KERNEL); - if (!buffer) - return ERR_PTR(-ENOMEM); - - ret = kernel_read(file, buffer, BINPRM_BUF_SIZE, &pos); - if (ret != BINPRM_BUF_SIZE) { - kfree(buffer); - if (ret < 0) - return ERR_PTR(ret); - return ERR_PTR(-EIO); - } - - return buffer; -} +// struct file *try_get_rto_file(struct file *file) +// { +// char *buffer, *rto_path; +// struct file *rto_file; + +// buffer = kmalloc(PATH_MAX, GFP_KERNEL); +// rto_path = file_path(file, buffer, PATH_MAX - 5); +// strcat(rto_path, ".rto"); +// rto_file = open_exec(rto_path); + +// kfree(buffer); +// return rto_file; +// } + +// void *load_bprm_buf(struct file *file) +// { +// ssize_t ret; +// char *buffer; +// loff_t pos = 0; + +// buffer = kmalloc(BINPRM_BUF_SIZE, GFP_KERNEL); +// if (!buffer) +// return ERR_PTR(-ENOMEM); + +// ret = kernel_read(file, buffer, BINPRM_BUF_SIZE, &pos); +// if (ret != BINPRM_BUF_SIZE) { +// kfree(buffer); +// if (ret < 0) +// return ERR_PTR(ret); +// return ERR_PTR(-EIO); +// } + +// return buffer; +// } static int prepare_rto(struct linux_binprm *bprm) { @@ -1114,14 +1200,17 @@ static inline int try_replace_file(struct linux_binprm *bprm) int ret; rto_file = try_get_rto_file(bprm->file); - if (IS_ERR(rto_file)) + if (IS_ERR(rto_file)) { + pr_info("try_get_rto_file fail %ld\n", PTR_ERR(rto_file)); return PTR_ERR(rto_file); + } original_file = bprm->file; bprm->file = rto_file; ret = prepare_rto(bprm); if (ret) { bprm->file = original_file; + pr_info("prepare_rto fail %d\n", ret); return ret; } @@ -1129,23 +1218,23 @@ static inline int try_replace_file(struct linux_binprm *bprm) return 0; } -#ifdef CONFIG_ARM64 -#ifdef start_thread -#undef start_thread -#endif +// #ifdef CONFIG_ARM64 +// #ifdef start_thread +// #undef start_thread +// #endif -#define start_thread ___start_thread +// #define start_thread ___start_thread -// arm64 start_thread is inline function, so copy it -static inline void ___start_thread(struct pt_regs *regs, unsigned long pc, - unsigned long sp) -{ - start_thread_common(regs, pc); - regs->pstate = PSR_MODE_EL0t; - rto_sym.spectre_v4_enable_task_mitigation(current); - regs->sp = sp; -} -#endif /* CONFIG_ARM64 */ +// // arm64 start_thread is inline function, so copy it +// static inline void ___start_thread(struct pt_regs *regs, unsigned long pc, +// unsigned long sp) +// { +// start_thread_common(regs, pc); +// regs->pstate = PSR_MODE_EL0t; +// rto_sym.spectre_v4_enable_task_mitigation(current); +// regs->sp = sp; +// } +// #endif /* CONFIG_ARM64 */ #endif /* CONFIG_ELF_SYSBOOST */ @@ -1177,7 +1266,7 @@ static int load_rto_binary(struct linux_binprm *bprm) unsigned long start_code, end_code, start_data, end_data; unsigned long reloc_func_desc __maybe_unused = 0; int executable_stack = EXSTACK_DEFAULT; - struct elfhdr *elf_ex; + struct elfhdr *elf_ex = (struct elfhdr *)bprm->buf; struct elfhdr *interp_elf_ex = NULL; struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE; struct mm_struct *mm; @@ -1190,36 +1279,51 @@ static int load_rto_binary(struct linux_binprm *bprm) struct loaded_rto *loaded_rto = NULL; struct list_head *preload_seg_pos = NULL; struct loaded_seg *loaded_seg; + bool using_hpage = false; +#endif + // TODO check rto inode! -load_rto: - elf_ex = (struct elfhdr *)bprm->buf; - is_rto_format = elf_ex->e_flags & OS_SPECIFIC_FLAG_RTO; - is_rto_symbolic_link = IS_SYSBOOST_RTO_SYMBOLIC_LINK(bprm->file->f_inode); +// load_rto: +// elf_ex = (struct elfhdr *)bprm->buf; +// is_rto_format = elf_ex->e_flags & OS_SPECIFIC_FLAG_RTO; +// is_rto_symbolic_link = IS_SYSBOOST_RTO_SYMBOLIC_LINK(bprm->file->f_inode); retval = -ENOEXEC; +#ifdef CONFIG_ELF_SYSBOOST /* close feature to rmmod this ko */ if (!use_rto) { goto out; } + +load_rto: + is_rto_symbolic_link = IS_SYSBOOST_RTO_SYMBOLIC_LINK(bprm->file->f_inode); + elf_ex = (struct elfhdr *)bprm->buf; + is_rto_format = elf_ex->e_flags & OS_SPECIFIC_FLAG_RTO; if (!is_rto_format && !is_rto_symbolic_link) { goto out; } /* replace app.rto file, then use binfmt */ - if (is_rto_symbolic_link) { - // struct inode *inode = bprm->file->f_inode; + // if (is_rto_symbolic_link) { + if (is_rto_symbolic_link && !is_rto_format) { + struct inode *inode = bprm->file->f_inode; int ret; - if (use_hpage) + + if (use_hpage) { loaded_rto = find_loaded_rto(bprm->file->f_inode); + using_hpage = true; + } ret = try_replace_file(bprm); if (ret) { /* limit print */ - printk("replace rto file fail, %d\n", ret); + pr_info("replace rto file fail, %d\n", ret); goto out; } - // pr_info("replace rto file success, loaded_rto: 0x%lx, inode: 0x%lx\n", - // loaded_rto, inode); + if (debug) { + pr_info("replace rto success, loaded_rto: %pK, inode: %pK\n", + loaded_rto, inode); + } goto load_rto; } @@ -1300,7 +1404,7 @@ load_rto: interp_elf_ex = kmalloc(sizeof(*interp_elf_ex), GFP_KERNEL); if (!interp_elf_ex) { retval = -ENOMEM; - goto out_free_ph; + goto out_free_file; } /* Get the exec headers */ @@ -1409,7 +1513,7 @@ out_free_interp: executable_stack); if (retval < 0) goto out_free_dentry; - + elf_bss = 0; elf_brk = 0; @@ -1418,25 +1522,29 @@ out_free_interp: start_data = 0; end_data = 0; +#ifdef CONFIG_ELF_SYSBOOST if (loaded_rto) preload_seg_pos = &loaded_rto->segs; +#endif /* Now we do a little grungy work by mmapping the ELF image into the correct location in memory. */ for(i = 0, elf_ppnt = elf_phdata; i < elf_ex->e_phnum; i++, elf_ppnt++) { - bool is_exec_seg = elf_ppnt->p_flags & PF_X; int elf_prot, elf_flags; unsigned long k, vaddr; unsigned long total_size = 0; unsigned long alignment; +#ifdef CONFIG_ELF_SYSBOOST unsigned long size, off; + bool is_exec_seg = !(elf_ppnt->p_flags & PF_W); +#endif if (elf_ppnt->p_type != PT_LOAD) continue; if (unlikely (elf_brk > elf_bss)) { unsigned long nbyte; - + /* There was a PT_LOAD segment with p_memsz > p_filesz before this one. Map anonymous pages, if needed, and clear the area. */ @@ -1522,11 +1630,37 @@ out_free_interp: * ELF vaddrs will be correctly offset. The result * is then page aligned. */ - // if (use_hpage) +#ifdef CONFIG_ELF_SYSBOOST + if (using_hpage) load_bias = ELF_HPAGESTART(load_bias - vaddr); + else + load_bias = ELF_PAGESTART(load_bias - vaddr); +#else + load_bias = ELF_PAGESTART(load_bias - vaddr); +#endif + // if (use_hpage) + // load_bias = ELF_HPAGESTART(load_bias - vaddr); // else // load_bias = ELF_PAGESTART(load_bias - vaddr); + /* + * Calculate the entire size of the ELF mapping + * (total_size), used for the initial mapping, + * due to load_addr_set which is set to true later + * once the initial mapping is performed. + * + * Note that this is only sensible when the LOAD + * segments are contiguous (or overlapping). If + * used for LOADs that are far apart, this would + * cause the holes between LOADs to be mapped, + * running the risk of having the mapping fail, + * as it would be larger than the ELF file itself. + * + * As a result, only ET_DYN does this, since + * some ET_EXEC (e.g. ia64) may have large virtual + * memory holes between LOADs. + * + */ total_size = total_mapping_size(elf_phdata, elf_ex->e_phnum); if (!total_size) { @@ -1535,17 +1669,24 @@ out_free_interp: } } +#ifdef CONFIG_ELF_SYSBOOST + error = elf_map(bprm->file, load_bias + vaddr, elf_ppnt, + elf_prot, elf_flags, total_size, using_hpage, is_exec_seg); +#else error = elf_map(bprm->file, load_bias + vaddr, elf_ppnt, - elf_prot, elf_flags, total_size, true, is_exec_seg); - // elf_prot, elf_flags, 0, use_hpage); + elf_prot, elf_flags, total_size); +#endif if (BAD_ADDR(error)) { +#ifdef CONFIG_ELF_SYSBOOST if (debug) - pr_info("lyt elf_map error: %ld\n", PTR_ERR((void*)error)); - retval = IS_ERR((void *)error) ? + pr_info("elf_map error: %ld\n", PTR_ERR((void*)error)); +#endif + retval = IS_ERR_VALUE(error) ? PTR_ERR((void*)error) : -EINVAL; goto out_free_dentry; } - if (use_hpage && preload_seg_pos) { +#ifdef CONFIG_ELF_SYSBOOST + if (using_hpage && preload_seg_pos) { preload_seg_pos = preload_seg_pos->next; BUG_ON(preload_seg_pos == &loaded_rto->segs); loaded_seg = list_entry(preload_seg_pos, @@ -1553,20 +1694,34 @@ out_free_interp: size = elf_ppnt->p_filesz + ELF_HPAGEOFFSET(elf_ppnt->p_vaddr); off = elf_ppnt->p_offset - ELF_HPAGEOFFSET(elf_ppnt->p_vaddr); size = ELF_HPAGEALIGN(size); - if (debug) - pr_info("lyt vaddr: 0x%lx, off: 0x%lx, size: 0x%lx\n", + if (debug){ + pr_info("loaded_seg: %pK\n", loaded_seg); + pr_info("elf_map vaddr: 0x%lx, off: 0x%lx, size: 0x%lx\n", error, off, size); + } if (is_exec_seg) rto_populate(bprm->file, error, off, size, loaded_seg); } +#endif if (!load_addr_set) { load_addr_set = 1; load_addr = (elf_ppnt->p_vaddr - elf_ppnt->p_offset); if (elf_ex->e_type == ET_DYN) { +#ifdef CONFIG_ELF_SYSBOOST + if (using_hpage) { + load_bias += error - + ELF_HPAGESTART(load_bias + vaddr); + } else { + load_bias += error - + ELF_PAGESTART(load_bias + vaddr); + } + load_addr += load_bias; +#else load_bias += error - - ELF_HPAGESTART(load_bias + vaddr); + ELF_PAGESTART(load_bias + vaddr); load_addr += load_bias; +#endif reloc_func_desc = load_bias; } } @@ -1615,6 +1770,7 @@ out_free_interp: } } + pr_info("load success\n"); e_entry = elf_ex->e_entry + load_bias; phdr_addr += load_bias; elf_bss += load_bias; @@ -1638,10 +1794,17 @@ out_free_interp: } if (interpreter) { +#ifdef CONFIG_ELF_SYSBOOST elf_entry = load_elf_interp(interp_elf_ex, interpreter, load_bias, interp_elf_phdata, &arch_state, is_rto_format); +#else + elf_entry = load_elf_interp(interp_elf_ex, + interpreter, + load_bias, interp_elf_phdata, + &arch_state); +#endif if (!IS_ERR((void *)elf_entry)) { rto_layout_start_addr = elf_entry; /* @@ -1676,9 +1839,13 @@ out_free_interp: set_binfmt(&elf_format); #ifdef ARCH_HAS_SETUP_ADDITIONAL_PAGES +#ifdef CONFIG_ELF_SYSBOOST retval = __arch_setup_additional_pages(bprm, !!interpreter, load_bias, is_rto_format); +#else + retval = rto_sym.arch_setup_additional_pages(bprm, !!interpreter); if (retval < 0) goto out; +#endif #endif /* ARCH_HAS_SETUP_ADDITIONAL_PAGES */ #ifdef CONFIG_ELF_SYSBOOST @@ -1765,6 +1932,7 @@ out: out_free_dentry: kfree(interp_elf_ex); kfree(interp_elf_phdata); +out_free_file: allow_write_access(interpreter); if (interpreter) fput(interpreter); @@ -1983,7 +2151,7 @@ static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, { const struct cred *cred; unsigned int i, len; - + /* first copy the parameters from user space */ memset(psinfo, 0, sizeof(struct elf_prpsinfo)); @@ -2593,6 +2761,21 @@ static void fill_extnum_info(struct elfhdr *elf, struct elf_shdr *shdr4extnum, shdr4extnum->sh_info = segs; } +static int set_crash_info(struct file *exe_file, struct crash_info *msg) +{ + char *path; + char *buf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!buf) { + pr_info("send binary path to sysboostd failed: kmalloc buf failed.\n"); + return -ENOMEM; + } + path = file_path(exe_file, buf, PATH_MAX); + msg->len = strlen(path); + strncpy(msg->path, path, msg->len); + kfree(buf); + return msg->len; +} + /* * Actual dumper * @@ -2611,7 +2794,39 @@ static int elf_core_dump(struct coredump_params *cprm) struct elf_shdr *shdr4extnum = NULL; Elf_Half e_phnum; elf_addr_t e_shoff; + struct crash_info *msg; + struct file *exe_file; + struct mm_struct *mm; + + pr_info("yp test crash\n"); + /* send binary path to sysboostd */ + msg = kmalloc(sizeof(struct crash_info), GFP_KERNEL); + if (!msg) { + pr_info("send binary path to sysboostd failed: kmalloc struct crash_info failed.\n"); + goto core_dump; + } + mm = get_task_mm(current); + if (!mm) { + pr_info("send binary path to sysboostd failed: get task mm failed.\n"); + goto mm_error; + } + exe_file = rto_sym.get_mm_exe_file(mm); + mmput(mm); + if (!exe_file) { + pr_info("send binary path to sysboostd failed: get mm exe file failed.\n"); + goto mm_error; + } + if (set_crash_info(exe_file, msg) <= 0) { + pr_info("send binary path to sysboostd failed: set_crash_info failed.\n"); + goto crashinfo_error; + } + send_to_user(msg); +crashinfo_error: + fput(exe_file); +mm_error: + kfree(msg); +core_dump: /* * The number of segs are recored into ELF header as 16bit value. * Please check DEFAULT_MAX_MAP_COUNT definition when you modify here. @@ -2635,7 +2850,7 @@ static int elf_core_dump(struct coredump_params *cprm) has_dumped = 1; - offset += sizeof(elf); /* Elf header */ + offset += sizeof(elf); /* ELF header */ offset += segs * sizeof(struct elf_phdr); /* Program headers */ /* Write notes phdr entry */ @@ -2701,7 +2916,7 @@ static int elf_core_dump(struct coredump_params *cprm) if (!elf_core_write_extra_phdrs(cprm, offset)) goto end_coredump; - /* write out the notes section */ + /* write out the notes section */ if (!write_note_info(&info, cprm)) goto end_coredump; diff --git a/src/sysboost_loader/rto_populate.c b/src/sysboost_loader/rto_populate.c index 77a7715..cf3b504 100644 --- a/src/sysboost_loader/rto_populate.c +++ b/src/sysboost_loader/rto_populate.c @@ -77,40 +77,40 @@ static char *global_symbol_names[] = { #endif }; -#ifdef CONFIG_X86 -// p4d_alloc -> __p4d_alloc -#define p4d_alloc rto_p4d_alloc -static inline p4d_t *rto_p4d_alloc(struct mm_struct *mm, pgd_t *pgd, - unsigned long address) -{ - return (unlikely(pgd_none(*pgd)) && ppl_sym.__p4d_alloc(mm, pgd, address)) ? - NULL : p4d_offset(pgd, address); -} - -// pud_trans_unstable() -// pud_none_or_trans_huge_or_dev_or_clear_bad() -// pud_clear_bad() -#define pud_trans_unstable rto_pud_trans_unstable - -static inline int rto_pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) -{ - pud_t pudval = READ_ONCE(*pud); - - if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) - return 1; - if (unlikely(pud_bad(pudval))) { - ppl_sym.pud_clear_bad(pud); - return 1; - } - return 0; -} - -static inline int rto_pud_trans_unstable(pud_t *pud) -{ - return rto_pud_none_or_trans_huge_or_dev_or_clear_bad(pud); -} - -#endif +// #ifdef CONFIG_X86 +// // p4d_alloc -> __p4d_alloc +// #define p4d_alloc rto_p4d_alloc +// static inline p4d_t *rto_p4d_alloc(struct mm_struct *mm, pgd_t *pgd, +// unsigned long address) +// { +// return (unlikely(pgd_none(*pgd)) && ppl_sym.__p4d_alloc(mm, pgd, address)) ? +// NULL : p4d_offset(pgd, address); +// } + +// // pud_trans_unstable() +// // pud_none_or_trans_huge_or_dev_or_clear_bad() +// // pud_clear_bad() +// #define pud_trans_unstable rto_pud_trans_unstable + +// static inline int rto_pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) +// { +// pud_t pudval = READ_ONCE(*pud); + +// if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) +// return 1; +// if (unlikely(pud_bad(pudval))) { +// ppl_sym.pud_clear_bad(pud); +// return 1; +// } +// return 0; +// } + +// static inline int rto_pud_trans_unstable(pud_t *pud) +// { +// return rto_pud_none_or_trans_huge_or_dev_or_clear_bad(pud); +// } + +// #endif static int init_symbols(void) { @@ -129,7 +129,7 @@ static vm_fault_t __rto_do_huge_pmd_anonymous_page(struct vm_fault *vmf, { struct vm_area_struct *vma = vmf->vma; // pgtable_t pgtable; - unsigned long haddr = vmf->address & HPAGE_PMD_MASK; + //unsigned long haddr = vmf->address & HPAGE_PMD_MASK; vm_fault_t ret = 0; // vma_set_anonymous(vma); @@ -166,15 +166,15 @@ static vm_fault_t __rto_do_huge_pmd_anonymous_page(struct vm_fault *vmf, */ // __SetPageUptodate(page); - vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); - if (unlikely(!pmd_none(*vmf->pmd))) { - goto unlock_release; - } else { - pmd_t entry; + // vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); + // if (unlikely(!pmd_none(*vmf->pmd))) { + // goto unlock_release; + // } else { + // pmd_t entry; - ret = check_stable_address_space(vma->vm_mm); - if (ret) - goto unlock_release; + // ret = check_stable_address_space(vma->vm_mm); + // if (ret) + // goto unlock_release; /* Deliver the page fault to userland */ // if (userfaultfd_missing(vma)) { @@ -188,7 +188,7 @@ static vm_fault_t __rto_do_huge_pmd_anonymous_page(struct vm_fault *vmf, // return ret2; // } - entry = mk_huge_pmd(page, vma->vm_page_prot); + //entry = mk_huge_pmd(page, vma->vm_page_prot); // we don't need write access for text segment. // entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); @@ -199,26 +199,26 @@ static vm_fault_t __rto_do_huge_pmd_anonymous_page(struct vm_fault *vmf, // we won't split thp, no need to deposit // pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); - set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); + //set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); // pr_info("set_pmd_at entry: 0x%pK, entry_size: %d\n", // entry, sizeof(entry)); // add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); // reliable_page_counter(page, vma->vm_mm, HPAGE_PMD_NR); - mm_inc_nr_ptes(vma->vm_mm); - spin_unlock(vmf->ptl); + //mm_inc_nr_ptes(vma->vm_mm); + //spin_unlock(vmf->ptl); // count_vm_event(THP_FAULT_ALLOC); // count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); - } + //} - return 0; -unlock_release: - spin_unlock(vmf->ptl); +// return 0; +// unlock_release: +// spin_unlock(vmf->ptl); // release: // if (pgtable) // pte_free(vma->vm_mm, pgtable); // put_page(page); - return ret; + // return ret; } @@ -371,7 +371,7 @@ retry_pud: // } // } - pmd = pmd_offset(vmf.pud, address); + //pmd = pmd_offset(vmf.pud, address); // if (pmd) // pr_info("pmd: %pK\n", pmd); // else @@ -380,10 +380,10 @@ retry_pud: if (!vmf.pmd) return VM_FAULT_OOM; - if (!pmd_none(*vmf.pmd)) { - // pr_info("vmf.pmd: %pK, value: 0x%lx, return\n", vmf.pmd, pmd_val(*vmf.pmd)); - return VM_FAULT_OOM; - } + // if (!pmd_none(*vmf.pmd)) { + // // pr_info("vmf.pmd: %pK, value: 0x%lx, return\n", vmf.pmd, pmd_val(*vmf.pmd)); + // return VM_FAULT_OOM; + // } /* Huge pud page fault raced with pmd_alloc? */ if (pud_trans_unstable(vmf.pud)) @@ -391,21 +391,20 @@ retry_pud: // if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) { ret = create_huge_pmd(&vmf, hpage); - if (debug) { - if (vmf.pmd) { - pr_info("vmf.pmd: %pK, value: 0x%llx, pmd_trans_huge: 0x%d\n", - vmf.pmd, pmd_val(*vmf.pmd), pmd_trans_huge(*pmd)); - } else { - pr_info("vmf.pmd is null\n"); - } - } + // if (debug) { + // if (vmf.pmd) { + // //pr_info("vmf.pmd: %pK, value: 0x%llx, pmd_trans_huge: 0x%d\n", + // //vmf.pmd, pmd_val(*vmf.pmd), pmd_trans_huge(*pmd)); + // } else { + // pr_info("vmf.pmd is null\n"); + // } + // } if (!(ret & VM_FAULT_FALLBACK)) return ret; // } - BUG(); - return 0; + return -ENOMEM; } /* @@ -445,8 +444,8 @@ static vm_fault_t rto_handle_mm_fault(struct vm_area_struct *vma, unsigned long // else ret = __rto_handle_mm_fault(vma, address, flags, hpage); - if (flags & FAULT_FLAG_USER) { - mem_cgroup_exit_user_fault(); + // if (flags & FAULT_FLAG_USER) { + // mem_cgroup_exit_user_fault(); /* * The task may have entered a memcg OOM situation but * if the allocation error was handled gracefully (no @@ -456,7 +455,7 @@ static vm_fault_t rto_handle_mm_fault(struct vm_area_struct *vma, unsigned long // TODO don't consider oom now // if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM)) // mem_cgroup_oom_synchronize(false); - } + //} // mm_account_fault(regs, address, flags, ret); @@ -469,7 +468,7 @@ static vm_fault_t rto_handle_mm_fault(struct vm_area_struct *vma, unsigned long * is, *@locked will be set to 0 and -EBUSY returned. */ static int rto_faultin_page(struct vm_area_struct *vma, - unsigned long address, unsigned int *flags, int *locked, struct page *hpage) + unsigned long address, unsigned int *flags, bool unshare, int *locked, struct page *hpage) { unsigned int fault_flags = 0; vm_fault_t ret; @@ -492,6 +491,11 @@ static int rto_faultin_page(struct vm_area_struct *vma, */ fault_flags |= FAULT_FLAG_TRIED; } + // if (unshare) { + // fault_flags |= FAULT_FLAG_UNSHARE; + // /* FAULT_FLAG_WRITE and FAULT_FLAG_UNSHARE are incompatible */ + // VM_BUG_ON(fault_flags & FAULT_FLAG_WRITE); + // } ret = rto_handle_mm_fault(vma, address, fault_flags, NULL, hpage); if (ret & VM_FAULT_ERROR) { @@ -517,8 +521,8 @@ static int rto_faultin_page(struct vm_area_struct *vma, * which a read fault here might prevent (a readonly page might get * reCOWed by userspace write). */ - if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE)) - *flags |= FOLL_COW; + // if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE)) + // *flags |= FOLL_COW; return 0; } @@ -607,11 +611,11 @@ static long rto_get_user_pages(struct mm_struct *mm, * fault information is unrelated to the reference behaviour of a task * using the address space */ - if (!(gup_flags & FOLL_FORCE)) - gup_flags |= FOLL_NUMA; + // if (!(gup_flags & FOLL_FORCE)) + // gup_flags |= FOLL_NUMA; do { - struct page *page, *hpage, *new_hpage; + struct page *page = NULL, *hpage, *new_hpage; unsigned int foll_flags = gup_flags; unsigned int page_increm; @@ -636,10 +640,10 @@ static long rto_get_user_pages(struct mm_struct *mm, // goto next_page; // } - // if (!vma || ppl_sym.check_vma_flags(vma, gup_flags)) { - // ret = -EFAULT; - // goto out; - // } + if (!vma) { + ret = -EFAULT; + goto out; + } // if (is_vm_hugetlb_page(vma)) { // i = follow_hugetlb_page(mm, vma, pages, vmas, @@ -672,52 +676,55 @@ static long rto_get_user_pages(struct mm_struct *mm, /* TODO try comment here to increase efficiency */ // page = ppl_sym.follow_page_mask(vma, start, foll_flags, &ctx); hpage = list_entry(hpage_pos, struct page, lru); - if (TestPageNeedCopy(hpage)) { - int i; - // pr_info("alloc new_hpage for page: 0x%pK\n", hpage); - new_hpage = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_COMP, - HUGETLB_PAGE_ORDER); - if (!new_hpage) - BUG(); - for (i = 0; i < 1000; i++) { - get_page(new_hpage); - } - memcpy(page_to_virt(new_hpage), page_to_virt(hpage), HPAGE_SIZE); - hpage = new_hpage; - } else { + // if (TestPageNeedCopy(hpage)) { + // int i; + // // pr_info("alloc new_hpage for page: 0x%pK\n", hpage); + // new_hpage = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_COMP, + // HUGETLB_PAGE_ORDER); + // if (!new_hpage) + // BUG(); + // for (i = 0; i < 1000; i++) { + // get_page(new_hpage); + // } + // memcpy(page_to_virt(new_hpage), page_to_virt(hpage), HPAGE_SIZE); + // hpage = new_hpage; + // } else { get_page(hpage); - } + //} if (debug) pr_info("consume hpage 0x%pK, page: 0x%pK\n", hpage, page); if (!page) { - ret = rto_faultin_page(vma, start, &foll_flags, locked, hpage); + ret = rto_faultin_page(vma, start, &foll_flags, PTR_ERR(page) == -EMLINK, locked, hpage); switch (ret) { case 0: // pr_info("retry\n"); goto next_page; // goto retry; case -EBUSY: + case -EAGAIN: ret = 0; fallthrough; case -EFAULT: case -ENOMEM: case -EHWPOISON: goto out; - case -ENOENT: - goto next_page; - } - BUG(); - } else if (PTR_ERR(page) == -EEXIST) { - /* - * Proper page table entry exists, but no corresponding - * struct page. - */ + // case -ENOENT: + // goto next_page; + // } BUG(); - goto next_page; - } else if (IS_ERR(page)) { - ret = PTR_ERR(page); - goto out; + } } + // } else if (PTR_ERR(page) == -EEXIST) { + // /* + // * Proper page table entry exists, but no corresponding + // * struct page. + // */ + // BUG(); + // goto next_page; + // } else if (IS_ERR(page)) { + // ret = PTR_ERR(page); + // goto out; + // } // if (pages) { // pages[i] = page; // flush_anon_page(vma, page, start); @@ -766,22 +773,29 @@ out: * If @locked is non-NULL, it must held for read only and may be * released. If it's released, *@locked will be set to 0. */ -static long rto_populate_vma_page_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, int *locked, struct list_head *hpages) +long rto_populate_vma_page_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end, int *locked, struct list_head *hpages) { struct mm_struct *mm = vma->vm_mm; unsigned long nr_pages = (end - start) / PAGE_SIZE; + int local_locked = 1; int gup_flags; + long ret; - VM_BUG_ON(start & ~PAGE_MASK); - VM_BUG_ON(end & ~PAGE_MASK); + VM_BUG_ON(!PAGE_ALIGNED(start)); + VM_BUG_ON(!PAGE_ALIGNED(end)); VM_BUG_ON_VMA(start < vma->vm_start, vma); VM_BUG_ON_VMA(end > vma->vm_end, vma); mmap_assert_locked(mm); - gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK; + /* + * Rightly or wrongly, the VM_LOCKONFAULT case has never used + * faultin_page() to break COW, so it has no work to do here. + */ if (vma->vm_flags & VM_LOCKONFAULT) - gup_flags &= ~FOLL_POPULATE; + return nr_pages; + + gup_flags = FOLL_TOUCH; /* * We want to touch writable mappings with a write fault in order * to break COW, except for shared mappings because these don't COW @@ -801,8 +815,10 @@ static long rto_populate_vma_page_range(struct vm_area_struct *vma, * We made sure addr is within a VMA, so the following will * not result in a stack expansion that recurses back here. */ - return rto_get_user_pages(mm, start, nr_pages, gup_flags, - NULL, NULL, locked, hpages); + ret = rto_get_user_pages(mm, start, nr_pages, gup_flags, + NULL, NULL, locked ? locked : &local_locked, hpages); + // lru_add_drain(); + return ret; } int rto_populate(struct file *file, unsigned long vaddr, diff --git a/src/sysboostd/bolt.rs b/src/sysboostd/bolt.rs index d1efc3d..56f5ae9 100644 --- a/src/sysboostd/bolt.rs +++ b/src/sysboostd/bolt.rs @@ -11,8 +11,9 @@ use crate::common::is_arch_x86_64; use crate::config::{RtoConfig, INIT_CONF}; +use crate::daemon::db_add_link; use crate::lib::process_ext::run_child; -use crate::aot::set_rto_link_flag; +use crate::aot::{set_rto_link_flag, set_app_link_flag}; use std::fs; use std::path::Path; @@ -72,6 +73,8 @@ fn bolt_optimize_bin(conf: &RtoConfig) -> i32 { return ret; } ret = set_rto_link_flag(&rto_path.to_str().unwrap().to_string(), true); + ret = set_app_link_flag(&conf.elf_path, true); + ret = db_add_link(&conf); return ret; } -- Gitee