[PATCHv3 0/3] pahole/kernel: Workaround dwarf bug for function encoding

[PATCHv3 0/3] pahole/kernel: Workaround dwarf bug for function encoding

[Jiri Olsa]

hi,
because of gcc bug [1] we can no longer rely on DW_AT_declaration
attribute to filter out declarations and end up with just
one copy of the function in the BTF data.

It seems this bug is not easy to fix, but regardless if the
it's coming soon, it's probably good idea not to depend so
much only on dwarf data and make some extra checks.

Thus for function encoding we are now doing following checks:
  - argument names are defined for the function
  - there's symbol and address defined for the function
  - function address belongs to ftrace locations (new in v2)
  - function is generated only once

v3 changes:
  - added Hao's ack for patch 1
  - fixed realloc memory leak [Andrii]
  - fixed addrs_cmp function [Andrii]
  - removed SET_SYMBOL macro [Andrii]
  - fixed the 'valid' function logic
  - added .init.bpf.preserve_type check
  - added iterator functions to new kernel section
    .init.bpf.preserve_type [Yonghong]

v2 changes:
  - add check ensuring functions belong to ftrace's mcount
    locations, this way we ensure to have in BTF only
    functions available for ftrace - patch 2 changelog
    describes all details
  - use collect* function names [Andrii]
  - use conventional size increase in realloc [Andrii]
  - drop elf_sym__is_function check
  - drop patch 3, it's not needed, because we follow ftrace
    locations

thanks,
jirka


[1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=97060

[PATCH 1/3] bpf: Move iterator functions into special init section

[Jiri Olsa]

With upcoming changes to pahole, that change the way how and
which kernel functions are stored in BTF data, we need a way
to recognize iterator functions.

Iterator functions need to be in BTF data, but have no real
body and are currently placed in .init.text section, so they
are freed after kernel init and are filtered out of BTF data
because of that.

The solution is to place these functions under new section:
  .init.bpf.preserve_type

And add 2 new symbols to mark that area:
  __init_bpf_preserve_type_begin
  __init_bpf_preserve_type_end

The code in pahole responsible for picking up the functions will
be able to recognize functions from this section and add them to
the BTF data and filter out all other .init.text functions.

Suggested-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Jiri Olsa <jolsa@redhat.com>
---
 include/asm-generic/vmlinux.lds.h | 16 +++++++++++++++-
 include/linux/bpf.h               |  8 +++++++-
 include/linux/init.h              |  1 +
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index cd14444bf600..e18e1030dabf 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -685,8 +685,21 @@
 	.BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) {			\
 		*(.BTF_ids)						\
 	}
+
+/*
+ * .init.bpf.preserve_type
+ *
+ * This section store special BPF function and marks them
+ * with begin/end symbols pair for the sake of pahole tool.
+ */
+#define INIT_BPF_PRESERVE_TYPE						\
+	__init_bpf_preserve_type_begin = .;                             \
+	*(.init.bpf.preserve_type)                                      \
+	__init_bpf_preserve_type_end = .;				\
+	MEM_DISCARD(init.bpf.preserve_type)
 #else
 #define BTF
+#define INIT_BPF_PRESERVE_TYPE
 #endif
 
 /*
@@ -740,7 +753,8 @@
 #define INIT_TEXT							\
 	*(.init.text .init.text.*)					\
 	*(.text.startup)						\
-	MEM_DISCARD(init.text*)
+	MEM_DISCARD(init.text*)						\
+	INIT_BPF_PRESERVE_TYPE
 
 #define EXIT_DATA							\
 	*(.exit.data .exit.data.*)					\
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 2fffd30e13ac..12ab39a034a3 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1276,10 +1276,16 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd);
 int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
 int bpf_obj_get_user(const char __user *pathname, int flags);
 
+#ifdef CONFIG_DEBUG_INFO_BTF
+#define BPF_INIT __init_bpf_preserve_type
+#else
+#define BPF_INIT __init
+#endif
+
 #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
 #define DEFINE_BPF_ITER_FUNC(target, args...)			\
 	extern int bpf_iter_ ## target(args);			\
-	int __init bpf_iter_ ## target(args) { return 0; }
+	int BPF_INIT bpf_iter_ ## target(args) { return 0; }
 
 struct bpf_iter_aux_info {
 	struct bpf_map *map;
diff --git a/include/linux/init.h b/include/linux/init.h
index 212fc9e2f691..133462863711 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -52,6 +52,7 @@
 #define __initconst	__section(.init.rodata)
 #define __exitdata	__section(.exit.data)
 #define __exit_call	__used __section(.exitcall.exit)
+#define __init_bpf_preserve_type __section(.init.bpf.preserve_type)
 
 /*
  * modpost check for section mismatches during the kernel build.
-- 
2.26.2

[PATCH 2/3] btf_encoder: Move find_all_percpu_vars in generic collect_symbols

[Jiri Olsa]

Moving find_all_percpu_vars under generic collect_symbols
function that walks over symbols and calls collect_percpu_var.

We will add another collect function that needs to go through
all the symbols, so it's better we go through them just once.

There's no functional change intended.

Acked-by: Hao Luo <haoluo@google.com>
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 btf_encoder.c | 124 +++++++++++++++++++++++++++-----------------------
 1 file changed, 67 insertions(+), 57 deletions(-)

diff --git a/btf_encoder.c b/btf_encoder.c
index 4c92908beab2..1866bb16a8ba 100644
--- a/btf_encoder.c
+++ b/btf_encoder.c
@@ -250,75 +250,85 @@ static bool percpu_var_exists(uint64_t addr, uint32_t *sz, const char **name)
 	return true;
 }
 
-static int find_all_percpu_vars(struct btf_elf *btfe)
+static int collect_percpu_var(struct btf_elf *btfe, GElf_Sym *sym)
 {
-	uint32_t core_id;
-	GElf_Sym sym;
+	const char *sym_name;
+	uint64_t addr;
+	uint32_t size;
 
-	/* cache variables' addresses, preparing for searching in symtab. */
-	percpu_var_cnt = 0;
+	/* compare a symbol's shndx to determine if it's a percpu variable */
+	if (elf_sym__section(sym) != btfe->percpu_shndx)
+		return 0;
+	if (elf_sym__type(sym) != STT_OBJECT)
+		return 0;
 
-	/* search within symtab for percpu variables */
-	elf_symtab__for_each_symbol(btfe->symtab, core_id, sym) {
-		const char *sym_name;
-		uint64_t addr;
-		uint32_t size;
+	addr = elf_sym__value(sym);
+	/*
+	 * Store only those symbols that have allocated space in the percpu section.
+	 * This excludes the following three types of symbols:
+	 *
+	 *  1. __ADDRESSABLE(sym), which are forcely emitted as symbols.
+	 *  2. __UNIQUE_ID(prefix), which are introduced to generate unique ids.
+	 *  3. __exitcall(fn), functions which are labeled as exit calls.
+	 *
+	 * In addition, the variables defined using DEFINE_PERCPU_FIRST are
+	 * also not included, which currently includes:
+	 *
+	 *  1. fixed_percpu_data
+	 */
+	if (!addr)
+		return 0;
 
-		/* compare a symbol's shndx to determine if it's a percpu variable */
-		if (elf_sym__section(&sym) != btfe->percpu_shndx)
-			continue;
-		if (elf_sym__type(&sym) != STT_OBJECT)
-			continue;
+	size = elf_sym__size(sym);
+	if (!size)
+		return 0; /* ignore zero-sized symbols */
 
-		addr = elf_sym__value(&sym);
-		/*
-		 * Store only those symbols that have allocated space in the percpu section.
-		 * This excludes the following three types of symbols:
-		 *
-		 *  1. __ADDRESSABLE(sym), which are forcely emitted as symbols.
-		 *  2. __UNIQUE_ID(prefix), which are introduced to generate unique ids.
-		 *  3. __exitcall(fn), functions which are labeled as exit calls.
-		 *
-		 * In addition, the variables defined using DEFINE_PERCPU_FIRST are
-		 * also not included, which currently includes:
-		 *
-		 *  1. fixed_percpu_data
-		 */
-		if (!addr)
-			continue;
+	sym_name = elf_sym__name(sym, btfe->symtab);
+	if (!btf_name_valid(sym_name)) {
+		dump_invalid_symbol("Found symbol of invalid name when encoding btf",
+				    sym_name, btf_elf__verbose, btf_elf__force);
+		if (btf_elf__force)
+			return 0;
+		return -1;
+	}
 
-		size = elf_sym__size(&sym);
-		if (!size)
-			continue; /* ignore zero-sized symbols */
+	if (btf_elf__verbose)
+		printf("Found per-CPU symbol '%s' at address 0x%lx\n", sym_name, addr);
 
-		sym_name = elf_sym__name(&sym, btfe->symtab);
-		if (!btf_name_valid(sym_name)) {
-			dump_invalid_symbol("Found symbol of invalid name when encoding btf",
-					    sym_name, btf_elf__verbose, btf_elf__force);
-			if (btf_elf__force)
-				continue;
-			return -1;
-		}
+	if (percpu_var_cnt == MAX_PERCPU_VAR_CNT) {
+		fprintf(stderr, "Reached the limit of per-CPU variables: %d\n",
+			MAX_PERCPU_VAR_CNT);
+		return -1;
+	}
+	percpu_vars[percpu_var_cnt].addr = addr;
+	percpu_vars[percpu_var_cnt].sz = size;
+	percpu_vars[percpu_var_cnt].name = sym_name;
+	percpu_var_cnt++;
 
-		if (btf_elf__verbose)
-			printf("Found per-CPU symbol '%s' at address 0x%lx\n", sym_name, addr);
+	return 0;
+}
+
+static int collect_symbols(struct btf_elf *btfe, bool collect_percpu_vars)
+{
+	uint32_t core_id;
+	GElf_Sym sym;
 
-		if (percpu_var_cnt == MAX_PERCPU_VAR_CNT) {
-			fprintf(stderr, "Reached the limit of per-CPU variables: %d\n",
-				MAX_PERCPU_VAR_CNT);
+	/* cache variables' addresses, preparing for searching in symtab. */
+	percpu_var_cnt = 0;
+
+	/* search within symtab for percpu variables */
+	elf_symtab__for_each_symbol(btfe->symtab, core_id, sym) {
+		if (collect_percpu_vars && collect_percpu_var(btfe, &sym))
 			return -1;
-		}
-		percpu_vars[percpu_var_cnt].addr = addr;
-		percpu_vars[percpu_var_cnt].sz = size;
-		percpu_vars[percpu_var_cnt].name = sym_name;
-		percpu_var_cnt++;
 	}
 
-	if (percpu_var_cnt)
-		qsort(percpu_vars, percpu_var_cnt, sizeof(percpu_vars[0]), percpu_var_cmp);
+	if (collect_percpu_vars) {
+		if (percpu_var_cnt)
+			qsort(percpu_vars, percpu_var_cnt, sizeof(percpu_vars[0]), percpu_var_cmp);
 
-	if (btf_elf__verbose)
-		printf("Found %d per-CPU variables!\n", percpu_var_cnt);
+		if (btf_elf__verbose)
+			printf("Found %d per-CPU variables!\n", percpu_var_cnt);
+	}
 	return 0;
 }
 
@@ -347,7 +357,7 @@ int cu__encode_btf(struct cu *cu, int verbose, bool force,
 		if (!btfe)
 			return -1;
 
-		if (!skip_encoding_vars && find_all_percpu_vars(btfe))
+		if (collect_symbols(btfe, !skip_encoding_vars))
 			goto out;
 
 		has_index_type = false;
-- 
2.26.2

[PATCH 3/3] btf_encoder: Change functions check due to broken dwarf

[Jiri Olsa]

We need to generate just single BTF instance for the
function, while DWARF data contains multiple instances
of DW_TAG_subprogram tag.

Unfortunately we can no longer rely on DW_AT_declaration
tag (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=97060)

Instead we apply following checks:
  - argument names are defined for the function
  - there's symbol and address defined for the function
  - function is generated only once

Also because we want to follow kernel's ftrace traceable
functions, this patchset is adding extra check that the
function is one of the ftrace's functions.

All ftrace functions addresses are stored in vmlinux
binary within symbols:
  __start_mcount_loc
  __stop_mcount_loc

During object preparation code we read those addresses,
sort them and use them as filter for all detected dwarf
functions.

We also filter out functions within .init section, ftrace
is doing that in runtime. At the same time we keep functions
from .init.bpf.preserve_type, because they are needed in BTF.

I can still see several differences to ftrace functions in
/sys/kernel/debug/tracing/available_filter_functions file:

  - available_filter_functions includes modules
  - available_filter_functions includes functions like:
      __acpi_match_device.part.0.constprop.0
      acpi_ns_check_sorted_list.constprop.0
      acpi_os_unmap_generic_address.part.0
      acpiphp_check_bridge.part.0

    which are not part of dwarf data
  - BTF includes multiple functions like:
      __clk_register_clkdev
      clk_register_clkdev

    which share same code so they appear just as single function
    in available_filter_functions, but dwarf keeps track of both
    of them

With this change I'm getting 38353 BTF functions, which
when added above functions to consideration gives same
amount of functions in available_filter_functions.

The patch still keeps the original function filter condition
(that uses current fn->declaration check) in case the object
does not contain *_mcount_loc symbol -> object is not vmlinux.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 btf_encoder.c | 261 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 259 insertions(+), 2 deletions(-)

diff --git a/btf_encoder.c b/btf_encoder.c
index 1866bb16a8ba..df89b4467e4c 100644
--- a/btf_encoder.c
+++ b/btf_encoder.c
@@ -26,6 +26,174 @@
  */
 #define KSYM_NAME_LEN 128
 
+struct symbols {
+	unsigned long start;
+	unsigned long stop;
+	unsigned long init_begin;
+	unsigned long init_end;
+	unsigned long init_bpf_begin;
+	unsigned long init_bpf_end;
+	unsigned long start_section;
+};
+
+struct elf_function {
+	const char	*name;
+	unsigned long	 addr;
+	bool		 generated;
+};
+
+static struct elf_function *functions;
+static int functions_alloc;
+static int functions_cnt;
+static int functions_valid;
+
+static int functions_cmp(const void *_a, const void *_b)
+{
+	const struct elf_function *a = _a;
+	const struct elf_function *b = _b;
+
+	return strcmp(a->name, b->name);
+}
+
+static void delete_functions(void)
+{
+	free(functions);
+	functions_alloc = functions_cnt = functions_valid = 0;
+}
+
+#ifndef max
+#define max(x, y) ((x) < (y) ? (y) : (x))
+#endif
+
+static int collect_function(struct btf_elf *btfe, GElf_Sym *sym)
+{
+	struct elf_function *new;
+
+	if (elf_sym__type(sym) != STT_FUNC)
+		return 0;
+	if (!elf_sym__value(sym))
+		return 0;
+
+	if (functions_cnt == functions_alloc) {
+		functions_alloc = max(1000, functions_alloc * 3 / 2);
+		new = realloc(functions, functions_alloc * sizeof(*functions));
+		if (!new) {
+			delete_functions();
+			return -1;
+		}
+		functions = new;
+	}
+
+	functions[functions_cnt].name = elf_sym__name(sym, btfe->symtab);
+	functions[functions_cnt].addr = elf_sym__value(sym);
+	functions[functions_cnt].generated = false;
+	functions_cnt++;
+	return 0;
+}
+
+static int addrs_cmp(const void *_a, const void *_b)
+{
+	const unsigned long *a = _a;
+	const unsigned long *b = _b;
+
+	if (*a == *b)
+		return 0;
+	return *a < *b ? -1 : 1;
+}
+
+static bool is_init(struct symbols *ms, unsigned long addr)
+{
+	return addr >= ms->init_begin && addr < ms->init_end;
+}
+
+static bool is_bpf_init(struct symbols *ms, unsigned long addr)
+{
+	return addr >= ms->init_bpf_begin && addr < ms->init_bpf_end;
+}
+
+static int filter_functions(struct btf_elf *btfe, struct symbols *ms)
+{
+	unsigned long *addrs, count, offset, i;
+	Elf_Data *data;
+	GElf_Shdr shdr;
+	Elf_Scn *sec;
+
+	/*
+	 * Find mcount addressed marked by __start_mcount_loc
+	 * and __stop_mcount_loc symbols and load them into
+	 * sorted array.
+	 */
+	sec = elf_getscn(btfe->elf, ms->start_section);
+	if (!sec || !gelf_getshdr(sec, &shdr)) {
+		fprintf(stderr, "Failed to get section(%lu) header.\n",
+			ms->start_section);
+		return -1;
+	}
+
+	offset = ms->start - shdr.sh_addr;
+	count  = (ms->stop - ms->start) / 8;
+
+	data = elf_getdata(sec, 0);
+	if (!data) {
+		fprintf(stderr, "Failed to get section(%lu) data.\n",
+			ms->start_section);
+		return -1;
+	}
+
+	addrs = malloc(count * sizeof(addrs[0]));
+	if (!addrs) {
+		fprintf(stderr, "Failed to allocate memory for ftrace addresses.\n");
+		return -1;
+	}
+
+	memcpy(addrs, data->d_buf + offset, count * sizeof(addrs[0]));
+	qsort(addrs, count, sizeof(addrs[0]), addrs_cmp);
+
+	/*
+	 * Let's got through all collected functions and filter
+	 * out those that are not in ftrace and init code.
+	 */
+	for (i = 0; i < functions_cnt; i++) {
+		struct elf_function *func = &functions[i];
+
+		/*
+		 * Do not enable .init section functions,
+		 * but keep .init.bpf.preserve_type functions.
+		 */
+		if (is_init(ms, func->addr) && !is_bpf_init(ms, func->addr))
+			continue;
+
+		/* Make sure function is within ftrace addresses. */
+		if (bsearch(&func->addr, addrs, count, sizeof(addrs[0]), addrs_cmp)) {
+			/*
+			 * We iterate over sorted array, so we can easily skip
+			 * not valid item and move following valid field into
+			 * its place, and still keep the 'new' array sorted.
+			 */
+			if (i != functions_valid)
+				functions[functions_valid] = functions[i];
+			functions_valid++;
+		}
+	}
+
+	free(addrs);
+	return 0;
+}
+
+static bool should_generate_function(const struct btf_elf *btfe, const char *name)
+{
+	struct elf_function *p;
+	struct elf_function key = { .name = name };
+
+	p = bsearch(&key, functions, functions_valid,
+		    sizeof(functions[0]), functions_cmp);
+	if (!p || p->generated)
+		return false;
+
+	p->generated = true;
+	return true;
+}
+
 static bool btf_name_char_ok(char c, bool first)
 {
 	if (c == '_' || c == '.')
@@ -207,6 +375,7 @@ int btf_encoder__encode()
 		btf_elf__add_datasec_type(btfe, PERCPU_SECTION, &btfe->percpu_secinfo);
 
 	err = btf_elf__encode(btfe, 0);
+	delete_functions();
 	btf_elf__delete(btfe);
 	btfe = NULL;
 
@@ -308,8 +477,45 @@ static int collect_percpu_var(struct btf_elf *btfe, GElf_Sym *sym)
 	return 0;
 }
 
+static void collect_symbol(GElf_Sym *sym, struct symbols *ms)
+{
+	if (!ms->start &&
+	    !strcmp("__start_mcount_loc", elf_sym__name(sym, btfe->symtab))) {
+		ms->start = sym->st_value;
+		ms->start_section = sym->st_shndx;
+	}
+
+	if (!ms->stop &&
+	    !strcmp("__stop_mcount_loc", elf_sym__name(sym, btfe->symtab)))
+		ms->stop = sym->st_value;
+
+	if (!ms->init_begin &&
+	    !strcmp("__init_begin", elf_sym__name(sym, btfe->symtab)))
+		ms->init_begin = sym->st_value;
+
+	if (!ms->init_end &&
+	    !strcmp("__init_end", elf_sym__name(sym, btfe->symtab)))
+		ms->init_end = sym->st_value;
+
+	if (!ms->init_bpf_begin &&
+	    !strcmp("__init_bpf_preserve_type_begin", elf_sym__name(sym, btfe->symtab)))
+		ms->init_bpf_begin = sym->st_value;
+
+	if (!ms->init_bpf_end &&
+	    !strcmp("__init_bpf_preserve_type_end", elf_sym__name(sym, btfe->symtab)))
+		ms->init_bpf_end = sym->st_value;
+}
+
+static int has_all_symbols(struct symbols *ms)
+{
+	return ms->start && ms->stop &&
+	       ms->init_begin && ms->init_end &&
+	       ms->init_bpf_begin && ms->init_bpf_end;
+}
+
 static int collect_symbols(struct btf_elf *btfe, bool collect_percpu_vars)
 {
+	struct symbols ms = { };
 	uint32_t core_id;
 	GElf_Sym sym;
 
@@ -320,6 +526,9 @@ static int collect_symbols(struct btf_elf *btfe, bool collect_percpu_vars)
 	elf_symtab__for_each_symbol(btfe->symtab, core_id, sym) {
 		if (collect_percpu_vars && collect_percpu_var(btfe, &sym))
 			return -1;
+		if (collect_function(btfe, &sym))
+			return -1;
+		collect_symbol(&sym, &ms);
 	}
 
 	if (collect_percpu_vars) {
@@ -329,9 +538,36 @@ static int collect_symbols(struct btf_elf *btfe, bool collect_percpu_vars)
 		if (btf_elf__verbose)
 			printf("Found %d per-CPU variables!\n", percpu_var_cnt);
 	}
+
+	if (functions_cnt && has_all_symbols(&ms)) {
+		qsort(functions, functions_cnt, sizeof(functions[0]), functions_cmp);
+		if (filter_functions(btfe, &ms)) {
+			fprintf(stderr, "Failed to filter dwarf functions\n");
+			return -1;
+		}
+		if (btf_elf__verbose)
+			printf("Found %d functions!\n", functions_valid);
+	} else {
+		if (btf_elf__verbose)
+			printf("vmlinux not detected, falling back to dwarf data\n");
+	}
+
 	return 0;
 }
 
+static bool has_arg_names(struct cu *cu, struct ftype *ftype)
+{
+	struct parameter *param;
+	const char *name;
+
+	ftype__for_each_parameter(ftype, param) {
+		name = dwarves__active_loader->strings__ptr(cu, param->name);
+		if (name == NULL)
+			return false;
+	}
+	return true;
+}
+
 int cu__encode_btf(struct cu *cu, int verbose, bool force,
 		   bool skip_encoding_vars)
 {
@@ -407,8 +643,28 @@ int cu__encode_btf(struct cu *cu, int verbose, bool force,
 		int btf_fnproto_id, btf_fn_id;
 		const char *name;
 
-		if (fn->declaration || !fn->external)
-			continue;
+		/*
+		 * The functions_valid != 0 means we parsed all necessary
+		 * kernel symbols and we are using ftrace location filter
+		 * for functions. If it's not available keep the current
+		 * dwarf declaration check.
+		 */
+		if (functions_valid) {
+			/*
+			 * We check following conditions:
+			 *   - argument names are defined
+			 *   - there's symbol and address defined for the function
+			 *   - function address belongs to ftrace locations
+			 *   - function is generated only once
+			 */
+			if (!has_arg_names(cu, &fn->proto))
+				continue;
+			if (!should_generate_function(btfe, function__name(fn, cu)))
+				continue;
+		} else {
+			if (fn->declaration || !fn->external)
+				continue;
+		}
 
 		btf_fnproto_id = btf_elf__add_func_proto(btfe, cu, &fn->proto, type_id_off);
 		name = dwarves__active_loader->strings__ptr(cu, fn->name);
@@ -492,6 +748,7 @@ int cu__encode_btf(struct cu *cu, int verbose, bool force,
 
 out:
 	if (err) {
+		delete_functions();
 		btf_elf__delete(btfe);
 		btfe = NULL;
 	}
-- 
2.26.2

Re: [PATCH 1/3] bpf: Move iterator functions into special init section

[Song Liu]

On Wed, Nov 4, 2020 at 2:02 PM Jiri Olsa <jolsa@kernel.org> wrote:
>
> With upcoming changes to pahole, that change the way how and
> which kernel functions are stored in BTF data, we need a way
> to recognize iterator functions.
>
> Iterator functions need to be in BTF data, but have no real
> body and are currently placed in .init.text section, so they
> are freed after kernel init and are filtered out of BTF data
> because of that.
>
> The solution is to place these functions under new section:
>   .init.bpf.preserve_type
>
> And add 2 new symbols to mark that area:
>   __init_bpf_preserve_type_begin
>   __init_bpf_preserve_type_end
>
> The code in pahole responsible for picking up the functions will
> be able to recognize functions from this section and add them to
> the BTF data and filter out all other .init.text functions.
>
> Suggested-by: Yonghong Song <yhs@fb.com>
> Signed-off-by: Jiri Olsa <jolsa@redhat.com>

Acked-by: Song Liu <songliubraving@fb.com>

[...]

Re: [PATCH 2/3] btf_encoder: Move find_all_percpu_vars in generic collect_symbols

[Andrii Nakryiko]

On Wed, Nov 4, 2020 at 2:02 PM Jiri Olsa <jolsa@kernel.org> wrote:
>
> Moving find_all_percpu_vars under generic collect_symbols
> function that walks over symbols and calls collect_percpu_var.
>
> We will add another collect function that needs to go through
> all the symbols, so it's better we go through them just once.
>
> There's no functional change intended.
>
> Acked-by: Hao Luo <haoluo@google.com>
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> ---

diff tool didn't do a great job here :( but code seems fine to me, so:

Acked-by: Andrii Nakryiko <andrii@kernel.org>

>  btf_encoder.c | 124 +++++++++++++++++++++++++++-----------------------
>  1 file changed, 67 insertions(+), 57 deletions(-)
>

[...]

Re: [PATCH 3/3] btf_encoder: Change functions check due to broken dwarf

[Andrii Nakryiko]

[

On Wed, Nov 4, 2020 at 2:01 PM Jiri Olsa <jolsa@kernel.org> wrote:
>
> We need to generate just single BTF instance for the
> function, while DWARF data contains multiple instances
> of DW_TAG_subprogram tag.
>
> Unfortunately we can no longer rely on DW_AT_declaration
> tag (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=97060)
>
> Instead we apply following checks:
>   - argument names are defined for the function
>   - there's symbol and address defined for the function
>   - function is generated only once
>
> Also because we want to follow kernel's ftrace traceable
> functions, this patchset is adding extra check that the
> function is one of the ftrace's functions.
>
> All ftrace functions addresses are stored in vmlinux
> binary within symbols:
>   __start_mcount_loc
>   __stop_mcount_loc
>
> During object preparation code we read those addresses,
> sort them and use them as filter for all detected dwarf
> functions.
>
> We also filter out functions within .init section, ftrace
> is doing that in runtime. At the same time we keep functions
> from .init.bpf.preserve_type, because they are needed in BTF.
>
> I can still see several differences to ftrace functions in
> /sys/kernel/debug/tracing/available_filter_functions file:
>
>   - available_filter_functions includes modules
>   - available_filter_functions includes functions like:
>       __acpi_match_device.part.0.constprop.0
>       acpi_ns_check_sorted_list.constprop.0
>       acpi_os_unmap_generic_address.part.0
>       acpiphp_check_bridge.part.0
>
>     which are not part of dwarf data
>   - BTF includes multiple functions like:
>       __clk_register_clkdev
>       clk_register_clkdev
>
>     which share same code so they appear just as single function
>     in available_filter_functions, but dwarf keeps track of both
>     of them
>
> With this change I'm getting 38353 BTF functions, which
> when added above functions to consideration gives same
> amount of functions in available_filter_functions.
>
> The patch still keeps the original function filter condition
> (that uses current fn->declaration check) in case the object
> does not contain *_mcount_loc symbol -> object is not vmlinux.
>
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> ---

Logic looks good, but the naming made it harder to understand what's
going on, had to jump back and forth more than usual.

Otherwise:

Acked-by: Andrii Nakryiko <andrii@kernel.org>

>  btf_encoder.c | 261 +++++++++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 259 insertions(+), 2 deletions(-)
>
> diff --git a/btf_encoder.c b/btf_encoder.c
> index 1866bb16a8ba..df89b4467e4c 100644
> --- a/btf_encoder.c
> +++ b/btf_encoder.c
> @@ -26,6 +26,174 @@
>   */
>  #define KSYM_NAME_LEN 128
>
> +struct symbols {

"symbols" is a bit generic name, something like funcs_layout seems to
convey the meaning a bit more precisely?


> +       unsigned long start;
> +       unsigned long stop;

start/stop mcount, right? mcount part is important is important

> +       unsigned long init_begin;
> +       unsigned long init_end;
> +       unsigned long init_bpf_begin;
> +       unsigned long init_bpf_end;
> +       unsigned long start_section;

start_section is quite ambiguous. That's the mcount section index, no?
mcount_sec_idx?

> +};
> +
> +struct elf_function {
> +       const char      *name;
> +       unsigned long    addr;
> +       bool             generated;
> +};
> +

[...]

> +       /*
> +        * Let's got through all collected functions and filter
> +        * out those that are not in ftrace and init code.
> +        */
> +       for (i = 0; i < functions_cnt; i++) {
> +               struct elf_function *func = &functions[i];
> +
> +               /*
> +                * Do not enable .init section functions,
> +                * but keep .init.bpf.preserve_type functions.
> +                */
> +               if (is_init(ms, func->addr) && !is_bpf_init(ms, func->addr))
> +                       continue;
> +
> +               /* Make sure function is within ftrace addresses. */
> +               if (bsearch(&func->addr, addrs, count, sizeof(addrs[0]), addrs_cmp)) {
> +                       /*
> +                        * We iterate over sorted array, so we can easily skip
> +                        * not valid item and move following valid field into
> +                        * its place, and still keep the 'new' array sorted.
> +                        */
> +                       if (i != functions_valid)
> +                               functions[functions_valid] = functions[i];
> +                       functions_valid++;
> +               }
> +       }

can we re-assign function_cnt = functions_valid here? and
functions_valid could be just a local temporary variable?

> +
> +       free(addrs);
> +       return 0;
> +}
> +

[...]

Re: [PATCH 3/3] btf_encoder: Change functions check due to broken dwarf

[Jiri Olsa]

On Thu, Nov 05, 2020 at 11:52:35AM -0800, Andrii Nakryiko wrote:

SNIP

> > +        * Let's got through all collected functions and filter
> > +        * out those that are not in ftrace and init code.
> > +        */
> > +       for (i = 0; i < functions_cnt; i++) {
> > +               struct elf_function *func = &functions[i];
> > +
> > +               /*
> > +                * Do not enable .init section functions,
> > +                * but keep .init.bpf.preserve_type functions.
> > +                */
> > +               if (is_init(ms, func->addr) && !is_bpf_init(ms, func->addr))
> > +                       continue;
> > +
> > +               /* Make sure function is within ftrace addresses. */
> > +               if (bsearch(&func->addr, addrs, count, sizeof(addrs[0]), addrs_cmp)) {
> > +                       /*
> > +                        * We iterate over sorted array, so we can easily skip
> > +                        * not valid item and move following valid field into
> > +                        * its place, and still keep the 'new' array sorted.
> > +                        */
> > +                       if (i != functions_valid)
> > +                               functions[functions_valid] = functions[i];
> > +                       functions_valid++;
> > +               }
> > +       }
> 
> can we re-assign function_cnt = functions_valid here? and
> functions_valid could be just a local temporary variable?

good idea, should be simpler.. will change

and ack to all naming changes above ;-)

thanks,
jirka

> 
> > +
> > +       free(addrs);
> > +       return 0;
> > +}
> > +
> 
> [...]
> 

Re: [PATCH 1/3] bpf: Move iterator functions into special init section

[Jiri Olsa]

On Mon, Nov 09, 2020 at 03:06:55PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Mon, Nov 09, 2020 at 03:05:00PM -0300, Arnaldo Carvalho de Melo escreveu:
> > Em Fri, Nov 06, 2020 at 11:25:10PM +0100, Jiri Olsa escreveu:
> > > With upcoming changes to pahole, that change the way how and
> > > which kernel functions are stored in BTF data, we need a way
> > > to recognize iterator functions.
> > > 
> > > Iterator functions need to be in BTF data, but have no real
> > > body and are currently placed in .init.text section, so they
> > > are freed after kernel init and are filtered out of BTF data
> > > because of that.
> > > 
> > > The solution is to place these functions under new section:
> > >   .init.bpf.preserve_type
> > > 
> > > And add 2 new symbols to mark that area:
> > >   __init_bpf_preserve_type_begin
> > >   __init_bpf_preserve_type_end
> > > 
> > > The code in pahole responsible for picking up the functions will
> > > be able to recognize functions from this section and add them to
> > > the BTF data and filter out all other .init.text functions.
> > 
> > This isn't applying on torvalds/master:
> > 
> > [acme@five linux]$ patch -p1 < /wb/1.patch
> > patching file include/asm-generic/vmlinux.lds.h
> > Hunk #2 succeeded at 754 (offset 1 line).
> > patching file include/linux/bpf.h
> > Hunk #1 succeeded at 1276 (offset -1 lines).
> > patching file include/linux/init.h
> > Hunk #1 FAILED at 52.
> > 1 out of 1 hunk FAILED -- saving rejects to file include/linux/init.h.rej
> > [acme@five linux]$
> > [acme@five linux]$ cat include/linux/init.h.rej
> > --- include/linux/init.h
> > +++ include/linux/init.h
> > @@ -52,6 +52,7 @@
> >  #define __initconst	__section(.init.rodata)
> >  #define __exitdata	__section(.exit.data)
> >  #define __exit_call	__used __section(.exitcall.exit)
> > +#define __init_bpf_preserve_type __section(.init.bpf.preserve_type)
> > 
> >  /*
> >   * modpost check for section mismatches during the kernel build.
> > [acme@five linux]$
> > 
> > 
> > I'm fixing it up by hand to try together with pahole's patches.
> 
> Due to:
> 
> 33def8498fdde180 ("treewide: Convert macro and uses of __section(foo) to __section("foo")")

ok, I'll send new version for the kernel patch

thanks,
jirka

Re: [PATCH 1/3] bpf: Move iterator functions into special init section

[Arnaldo Carvalho de Melo]

Em Mon, Nov 09, 2020 at 03:06:55PM -0300, Arnaldo Carvalho de Melo escreveu:
> > I'm fixing it up by hand to try together with pahole's patches.
 
> Due to:
 
> 33def8498fdde180 ("treewide: Convert macro and uses of __section(foo) to __section("foo")")
> 

For convenience:

 asm-generic/vmlinux.lds.h |   16 +++++++++++++++-
 linux/bpf.h               |    8 +++++++-
 linux/init.h              |    1 +
 3 files changed, 23 insertions(+), 2 deletions(-)

---

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index b2b3d81b1535a5ab..f91029b3443bf0d2 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -685,8 +685,21 @@
 	.BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) {			\
 		*(.BTF_ids)						\
 	}
+
+/*
+ * .init.bpf.preserve_type
+ *
+ * This section store special BPF function and marks them
+ * with begin/end symbols pair for the sake of pahole tool.
+ */
+#define INIT_BPF_PRESERVE_TYPE						\
+	__init_bpf_preserve_type_begin = .;                             \
+	*(.init.bpf.preserve_type)                                      \
+	__init_bpf_preserve_type_end = .;				\
+	MEM_DISCARD(init.bpf.preserve_type)
 #else
 #define BTF
+#define INIT_BPF_PRESERVE_TYPE
 #endif
 
 /*
@@ -741,7 +754,8 @@
 #define INIT_TEXT							\
 	*(.init.text .init.text.*)					\
 	*(.text.startup)						\
-	MEM_DISCARD(init.text*)
+	MEM_DISCARD(init.text*)						\
+	INIT_BPF_PRESERVE_TYPE
 
 #define EXIT_DATA							\
 	*(.exit.data .exit.data.*)					\
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 2b16bf48aab61a1f..73e8ededde3e9c09 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1276,10 +1276,16 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd);
 int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
 int bpf_obj_get_user(const char __user *pathname, int flags);
 
+#ifdef CONFIG_DEBUG_INFO_BTF
+#define BPF_INIT __init_bpf_preserve_type
+#else
+#define BPF_INIT __init
+#endif
+
 #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
 #define DEFINE_BPF_ITER_FUNC(target, args...)			\
 	extern int bpf_iter_ ## target(args);			\
-	int __init bpf_iter_ ## target(args) { return 0; }
+	int BPF_INIT bpf_iter_ ## target(args) { return 0; }
 
 struct bpf_iter_aux_info {
 	struct bpf_map *map;
diff --git a/include/linux/init.h b/include/linux/init.h
index 7b53cb3092ee9956..a7c71e3b5f9a1d65 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -52,6 +52,7 @@
 #define __initconst	__section(".init.rodata")
 #define __exitdata	__section(".exit.data")
 #define __exit_call	__used __section(".exitcall.exit")
+#define __init_bpf_preserve_type __section(".init.bpf.preserve_type")
 
 /*
  * modpost check for section mismatches during the kernel build.

Re: [PATCH 1/3] bpf: Move iterator functions into special init section

[Arnaldo Carvalho de Melo]

Em Mon, Nov 09, 2020 at 03:05:00PM -0300, Arnaldo Carvalho de Melo escreveu:
> Em Fri, Nov 06, 2020 at 11:25:10PM +0100, Jiri Olsa escreveu:
> > With upcoming changes to pahole, that change the way how and
> > which kernel functions are stored in BTF data, we need a way
> > to recognize iterator functions.
> > 
> > Iterator functions need to be in BTF data, but have no real
> > body and are currently placed in .init.text section, so they
> > are freed after kernel init and are filtered out of BTF data
> > because of that.
> > 
> > The solution is to place these functions under new section:
> >   .init.bpf.preserve_type
> > 
> > And add 2 new symbols to mark that area:
> >   __init_bpf_preserve_type_begin
> >   __init_bpf_preserve_type_end
> > 
> > The code in pahole responsible for picking up the functions will
> > be able to recognize functions from this section and add them to
> > the BTF data and filter out all other .init.text functions.
> 
> This isn't applying on torvalds/master:
> 
> [acme@five linux]$ patch -p1 < /wb/1.patch
> patching file include/asm-generic/vmlinux.lds.h
> Hunk #2 succeeded at 754 (offset 1 line).
> patching file include/linux/bpf.h
> Hunk #1 succeeded at 1276 (offset -1 lines).
> patching file include/linux/init.h
> Hunk #1 FAILED at 52.
> 1 out of 1 hunk FAILED -- saving rejects to file include/linux/init.h.rej
> [acme@five linux]$
> [acme@five linux]$ cat include/linux/init.h.rej
> --- include/linux/init.h
> +++ include/linux/init.h
> @@ -52,6 +52,7 @@
>  #define __initconst	__section(.init.rodata)
>  #define __exitdata	__section(.exit.data)
>  #define __exit_call	__used __section(.exitcall.exit)
> +#define __init_bpf_preserve_type __section(.init.bpf.preserve_type)
> 
>  /*
>   * modpost check for section mismatches during the kernel build.
> [acme@five linux]$
> 
> 
> I'm fixing it up by hand to try together with pahole's patches.

Due to:

33def8498fdde180 ("treewide: Convert macro and uses of __section(foo) to __section("foo")")

I'm using this now:

diff --git a/include/linux/init.h b/include/linux/init.h
index 7b53cb3092ee9956..a7c71e3b5f9a1d65 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -52,6 +52,7 @@
 #define __initconst	__section(".init.rodata")
 #define __exitdata	__section(".exit.data")
 #define __exit_call	__used __section(".exitcall.exit")
+#define __init_bpf_preserve_type __section(".init.bpf.preserve_type")
 
 /*
  * modpost check for section mismatches during the kernel build.

Re: [PATCH 1/3] bpf: Move iterator functions into special init section

[Arnaldo Carvalho de Melo]

Em Fri, Nov 06, 2020 at 11:25:10PM +0100, Jiri Olsa escreveu:
> With upcoming changes to pahole, that change the way how and
> which kernel functions are stored in BTF data, we need a way
> to recognize iterator functions.
> 
> Iterator functions need to be in BTF data, but have no real
> body and are currently placed in .init.text section, so they
> are freed after kernel init and are filtered out of BTF data
> because of that.
> 
> The solution is to place these functions under new section:
>   .init.bpf.preserve_type
> 
> And add 2 new symbols to mark that area:
>   __init_bpf_preserve_type_begin
>   __init_bpf_preserve_type_end
> 
> The code in pahole responsible for picking up the functions will
> be able to recognize functions from this section and add them to
> the BTF data and filter out all other .init.text functions.

This isn't applying on torvalds/master:

[acme@five linux]$ patch -p1 < /wb/1.patch
patching file include/asm-generic/vmlinux.lds.h
Hunk #2 succeeded at 754 (offset 1 line).
patching file include/linux/bpf.h
Hunk #1 succeeded at 1276 (offset -1 lines).
patching file include/linux/init.h
Hunk #1 FAILED at 52.
1 out of 1 hunk FAILED -- saving rejects to file include/linux/init.h.rej
[acme@five linux]$
[acme@five linux]$ cat include/linux/init.h.rej
--- include/linux/init.h
+++ include/linux/init.h
@@ -52,6 +52,7 @@
 #define __initconst	__section(.init.rodata)
 #define __exitdata	__section(.exit.data)
 #define __exit_call	__used __section(.exitcall.exit)
+#define __init_bpf_preserve_type __section(.init.bpf.preserve_type)

 /*
  * modpost check for section mismatches during the kernel build.
[acme@five linux]$


I'm fixing it up by hand to try together with pahole's patches.

- Arnaldo
 
> Suggested-by: Yonghong Song <yhs@fb.com>
> Acked-by: Song Liu <songliubraving@fb.com>
> Signed-off-by: Jiri Olsa <jolsa@redhat.com>
> ---
>  include/asm-generic/vmlinux.lds.h | 16 +++++++++++++++-
>  include/linux/bpf.h               |  8 +++++++-
>  include/linux/init.h              |  1 +
>  3 files changed, 23 insertions(+), 2 deletions(-)
> 
> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
> index cd14444bf600..e18e1030dabf 100644
> --- a/include/asm-generic/vmlinux.lds.h
> +++ b/include/asm-generic/vmlinux.lds.h
> @@ -685,8 +685,21 @@
>  	.BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) {			\
>  		*(.BTF_ids)						\
>  	}
> +
> +/*
> + * .init.bpf.preserve_type
> + *
> + * This section store special BPF function and marks them
> + * with begin/end symbols pair for the sake of pahole tool.
> + */
> +#define INIT_BPF_PRESERVE_TYPE						\
> +	__init_bpf_preserve_type_begin = .;                             \
> +	*(.init.bpf.preserve_type)                                      \
> +	__init_bpf_preserve_type_end = .;				\
> +	MEM_DISCARD(init.bpf.preserve_type)
>  #else
>  #define BTF
> +#define INIT_BPF_PRESERVE_TYPE
>  #endif
>  
>  /*
> @@ -740,7 +753,8 @@
>  #define INIT_TEXT							\
>  	*(.init.text .init.text.*)					\
>  	*(.text.startup)						\
> -	MEM_DISCARD(init.text*)
> +	MEM_DISCARD(init.text*)						\
> +	INIT_BPF_PRESERVE_TYPE
>  
>  #define EXIT_DATA							\
>  	*(.exit.data .exit.data.*)					\
> diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> index 73d5381a5d5c..894f66c7703e 100644
> --- a/include/linux/bpf.h
> +++ b/include/linux/bpf.h
> @@ -1277,10 +1277,16 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd);
>  int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
>  int bpf_obj_get_user(const char __user *pathname, int flags);
>  
> +#ifdef CONFIG_DEBUG_INFO_BTF
> +#define BPF_INIT __init_bpf_preserve_type
> +#else
> +#define BPF_INIT __init
> +#endif
> +
>  #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
>  #define DEFINE_BPF_ITER_FUNC(target, args...)			\
>  	extern int bpf_iter_ ## target(args);			\
> -	int __init bpf_iter_ ## target(args) { return 0; }
> +	int BPF_INIT bpf_iter_ ## target(args) { return 0; }
>  
>  struct bpf_iter_aux_info {
>  	struct bpf_map *map;
> diff --git a/include/linux/init.h b/include/linux/init.h
> index 212fc9e2f691..133462863711 100644
> --- a/include/linux/init.h
> +++ b/include/linux/init.h
> @@ -52,6 +52,7 @@
>  #define __initconst	__section(.init.rodata)
>  #define __exitdata	__section(.exit.data)
>  #define __exit_call	__used __section(.exitcall.exit)
> +#define __init_bpf_preserve_type __section(.init.bpf.preserve_type)
>  
>  /*
>   * modpost check for section mismatches during the kernel build.
> -- 
> 2.26.2
> 

-- 

- Arnaldo

[PATCH 1/3] bpf: Move iterator functions into special init section

[Jiri Olsa]

With upcoming changes to pahole, that change the way how and
which kernel functions are stored in BTF data, we need a way
to recognize iterator functions.

Iterator functions need to be in BTF data, but have no real
body and are currently placed in .init.text section, so they
are freed after kernel init and are filtered out of BTF data
because of that.

The solution is to place these functions under new section:
  .init.bpf.preserve_type

And add 2 new symbols to mark that area:
  __init_bpf_preserve_type_begin
  __init_bpf_preserve_type_end

The code in pahole responsible for picking up the functions will
be able to recognize functions from this section and add them to
the BTF data and filter out all other .init.text functions.

Suggested-by: Yonghong Song <yhs@fb.com>
Acked-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Jiri Olsa <jolsa@redhat.com>
---
 include/asm-generic/vmlinux.lds.h | 16 +++++++++++++++-
 include/linux/bpf.h               |  8 +++++++-
 include/linux/init.h              |  1 +
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index cd14444bf600..e18e1030dabf 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -685,8 +685,21 @@
 	.BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) {			\
 		*(.BTF_ids)						\
 	}
+
+/*
+ * .init.bpf.preserve_type
+ *
+ * This section store special BPF function and marks them
+ * with begin/end symbols pair for the sake of pahole tool.
+ */
+#define INIT_BPF_PRESERVE_TYPE						\
+	__init_bpf_preserve_type_begin = .;                             \
+	*(.init.bpf.preserve_type)                                      \
+	__init_bpf_preserve_type_end = .;				\
+	MEM_DISCARD(init.bpf.preserve_type)
 #else
 #define BTF
+#define INIT_BPF_PRESERVE_TYPE
 #endif
 
 /*
@@ -740,7 +753,8 @@
 #define INIT_TEXT							\
 	*(.init.text .init.text.*)					\
 	*(.text.startup)						\
-	MEM_DISCARD(init.text*)
+	MEM_DISCARD(init.text*)						\
+	INIT_BPF_PRESERVE_TYPE
 
 #define EXIT_DATA							\
 	*(.exit.data .exit.data.*)					\
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 73d5381a5d5c..894f66c7703e 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1277,10 +1277,16 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd);
 int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
 int bpf_obj_get_user(const char __user *pathname, int flags);
 
+#ifdef CONFIG_DEBUG_INFO_BTF
+#define BPF_INIT __init_bpf_preserve_type
+#else
+#define BPF_INIT __init
+#endif
+
 #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
 #define DEFINE_BPF_ITER_FUNC(target, args...)			\
 	extern int bpf_iter_ ## target(args);			\
-	int __init bpf_iter_ ## target(args) { return 0; }
+	int BPF_INIT bpf_iter_ ## target(args) { return 0; }
 
 struct bpf_iter_aux_info {
 	struct bpf_map *map;
diff --git a/include/linux/init.h b/include/linux/init.h
index 212fc9e2f691..133462863711 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -52,6 +52,7 @@
 #define __initconst	__section(.init.rodata)
 #define __exitdata	__section(.exit.data)
 #define __exit_call	__used __section(.exitcall.exit)
+#define __init_bpf_preserve_type __section(.init.bpf.preserve_type)
 
 /*
  * modpost check for section mismatches during the kernel build.
-- 
2.26.2