Netdev Archive on lore.kernel.org
 help / color / Atom feed
From: Sasha Levin <sashal@kernel.org>
To: linux-kernel@vger.kernel.org, stable@vger.kernel.org
Cc: Matt Mullins <mmullins@fb.com>, Andrii Nakryiko <andriin@fb.com>,
	Daniel Borkmann <daniel@iogearbox.net>,
	Alexei Starovoitov <ast@kernel.org>,
	Sasha Levin <sashal@kernel.org>,
	netdev@vger.kernel.org, bpf@vger.kernel.org
Subject: [PATCH AUTOSEL 4.19 49/60] bpf: fix nested bpf tracepoints with per-cpu data
Date: Wed, 26 Jun 2019 20:36:04 -0400
Message-ID: <20190627003616.20767-49-sashal@kernel.org> (raw)
In-Reply-To: <20190627003616.20767-1-sashal@kernel.org>

From: Matt Mullins <mmullins@fb.com>

[ Upstream commit 9594dc3c7e71b9f52bee1d7852eb3d4e3aea9e99 ]

BPF_PROG_TYPE_RAW_TRACEPOINTs can be executed nested on the same CPU, as
they do not increment bpf_prog_active while executing.

This enables three levels of nesting, to support
  - a kprobe or raw tp or perf event,
  - another one of the above that irq context happens to call, and
  - another one in nmi context
(at most one of which may be a kprobe or perf event).

Fixes: 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")
Signed-off-by: Matt Mullins <mmullins@fb.com>
Acked-by: Andrii Nakryiko <andriin@fb.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
 kernel/trace/bpf_trace.c | 100 ++++++++++++++++++++++++++++++++-------
 1 file changed, 84 insertions(+), 16 deletions(-)

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 6c28d519447d..83c4e76f513a 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -365,8 +365,6 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
 	.arg4_type	= ARG_CONST_SIZE,
 };
 
-static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd);
-
 static __always_inline u64
 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
 			u64 flags, struct perf_sample_data *sd)
@@ -398,24 +396,50 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
 	return 0;
 }
 
+/*
+ * Support executing tracepoints in normal, irq, and nmi context that each call
+ * bpf_perf_event_output
+ */
+struct bpf_trace_sample_data {
+	struct perf_sample_data sds[3];
+};
+
+static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds);
+static DEFINE_PER_CPU(int, bpf_trace_nest_level);
 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
 	   u64, flags, void *, data, u64, size)
 {
-	struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd);
+	struct bpf_trace_sample_data *sds = this_cpu_ptr(&bpf_trace_sds);
+	int nest_level = this_cpu_inc_return(bpf_trace_nest_level);
 	struct perf_raw_record raw = {
 		.frag = {
 			.size = size,
 			.data = data,
 		},
 	};
+	struct perf_sample_data *sd;
+	int err;
 
-	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
-		return -EINVAL;
+	if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) {
+		err = -EBUSY;
+		goto out;
+	}
+
+	sd = &sds->sds[nest_level - 1];
+
+	if (unlikely(flags & ~(BPF_F_INDEX_MASK))) {
+		err = -EINVAL;
+		goto out;
+	}
 
 	perf_sample_data_init(sd, 0, 0);
 	sd->raw = &raw;
 
-	return __bpf_perf_event_output(regs, map, flags, sd);
+	err = __bpf_perf_event_output(regs, map, flags, sd);
+
+out:
+	this_cpu_dec(bpf_trace_nest_level);
+	return err;
 }
 
 static const struct bpf_func_proto bpf_perf_event_output_proto = {
@@ -772,16 +796,48 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 /*
  * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
  * to avoid potential recursive reuse issue when/if tracepoints are added
- * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack
+ * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack.
+ *
+ * Since raw tracepoints run despite bpf_prog_active, support concurrent usage
+ * in normal, irq, and nmi context.
  */
-static DEFINE_PER_CPU(struct pt_regs, bpf_raw_tp_regs);
+struct bpf_raw_tp_regs {
+	struct pt_regs regs[3];
+};
+static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs);
+static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level);
+static struct pt_regs *get_bpf_raw_tp_regs(void)
+{
+	struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs);
+	int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level);
+
+	if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) {
+		this_cpu_dec(bpf_raw_tp_nest_level);
+		return ERR_PTR(-EBUSY);
+	}
+
+	return &tp_regs->regs[nest_level - 1];
+}
+
+static void put_bpf_raw_tp_regs(void)
+{
+	this_cpu_dec(bpf_raw_tp_nest_level);
+}
+
 BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args,
 	   struct bpf_map *, map, u64, flags, void *, data, u64, size)
 {
-	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+	struct pt_regs *regs = get_bpf_raw_tp_regs();
+	int ret;
+
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
 
 	perf_fetch_caller_regs(regs);
-	return ____bpf_perf_event_output(regs, map, flags, data, size);
+	ret = ____bpf_perf_event_output(regs, map, flags, data, size);
+
+	put_bpf_raw_tp_regs();
+	return ret;
 }
 
 static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
@@ -798,12 +854,18 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
 BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args,
 	   struct bpf_map *, map, u64, flags)
 {
-	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+	struct pt_regs *regs = get_bpf_raw_tp_regs();
+	int ret;
+
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
 
 	perf_fetch_caller_regs(regs);
 	/* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */
-	return bpf_get_stackid((unsigned long) regs, (unsigned long) map,
-			       flags, 0, 0);
+	ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map,
+			      flags, 0, 0);
+	put_bpf_raw_tp_regs();
+	return ret;
 }
 
 static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
@@ -818,11 +880,17 @@ static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
 BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args,
 	   void *, buf, u32, size, u64, flags)
 {
-	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+	struct pt_regs *regs = get_bpf_raw_tp_regs();
+	int ret;
+
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
 
 	perf_fetch_caller_regs(regs);
-	return bpf_get_stack((unsigned long) regs, (unsigned long) buf,
-			     (unsigned long) size, flags, 0);
+	ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf,
+			    (unsigned long) size, flags, 0);
+	put_bpf_raw_tp_regs();
+	return ret;
 }
 
 static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
-- 
2.20.1


  parent reply index

Thread overview: 41+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
     [not found] <20190627003616.20767-1-sashal@kernel.org>
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 04/60] samples, bpf: fix to change the buffer size for read() Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 05/60] samples, bpf: suppress compiler warning Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 06/60] mac80211: fix rate reporting inside cfg80211_calculate_bitrate_he() Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 07/60] bpf: sockmap, fix use after free from sleep in psock backlog workqueue Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 11/60] mac80211: mesh: fix RCU warning Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 12/60] mac80211: free peer keys before vif down in mesh Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 13/60] mwifiex: Fix possible buffer overflows at parsing bss descriptor Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 14/60] mwifiex: Abort at too short BSS descriptor element Sasha Levin
2019-06-28 22:58   ` Brian Norris
2019-07-10 14:51     ` Sasha Levin
2019-07-10 21:12       ` Brian Norris
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 15/60] iwlwifi: Fix double-free problems in iwl_req_fw_callback() Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 16/60] mwifiex: Fix heap overflow in mwifiex_uap_parse_tail_ies() Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 17/60] bpf: udp: Avoid calling reuseport's bpf_prog from udp_gro Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 18/60] netfilter: ipv6: nf_defrag: fix leakage of unqueued fragments Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 20/60] netfilter: ipv6: nf_defrag: accept duplicate fragments again Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 21/60] dt-bindings: can: mcp251x: add mcp25625 support Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 22/60] can: mcp251x: add support for mcp25625 Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 23/60] can: m_can: implement errata "Needless activation of MRAF irq" Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 24/60] can: af_can: Fix error path of can_init() Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 25/60] net: phy: rename Asix Electronics PHY driver Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 26/60] ibmvnic: Do not close unopened driver during reset Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 27/60] ibmvnic: Refresh device multicast list after reset Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 28/60] ibmvnic: Fix unchecked return codes of memory allocations Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 31/60] bpf: lpm_trie: check left child of last leftmost node for NULL Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 37/60] mlxsw: spectrum: Disallow prio-tagged packets when PVID is removed Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 40/60] mac80211: only warn once on chanctx_conf being NULL Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 41/60] mac80211: do not start any work during reconfigure flow Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 42/60] bpf, devmap: Fix premature entry free on destroying map Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 43/60] bpf, devmap: Add missing bulk queue free Sasha Levin
2019-06-27  0:35 ` [PATCH AUTOSEL 4.19 44/60] bpf, devmap: Add missing RCU read lock on flush Sasha Levin
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 45/60] bpf, x64: fix stack layout of JITed bpf code Sasha Levin
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 46/60] qmi_wwan: add support for QMAP padding in the RX path Sasha Levin
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 47/60] qmi_wwan: avoid RCU stalls on device disconnect when in QMAP mode Sasha Levin
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 48/60] qmi_wwan: extend permitted QMAP mux_id value range Sasha Levin
2019-06-27  0:36 ` Sasha Levin [this message]
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 52/60] bnx2x: Check if transceiver implements DDM before access Sasha Levin
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 54/60] ip6_tunnel: allow not to count pkts on tstats by passing dev as NULL Sasha Levin
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 55/60] net: lio_core: fix potential sign-extension overflow on large shift Sasha Levin
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 58/60] net: dsa: mv88e6xxx: fix shift of FID bits in mv88e6185_g1_vtu_loadpurge() Sasha Levin
2019-06-27  0:36 ` [PATCH AUTOSEL 4.19 60/60] net :sunrpc :clnt :Fix xps refcount imbalance on the error path Sasha Levin

Reply instructions:

You may reply publically to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20190627003616.20767-49-sashal@kernel.org \
    --to=sashal@kernel.org \
    --cc=andriin@fb.com \
    --cc=ast@kernel.org \
    --cc=bpf@vger.kernel.org \
    --cc=daniel@iogearbox.net \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mmullins@fb.com \
    --cc=netdev@vger.kernel.org \
    --cc=stable@vger.kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Netdev Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/netdev/0 netdev/git/0.git
	git clone --mirror https://lore.kernel.org/netdev/1 netdev/git/1.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 netdev netdev/ https://lore.kernel.org/netdev \
		netdev@vger.kernel.org netdev@archiver.kernel.org
	public-inbox-index netdev

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.netdev


AGPL code for this site: git clone https://public-inbox.org/ public-inbox