From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
To: linux-kernel@vger.kernel.org
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
stable@vger.kernel.org, Jann Horn <jannh@google.com>,
Linus Torvalds <torvalds@linux-foundation.org>,
"Peter Zijlstra (Intel)" <peterz@infradead.org>
Subject: [PATCH 4.19 56/65] futex: Fix inode life-time issue
Date: Tue, 24 Mar 2020 14:11:17 +0100 [thread overview]
Message-ID: <20200324130803.865203591@linuxfoundation.org> (raw)
In-Reply-To: <20200324130756.679112147@linuxfoundation.org>
From: Peter Zijlstra <peterz@infradead.org>
commit 8019ad13ef7f64be44d4f892af9c840179009254 upstream.
As reported by Jann, ihold() does not in fact guarantee inode
persistence. And instead of making it so, replace the usage of inode
pointers with a per boot, machine wide, unique inode identifier.
This sequence number is global, but shared (file backed) futexes are
rare enough that this should not become a performance issue.
Reported-by: Jann Horn <jannh@google.com>
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
fs/inode.c | 1
include/linux/fs.h | 1
include/linux/futex.h | 17 +++++----
kernel/futex.c | 89 +++++++++++++++++++++++++++++---------------------
4 files changed, 65 insertions(+), 43 deletions(-)
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -136,6 +136,7 @@ int inode_init_always(struct super_block
inode->i_sb = sb;
inode->i_blkbits = sb->s_blocksize_bits;
inode->i_flags = 0;
+ atomic64_set(&inode->i_sequence, 0);
atomic_set(&inode->i_count, 1);
inode->i_op = &empty_iops;
inode->i_fop = &no_open_fops;
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -663,6 +663,7 @@ struct inode {
struct rcu_head i_rcu;
};
atomic64_t i_version;
+ atomic64_t i_sequence; /* see futex */
atomic_t i_count;
atomic_t i_dio_count;
atomic_t i_writecount;
--- a/include/linux/futex.h
+++ b/include/linux/futex.h
@@ -29,23 +29,26 @@ struct task_struct;
union futex_key {
struct {
+ u64 i_seq;
unsigned long pgoff;
- struct inode *inode;
- int offset;
+ unsigned int offset;
} shared;
struct {
+ union {
+ struct mm_struct *mm;
+ u64 __tmp;
+ };
unsigned long address;
- struct mm_struct *mm;
- int offset;
+ unsigned int offset;
} private;
struct {
+ u64 ptr;
unsigned long word;
- void *ptr;
- int offset;
+ unsigned int offset;
} both;
};
-#define FUTEX_KEY_INIT (union futex_key) { .both = { .ptr = NULL } }
+#define FUTEX_KEY_INIT (union futex_key) { .both = { .ptr = 0ULL } }
#ifdef CONFIG_FUTEX
extern void exit_robust_list(struct task_struct *curr);
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -439,7 +439,7 @@ static void get_futex_key_refs(union fut
switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
case FUT_OFF_INODE:
- ihold(key->shared.inode); /* implies smp_mb(); (B) */
+ smp_mb(); /* explicit smp_mb(); (B) */
break;
case FUT_OFF_MMSHARED:
futex_get_mm(key); /* implies smp_mb(); (B) */
@@ -473,7 +473,6 @@ static void drop_futex_key_refs(union fu
switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
case FUT_OFF_INODE:
- iput(key->shared.inode);
break;
case FUT_OFF_MMSHARED:
mmdrop(key->private.mm);
@@ -481,6 +480,46 @@ static void drop_futex_key_refs(union fu
}
}
+/*
+ * Generate a machine wide unique identifier for this inode.
+ *
+ * This relies on u64 not wrapping in the life-time of the machine; which with
+ * 1ns resolution means almost 585 years.
+ *
+ * This further relies on the fact that a well formed program will not unmap
+ * the file while it has a (shared) futex waiting on it. This mapping will have
+ * a file reference which pins the mount and inode.
+ *
+ * If for some reason an inode gets evicted and read back in again, it will get
+ * a new sequence number and will _NOT_ match, even though it is the exact same
+ * file.
+ *
+ * It is important that match_futex() will never have a false-positive, esp.
+ * for PI futexes that can mess up the state. The above argues that false-negatives
+ * are only possible for malformed programs.
+ */
+static u64 get_inode_sequence_number(struct inode *inode)
+{
+ static atomic64_t i_seq;
+ u64 old;
+
+ /* Does the inode already have a sequence number? */
+ old = atomic64_read(&inode->i_sequence);
+ if (likely(old))
+ return old;
+
+ for (;;) {
+ u64 new = atomic64_add_return(1, &i_seq);
+ if (WARN_ON_ONCE(!new))
+ continue;
+
+ old = atomic64_cmpxchg_relaxed(&inode->i_sequence, 0, new);
+ if (old)
+ return old;
+ return new;
+ }
+}
+
/**
* get_futex_key() - Get parameters which are the keys for a futex
* @uaddr: virtual address of the futex
@@ -493,9 +532,15 @@ static void drop_futex_key_refs(union fu
*
* The key words are stored in @key on success.
*
- * For shared mappings, it's (page->index, file_inode(vma->vm_file),
- * offset_within_page). For private mappings, it's (uaddr, current->mm).
- * We can usually work out the index without swapping in the page.
+ * For shared mappings (when @fshared), the key is:
+ * ( inode->i_sequence, page->index, offset_within_page )
+ * [ also see get_inode_sequence_number() ]
+ *
+ * For private mappings (or when !@fshared), the key is:
+ * ( current->mm, address, 0 )
+ *
+ * This allows (cross process, where applicable) identification of the futex
+ * without keeping the page pinned for the duration of the FUTEX_WAIT.
*
* lock_page() might sleep, the caller should not hold a spinlock.
*/
@@ -635,8 +680,6 @@ again:
key->private.mm = mm;
key->private.address = address;
- get_futex_key_refs(key); /* implies smp_mb(); (B) */
-
} else {
struct inode *inode;
@@ -668,40 +711,14 @@ again:
goto again;
}
- /*
- * Take a reference unless it is about to be freed. Previously
- * this reference was taken by ihold under the page lock
- * pinning the inode in place so i_lock was unnecessary. The
- * only way for this check to fail is if the inode was
- * truncated in parallel which is almost certainly an
- * application bug. In such a case, just retry.
- *
- * We are not calling into get_futex_key_refs() in file-backed
- * cases, therefore a successful atomic_inc return below will
- * guarantee that get_futex_key() will still imply smp_mb(); (B).
- */
- if (!atomic_inc_not_zero(&inode->i_count)) {
- rcu_read_unlock();
- put_page(page);
-
- goto again;
- }
-
- /* Should be impossible but lets be paranoid for now */
- if (WARN_ON_ONCE(inode->i_mapping != mapping)) {
- err = -EFAULT;
- rcu_read_unlock();
- iput(inode);
-
- goto out;
- }
-
key->both.offset |= FUT_OFF_INODE; /* inode-based key */
- key->shared.inode = inode;
+ key->shared.i_seq = get_inode_sequence_number(inode);
key->shared.pgoff = basepage_index(tail);
rcu_read_unlock();
}
+ get_futex_key_refs(key); /* implies smp_mb(); (B) */
+
out:
put_page(page);
return err;
next prev parent reply other threads:[~2020-03-24 13:15 UTC|newest]
Thread overview: 70+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-03-24 13:10 [PATCH 4.19 00/65] 4.19.113-rc1 review Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 01/65] drm/mediatek: Find the cursor plane instead of hard coding it Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 02/65] spi: qup: call spi_qup_pm_resume_runtime before suspending Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 03/65] powerpc: Include .BTF section Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 04/65] ARM: dts: dra7: Add "dma-ranges" property to PCIe RC DT nodes Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 05/65] spi: pxa2xx: Add CS control clock quirk Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 06/65] spi/zynqmp: remove entry that causes a cs glitch Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 07/65] drm/exynos: dsi: propagate error value and silence meaningless warning Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 08/65] drm/exynos: dsi: fix workaround for the legacy clock name Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 09/65] drivers/perf: arm_pmu_acpi: Fix incorrect checking of gicc pointer Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 10/65] altera-stapl: altera_get_note: prevent write beyond end of key Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 11/65] dm bio record: save/restore bi_end_io and bi_integrity Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 12/65] dm integrity: use dm_bio_record and dm_bio_restore Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 13/65] riscv: avoid the PIC offset of static percpu data in module beyond 2G limits Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 14/65] drm/amd/display: Clear link settings on MST disable connector Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 15/65] drm/amd/display: fix dcc swath size calculations on dcn1 Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 16/65] xenbus: req->body should be updated before req->state Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 17/65] xenbus: req->err " Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 18/65] block, bfq: fix overwrite of bfq_group pointer in bfq_find_set_group() Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 19/65] parse-maintainers: Mark as executable Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 20/65] USB: Disable LPM on WD19s Realtek Hub Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 21/65] usb: quirks: add NO_LPM quirk for RTL8153 based ethernet adapters Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 22/65] USB: serial: option: add ME910G1 ECM composition 0x110b Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 23/65] usb: host: xhci-plat: add a shutdown Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 24/65] USB: serial: pl2303: add device-id for HP LD381 Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 25/65] usb: xhci: apply XHCI_SUSPEND_DELAY to AMD XHCI controller 1022:145c Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 26/65] ALSA: line6: Fix endless MIDI read loop Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 27/65] ALSA: seq: virmidi: Fix running status after receiving sysex Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 28/65] ALSA: seq: oss: " Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 29/65] ALSA: pcm: oss: Avoid plugin buffer overflow Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 30/65] ALSA: pcm: oss: Remove WARNING from snd_pcm_plug_alloc() checks Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 31/65] iio: st_sensors: remap SMO8840 to LIS2DH12 Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 32/65] iio: trigger: stm32-timer: disable master mode when stopping Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 33/65] iio: magnetometer: ak8974: Fix negative raw values in sysfs Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 34/65] iio: adc: at91-sama5d2_adc: fix differential channels in triggered mode Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 35/65] mmc: rtsx_pci: Fix support for speed-modes that relies on tuning Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 36/65] mmc: sdhci-of-at91: fix cd-gpios for SAMA5D2 Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 37/65] staging: rtl8188eu: Add device id for MERCUSYS MW150US v2 Greg Kroah-Hartman
2020-03-24 13:10 ` [PATCH 4.19 38/65] staging: greybus: loopback_test: fix poll-mask build breakage Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 39/65] staging/speakup: fix get_word non-space look-ahead Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 40/65] intel_th: Fix user-visible error codes Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 41/65] intel_th: pci: Add Elkhart Lake CPU support Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 42/65] rtc: max8907: add missing select REGMAP_IRQ Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 43/65] xhci: Do not open code __print_symbolic() in xhci trace events Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 44/65] btrfs: fix log context list corruption after rename whiteout error Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 45/65] drm/amd/amdgpu: Fix GPR read from debugfs (v2) Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 46/65] drm/lease: fix WARNING in idr_destroy Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 47/65] memcg: fix NULL pointer dereference in __mem_cgroup_usage_unregister_event Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 48/65] mm: slub: be more careful about the double cmpxchg of freelist Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 49/65] mm, slub: prevent kmalloc_node crashes and memory leaks Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 50/65] page-flags: fix a crash at SetPageError(THP_SWAP) Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 51/65] x86/mm: split vmalloc_sync_all() Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 52/65] USB: cdc-acm: fix close_delay and closing_wait units in TIOCSSERIAL Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 53/65] USB: cdc-acm: fix rounding error " Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 54/65] iio: light: vcnl4000: update sampling periods for vcnl4200 Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 55/65] kbuild: Disable -Wpointer-to-enum-cast Greg Kroah-Hartman
2020-03-24 13:11 ` Greg Kroah-Hartman [this message]
2020-03-24 13:11 ` [PATCH 4.19 57/65] futex: Unbreak futex hashing Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 58/65] Revert "vrf: mark skb for multicast or link-local as enslaved to VRF" Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 59/65] Revert "ipv6: Fix handling of LLA with VRF and sockets bound " Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 60/65] ALSA: hda/realtek: Fix pop noise on ALC225 Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 61/65] arm64: smp: fix smp_send_stop() behaviour Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 62/65] arm64: smp: fix crash_smp_send_stop() behaviour Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 63/65] drm/bridge: dw-hdmi: fix AVI frame colorimetry Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 64/65] staging: greybus: loopback_test: fix potential path truncation Greg Kroah-Hartman
2020-03-24 13:11 ` [PATCH 4.19 65/65] staging: greybus: loopback_test: fix potential path truncations Greg Kroah-Hartman
2020-03-24 15:53 ` [PATCH 4.19 00/65] 4.19.113-rc1 review Chris Paterson
2020-03-24 19:51 ` shuah
2020-03-24 20:55 ` Guenter Roeck
2020-03-25 4:50 ` Naresh Kamboju
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200324130803.865203591@linuxfoundation.org \
--to=gregkh@linuxfoundation.org \
--cc=jannh@google.com \
--cc=linux-kernel@vger.kernel.org \
--cc=peterz@infradead.org \
--cc=stable@vger.kernel.org \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).