Linux-PCI Archive on lore.kernel.org
 help / color / Atom feed
From: Sergey Miroshnichenko <s.miroshnichenko@yadro.com>
To: <linux-pci@vger.kernel.org>, <linuxppc-dev@lists.ozlabs.org>
Cc: Bjorn Helgaas <helgaas@kernel.org>, <linux@yadro.com>,
	Sergey Miroshnichenko <s.miroshnichenko@yadro.com>
Subject: [PATCH RFC v4 02/21] PCI: Fix race condition in pci_enable/disable_device()
Date: Mon, 11 Mar 2019 16:31:03 +0300
Message-ID: <20190311133122.11417-3-s.miroshnichenko@yadro.com> (raw)
In-Reply-To: <20190311133122.11417-1-s.miroshnichenko@yadro.com>

 CPU0                                      CPU1

 pci_enable_device_mem()                   pci_enable_device_mem()
   pci_enable_bridge()                       pci_enable_bridge()
     pci_is_enabled()
       return false;
     atomic_inc_return(enable_cnt)
     Start actual enabling the bridge
     ...                                       pci_is_enabled()
     ...                                         return true;
     ...                                   Start memory requests <-- FAIL
     ...
     Set the PCI_COMMAND_MEMORY bit <-- Must wait for this

This patch protects the pci_enable/disable_device() and pci_enable_bridge()
with mutexes.

Signed-off-by: Sergey Miroshnichenko <s.miroshnichenko@yadro.com>
---
 drivers/pci/pci.c   | 26 ++++++++++++++++++++++----
 drivers/pci/probe.c |  1 +
 include/linux/pci.h |  1 +
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index f006068be209..895201d4c9e6 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -1615,6 +1615,8 @@ static void pci_enable_bridge(struct pci_dev *dev)
 	struct pci_dev *bridge;
 	int retval;
 
+	mutex_lock(&dev->enable_mutex);
+
 	bridge = pci_upstream_bridge(dev);
 	if (bridge)
 		pci_enable_bridge(bridge);
@@ -1622,6 +1624,7 @@ static void pci_enable_bridge(struct pci_dev *dev)
 	if (pci_is_enabled(dev)) {
 		if (!dev->is_busmaster)
 			pci_set_master(dev);
+		mutex_unlock(&dev->enable_mutex);
 		return;
 	}
 
@@ -1630,11 +1633,14 @@ static void pci_enable_bridge(struct pci_dev *dev)
 		pci_err(dev, "Error enabling bridge (%d), continuing\n",
 			retval);
 	pci_set_master(dev);
+	mutex_unlock(&dev->enable_mutex);
 }
 
 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
 {
 	struct pci_dev *bridge;
+	/* Enable-locking of bridges is performed within the pci_enable_bridge() */
+	bool need_lock = !dev->subordinate;
 	int err;
 	int i, bars = 0;
 
@@ -1650,8 +1656,13 @@ static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
 		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
 	}
 
-	if (atomic_inc_return(&dev->enable_cnt) > 1)
+	if (need_lock)
+		mutex_lock(&dev->enable_mutex);
+	if (pci_is_enabled(dev)) {
+		if (need_lock)
+			mutex_unlock(&dev->enable_mutex);
 		return 0;		/* already enabled */
+	}
 
 	bridge = pci_upstream_bridge(dev);
 	if (bridge)
@@ -1666,8 +1677,10 @@ static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
 			bars |= (1 << i);
 
 	err = do_pci_enable_device(dev, bars);
-	if (err < 0)
-		atomic_dec(&dev->enable_cnt);
+	if (err >= 0)
+		atomic_inc(&dev->enable_cnt);
+	if (need_lock)
+		mutex_unlock(&dev->enable_mutex);
 	return err;
 }
 
@@ -1910,15 +1923,20 @@ void pci_disable_device(struct pci_dev *dev)
 	if (dr)
 		dr->enabled = 0;
 
+	mutex_lock(&dev->enable_mutex);
 	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
 		      "disabling already-disabled device");
 
-	if (atomic_dec_return(&dev->enable_cnt) != 0)
+	if (atomic_dec_return(&dev->enable_cnt) != 0) {
+		mutex_unlock(&dev->enable_mutex);
 		return;
+	}
 
 	do_pci_disable_device(dev);
 
 	dev->is_busmaster = 0;
+
+	mutex_unlock(&dev->enable_mutex);
 }
 EXPORT_SYMBOL(pci_disable_device);
 
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 2ec0df04e0dc..977a127ce791 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -2267,6 +2267,7 @@ struct pci_dev *pci_alloc_dev(struct pci_bus *bus)
 	INIT_LIST_HEAD(&dev->bus_list);
 	dev->dev.type = &pci_dev_type;
 	dev->bus = pci_bus_get(bus);
+	mutex_init(&dev->enable_mutex);
 
 	return dev;
 }
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 77448215ef5b..cb2760a31fe2 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -419,6 +419,7 @@ struct pci_dev {
 	unsigned int	no_vf_scan:1;		/* Don't scan for VFs after IOV enablement */
 	pci_dev_flags_t dev_flags;
 	atomic_t	enable_cnt;	/* pci_enable_device has been called */
+	struct mutex	enable_mutex;
 
 	u32		saved_config_space[16]; /* Config space saved at suspend time */
 	struct hlist_head saved_cap_space;
-- 
2.20.1


  parent reply index

Thread overview: 44+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-03-11 13:31 [PATCH RFC v4 00/21] PCI: Allow BAR movement during hotplug Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 01/21] PCI: Fix writing invalid BARs during pci_restore_state() Sergey Miroshnichenko
2019-03-26 14:02   ` Bjorn Helgaas
2019-03-11 13:31 ` Sergey Miroshnichenko [this message]
2019-03-26 19:00   ` [PATCH RFC v4 02/21] PCI: Fix race condition in pci_enable/disable_device() Bjorn Helgaas
2019-03-27 17:11     ` Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 03/21] PCI: Enable bridge's I/O and MEM access for hotplugged devices Sergey Miroshnichenko
2019-03-26 19:13   ` Bjorn Helgaas
2019-03-27 17:13     ` Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 04/21] PCI: Define PCI-specific version of the release_child_resources() Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 05/21] PCI: hotplug: Add a flag for the movable BARs feature Sergey Miroshnichenko
2019-03-26 19:24   ` Bjorn Helgaas
2019-03-27 17:16     ` Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 06/21] PCI: Pause the devices with movable BARs during rescan Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 07/21] PCI: Wake up bridges during rescan when movable BARs enabled Sergey Miroshnichenko
2019-03-26 19:28   ` Bjorn Helgaas
2019-03-11 13:31 ` [PATCH RFC v4 08/21] nvme-pci: Handle movable BARs Sergey Miroshnichenko
2019-03-26 20:20   ` Bjorn Helgaas
2019-03-27 17:30     ` Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 09/21] PCI: Mark immovable BARs with PCI_FIXED Sergey Miroshnichenko
2019-03-26 20:28   ` Bjorn Helgaas
2019-03-27 17:03     ` David Laight
2019-03-27 17:39       ` Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 10/21] PCI: Fix assigning of fixed prefetchable resources Sergey Miroshnichenko
2019-03-26 20:37   ` Bjorn Helgaas
2019-03-11 13:31 ` [PATCH RFC v4 11/21] PCI: Release and reassign the root bridge resources during rescan Sergey Miroshnichenko
2019-03-26 20:41   ` Bjorn Helgaas
2019-03-27 17:40     ` Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 12/21] PCI: Don't allow hotplugged devices to steal resources Sergey Miroshnichenko
2019-03-26 20:55   ` Bjorn Helgaas
2019-03-27 18:02     ` Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 13/21] PCI: Include fixed BARs into the bus size calculating Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 14/21] PCI: Don't reserve memory for hotplug when enabled movable BARs Sergey Miroshnichenko
2019-03-26 20:57   ` Bjorn Helgaas
2019-03-11 13:31 ` [PATCH RFC v4 15/21] PCI: Allow the failed resources to be reassigned later Sergey Miroshnichenko
2019-03-26 20:58   ` Bjorn Helgaas
2019-03-11 13:31 ` [PATCH RFC v4 16/21] PCI: Calculate fixed areas of bridge windows based on fixed BARs Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 17/21] PCI: Calculate boundaries for bridge windows Sergey Miroshnichenko
2019-03-26 21:01   ` Bjorn Helgaas
2019-03-11 13:31 ` [PATCH RFC v4 18/21] PCI: Make sure bridge windows include their fixed BARs Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 19/21] PCI: Prioritize fixed BAR assigning over the movable ones Sergey Miroshnichenko
2019-03-11 13:31 ` [PATCH RFC v4 20/21] PCI: pciehp: Add support for the movable BARs feature Sergey Miroshnichenko
2019-03-26 21:11   ` Bjorn Helgaas
2019-03-11 13:31 ` [PATCH RFC v4 21/21] powerpc/pci: Fix crash with enabled movable BARs Sergey Miroshnichenko

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20190311133122.11417-3-s.miroshnichenko@yadro.com \
    --to=s.miroshnichenko@yadro.com \
    --cc=helgaas@kernel.org \
    --cc=linux-pci@vger.kernel.org \
    --cc=linux@yadro.com \
    --cc=linuxppc-dev@lists.ozlabs.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Linux-PCI Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/linux-pci/0 linux-pci/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 linux-pci linux-pci/ https://lore.kernel.org/linux-pci \
		linux-pci@vger.kernel.org
	public-inbox-index linux-pci

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-pci


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git