linux-kernel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
* RE: [PATCH 02/22] NTB: Add peer indexed ports NTB API
@ 2016-12-04  0:04 Allen Hubbe
  2016-12-07 22:56 ` Allen Hubbe
  0 siblings, 1 reply; 6+ messages in thread
From: Allen Hubbe @ 2016-12-04  0:04 UTC (permalink / raw)
  To: 'Serge Semin', jdmason, dave.jiang, Xiangliang.Yu
  Cc: Sergey.Semin, linux-ntb, linux-kernel

From: Serge Semin
> Add new port-index NTB API. Additionally lets get rid of Primary and
> Secondary topologies, since port-number can be effectively used instead.

Split into two patches please.

I see no harm to the TOPO changes, though I wonder if they are necessary.

> Signed-off-by: Serge Semin <fancer.lancer@gmail.com>
> 
> ---
>  include/linux/ntb.h | 101 ++++++++++++++++++++++++++++++++++++++++------------
>  1 file changed, 79 insertions(+), 22 deletions(-)
> 
> diff --git a/include/linux/ntb.h b/include/linux/ntb.h
> index 5d1f260..0941a43 100644
> --- a/include/linux/ntb.h
> +++ b/include/linux/ntb.h
> @@ -64,37 +64,21 @@ struct pci_dev;
>  /**
>   * enum ntb_topo - NTB connection topology
>   * @NTB_TOPO_NONE:	Topology is unknown or invalid.
> - * @NTB_TOPO_PRI:	On primary side of local ntb.
> - * @NTB_TOPO_SEC:	On secondary side of remote ntb.
> - * @NTB_TOPO_B2B_USD:	On primary side of local ntb upstream of remote ntb.
> - * @NTB_TOPO_B2B_DSD:	On primary side of local ntb downstream of remote ntb.
> + * @NTB_TOPO_P2P:	Simple port-to-port NTB topology
> + * @NTB_TOPO_B2B:	Bridge-to-bridge NTB topology
>   */
>  enum ntb_topo {
>  	NTB_TOPO_NONE = -1,
> -	NTB_TOPO_PRI,
> -	NTB_TOPO_SEC,
> -	NTB_TOPO_B2B_USD,
> -	NTB_TOPO_B2B_DSD,
> +	NTB_TOPO_P2P,
> +	NTB_TOPO_B2B
>  };
> 
> -static inline int ntb_topo_is_b2b(enum ntb_topo topo)
> -{
> -	switch ((int)topo) {
> -	case NTB_TOPO_B2B_USD:
> -	case NTB_TOPO_B2B_DSD:
> -		return 1;
> -	}
> -	return 0;
> -}
> -
>  static inline char *ntb_topo_string(enum ntb_topo topo)
>  {
>  	switch (topo) {
>  	case NTB_TOPO_NONE:	return "NTB_TOPO_NONE";
> -	case NTB_TOPO_PRI:	return "NTB_TOPO_PRI";
> -	case NTB_TOPO_SEC:	return "NTB_TOPO_SEC";
> -	case NTB_TOPO_B2B_USD:	return "NTB_TOPO_B2B_USD";
> -	case NTB_TOPO_B2B_DSD:	return "NTB_TOPO_B2B_DSD";
> +	case NTB_TOPO_P2P:	return "NTB_TOPO_P2P";
> +	case NTB_TOPO_B2B:	return "NTB_TOPO_B2B";
>  	}
>  	return "NTB_TOPO_INVALID";
>  }
> @@ -179,6 +163,10 @@ static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
> 
>  /**
>   * struct ntb_ctx_ops - ntb device operations
> + * @port_number:	See ntb_port_number().
> + * @peer_port_count:	See ntb_peer_port_count().
> + * @peer_port_number:	See ntb_peer_port_number().
> + * @peer_port_idx:	See ntb_peer_port_idx().
>   * @link_is_up:		See ntb_link_is_up().
>   * @link_enable:	See ntb_link_enable().
>   * @link_disable:	See ntb_link_disable().
> @@ -212,6 +200,11 @@ static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
>   * @peer_spad_write:	See ntb_peer_spad_write().
>   */
>  struct ntb_dev_ops {
> +	int (*port_number)(struct ntb_dev *ntb);
> +	int (*peer_port_count)(struct ntb_dev *ntb);
> +	int (*peer_port_number)(struct ntb_dev *ntb, int pidx);
> +	int (*peer_port_idx)(struct ntb_dev *ntb, int port);
> +
>  	int (*link_is_up)(struct ntb_dev *ntb,
>  			  enum ntb_speed *speed, enum ntb_width *width);
>  	int (*link_enable)(struct ntb_dev *ntb,
> @@ -265,6 +258,10 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
>  {
>  	/* commented callbacks are not required: */
>  	return
> +		ops->port_number			&&
> +		ops->peer_port_count			&&
> +		ops->peer_port_number			&&
> +		ops->peer_port_idx			&&
>  		ops->link_is_up				&&
>  		ops->link_enable			&&
>  		ops->link_disable			&&
> @@ -319,6 +316,7 @@ struct ntb_client {
>   * @dev:		Linux device object.
>   * @pdev:		Pci device entry of the ntb.
>   * @topo:		Detected topology of the ntb.
> + * @port:		Local port of the ntb.
>   * @ops:		See &ntb_dev_ops.
>   * @ctx:		See &ntb_ctx_ops.
>   * @ctx_ops:		See &ntb_ctx_ops.
> @@ -327,6 +325,7 @@ struct ntb_dev {
>  	struct device			dev;
>  	struct pci_dev			*pdev;
>  	enum ntb_topo			topo;
> +	int				port;
>  	const struct ntb_dev_ops	*ops;
>  	void				*ctx;
>  	const struct ntb_ctx_ops	*ctx_ops;
> @@ -442,6 +441,64 @@ void ntb_link_event(struct ntb_dev *ntb);
>  void ntb_db_event(struct ntb_dev *ntb, int vector);
> 
>  /**
> + * ntb_port_number() - get the local port number
> + * @ntb:	NTB device context.
> + *
> + * Hardware must support at least simple two-ports topology
> + *
> + * Return: the local port number
> + */
> +static inline int ntb_port_number(struct ntb_dev *ntb)
> +{
> +	return ntb->ops->port_number(ntb);
> +}
> +
> +/**
> + * ntb_peer_port_count() - get the number of peer device ports
> + * @ntb:	NTB device context.
> + *
> + * Hardware may support an access to memory of several remote domains
> + * over multi-port NTB devices. This method returns the number of peers,
> + * local device can have shared memory with.
> + *
> + * Return: the number of peer ports
> + */
> +static inline int ntb_peer_port_count(struct ntb_dev *ntb)
> +{
> +	return ntb->ops->peer_port_count(ntb);
> +}
> +
> +/**
> + * ntb_peer_port_number() - get the peer port by given index
> + * @ntb:	NTB device context.
> + * @pidx:	Peer port index.
> + *
> + * Peer ports are continuously enumerated by NTB API logic, so this methods
> + * lets to retrieve port real number by its index.
> + *
> + * Return: the peer device port or negative value indicating an error
> + */
> +static inline int ntb_peer_port_number(struct ntb_dev *ntb, int pidx)
> +{
> +	return ntb->ops->peer_port_number(ntb, pidx);
> +}
> +
> +/**
> + * ntb_peer_port_idx() - get the peer device port index by given port number
> + * @ntb:	NTB device context.
> + * @port:	Peer port number.
> + *
> + * Inverse operation of ntb_peer_port_number(), so one can get port index
> + * by its port number.
> + *
> + * Return: the peer port index or negative value indicating an error
> + */
> +static inline int ntb_peer_port_idx(struct ntb_dev *ntb, int port)
> +{
> +	return ntb->ops->peer_port_idx(ntb, port);
> +}
> +
> +/**
>   * ntb_link_is_up() - get the current ntb link state
>   * @ntb:	NTB device context.
>   * @speed:	OUT - The link speed expressed as PCIe generation number.
> --
> 2.6.6

^ permalink raw reply	[flat|nested] 6+ messages in thread
* [PATCH 00/22] NTB: Alter kernel API to support multi-port devices
@ 2016-11-29 17:15 Serge Semin
  2016-11-29 17:15 ` [PATCH 02/22] NTB: Add peer indexed ports NTB API Serge Semin
  0 siblings, 1 reply; 6+ messages in thread
From: Serge Semin @ 2016-11-29 17:15 UTC (permalink / raw)
  To: jdmason, dave.jiang, Allen.Hubbe, Xiangliang.Yu
  Cc: Sergey.Semin, linux-ntb, linux-kernel, Serge Semin

There are devices, like IDT PCIe-switches, which have more than just two ports.
Particularly one device can have up to eight ports with NTB-function activated.
In order to support such devices, NTB kernel API should be altered since
currently it's optimized to work with two-ports devices only.

Here are the changes I made to conform the design we discussed a few months ago:

1) Port-index-related methods are added to KAPI
 ntb_port_number();
 ntb_peer_port_count();
 ntb_peer_port_number(pdix);
 ntb_peer_port_idx(port);

2) Link state method returns bitfield of link states for each reachable port
 u64 ntb_link_is_up();
 
3) Link enable/disable methods work with local link side of NTB
 ntb_link_enable()/ntb_link_disable();
 
4) NTB memory window related interface does the following things
 ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated
for shared buffer with specified peer device.
 ntb_mw_get_align(pidx, widx); - get alignment and size restrition parameters
to properly allocate inbound memory region.
 ntb_peer_mw_count(); - get number of outbound memory windows.
 ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window 

Inbound MW based hardware:
 ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound
memory window so a peer device could access it.
 ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound
memory window.

Outbound MW based hardware:
 ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory
window retrieved from a peer device
 ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an
outbound memory window

5) Scratchpad interface needs to support multi-port devices as well
 ntb_spad_count() - return number of Scratchpad per each port
 ntb_peer_spad_addr(pidx, sidx) - address of Scratchpad register of the
peer device with pidx-index
 ntb_peer_spad_read(pidx, sidx) - read specified Scratchpad register of the
peer with pidx-index
 ntb_peer_spad_write(pidx, sidx) - write data to Scratchpad register of the
peer with pidx-index

6) Introduce new messaging interface of NTB KAPI
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_recv(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_send(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event
 
7) Topology reduced to be either P2P (port-to-port) or B2B (bridge-to-bridge).
Since there is port number introduced to be part of ntb_dev structure, real
port number can be used to determine Primary and Secondary sides. Intel and AMD
driver are altered to support this novation.

8) Standard test drivers: PingPong, Debugging tool and Raw Perf as well as NTB
Transport drivers don't support multi-port devices at the moment.
Since we haven't got any real multi-port hadrware driver, it's dangerous to
make any serious alterations in the debugging tools. So I have made those
drivers to work the way they wokred before, but using the new NTB API.
The situation will change when I finish porting my current IDT NTB driver to
support new API. Then I'll be able to refactor the tools and test them using
real multi-port hardware.

The changes are made on top of the NTB-fork of the kernel:
https://github.com/jonmason/ntb
of "ntb"-branch with last commit:
9c763584b7c8911106bb77af7e648bef09af9d80 Linux 4.9-rc6

Signed-off-by: Serge Semin <fancer.lancer@gmail.com>

Serge Semin (22):
  NTB: Move link state API being first in sources
  NTB: Add peer indexed ports NTB API
  NTB: Alter NTB API to support both inbound and outbound MW based
    interfaces
  NTB: Add messaging NTB API
  NTB: Alter Scratchpads NTB API to support multi-ports interface
  NTB: Slightly alter link state NTB API
  NTB: Fix a few ntb.h issues
  NTB: Add T-Platforms copyrights to NTB API
  NTB Intel: Move link-related methods being first in the driver
  NTB Intel: Add port-related NTB API callback methods
  NTB Intel: Alter MW interface to fit new NTB API
  NTB Intel: Alter Scratchpads interface to fit new NTB API
  NTB Intel: Add T-Platforms copyrights to Intel NTB driver
  NTB AMD: Move link-related methods being first in the driver
  NTB AMD: Add port-related NTB API callback methods
  NTB AMD: Alter MW interface to fit new NTB API
  NTB AMD: Alter Scratchpads interface to fit new NTB API
  NTB AMD: Add T-Platforms copyrights to AMD NTB driver
  NTB PingPong: Alter driver to work with two-ports NTB API
  NTB Tool: Alter driver to work with two-ports NTB API
  NTB Perf: Alter driver to work with two-ports NTB API
  NTB Transport: Alter driver to work with two-ports NTB API

 drivers/ntb/hw/amd/ntb_hw_amd.c     | 310 +++++++++------
 drivers/ntb/hw/amd/ntb_hw_amd.h     |  12 +
 drivers/ntb/hw/intel/ntb_hw_intel.c | 420 ++++++++++++--------
 drivers/ntb/hw/intel/ntb_hw_intel.h |  12 +
 drivers/ntb/ntb.c                   |  15 +
 drivers/ntb/ntb_transport.c         |  43 +-
 drivers/ntb/test/ntb_perf.c         |  27 +-
 drivers/ntb/test/ntb_pingpong.c     |  11 +-
 drivers/ntb/test/ntb_tool.c         |  87 +++--
 include/linux/ntb.h                 | 753 +++++++++++++++++++++++++++++-------
 10 files changed, 1221 insertions(+), 469 deletions(-)

-- 
2.6.6

^ permalink raw reply	[flat|nested] 6+ messages in thread

end of thread, other threads:[~2016-12-07 22:56 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2016-12-04  0:04 [PATCH 02/22] NTB: Add peer indexed ports NTB API Allen Hubbe
2016-12-07 22:56 ` Allen Hubbe
  -- strict thread matches above, loose matches on Subject: below --
2016-11-29 17:15 [PATCH 00/22] NTB: Alter kernel API to support multi-port devices Serge Semin
2016-11-29 17:15 ` [PATCH 02/22] NTB: Add peer indexed ports NTB API Serge Semin
2016-11-30 18:40   ` kbuild test robot
2016-11-30 19:12   ` kbuild test robot
2016-11-30 20:04   ` kbuild test robot

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).