Re: IOAT DMA w/IOMMU

[Kit Chow <kchow@gigaio.com>]

Turns out there is no dma_map_resource routine on x86. get_dma_ops 
returns intel_dma_ops which has map_resource pointing to NULL.

(gdb) p intel_dma_ops
$7 = {alloc = 0xffffffff8150f310 <intel_alloc_coherent>,
   free = 0xffffffff8150ec20 <intel_free_coherent>,
   mmap = 0x0 <irq_stack_union>, get_sgtable = 0x0 <irq_stack_union>,
   map_page = 0xffffffff8150f2d0 <intel_map_page>,
   unmap_page = 0xffffffff8150ec10 <intel_unmap_page>,
   map_sg = 0xffffffff8150ef40 <intel_map_sg>,
   unmap_sg = 0xffffffff8150eb80 <intel_unmap_sg>,
   map_resource = 0x0 <irq_stack_union>,
   unmap_resource = 0x0 <irq_stack_union>,
   sync_single_for_cpu = 0x0 <irq_stack_union>,
   sync_single_for_device = 0x0 <irq_stack_union>,
   sync_sg_for_cpu = 0x0 <irq_stack_union>,
   sync_sg_for_device = 0x0 <irq_stack_union>,
   cache_sync = 0x0 <irq_stack_union>,
   mapping_error = 0xffffffff815095f0 <intel_mapping_error>,
   dma_supported = 0xffffffff81033830 <x86_dma_supported>, is_phys = 0}

Will poke around some in the intel_map_page code but can you actually 
get a valid struct page for a pci bar address (dma_map_single calls 
virt_to_page)?  If not, does a map_resource routine that can properly 
map a pci bar address need to be implemented?

Kit

---


static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
                                               size_t size,
                                               enum dma_data_direction dir,
                                               unsigned long attrs)
{
         const struct dma_map_ops *ops = get_dma_ops(dev);
         dma_addr_t addr;

         BUG_ON(!valid_dma_direction(dir));
         addr = ops->map_page(dev, virt_to_page(ptr),
                              offset_in_page(ptr), size,
                              dir, attrs);
         debug_dma_map_page(dev, virt_to_page(ptr),
                            offset_in_page(ptr), size,
                            dir, addr, true);
         return addr;
}






On 08/09/2018 04:00 PM, Kit Chow wrote:
>
>
> On 08/09/2018 03:50 PM, Logan Gunthorpe wrote:
>>
>> On 09/08/18 04:48 PM, Kit Chow wrote:
>>> Based on Logan's comments, I am very hopeful that the dma_map_resource
>>> will make things work on the older platforms...
>> Well, I *think* dma_map_single() would still work. So I'm not that
>> confident that's the root of your problem. I'd still like to see the
>> actual code snippet you are using.
>>
>> Logan
> Here's the code snippet - (ntbdebug & 4) path does dma_map_resource of 
> the pci bar address.
>
> It was:
>                 unmap->addr[1] = dma_map_single(device->dev, (void 
> *)dest, len,
>                     DMA_TO_DEVICE);
>
> Kit
> ---
>
>
> static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
>                                struct ntb_queue_entry *entry)
> {
>         struct dma_async_tx_descriptor *txd;
>         struct dma_chan *chan = qp->tx_dma_chan;
>         struct dma_device *device;
>         size_t len = entry->len;
>         void *buf = entry->buf;
>         size_t dest_off, buff_off;
>         struct dmaengine_unmap_data *unmap;
>         dma_addr_t dest;
>         dma_cookie_t cookie;
>         int     unmapcnt;
>
>         device = chan->device;
>
>         dest = qp->tx_mw_phys + qp->tx_max_frame * entry->tx_index;
>
>         buff_off = (size_t)buf & ~PAGE_MASK;
>         dest_off = (size_t)dest & ~PAGE_MASK;
>
>         if (!is_dma_copy_aligned(device, buff_off, dest_off, len))
>                 goto err;
>
>
>         if (ntbdebug & 0x4) {
>                 unmapcnt = 2;
>         } else {
>                 unmapcnt = 1;
>         }
>
>         unmap = dmaengine_get_unmap_data(device->dev, unmapcnt, 
> GFP_NOWAIT);
>         if (!unmap)
>                 goto err;
>
>         unmap->len = len;
>         unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf),
>                                       buff_off, len, DMA_TO_DEVICE);
>         if (dma_mapping_error(device->dev, unmap->addr[0]))
>                 goto err_get_unmap;
>
>         if (ntbdebug & 0x4) {
>                 unmap->addr[1] = dma_map_resource(device->dev,
>                     (phys_addr_t)dest, len, DMA_TO_DEVICE, 0);
>                 if (dma_mapping_error(device->dev, unmap->addr[1]))
>                         goto err_get_unmap;
>                 unmap->to_cnt = 2;
>         } else {
>                 unmap->addr[1] = dest;
>                 unmap->to_cnt = 1;
>         }
>
>         txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
>             unmap->addr[0], len, DMA_PREP_INTERRUPT);
>
>         if (!txd)
>                 goto err_get_unmap;
>
>         txd->callback_result = ntb_tx_copy_callback;
>         txd->callback_param = entry;
>         dma_set_unmap(txd, unmap);
>
>         cookie = dmaengine_submit(txd);
>         if (dma_submit_error(cookie))
>                 goto err_set_unmap;
>
>         dmaengine_unmap_put(unmap);
>
>         dma_async_issue_pending(chan);
>
>         return 0;
>
> err_set_unmap:
>         dma_descriptor_unmap(txd);
>         txd->desc_free(txd);
> err_get_unmap:
>         dmaengine_unmap_put(unmap);
> err:
>         return -ENXIO;
> }
>