The interrupt handler reads and acknowledges channel status before taking the channel lock. Termination can finish and a new transfer can start before the handler takes that lock. The saved completion or error status then applies to the new descriptor, stopping it and completing its cookie before its DMA transfer has finished. Termination also leaves completion and error status pending. A subsequent interrupt can consume that status after the channel has been reused, even if no handler was running when the old transfer was stopped. Read and acknowledge status under the channel lock and keep it held while processing the event. Move locking out of the completion and error helpers into their sole caller, and clear all channel status after a successful stop. Status collection, descriptor ownership and channel reuse then share the same serialization. Initialize the virtual channels before requesting the shared IRQs so every lock is ready before the interrupt handler can run. Fixes: 1fe20f1b8454 ("dmaengine: Introduce DW AXI DMAC driver") Cc: stable@vger.kernel.org Signed-off-by: James Hilliard --- drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c | 31 ++++++++++++-------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 97098367535f..0116c44d3bef 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1071,9 +1072,8 @@ static void axi_chan_list_dump_lli(struct axi_dma_chan *chan, static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) { struct virt_dma_desc *vd; - unsigned long flags; - spin_lock_irqsave(&chan->vc.lock, flags); + lockdep_assert_held(&chan->vc.lock); axi_chan_disable(chan); @@ -1082,7 +1082,7 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) if (!vd) { dev_err(vchan_chan_dev(&chan->vc), "BUG: %s, IRQ with no descriptors\n", axi_chan_name(chan)); - goto out; + return; } /* Remove the completed descriptor from issued list */ list_del(&vd->node); @@ -1099,9 +1099,6 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) chan->desc = NULL; chan->cyclic_restart = false; axi_chan_start_first_queued(chan); - -out: - spin_unlock_irqrestore(&chan->vc.lock, flags); } static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) @@ -1110,11 +1107,11 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) struct axi_dma_hw_desc *hw_desc; struct axi_dma_desc *desc; struct virt_dma_desc *vd; - unsigned long flags; u64 llp; int i; - spin_lock_irqsave(&chan->vc.lock, flags); + lockdep_assert_held(&chan->vc.lock); + if (unlikely(axi_chan_is_hw_enable(chan))) { dev_err(vchan_chan_dev(&chan->vc), "BUG: %s caught DWAXIDMAC_IRQ_DMA_TRF, but channel not idle!\n", axi_chan_name(chan)); @@ -1126,7 +1123,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) if (!vd) { dev_err(vchan_chan_dev(&chan->vc), "BUG: %s, IRQ with no descriptors\n", axi_chan_name(chan)); - goto out; + return; } desc = vd_to_axi_desc(vd); @@ -1158,9 +1155,6 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) chan->desc = NULL; axi_chan_start_first_queued(chan); } - -out: - spin_unlock_irqrestore(&chan->vc.lock, flags); } static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) @@ -1168,6 +1162,7 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) struct axi_dma_chip *chip = dev_id; struct dw_axi_dma *dw = chip->dw; struct axi_dma_chan *chan; + unsigned long flags; u32 status, i; @@ -1177,6 +1172,7 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) /* Poll, clear and process every channel interrupt status */ for (i = 0; i < dw->hdata->nr_channels; i++) { chan = &dw->chan[i]; + spin_lock_irqsave(&chan->vc.lock, flags); status = axi_chan_irq_read(chan); /* Leave suspension status for the pause and resume paths. */ status &= ~DWAXIDMAC_IRQ_SUSPENDED; @@ -1189,6 +1185,7 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) axi_chan_handle_err(chan, status); else if (status & DWAXIDMAC_IRQ_DMA_TRF) axi_chan_block_xfer_complete(chan); + spin_unlock_irqrestore(&chan->vc.lock, flags); } /* Re-enable interrupts */ @@ -1225,7 +1222,7 @@ static int dma_chan_terminate_all(struct dma_chan *dchan) vchan_terminate_all(&chan->vc); - axi_chan_irq_clear(chan, DWAXIDMAC_IRQ_SUSPENDED); + axi_chan_irq_clear(chan, DWAXIDMAC_IRQ_ALL); chan->desc = NULL; chan->pause_requested = false; chan->is_paused = false; @@ -1599,10 +1596,6 @@ static int dw_probe(struct platform_device *pdev) if (!dw->chan) return -ENOMEM; - ret = axi_req_irqs(pdev, chip); - if (ret) - return ret; - INIT_LIST_HEAD(&dw->dma.channels); for (i = 0; i < hdata->nr_channels; i++) { struct axi_dma_chan *chan = &dw->chan[i]; @@ -1616,6 +1609,10 @@ static int dw_probe(struct platform_device *pdev) vchan_init(&chan->vc, &dw->dma); } + ret = axi_req_irqs(pdev, chip); + if (ret) + return ret; + /* Set capabilities */ dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask); dma_cap_set(DMA_SLAVE, dw->dma.cap_mask); -- 2.53.0