From: Serge Semin The descriptor scanner mixes transfer-complete handling with hardware-LLP residue calculation. A status query needs the latter for the requested cookie even when no transfer-complete event is pending. Move hardware-LLP residue calculation into its own helper and call it from the status getter under the channel lock. Keep software-LLP progress in the scanner and subtract the current block's progress when reporting its residue. The scanner now handles transfer-complete events only, retaining the distinction between zero source residue and destination completion. Also search the queued descriptors so a transfer waiting to start reports its full length instead of zero residue. Fixes: 3bfb1d20b547 ("dmaengine: Driver for the Synopsys DesignWare DMA controller") Cc: stable@vger.kernel.org Signed-off-by: Serge Semin [james: retain polled completion and report queued residue] Signed-off-by: James Hilliard --- Changelog RFC: - Instead of just dropping the dwc_scan_descriptors() method invocation calculate the residue in the Tx-status getter. --- drivers/dma/dw/core.c | 166 +++++++++++++++++++++----------------------------- 1 file changed, 68 insertions(+), 98 deletions(-) diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index 1f7afcd01565..1d6419e1cd3c 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -274,123 +274,60 @@ static inline u32 dwc_get_sent(struct dw_dma_chan *dwc) static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) { - dma_addr_t llp; struct dw_desc *desc, *_desc; struct dw_desc *child; - u32 status_xfer; unsigned long flags; LIST_HEAD(list); spin_lock_irqsave(&dwc->lock, flags); - llp = channel_readl(dwc, LLP); - status_xfer = dma_readl(dw, RAW.XFER); - if (status_xfer & dwc->mask) { - /* Everything we've submitted is done */ - dma_writel(dw, CLEAR.XFER, dwc->mask); - if (list_empty(&dwc->active_list)) { - spin_unlock_irqrestore(&dwc->lock, flags); - return; - } - - if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { - struct list_head *head, *active = dwc->tx_node_active; - - /* - * We are inside first active descriptor. - * Otherwise something is really wrong. - */ - desc = dwc_first_active(dwc); - - head = &desc->tx_list; - if (active != head) { - /* Update residue to reflect last sent descriptor */ - if (active == head->next) - desc->residue -= desc->len; - else - desc->residue -= to_dw_desc(active->prev)->len; - - child = to_dw_desc(active); + if (!(dma_readl(dw, RAW.XFER) & dwc->mask)) + goto unlock; - /* Submit next block */ - dwc_do_single_block(dwc, child); + dma_writel(dw, CLEAR.XFER, dwc->mask); + if (list_empty(&dwc->active_list)) + goto unlock; - spin_unlock_irqrestore(&dwc->lock, flags); - return; - } + desc = dwc_first_active(dwc); + if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { + struct list_head *head = &desc->tx_list; + struct list_head *active = dwc->tx_node_active; - /* We are done here */ - clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags); - } + if (active != head) { + /* Update residue to reflect last sent descriptor. */ + if (active == head->next) + desc->residue -= desc->len; + else + desc->residue -= to_dw_desc(active->prev)->len; - if (dma_readl(dw, CH_EN) & dwc->mask) { - dev_err(chan2dev(&dwc->chan), - "BUG: XFER bit set, but channel not idle!\n"); + child = to_dw_desc(active); - /* Try to continue after resetting the channel... */ - dwc_chan_disable(dw, dwc); + /* Submit next block, including for polled transfers. */ + dwc_do_single_block(dwc, child); + goto unlock; } - /* Detach the completed transfer before starting queued work. */ - list_splice_init(&dwc->active_list, &list); - list_for_each_entry(desc, &list, desc_node) - dma_cookie_complete(&desc->txd); - dwc_dostart_first_queued(dwc); - spin_unlock_irqrestore(&dwc->lock, flags); - - list_for_each_entry_safe(desc, _desc, &list, desc_node) { - list_del_init(&desc->desc_node); - dwc_descriptor_free(dwc, desc, true); - } - return; + clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags); } - if (list_empty(&dwc->active_list)) { - spin_unlock_irqrestore(&dwc->lock, flags); - return; - } + if (dma_readl(dw, CH_EN) & dwc->mask) { + dev_err(chan2dev(&dwc->chan), + "BUG: XFER bit set, but channel not idle!\n"); - if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { - dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__); - spin_unlock_irqrestore(&dwc->lock, flags); - return; + /* Try to continue after resetting the channel... */ + dwc_chan_disable(dw, dwc); } - dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp); - - list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) { - /* Initial residue value */ - desc->residue = desc->total_len; - - /* Check first descriptors addr */ - if (desc->txd.phys == DWC_LLP_LOC(llp)) { - spin_unlock_irqrestore(&dwc->lock, flags); - return; - } - - /* Check first descriptors llp */ - if (lli_read(desc, llp) == llp) { - /* This one is currently in progress */ - desc->residue -= dwc_get_sent(dwc); - spin_unlock_irqrestore(&dwc->lock, flags); - return; - } - - desc->residue -= desc->len; - list_for_each_entry(child, &desc->tx_list, desc_node) { - if (lli_read(child, llp) == llp) { - /* Currently in progress */ - desc->residue -= dwc_get_sent(dwc); - spin_unlock_irqrestore(&dwc->lock, flags); - return; - } - desc->residue -= child->len; - } + dma_cookie_complete(&desc->txd); + list_move_tail(&desc->desc_node, &list); + dwc_dostart_first_queued(dwc); +unlock: + spin_unlock_irqrestore(&dwc->lock, flags); + list_for_each_entry_safe(desc, _desc, &list, desc_node) { + list_del_init(&desc->desc_node); + dwc_descriptor_free(dwc, desc, true); } - - /* Residue describes source reads; only RAW.XFER completes a transfer. */ - spin_unlock_irqrestore(&dwc->lock, flags); } static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc) @@ -969,9 +906,39 @@ static struct dw_desc *dwc_find_desc(struct dw_dma_chan *dwc, dma_cookie_t c) if (desc->txd.cookie == c) return desc; + list_for_each_entry(desc, &dwc->queue, desc_node) + if (desc->txd.cookie == c) + return desc; + return NULL; } +static u32 dwc_get_hard_llp_desc_residue(struct dw_dma_chan *dwc, struct dw_desc *desc) +{ + u32 residue = desc->total_len; + struct dw_desc *child; + dma_addr_t llp; + + llp = channel_readl(dwc, LLP); + + /* The first descriptor may still be waiting to be fetched. */ + if (desc->txd.phys == DWC_LLP_LOC(llp)) + return residue; + + if (lli_read(desc, llp) == llp) + return residue - dwc_get_sent(dwc); + + residue -= desc->len; + list_for_each_entry(child, &desc->tx_list, desc_node) { + if (lli_read(child, llp) == llp) + return residue - dwc_get_sent(dwc); + + residue -= child->len; + } + + return residue; +} + static u32 dwc_get_residue_and_status(struct dw_dma_chan *dwc, dma_cookie_t cookie, enum dma_status *status) { @@ -984,9 +951,12 @@ static u32 dwc_get_residue_and_status(struct dw_dma_chan *dwc, dma_cookie_t cook desc = dwc_find_desc(dwc, cookie); if (desc) { if (desc == dwc_first_active(dwc)) { - residue = desc->residue; - if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue) + if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { + residue = desc->residue; residue -= dwc_get_sent(dwc); + } else { + residue = dwc_get_hard_llp_desc_residue(dwc, desc); + } if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags)) *status = DMA_PAUSED; } else { -- 2.53.0