On R10000-class CPUs, a store prefetch while copying the preceding vector can leave a dirty line whose write-back overwrites the uncached cache error handler. Write back and invalidate that line in set_uncached_handler() before the copy, through a new flush_dcache_size() hook. c-r4k and c-r3k provide it with r4k_dma_cache_wback_inv() and r3k_dma_cache_wback_inv(), renamed as they are no longer DMA-only. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Cc: stable@vger.kernel.org Signed-off-by: Imre Kaloz --- Changes in v3: - Renamed the hook to flush_dcache_size() and the c-r4k implementation to r4k_flush_dcache_size(). - r3k_dma_cache_wback_inv() becomes r3k_flush_dcache_size() and is installed as the hook as well. Changes in v2: - Fixes: 1da177e4c3f4 instead of e01402b115cc. - Moved the write-back into set_uncached_handler(), for every caller. - Renamed r4k_dma_cache_wback_inv() to r4k_cache_wback_inv(). v2: https://lore.kernel.org/r/20261026120430.13238-1-kaloz@kernel.org arch/mips/include/asm/cacheflush.h | 1 + arch/mips/kernel/traps.c | 6 ++++++ arch/mips/mm/c-r3k.c | 9 +++++---- arch/mips/mm/c-r4k.c | 11 ++++++----- arch/mips/mm/cache.c | 1 + 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/arch/mips/include/asm/cacheflush.h b/arch/mips/include/asm/cacheflush.h index 5099c1b65a58..fb8eeb205897 100644 --- a/arch/mips/include/asm/cacheflush.h +++ b/arch/mips/include/asm/cacheflush.h @@ -120,6 +120,7 @@ extern void copy_from_user_page(struct vm_area_struct *vma, extern void (*flush_icache_all)(void); extern void (*flush_data_cache_page)(unsigned long addr); +extern void (*flush_dcache_size)(unsigned long start, unsigned long size); /* Run kernel code uncached, useful for cache probing functions. */ unsigned long run_uncached(void *func); diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 8ec1e185b35c..19a8946e14ae 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -2305,6 +2305,12 @@ void set_uncached_handler(unsigned long offset, void *addr, if (!addr) panic(panic_null_cerr); + /* + * A store prefetch or speculative store may have left a dirty line of + * the cached view; write it back and invalidate it first, or its later + * write-back overwrites the handler. + */ + flush_dcache_size(ebase + offset, size); memcpy((void *)(uncached_ebase + offset), addr, size); } diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c index 5869df848fab..a1be13775d2c 100644 --- a/arch/mips/mm/c-r3k.c +++ b/arch/mips/mm/c-r3k.c @@ -270,7 +270,7 @@ static void r3k_flush_kernel_vmap_range(unsigned long vaddr, int size) BUG(); } -static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size) +static void r3k_flush_dcache_size(unsigned long start, unsigned long size) { /* Catch bad driver code */ BUG_ON(size == 0); @@ -299,10 +299,11 @@ void r3k_cache_init(void) __flush_kernel_vmap_range = r3k_flush_kernel_vmap_range; flush_data_cache_page = r3k_flush_data_cache_page; + flush_dcache_size = r3k_flush_dcache_size; - _dma_cache_wback_inv = r3k_dma_cache_wback_inv; - _dma_cache_wback = r3k_dma_cache_wback_inv; - _dma_cache_inv = r3k_dma_cache_wback_inv; + _dma_cache_wback_inv = r3k_flush_dcache_size; + _dma_cache_wback = r3k_flush_dcache_size; + _dma_cache_inv = r3k_flush_dcache_size; pr_info("Primary instruction cache %ldkB, linesize %ld bytes.\n", icache_size >> 10, icache_lsize); diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index d8aadd3dc057..8b600de64dff 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -743,9 +743,7 @@ static void r4k_flush_icache_user_range(unsigned long start, unsigned long end) return __r4k_flush_icache_range(start, end, true); } -#ifdef CONFIG_DMA_NONCOHERENT - -static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) +static void r4k_flush_dcache_size(unsigned long addr, unsigned long size) { /* Catch bad driver code */ if (WARN_ON(size == 0)) @@ -786,6 +784,8 @@ static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) __sync(); } +#ifdef CONFIG_DMA_NONCOHERENT + static void prefetch_cache_inv(unsigned long addr, unsigned long size) { unsigned int linesz = cpu_scache_line_size(); @@ -1743,14 +1743,15 @@ void r4k_cache_init(void) flush_icache_all = r4k_flush_icache_all; flush_data_cache_page = r4k_flush_data_cache_page; + flush_dcache_size = r4k_flush_dcache_size; flush_icache_range = r4k_flush_icache_range; local_flush_icache_range = local_r4k_flush_icache_range; __flush_icache_user_range = r4k_flush_icache_user_range; __local_flush_icache_user_range = local_r4k_flush_icache_user_range; #ifdef CONFIG_DMA_NONCOHERENT - _dma_cache_wback_inv = r4k_dma_cache_wback_inv; - _dma_cache_wback = r4k_dma_cache_wback_inv; + _dma_cache_wback_inv = r4k_flush_dcache_size; + _dma_cache_wback = r4k_flush_dcache_size; _dma_cache_inv = r4k_dma_cache_inv; #endif /* CONFIG_DMA_NONCOHERENT */ diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index ad9b0430a28e..b69b87c6c0cb 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -51,6 +51,7 @@ EXPORT_SYMBOL_GPL(__flush_kernel_vmap_range); /* MIPS specific cache operations */ void (*flush_data_cache_page)(unsigned long addr); void (*flush_icache_all)(void); +void (*flush_dcache_size)(unsigned long start, unsigned long size); EXPORT_SYMBOL(flush_data_cache_page); EXPORT_SYMBOL(flush_icache_all); base-commit: 24a19b953e2a3d0ea5a8c4951a0ab8f32720c907 -- 2.47.3