Nothing in "system/ram_addr.h" requires definitions fromi "exec/cpu-common.h", remove it. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 6b528338efc..f74a0ecee56 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -29,7 +29,6 @@ #include "qemu/rcu.h" #include "exec/hwaddr.h" -#include "exec/cpu-common.h" extern uint64_t total_dirty_pages; -- 2.51.0 The "exec/target_page.h" header is indirectly pulled from "system/ram_addr.h". Include it explicitly, in order to avoid unrelated issues when refactoring "system/ram_addr.h": accel/kvm/kvm-all.c: In function ‘kvm_init’: accel/kvm/kvm-all.c:2636:12: error: ‘TARGET_PAGE_SIZE’ undeclared (first use in this function); did you mean ‘TARGET_PAGE_BITS’? 2636 | assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size()); | ^~~~~~~~~~~~~~~~ Signed-off-by: Philippe Mathieu-Daudé --- accel/kvm/kvm-all.c | 1 + 1 file changed, 1 insertion(+) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index e3c84723406..08b2b5a371c 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -36,6 +36,7 @@ #include "accel/accel-ops.h" #include "qemu/bswap.h" #include "exec/tswap.h" +#include "exec/target_page.h" #include "system/memory.h" #include "system/ram_addr.h" #include "qemu/event_notifier.h" -- 2.51.0 The "exec/target_page.h" header is indirectly pulled from "system/ram_addr.h". Include it explicitly, in order to avoid unrelated issues when refactoring "system/ram_addr.h": hw/s390x/s390-stattrib-kvm.c: In function ‘kvm_s390_stattrib_set_stattr’: hw/s390x/s390-stattrib-kvm.c:89:57: error: ‘TARGET_PAGE_SIZE’ undeclared (first use in this function); did you mean ‘TARGET_PAGE_BITS’? 89 | unsigned long max = s390_get_memory_limit(s390ms) / TARGET_PAGE_SIZE; | ^~~~~~~~~~~~~~~~ | TARGET_PAGE_BITS Since "system/ram_addr.h" is actually not needed, remove it. Signed-off-by: Philippe Mathieu-Daudé --- hw/s390x/s390-stattrib-kvm.c | 2 +- hw/s390x/s390-stattrib.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/s390x/s390-stattrib-kvm.c b/hw/s390x/s390-stattrib-kvm.c index e1fee361dc3..73df1f600b9 100644 --- a/hw/s390x/s390-stattrib-kvm.c +++ b/hw/s390x/s390-stattrib-kvm.c @@ -10,13 +10,13 @@ */ #include "qemu/osdep.h" +#include "exec/target_page.h" #include "hw/s390x/s390-virtio-ccw.h" #include "migration/qemu-file.h" #include "hw/s390x/storage-attributes.h" #include "qemu/error-report.h" #include "system/kvm.h" #include "system/memory_mapping.h" -#include "system/ram_addr.h" #include "kvm/kvm_s390x.h" #include "qapi/error.h" diff --git a/hw/s390x/s390-stattrib.c b/hw/s390x/s390-stattrib.c index 13a678a8037..aa185372914 100644 --- a/hw/s390x/s390-stattrib.c +++ b/hw/s390x/s390-stattrib.c @@ -11,12 +11,12 @@ #include "qemu/osdep.h" #include "qemu/units.h" +#include "exec/target_page.h" #include "migration/qemu-file.h" #include "migration/register.h" #include "hw/qdev-properties.h" #include "hw/s390x/storage-attributes.h" #include "qemu/error-report.h" -#include "system/ram_addr.h" #include "qapi/error.h" #include "qobject/qdict.h" #include "cpu.h" -- 2.51.0 The "exec/target_page.h" header is indirectly pulled from "system/ram_addr.h". Include it explicitly, in order to avoid unrelated issues when refactoring "system/ram_addr.h": hw/vfio/listener.c: In function ‘vfio_ram_discard_register_listener’: hw/vfio/listener.c:258:28: error: implicit declaration of function ‘qemu_target_page_size’; did you mean ‘qemu_ram_pagesize’? 258 | int target_page_size = qemu_target_page_size(); | ^~~~~~~~~~~~~~~~~~~~~ | qemu_ram_pagesize Signed-off-by: Philippe Mathieu-Daudé --- hw/vfio/listener.c | 1 + 1 file changed, 1 insertion(+) diff --git a/hw/vfio/listener.c b/hw/vfio/listener.c index a2c19a3cec1..b5cefc9395c 100644 --- a/hw/vfio/listener.c +++ b/hw/vfio/listener.c @@ -25,6 +25,7 @@ #endif #include +#include "exec/target_page.h" #include "hw/vfio/vfio-device.h" #include "hw/vfio/pci.h" #include "system/address-spaces.h" -- 2.51.0 The "exec/target_page.h" header is indirectly pulled from "system/ram_addr.h". Include it explicitly, in order to avoid unrelated issues when refactoring "system/ram_addr.h": target/arm/tcg/mte_helper.c:815:23: error: use of undeclared identifier 'TARGET_PAGE_MASK' 815 | prev_page = ptr & TARGET_PAGE_MASK; | ^ target/arm/tcg/mte_helper.c:816:29: error: use of undeclared identifier 'TARGET_PAGE_SIZE' 816 | next_page = prev_page + TARGET_PAGE_SIZE; | ^ Signed-off-by: Philippe Mathieu-Daudé --- target/arm/tcg/mte_helper.c | 1 + 1 file changed, 1 insertion(+) diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index 302e899287c..7d80244788e 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -21,6 +21,7 @@ #include "qemu/log.h" #include "cpu.h" #include "internals.h" +#include "exec/target_page.h" #include "exec/page-protection.h" #ifdef CONFIG_USER_ONLY #include "user/cpu_loop.h" -- 2.51.0 None of these files require definition exposed by "system/ram_addr.h", remove its inclusion. Signed-off-by: Philippe Mathieu-Daudé --- hw/ppc/spapr.c | 1 - hw/ppc/spapr_caps.c | 1 - hw/ppc/spapr_pci.c | 1 - hw/remote/memory.c | 1 - hw/remote/proxy-memory-listener.c | 1 - hw/s390x/s390-virtio-ccw.c | 1 - hw/vfio/spapr.c | 1 - hw/virtio/virtio-mem.c | 1 - 8 files changed, 8 deletions(-) diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index eb22333404d..15d09ef9618 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -77,7 +77,6 @@ #include "hw/virtio/virtio-scsi.h" #include "hw/virtio/vhost-scsi-common.h" -#include "system/ram_addr.h" #include "system/confidential-guest-support.h" #include "hw/usb.h" #include "qemu/config-file.h" diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c index f2f5722d8ad..0f94c192fd4 100644 --- a/hw/ppc/spapr_caps.c +++ b/hw/ppc/spapr_caps.c @@ -27,7 +27,6 @@ #include "qapi/error.h" #include "qapi/visitor.h" #include "system/hw_accel.h" -#include "system/ram_addr.h" #include "target/ppc/cpu.h" #include "target/ppc/mmu-hash64.h" #include "cpu-models.h" diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c index 1ac1185825e..f9095552e86 100644 --- a/hw/ppc/spapr_pci.c +++ b/hw/ppc/spapr_pci.c @@ -34,7 +34,6 @@ #include "hw/pci/pci_host.h" #include "hw/ppc/spapr.h" #include "hw/pci-host/spapr.h" -#include "system/ram_addr.h" #include #include "trace.h" #include "qemu/error-report.h" diff --git a/hw/remote/memory.c b/hw/remote/memory.c index 00193a552fa..8195aa5fb83 100644 --- a/hw/remote/memory.c +++ b/hw/remote/memory.c @@ -11,7 +11,6 @@ #include "qemu/osdep.h" #include "hw/remote/memory.h" -#include "system/ram_addr.h" #include "qapi/error.h" static void remote_sysmem_reset(void) diff --git a/hw/remote/proxy-memory-listener.c b/hw/remote/proxy-memory-listener.c index 30ac74961dd..e1a52d24f0b 100644 --- a/hw/remote/proxy-memory-listener.c +++ b/hw/remote/proxy-memory-listener.c @@ -12,7 +12,6 @@ #include "qemu/range.h" #include "system/memory.h" #include "exec/cpu-common.h" -#include "system/ram_addr.h" #include "qapi/error.h" #include "qemu/error-report.h" #include "hw/remote/mpqemu-link.h" diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index d0c6e80cb05..ad2c48188a8 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -13,7 +13,6 @@ #include "qemu/osdep.h" #include "qapi/error.h" -#include "system/ram_addr.h" #include "system/confidential-guest-support.h" #include "hw/boards.h" #include "hw/s390x/sclp.h" diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c index 8d9d68da4ec..0f23681a3f9 100644 --- a/hw/vfio/spapr.c +++ b/hw/vfio/spapr.c @@ -17,7 +17,6 @@ #include "hw/vfio/vfio-container-legacy.h" #include "hw/hw.h" -#include "system/ram_addr.h" #include "qemu/error-report.h" #include "qapi/error.h" #include "trace.h" diff --git a/hw/virtio/virtio-mem.c b/hw/virtio/virtio-mem.c index 1de2d3de521..15ba6799f22 100644 --- a/hw/virtio/virtio-mem.c +++ b/hw/virtio/virtio-mem.c @@ -25,7 +25,6 @@ #include "hw/virtio/virtio-mem.h" #include "qapi/error.h" #include "qapi/visitor.h" -#include "system/ram_addr.h" #include "migration/misc.h" #include "hw/boards.h" #include "hw/qdev-properties.h" -- 2.51.0 Signed-off-by: Philippe Mathieu-Daudé --- include/exec/cputlb.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/exec/cputlb.h b/include/exec/cputlb.h index 9bec0e78909..db7cbf97826 100644 --- a/include/exec/cputlb.h +++ b/include/exec/cputlb.h @@ -31,6 +31,7 @@ void tlb_unprotect_code(ram_addr_t ram_addr); #endif #ifndef CONFIG_USER_ONLY +/* Called with rcu_read_lock held. */ void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length); void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length); #endif -- 2.51.0 Generally we want to clarify terminology and avoid confusions, prefering @start with (exclusive) @end, and base @addr with @length (for inclusive range). Here as tlb_reset_dirty() and tlb_reset_dirty_range_all() operate on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé --- include/exec/cputlb.h | 4 ++-- accel/tcg/cputlb.c | 6 +++--- system/physmem.c | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/exec/cputlb.h b/include/exec/cputlb.h index db7cbf97826..995fe31ef75 100644 --- a/include/exec/cputlb.h +++ b/include/exec/cputlb.h @@ -32,8 +32,8 @@ void tlb_unprotect_code(ram_addr_t ram_addr); #ifndef CONFIG_USER_ONLY /* Called with rcu_read_lock held. */ -void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length); -void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length); +void tlb_reset_dirty(CPUState *cpu, uintptr_t addr, uintptr_t length); +void tlb_reset_dirty_range_all(ram_addr_t addr, ram_addr_t length); #endif /** diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 2a6aa01c57c..6807328df82 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -916,7 +916,7 @@ static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s) * We must take tlb_c.lock to avoid racing with another vCPU update. The only * thing actually updated is the target TLB entry ->addr_write flags. */ -void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length) +void tlb_reset_dirty(CPUState *cpu, uintptr_t addr, uintptr_t length) { int mmu_idx; @@ -929,12 +929,12 @@ void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length) for (i = 0; i < n; i++) { tlb_reset_dirty_range_locked(&desc->fulltlb[i], &fast->table[i], - start, length); + addr, length); } for (i = 0; i < CPU_VTLB_SIZE; i++) { tlb_reset_dirty_range_locked(&desc->vfulltlb[i], &desc->vtable[i], - start, length); + addr, length); } } qemu_spin_unlock(&cpu->neg.tlb.c.lock); diff --git a/system/physmem.c b/system/physmem.c index 000bde90c2e..098824ad1d2 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -882,16 +882,16 @@ found: return block; } -void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length) +void tlb_reset_dirty_range_all(ram_addr_t addr, ram_addr_t length) { CPUState *cpu; - ram_addr_t start1; + ram_addr_t start, start1; RAMBlock *block; ram_addr_t end; assert(tcg_enabled()); - end = TARGET_PAGE_ALIGN(start + length); - start &= TARGET_PAGE_MASK; + end = TARGET_PAGE_ALIGN(addr + length); + start = addr & TARGET_PAGE_MASK; RCU_READ_LOCK_GUARD(); block = qemu_get_ram_block(start); -- 2.51.0 Generally we want to clarify terminology and avoid confusions, prefering @start with (exclusive) @end, and base @addr with @length (for inclusive range). Here as cpu_physical_memory_get_dirty() operates on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index f74a0ecee56..585ed78c767 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -142,7 +142,7 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, +static inline bool cpu_physical_memory_get_dirty(ram_addr_t addr, ram_addr_t length, unsigned client) { @@ -153,8 +153,8 @@ static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, assert(client < DIRTY_MEMORY_NUM); - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; + end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; + page = addr >> TARGET_PAGE_BITS; WITH_RCU_READ_LOCK_GUARD() { blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); -- 2.51.0 Avoid maintaining large functions in header, rely on the linker to optimize at linking time. cpu_physical_memory_get_dirty() doesn't involve any CPU, remove the 'cpu_' prefix. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 46 +-------------------------------------- system/physmem.c | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 585ed78c767..f8a307d1a3d 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -142,46 +142,6 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -static inline bool cpu_physical_memory_get_dirty(ram_addr_t addr, - ram_addr_t length, - unsigned client) -{ - DirtyMemoryBlocks *blocks; - unsigned long end, page; - unsigned long idx, offset, base; - bool dirty = false; - - assert(client < DIRTY_MEMORY_NUM); - - end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; - page = addr >> TARGET_PAGE_BITS; - - WITH_RCU_READ_LOCK_GUARD() { - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - unsigned long num = next - base; - unsigned long found = find_next_bit(blocks->blocks[idx], - num, offset); - if (found < num) { - dirty = true; - break; - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - } - - return dirty; -} - static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, ram_addr_t length, unsigned client) @@ -221,11 +181,7 @@ static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, return dirty; } -static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, - unsigned client) -{ - return cpu_physical_memory_get_dirty(addr, 1, client); -} +bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) { diff --git a/system/physmem.c b/system/physmem.c index 098824ad1d2..7973448b3f8 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -902,6 +902,50 @@ void tlb_reset_dirty_range_all(ram_addr_t addr, ram_addr_t length) } } +static bool physical_memory_get_dirty(ram_addr_t addr, ram_addr_t length, + unsigned client) +{ + DirtyMemoryBlocks *blocks; + unsigned long end, page; + unsigned long idx, offset, base; + bool dirty = false; + + assert(client < DIRTY_MEMORY_NUM); + + end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; + page = addr >> TARGET_PAGE_BITS; + + WITH_RCU_READ_LOCK_GUARD() { + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); + + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + base = page - offset; + while (page < end) { + unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); + unsigned long num = next - base; + unsigned long found = find_next_bit(blocks->blocks[idx], + num, offset); + if (found < num) { + dirty = true; + break; + } + + page = next; + idx++; + offset = 0; + base += DIRTY_MEMORY_BLOCK_SIZE; + } + } + + return dirty; +} + +bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client) +{ + return physical_memory_get_dirty(addr, 1, client); +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, -- 2.51.0 Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 9 +-------- system/physmem.c | 9 +++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index f8a307d1a3d..cdf25c315be 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -183,14 +183,7 @@ static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); -static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) -{ - bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); - bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); - bool migration = - cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); - return !(vga && code && migration); -} +bool cpu_physical_memory_is_clean(ram_addr_t addr); static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, ram_addr_t length, diff --git a/system/physmem.c b/system/physmem.c index 7973448b3f8..b27519c3075 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -946,6 +946,15 @@ bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client) return physical_memory_get_dirty(addr, 1, client); } +bool cpu_physical_memory_is_clean(ram_addr_t addr) +{ + bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); + bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); + bool migration = + cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); + return !(vga && code && migration); +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, -- 2.51.0 Generally we want to clarify terminology and avoid confusions, prefering @start with (exclusive) @end, and base @addr with @length (for inclusive range). Here as cpu_physical_memory_all_dirty() operates on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index cdf25c315be..7a9e8f86be0 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -142,7 +142,7 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, +static inline bool cpu_physical_memory_all_dirty(ram_addr_t addr, ram_addr_t length, unsigned client) { @@ -153,8 +153,8 @@ static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, assert(client < DIRTY_MEMORY_NUM); - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; + end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; + page = addr >> TARGET_PAGE_BITS; RCU_READ_LOCK_GUARD(); -- 2.51.0 Generally we want to clarify terminology and avoid confusions, prefering @start with (exclusive) @end, and base @addr with @length (for inclusive range). Here as cpu_physical_memory_range_includes_clean() operates on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 7a9e8f86be0..e06cc4d0c52 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -185,22 +185,22 @@ bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); bool cpu_physical_memory_is_clean(ram_addr_t addr); -static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, +static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, ram_addr_t length, uint8_t mask) { uint8_t ret = 0; if (mask & (1 << DIRTY_MEMORY_VGA) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { + !cpu_physical_memory_all_dirty(addr, length, DIRTY_MEMORY_VGA)) { ret |= (1 << DIRTY_MEMORY_VGA); } if (mask & (1 << DIRTY_MEMORY_CODE) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { + !cpu_physical_memory_all_dirty(addr, length, DIRTY_MEMORY_CODE)) { ret |= (1 << DIRTY_MEMORY_CODE); } if (mask & (1 << DIRTY_MEMORY_MIGRATION) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { + !cpu_physical_memory_all_dirty(addr, length, DIRTY_MEMORY_MIGRATION)) { ret |= (1 << DIRTY_MEMORY_MIGRATION); } return ret; -- 2.51.0 Avoid maintaining large functions in header, rely on the linker to optimize at linking time. cpu_physical_memory_all_dirty() doesn't involve any CPU, remove the 'cpu_' prefix. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 62 ++------------------------------------- system/physmem.c | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 59 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index e06cc4d0c52..809169b9903 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -142,69 +142,13 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -static inline bool cpu_physical_memory_all_dirty(ram_addr_t addr, - ram_addr_t length, - unsigned client) -{ - DirtyMemoryBlocks *blocks; - unsigned long end, page; - unsigned long idx, offset, base; - bool dirty = true; - - assert(client < DIRTY_MEMORY_NUM); - - end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; - page = addr >> TARGET_PAGE_BITS; - - RCU_READ_LOCK_GUARD(); - - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - unsigned long num = next - base; - unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset); - if (found < num) { - dirty = false; - break; - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - - return dirty; -} - bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); bool cpu_physical_memory_is_clean(ram_addr_t addr); -static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, - ram_addr_t length, - uint8_t mask) -{ - uint8_t ret = 0; - - if (mask & (1 << DIRTY_MEMORY_VGA) && - !cpu_physical_memory_all_dirty(addr, length, DIRTY_MEMORY_VGA)) { - ret |= (1 << DIRTY_MEMORY_VGA); - } - if (mask & (1 << DIRTY_MEMORY_CODE) && - !cpu_physical_memory_all_dirty(addr, length, DIRTY_MEMORY_CODE)) { - ret |= (1 << DIRTY_MEMORY_CODE); - } - if (mask & (1 << DIRTY_MEMORY_MIGRATION) && - !cpu_physical_memory_all_dirty(addr, length, DIRTY_MEMORY_MIGRATION)) { - ret |= (1 << DIRTY_MEMORY_MIGRATION); - } - return ret; -} +uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, + ram_addr_t length, + uint8_t mask); static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) diff --git a/system/physmem.c b/system/physmem.c index b27519c3075..11b08570b62 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -955,6 +955,66 @@ bool cpu_physical_memory_is_clean(ram_addr_t addr) return !(vga && code && migration); } +static bool physical_memory_all_dirty(ram_addr_t addr, ram_addr_t length, + unsigned client) +{ + DirtyMemoryBlocks *blocks; + unsigned long end, page; + unsigned long idx, offset, base; + bool dirty = true; + + assert(client < DIRTY_MEMORY_NUM); + + end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; + page = addr >> TARGET_PAGE_BITS; + + RCU_READ_LOCK_GUARD(); + + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); + + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + base = page - offset; + while (page < end) { + unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); + unsigned long num = next - base; + unsigned long found = find_next_zero_bit(blocks->blocks[idx], + num, offset); + if (found < num) { + dirty = false; + break; + } + + page = next; + idx++; + offset = 0; + base += DIRTY_MEMORY_BLOCK_SIZE; + } + + return dirty; +} + +uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, + ram_addr_t length, + uint8_t mask) +{ + uint8_t ret = 0; + + if (mask & (1 << DIRTY_MEMORY_VGA) && + !physical_memory_all_dirty(addr, length, DIRTY_MEMORY_VGA)) { + ret |= (1 << DIRTY_MEMORY_VGA); + } + if (mask & (1 << DIRTY_MEMORY_CODE) && + !physical_memory_all_dirty(addr, length, DIRTY_MEMORY_CODE)) { + ret |= (1 << DIRTY_MEMORY_CODE); + } + if (mask & (1 << DIRTY_MEMORY_MIGRATION) && + !physical_memory_all_dirty(addr, length, DIRTY_MEMORY_MIGRATION)) { + ret |= (1 << DIRTY_MEMORY_MIGRATION); + } + return ret; +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, -- 2.51.0 Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 19 +------------------ system/physmem.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 809169b9903..6ed17b455b4 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -150,24 +150,7 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, ram_addr_t length, uint8_t mask); -static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, - unsigned client) -{ - unsigned long page, idx, offset; - DirtyMemoryBlocks *blocks; - - assert(client < DIRTY_MEMORY_NUM); - - page = addr >> TARGET_PAGE_BITS; - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - - RCU_READ_LOCK_GUARD(); - - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - set_bit_atomic(offset, blocks->blocks[idx]); -} +void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, diff --git a/system/physmem.c b/system/physmem.c index 11b08570b62..cb0efbeabb2 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1015,6 +1015,24 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, return ret; } +void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) +{ + unsigned long page, idx, offset; + DirtyMemoryBlocks *blocks; + + assert(client < DIRTY_MEMORY_NUM); + + page = addr >> TARGET_PAGE_BITS; + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + + RCU_READ_LOCK_GUARD(); + + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); + + set_bit_atomic(offset, blocks->blocks[idx]); +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, -- 2.51.0 Generally we want to clarify terminology and avoid confusions, prefering @start with (exclusive) @end, and base @addr with @length (for inclusive range). Here as cpu_physical_memory_set_dirty_range() operates on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 6ed17b455b4..84a8b5c003d 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -152,7 +152,7 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); -static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, +static inline void cpu_physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, uint8_t mask) { @@ -165,8 +165,8 @@ static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, return; } - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; + end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; + page = addr >> TARGET_PAGE_BITS; WITH_RCU_READ_LOCK_GUARD() { for (i = 0; i < DIRTY_MEMORY_NUM; i++) { @@ -200,7 +200,7 @@ static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, } if (xen_enabled()) { - xen_hvm_modified_memory(start, length); + xen_hvm_modified_memory(addr, length); } } -- 2.51.0 Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 53 ++------------------------------------- system/physmem.c | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 84a8b5c003d..6377dd19a2f 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -152,57 +152,8 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); -static inline void cpu_physical_memory_set_dirty_range(ram_addr_t addr, - ram_addr_t length, - uint8_t mask) -{ - DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; - unsigned long end, page; - unsigned long idx, offset, base; - int i; - - if (!mask && !xen_enabled()) { - return; - } - - end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; - page = addr >> TARGET_PAGE_BITS; - - WITH_RCU_READ_LOCK_GUARD() { - for (i = 0; i < DIRTY_MEMORY_NUM; i++) { - blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]); - } - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - - if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], - offset, next - page); - } - if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], - offset, next - page); - } - if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], - offset, next - page); - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - } - - if (xen_enabled()) { - xen_hvm_modified_memory(addr, length); - } -} +void cpu_physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, + uint8_t mask); #if !defined(_WIN32) diff --git a/system/physmem.c b/system/physmem.c index cb0efbeabb2..383aecb391f 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1033,6 +1033,57 @@ void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) set_bit_atomic(offset, blocks->blocks[idx]); } +void cpu_physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, + uint8_t mask) +{ + DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; + unsigned long end, page; + unsigned long idx, offset, base; + int i; + + if (!mask && !xen_enabled()) { + return; + } + + end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; + page = addr >> TARGET_PAGE_BITS; + + WITH_RCU_READ_LOCK_GUARD() { + for (i = 0; i < DIRTY_MEMORY_NUM; i++) { + blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]); + } + + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + base = page - offset; + while (page < end) { + unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); + + if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], + offset, next - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], + offset, next - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], + offset, next - page); + } + + page = next; + idx++; + offset = 0; + base += DIRTY_MEMORY_BLOCK_SIZE; + } + } + + if (xen_enabled()) { + xen_hvm_modified_memory(addr, length); + } +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, -- 2.51.0 Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Remove the now unneeded "system/xen.h" header. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 103 +------------------------------------ system/physmem.c | 105 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 102 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 6377dd19a2f..4c227fee412 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -19,7 +19,6 @@ #ifndef SYSTEM_RAM_ADDR_H #define SYSTEM_RAM_ADDR_H -#include "system/xen.h" #include "system/tcg.h" #include "exec/cputlb.h" #include "exec/ramlist.h" @@ -156,115 +155,15 @@ void cpu_physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, uint8_t mask); #if !defined(_WIN32) - /* * Contrary to cpu_physical_memory_sync_dirty_bitmap() this function returns * the number of dirty pages in @bitmap passed as argument. On the other hand, * cpu_physical_memory_sync_dirty_bitmap() returns newly dirtied pages that * weren't set in the global migration bitmap. */ -static inline uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t start, - ram_addr_t pages) -{ - unsigned long i, j; - unsigned long page_number, c, nbits; - hwaddr addr; - ram_addr_t ram_addr; - uint64_t num_dirty = 0; - unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; - unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE; - unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); - - /* start address is aligned at the start of a word? */ - if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && - (hpratio == 1)) { - unsigned long **blocks[DIRTY_MEMORY_NUM]; - unsigned long idx; - unsigned long offset; - long k; - long nr = BITS_TO_LONGS(pages); - - idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; - offset = BIT_WORD((start >> TARGET_PAGE_BITS) % - DIRTY_MEMORY_BLOCK_SIZE); - - WITH_RCU_READ_LOCK_GUARD() { - for (i = 0; i < DIRTY_MEMORY_NUM; i++) { - blocks[i] = - qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks; - } - - for (k = 0; k < nr; k++) { - if (bitmap[k]) { - unsigned long temp = leul_to_cpu(bitmap[k]); - - nbits = ctpopl(temp); - qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); - - if (global_dirty_tracking) { - qatomic_or( - &blocks[DIRTY_MEMORY_MIGRATION][idx][offset], - temp); - if (unlikely( - global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { - total_dirty_pages += nbits; - } - } - - num_dirty += nbits; - - if (tcg_enabled()) { - qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], - temp); - } - } - - if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { - offset = 0; - idx++; - } - } - } - - if (xen_enabled()) { - xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); - } - } else { - uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; - - if (!global_dirty_tracking) { - clients &= ~(1 << DIRTY_MEMORY_MIGRATION); - } - - /* - * bitmap-traveling is faster than memory-traveling (for addr...) - * especially when most of the memory is not dirty. - */ - for (i = 0; i < len; i++) { - if (bitmap[i] != 0) { - c = leul_to_cpu(bitmap[i]); - nbits = ctpopl(c); - if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { - total_dirty_pages += nbits; - } - num_dirty += nbits; - do { - j = ctzl(c); - c &= ~(1ul << j); - page_number = (i * HOST_LONG_BITS + j) * hpratio; - addr = page_number * TARGET_PAGE_SIZE; - ram_addr = start + addr; - cpu_physical_memory_set_dirty_range(ram_addr, - TARGET_PAGE_SIZE * hpratio, clients); - } while (c != 0); - } - } - } - - return num_dirty; -} + ram_addr_t pages); #endif /* not _WIN32 */ static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, diff --git a/system/physmem.c b/system/physmem.c index 383aecb391f..e78ca410ebf 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1208,6 +1208,111 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, return false; } +#if !defined(_WIN32) +uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, + ram_addr_t start, + ram_addr_t pages) +{ + unsigned long i, j; + unsigned long page_number, c, nbits; + hwaddr addr; + ram_addr_t ram_addr; + uint64_t num_dirty = 0; + unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; + unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE; + unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); + + /* start address is aligned at the start of a word? */ + if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && + (hpratio == 1)) { + unsigned long **blocks[DIRTY_MEMORY_NUM]; + unsigned long idx; + unsigned long offset; + long k; + long nr = BITS_TO_LONGS(pages); + + idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; + offset = BIT_WORD((start >> TARGET_PAGE_BITS) % + DIRTY_MEMORY_BLOCK_SIZE); + + WITH_RCU_READ_LOCK_GUARD() { + for (i = 0; i < DIRTY_MEMORY_NUM; i++) { + blocks[i] = + qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks; + } + + for (k = 0; k < nr; k++) { + if (bitmap[k]) { + unsigned long temp = leul_to_cpu(bitmap[k]); + + nbits = ctpopl(temp); + qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); + + if (global_dirty_tracking) { + qatomic_or( + &blocks[DIRTY_MEMORY_MIGRATION][idx][offset], + temp); + if (unlikely( + global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { + total_dirty_pages += nbits; + } + } + + num_dirty += nbits; + + if (tcg_enabled()) { + qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], + temp); + } + } + + if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { + offset = 0; + idx++; + } + } + } + + if (xen_enabled()) { + xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); + } + } else { + uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL + : DIRTY_CLIENTS_NOCODE; + + if (!global_dirty_tracking) { + clients &= ~(1 << DIRTY_MEMORY_MIGRATION); + } + + /* + * bitmap-traveling is faster than memory-traveling (for addr...) + * especially when most of the memory is not dirty. + */ + for (i = 0; i < len; i++) { + if (bitmap[i] != 0) { + c = leul_to_cpu(bitmap[i]); + nbits = ctpopl(c); + if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { + total_dirty_pages += nbits; + } + num_dirty += nbits; + do { + j = ctzl(c); + c &= ~(1ul << j); + page_number = (i * HOST_LONG_BITS + j) * hpratio; + addr = page_number * TARGET_PAGE_SIZE; + ram_addr = start + addr; + cpu_physical_memory_set_dirty_range(ram_addr, + TARGET_PAGE_SIZE * hpratio, clients); + } while (c != 0); + } + } + } + + return num_dirty; +} +#endif /* not _WIN32 */ + static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end, uint16_t section); static subpage_t *subpage_init(FlatView *fv, hwaddr base); -- 2.51.0 Generally we want to clarify terminology and avoid confusions, prefering @start with (exclusive) @end, and base @addr with @length (for inclusive range). Here as cpu_physical_memory_dirty_bits_cleared() operates on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 4c227fee412..e65f479e266 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -166,13 +166,12 @@ uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t pages); #endif /* not _WIN32 */ -static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, +static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length) { if (tcg_enabled()) { - tlb_reset_dirty_range_all(start, length); + tlb_reset_dirty_range_all(addr, length); } - } bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, -- 2.51.0 Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 11 ++--------- system/physmem.c | 7 +++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index e65f479e266..7197913d761 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -19,8 +19,6 @@ #ifndef SYSTEM_RAM_ADDR_H #define SYSTEM_RAM_ADDR_H -#include "system/tcg.h" -#include "exec/cputlb.h" #include "exec/ramlist.h" #include "system/ramblock.h" #include "system/memory.h" @@ -166,13 +164,8 @@ uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t pages); #endif /* not _WIN32 */ -static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t addr, - ram_addr_t length) -{ - if (tcg_enabled()) { - tlb_reset_dirty_range_all(addr, length); - } -} +void cpu_physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length); + bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, unsigned client); diff --git a/system/physmem.c b/system/physmem.c index e78ca410ebf..c475ce0a5db 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -902,6 +902,13 @@ void tlb_reset_dirty_range_all(ram_addr_t addr, ram_addr_t length) } } +void cpu_physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length) +{ + if (tcg_enabled()) { + tlb_reset_dirty_range_all(addr, length); + } +} + static bool physical_memory_get_dirty(ram_addr_t addr, ram_addr_t length, unsigned client) { -- 2.51.0 Generally we want to clarify terminology and avoid confusions, prefering @start with (exclusive) @end, and base @addr with @length (for inclusive range). Here as cpu_physical_memory_test_and_clear_dirty() operates on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 2 +- system/physmem.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 7197913d761..3899c084076 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -166,7 +166,7 @@ uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, void cpu_physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length); -bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, +bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t addr, ram_addr_t length, unsigned client); diff --git a/system/physmem.c b/system/physmem.c index c475ce0a5db..9e36748dc4a 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1092,7 +1092,7 @@ void cpu_physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, } /* Note: start and end must be within the same ram block. */ -bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, +bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t addr, ram_addr_t length, unsigned client) { @@ -1106,16 +1106,16 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, return false; } - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - start_page = start >> TARGET_PAGE_BITS; + end = TARGET_PAGE_ALIGN(addr + length) >> TARGET_PAGE_BITS; + start_page = addr >> TARGET_PAGE_BITS; page = start_page; WITH_RCU_READ_LOCK_GUARD() { blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - ramblock = qemu_get_ram_block(start); + ramblock = qemu_get_ram_block(addr); /* Range sanity check on the ramblock */ - assert(start >= ramblock->offset && - start + length <= ramblock->offset + ramblock->used_length); + assert(addr >= ramblock->offset && + addr + length <= ramblock->offset + ramblock->used_length); while (page < end) { unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE; @@ -1134,7 +1134,7 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, } if (dirty) { - cpu_physical_memory_dirty_bits_cleared(start, length); + cpu_physical_memory_dirty_bits_cleared(addr, length); } return dirty; -- 2.51.0 cpu_physical_memory_clear_dirty_range() is now only called within system/physmem.c, by qemu_ram_resize(). Reduce its scope by making it internal to this file. Since it doesn't involve any CPU, remove the 'cpu_' prefix. As it operates on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 9 --------- system/physmem.c | 9 ++++++++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 3899c084076..c55e3849b0e 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -177,15 +177,6 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, ram_addr_t length); -static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, - ram_addr_t length) -{ - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); -} - - /* Called with RCU critical section */ static inline uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, diff --git a/system/physmem.c b/system/physmem.c index 9e36748dc4a..40ec67572b0 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1140,6 +1140,13 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t addr, return dirty; } +static void physical_memory_clear_dirty_range(ram_addr_t addr, ram_addr_t length) +{ + cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_MIGRATION); + cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_VGA); + cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_CODE); +} + DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client) { @@ -2076,7 +2083,7 @@ int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp) ram_block_notify_resize(block->host, oldsize, newsize); } - cpu_physical_memory_clear_dirty_range(block->offset, block->used_length); + physical_memory_clear_dirty_range(block->offset, block->used_length); block->used_length = newsize; cpu_physical_memory_set_dirty_range(block->offset, block->used_length, DIRTY_CLIENTS_ALL); -- 2.51.0 cpu_physical_memory_sync_dirty_bitmap() is now only called within system/physmem.c, by ramblock_sync_dirty_bitmap(). Reduce its scope by making it internal to this file. Since it doesn't involve any CPU, remove the 'cpu_' prefix. Remove the now unneeded "qemu/rcu.h" and "system/memory.h" headers. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 79 --------------------------------------- migration/ram.c | 78 +++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 80 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index c55e3849b0e..24dba1fba1c 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -21,10 +21,7 @@ #include "exec/ramlist.h" #include "system/ramblock.h" -#include "system/memory.h" #include "exec/target_page.h" -#include "qemu/rcu.h" - #include "exec/hwaddr.h" extern uint64_t total_dirty_pages; @@ -177,80 +174,4 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, ram_addr_t length); -/* Called with RCU critical section */ -static inline -uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, - ram_addr_t start, - ram_addr_t length) -{ - ram_addr_t addr; - unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); - uint64_t num_dirty = 0; - unsigned long *dest = rb->bmap; - - /* start address and length is aligned at the start of a word? */ - if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) == - (start + rb->offset) && - !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) { - int k; - int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); - unsigned long * const *src; - unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; - unsigned long offset = BIT_WORD((word * BITS_PER_LONG) % - DIRTY_MEMORY_BLOCK_SIZE); - unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); - - src = qatomic_rcu_read( - &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; - - for (k = page; k < page + nr; k++) { - if (src[idx][offset]) { - unsigned long bits = qatomic_xchg(&src[idx][offset], 0); - unsigned long new_dirty; - new_dirty = ~dest[k]; - dest[k] |= bits; - new_dirty &= bits; - num_dirty += ctpopl(new_dirty); - } - - if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { - offset = 0; - idx++; - } - } - if (num_dirty) { - cpu_physical_memory_dirty_bits_cleared(start, length); - } - - if (rb->clear_bmap) { - /* - * Postpone the dirty bitmap clear to the point before we - * really send the pages, also we will split the clear - * dirty procedure into smaller chunks. - */ - clear_bmap_set(rb, start >> TARGET_PAGE_BITS, - length >> TARGET_PAGE_BITS); - } else { - /* Slow path - still do that in a huge chunk */ - memory_region_clear_dirty_bitmap(rb->mr, start, length); - } - } else { - ram_addr_t offset = rb->offset; - - for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { - if (cpu_physical_memory_test_and_clear_dirty( - start + addr + offset, - TARGET_PAGE_SIZE, - DIRTY_MEMORY_MIGRATION)) { - long k = (start + addr) >> TARGET_PAGE_BITS; - if (!test_and_set_bit(k, dest)) { - num_dirty++; - } - } - } - } - - return num_dirty; -} - #endif diff --git a/migration/ram.c b/migration/ram.c index 91e65be83d8..e797c0710d3 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -935,11 +935,87 @@ bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start) return false; } +/* Called with RCU critical section */ +static inline +uint64_t physical_memory_sync_dirty_bitmap(RAMBlock *rb, + ram_addr_t start, + ram_addr_t length) +{ + ram_addr_t addr; + unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); + uint64_t num_dirty = 0; + unsigned long *dest = rb->bmap; + + /* start address and length is aligned at the start of a word? */ + if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) == + (start + rb->offset) && + !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) { + int k; + int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); + unsigned long * const *src; + unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; + unsigned long offset = BIT_WORD((word * BITS_PER_LONG) % + DIRTY_MEMORY_BLOCK_SIZE); + unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); + + src = qatomic_rcu_read( + &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; + + for (k = page; k < page + nr; k++) { + if (src[idx][offset]) { + unsigned long bits = qatomic_xchg(&src[idx][offset], 0); + unsigned long new_dirty; + new_dirty = ~dest[k]; + dest[k] |= bits; + new_dirty &= bits; + num_dirty += ctpopl(new_dirty); + } + + if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { + offset = 0; + idx++; + } + } + if (num_dirty) { + cpu_physical_memory_dirty_bits_cleared(start, length); + } + + if (rb->clear_bmap) { + /* + * Postpone the dirty bitmap clear to the point before we + * really send the pages, also we will split the clear + * dirty procedure into smaller chunks. + */ + clear_bmap_set(rb, start >> TARGET_PAGE_BITS, + length >> TARGET_PAGE_BITS); + } else { + /* Slow path - still do that in a huge chunk */ + memory_region_clear_dirty_bitmap(rb->mr, start, length); + } + } else { + ram_addr_t offset = rb->offset; + + for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { + if (cpu_physical_memory_test_and_clear_dirty( + start + addr + offset, + TARGET_PAGE_SIZE, + DIRTY_MEMORY_MIGRATION)) { + long k = (start + addr) >> TARGET_PAGE_BITS; + if (!test_and_set_bit(k, dest)) { + num_dirty++; + } + } + } + } + + return num_dirty; +} + /* Called with RCU critical section */ static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb) { uint64_t new_dirty_pages = - cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length); + physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length); rs->migration_dirty_pages += new_dirty_pages; rs->num_dirty_pages_period += new_dirty_pages; -- 2.51.0 The functions related to the Physical Memory API declared in "system/ram_addr.h" do not operate on vCPU. Remove the 'cpu_' prefix. Signed-off-by: Philippe Mathieu-Daudé --- include/system/ram_addr.h | 24 +++++++++---------- accel/kvm/kvm-all.c | 2 +- accel/tcg/cputlb.c | 12 +++++----- hw/vfio/container-legacy.c | 8 +++---- hw/vfio/container.c | 4 ++-- migration/ram.c | 4 ++-- system/memory.c | 8 +++---- system/physmem.c | 48 ++++++++++++++++++------------------- target/arm/tcg/mte_helper.c | 2 +- system/memory_ldst.c.inc | 2 +- tests/tsan/ignore.tsan | 4 ++-- 11 files changed, 59 insertions(+), 59 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 24dba1fba1c..015f943603b 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -136,41 +136,41 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); +bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); -bool cpu_physical_memory_is_clean(ram_addr_t addr); +bool physical_memory_is_clean(ram_addr_t addr); -uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, +uint8_t physical_memory_range_includes_clean(ram_addr_t addr, ram_addr_t length, uint8_t mask); -void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); +void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); -void cpu_physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, +void physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, uint8_t mask); #if !defined(_WIN32) /* - * Contrary to cpu_physical_memory_sync_dirty_bitmap() this function returns + * Contrary to physical_memory_sync_dirty_bitmap() this function returns * the number of dirty pages in @bitmap passed as argument. On the other hand, - * cpu_physical_memory_sync_dirty_bitmap() returns newly dirtied pages that + * physical_memory_sync_dirty_bitmap() returns newly dirtied pages that * weren't set in the global migration bitmap. */ -uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, +uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t start, ram_addr_t pages); #endif /* not _WIN32 */ -void cpu_physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length); +void physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length); -bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t addr, +bool physical_memory_test_and_clear_dirty(ram_addr_t addr, ram_addr_t length, unsigned client); -DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty +DirtyBitmapSnapshot *physical_memory_snapshot_and_clear_dirty (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); -bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, +bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, ram_addr_t length); diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 08b2b5a371c..a7ece7db964 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -758,7 +758,7 @@ static void kvm_slot_sync_dirty_pages(KVMSlot *slot) ram_addr_t start = slot->ram_start_offset; ram_addr_t pages = slot->memory_size / qemu_real_host_page_size(); - cpu_physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages); + physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages); } static void kvm_slot_reset_dirty_pages(KVMSlot *slot) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 6807328df82..1f5dd023a0a 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -858,7 +858,7 @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, can be detected */ void tlb_protect_code(ram_addr_t ram_addr) { - cpu_physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK, + physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK, TARGET_PAGE_SIZE, DIRTY_MEMORY_CODE); } @@ -867,7 +867,7 @@ void tlb_protect_code(ram_addr_t ram_addr) tested for self modifying code */ void tlb_unprotect_code(ram_addr_t ram_addr) { - cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE); + physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE); } @@ -1085,7 +1085,7 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx, if (prot & PAGE_WRITE) { if (section->readonly) { write_flags |= TLB_DISCARD_WRITE; - } else if (cpu_physical_memory_is_clean(iotlb)) { + } else if (physical_memory_is_clean(iotlb)) { write_flags |= TLB_NOTDIRTY; } } @@ -1341,7 +1341,7 @@ static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size); - if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) { + if (!physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) { tb_invalidate_phys_range_fast(cpu, ram_addr, size, retaddr); } @@ -1349,10 +1349,10 @@ static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, * Set both VGA and migration bits for simplicity and to remove * the notdirty callback faster. */ - cpu_physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE); + physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE); /* We remove the notdirty callback only if the code has been flushed. */ - if (!cpu_physical_memory_is_clean(ram_addr)) { + if (!physical_memory_is_clean(ram_addr)) { trace_memory_notdirty_set_dirty(mem_vaddr); tlb_set_dirty(cpu, mem_vaddr); } diff --git a/hw/vfio/container-legacy.c b/hw/vfio/container-legacy.c index 3a710d8265c..eb9911eaeaf 100644 --- a/hw/vfio/container-legacy.c +++ b/hw/vfio/container-legacy.c @@ -92,7 +92,7 @@ static int vfio_dma_unmap_bitmap(const VFIOLegacyContainer *container, bitmap = (struct vfio_bitmap *)&unmap->data; /* - * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of + * physical_memory_set_dirty_lebitmap() supports pages in bitmap of * qemu_real_host_page_size to mark those dirty. Hence set bitmap_pgsize * to qemu_real_host_page_size. */ @@ -108,7 +108,7 @@ static int vfio_dma_unmap_bitmap(const VFIOLegacyContainer *container, ret = ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, unmap); if (!ret) { - cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, + physical_memory_set_dirty_lebitmap(vbmap.bitmap, iotlb->translated_addr, vbmap.pages); } else { error_report("VFIO_UNMAP_DMA with DIRTY_BITMAP : %m"); @@ -284,7 +284,7 @@ static int vfio_legacy_query_dirty_bitmap(const VFIOContainer *bcontainer, range->size = size; /* - * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of + * physical_memory_set_dirty_lebitmap() supports pages in bitmap of * qemu_real_host_page_size to mark those dirty. Hence set bitmap's pgsize * to qemu_real_host_page_size. */ @@ -503,7 +503,7 @@ static void vfio_get_iommu_info_migration(VFIOLegacyContainer *container, header); /* - * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of + * physical_memory_set_dirty_lebitmap() supports pages in bitmap of * qemu_real_host_page_size to mark those dirty. */ if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) { diff --git a/hw/vfio/container.c b/hw/vfio/container.c index 41de3439246..3fb19a1c8ad 100644 --- a/hw/vfio/container.c +++ b/hw/vfio/container.c @@ -255,7 +255,7 @@ int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, int ret; if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) { - cpu_physical_memory_set_dirty_range(translated_addr, size, + physical_memory_set_dirty_range(translated_addr, size, tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE); return 0; @@ -280,7 +280,7 @@ int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, goto out; } - dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, + dirty_pages = physical_memory_set_dirty_lebitmap(vbmap.bitmap, translated_addr, vbmap.pages); diff --git a/migration/ram.c b/migration/ram.c index e797c0710d3..db745f2a028 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -977,7 +977,7 @@ uint64_t physical_memory_sync_dirty_bitmap(RAMBlock *rb, } } if (num_dirty) { - cpu_physical_memory_dirty_bits_cleared(start, length); + physical_memory_dirty_bits_cleared(start, length); } if (rb->clear_bmap) { @@ -996,7 +996,7 @@ uint64_t physical_memory_sync_dirty_bitmap(RAMBlock *rb, ram_addr_t offset = rb->offset; for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { - if (cpu_physical_memory_test_and_clear_dirty( + if (physical_memory_test_and_clear_dirty( start + addr + offset, TARGET_PAGE_SIZE, DIRTY_MEMORY_MIGRATION)) { diff --git a/system/memory.c b/system/memory.c index cf8cad69611..dd045da60c0 100644 --- a/system/memory.c +++ b/system/memory.c @@ -2275,7 +2275,7 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, hwaddr size) { assert(mr->ram_block); - cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, + physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, size, memory_region_get_dirty_log_mask(mr)); } @@ -2379,7 +2379,7 @@ DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *snapshot; assert(mr->ram_block); memory_region_sync_dirty_bitmap(mr, false); - snapshot = cpu_physical_memory_snapshot_and_clear_dirty(mr, addr, size, client); + snapshot = physical_memory_snapshot_and_clear_dirty(mr, addr, size, client); memory_global_after_dirty_log_sync(); return snapshot; } @@ -2388,7 +2388,7 @@ bool memory_region_snapshot_get_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *sna hwaddr addr, hwaddr size) { assert(mr->ram_block); - return cpu_physical_memory_snapshot_get_dirty(snap, + return physical_memory_snapshot_get_dirty(snap, memory_region_get_ram_addr(mr) + addr, size); } @@ -2426,7 +2426,7 @@ void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr, hwaddr size, unsigned client) { assert(mr->ram_block); - cpu_physical_memory_test_and_clear_dirty( + physical_memory_test_and_clear_dirty( memory_region_get_ram_addr(mr) + addr, size, client); } diff --git a/system/physmem.c b/system/physmem.c index 40ec67572b0..784c2810964 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -902,7 +902,7 @@ void tlb_reset_dirty_range_all(ram_addr_t addr, ram_addr_t length) } } -void cpu_physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length) +void physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length) { if (tcg_enabled()) { tlb_reset_dirty_range_all(addr, length); @@ -948,17 +948,17 @@ static bool physical_memory_get_dirty(ram_addr_t addr, ram_addr_t length, return dirty; } -bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client) +bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client) { return physical_memory_get_dirty(addr, 1, client); } -bool cpu_physical_memory_is_clean(ram_addr_t addr) +bool physical_memory_is_clean(ram_addr_t addr) { - bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); - bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); + bool vga = physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); + bool code = physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); bool migration = - cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); + physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); return !(vga && code && migration); } @@ -1001,7 +1001,7 @@ static bool physical_memory_all_dirty(ram_addr_t addr, ram_addr_t length, return dirty; } -uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, +uint8_t physical_memory_range_includes_clean(ram_addr_t addr, ram_addr_t length, uint8_t mask) { @@ -1022,7 +1022,7 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t addr, return ret; } -void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) +void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) { unsigned long page, idx, offset; DirtyMemoryBlocks *blocks; @@ -1040,7 +1040,7 @@ void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) set_bit_atomic(offset, blocks->blocks[idx]); } -void cpu_physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, +void physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, uint8_t mask) { DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; @@ -1092,7 +1092,7 @@ void cpu_physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, } /* Note: start and end must be within the same ram block. */ -bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t addr, +bool physical_memory_test_and_clear_dirty(ram_addr_t addr, ram_addr_t length, unsigned client) { @@ -1134,7 +1134,7 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t addr, } if (dirty) { - cpu_physical_memory_dirty_bits_cleared(addr, length); + physical_memory_dirty_bits_cleared(addr, length); } return dirty; @@ -1142,12 +1142,12 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t addr, static void physical_memory_clear_dirty_range(ram_addr_t addr, ram_addr_t length) { - cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_MIGRATION); - cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_VGA); - cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_CODE); + physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_MIGRATION); + physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_VGA); + physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_CODE); } -DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty +DirtyBitmapSnapshot *physical_memory_snapshot_and_clear_dirty (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client) { DirtyMemoryBlocks *blocks; @@ -1194,14 +1194,14 @@ DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty } } - cpu_physical_memory_dirty_bits_cleared(start, length); + physical_memory_dirty_bits_cleared(start, length); memory_region_clear_dirty_bitmap(mr, offset, length); return snap; } -bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, +bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, ram_addr_t length) { @@ -1223,7 +1223,7 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, } #if !defined(_WIN32) -uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, +uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t start, ram_addr_t pages) { @@ -1316,7 +1316,7 @@ uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, page_number = (i * HOST_LONG_BITS + j) * hpratio; addr = page_number * TARGET_PAGE_SIZE; ram_addr = start + addr; - cpu_physical_memory_set_dirty_range(ram_addr, + physical_memory_set_dirty_range(ram_addr, TARGET_PAGE_SIZE * hpratio, clients); } while (c != 0); } @@ -2085,7 +2085,7 @@ int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp) physical_memory_clear_dirty_range(block->offset, block->used_length); block->used_length = newsize; - cpu_physical_memory_set_dirty_range(block->offset, block->used_length, + physical_memory_set_dirty_range(block->offset, block->used_length, DIRTY_CLIENTS_ALL); memory_region_set_size(block->mr, unaligned_size); if (block->resized) { @@ -2290,7 +2290,7 @@ static void ram_block_add(RAMBlock *new_block, Error **errp) ram_list.version++; qemu_mutex_unlock_ramlist(); - cpu_physical_memory_set_dirty_range(new_block->offset, + physical_memory_set_dirty_range(new_block->offset, new_block->used_length, DIRTY_CLIENTS_ALL); @@ -3139,19 +3139,19 @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, addr += ramaddr; /* No early return if dirty_log_mask is or becomes 0, because - * cpu_physical_memory_set_dirty_range will still call + * physical_memory_set_dirty_range will still call * xen_modified_memory. */ if (dirty_log_mask) { dirty_log_mask = - cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask); + physical_memory_range_includes_clean(addr, length, dirty_log_mask); } if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { assert(tcg_enabled()); tb_invalidate_phys_range(NULL, addr, addr + length - 1); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } - cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); + physical_memory_set_dirty_range(addr, length, dirty_log_mask); } void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size) diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index 7d80244788e..077ff4b2b2c 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -189,7 +189,7 @@ uint8_t *allocation_tag_mem_probe(CPUARMState *env, int ptr_mmu_idx, */ if (tag_access == MMU_DATA_STORE) { ram_addr_t tag_ra = memory_region_get_ram_addr(mr) + xlat; - cpu_physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION); + physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION); } return memory_region_get_ram_ptr(mr) + xlat; diff --git a/system/memory_ldst.c.inc b/system/memory_ldst.c.inc index 7f32d3d9ff3..333da209d1a 100644 --- a/system/memory_ldst.c.inc +++ b/system/memory_ldst.c.inc @@ -287,7 +287,7 @@ void glue(address_space_stl_notdirty, SUFFIX)(ARG1_DECL, dirty_log_mask = memory_region_get_dirty_log_mask(mr); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); - cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, + physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, 4, dirty_log_mask); r = MEMTX_OK; } diff --git a/tests/tsan/ignore.tsan b/tests/tsan/ignore.tsan index 423e482d2f9..8fa00a2c49b 100644 --- a/tests/tsan/ignore.tsan +++ b/tests/tsan/ignore.tsan @@ -4,7 +4,7 @@ # The eventual goal would be to fix these warnings. # TSan is not happy about setting/getting of dirty bits, -# for example, cpu_physical_memory_set_dirty_range, -# and cpu_physical_memory_get_dirty. +# for example, physical_memory_set_dirty_range, +# and physical_memory_get_dirty. src:bitops.c src:bitmap.c -- 2.51.0 Very few files use the Physical Memory API. Declare its methods in their own header: "system/physmem.h". Signed-off-by: Philippe Mathieu-Daudé --- MAINTAINERS | 1 + include/system/physmem.h | 56 +++++++++++++++++++++++++++++++++++++ include/system/ram_addr.h | 42 ---------------------------- accel/kvm/kvm-all.c | 2 +- accel/tcg/cputlb.c | 1 + hw/vfio/container-legacy.c | 2 +- hw/vfio/container.c | 1 + hw/vfio/listener.c | 1 - migration/ram.c | 1 + system/memory.c | 1 + system/physmem.c | 1 + target/arm/tcg/mte_helper.c | 2 +- 12 files changed, 65 insertions(+), 46 deletions(-) create mode 100644 include/system/physmem.h diff --git a/MAINTAINERS b/MAINTAINERS index 7d134a85e66..866b43434c7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3206,6 +3206,7 @@ S: Supported F: include/system/ioport.h F: include/exec/memop.h F: include/system/memory.h +F: include/system/physmem.h F: include/system/ram_addr.h F: include/system/ramblock.h F: include/system/memory_mapping.h diff --git a/include/system/physmem.h b/include/system/physmem.h new file mode 100644 index 00000000000..7ae266729d2 --- /dev/null +++ b/include/system/physmem.h @@ -0,0 +1,56 @@ +/* + * QEMU physical memory interfaces (target independent). + * + * Copyright (c) 2003 Fabrice Bellard + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#ifndef QEMU_SYSTEM_PHYSMEM_H +#define QEMU_SYSTEM_PHYSMEM_H + +#include "exec/hwaddr.h" +#include "exec/ramlist.h" + +#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) +#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) + +bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); + +bool physical_memory_is_clean(ram_addr_t addr); + +uint8_t physical_memory_range_includes_clean(ram_addr_t addr, + ram_addr_t length, + uint8_t mask); + +void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); + +void physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, + uint8_t mask); + +#if !defined(_WIN32) +/* + * Contrary to physical_memory_sync_dirty_bitmap() this function returns + * the number of dirty pages in @bitmap passed as argument. On the other hand, + * physical_memory_sync_dirty_bitmap() returns newly dirtied pages that + * weren't set in the global migration bitmap. + */ +uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, + ram_addr_t start, + ram_addr_t pages); +#endif /* not _WIN32 */ + +void physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length); + +bool physical_memory_test_and_clear_dirty(ram_addr_t addr, + ram_addr_t length, + unsigned client); + +DirtyBitmapSnapshot * +physical_memory_snapshot_and_clear_dirty(MemoryRegion *mr, hwaddr offset, + hwaddr length, unsigned client); + +bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, + ram_addr_t start, + ram_addr_t length); + +#endif diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 015f943603b..683485980ce 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -19,7 +19,6 @@ #ifndef SYSTEM_RAM_ADDR_H #define SYSTEM_RAM_ADDR_H -#include "exec/ramlist.h" #include "system/ramblock.h" #include "exec/target_page.h" #include "exec/hwaddr.h" @@ -133,45 +132,4 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) qemu_ram_msync(block, 0, block->used_length); } -#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) -#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) - -bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); - -bool physical_memory_is_clean(ram_addr_t addr); - -uint8_t physical_memory_range_includes_clean(ram_addr_t addr, - ram_addr_t length, - uint8_t mask); - -void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); - -void physical_memory_set_dirty_range(ram_addr_t addr, ram_addr_t length, - uint8_t mask); - -#if !defined(_WIN32) -/* - * Contrary to physical_memory_sync_dirty_bitmap() this function returns - * the number of dirty pages in @bitmap passed as argument. On the other hand, - * physical_memory_sync_dirty_bitmap() returns newly dirtied pages that - * weren't set in the global migration bitmap. - */ -uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, - ram_addr_t start, - ram_addr_t pages); -#endif /* not _WIN32 */ - -void physical_memory_dirty_bits_cleared(ram_addr_t addr, ram_addr_t length); - -bool physical_memory_test_and_clear_dirty(ram_addr_t addr, - ram_addr_t length, - unsigned client); - -DirtyBitmapSnapshot *physical_memory_snapshot_and_clear_dirty - (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); - -bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, - ram_addr_t start, - ram_addr_t length); - #endif diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index a7ece7db964..58802f7c3cc 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -32,13 +32,13 @@ #include "system/runstate.h" #include "system/cpus.h" #include "system/accel-blocker.h" +#include "system/physmem.h" #include "system/ramblock.h" #include "accel/accel-ops.h" #include "qemu/bswap.h" #include "exec/tswap.h" #include "exec/target_page.h" #include "system/memory.h" -#include "system/ram_addr.h" #include "qemu/event_notifier.h" #include "qemu/main-loop.h" #include "trace.h" diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 1f5dd023a0a..8e1ec7ab1af 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -25,6 +25,7 @@ #include "accel/tcg/probe.h" #include "exec/page-protection.h" #include "system/memory.h" +#include "system/physmem.h" #include "accel/tcg/cpu-ldst-common.h" #include "accel/tcg/cpu-mmu-index.h" #include "exec/cputlb.h" diff --git a/hw/vfio/container-legacy.c b/hw/vfio/container-legacy.c index eb9911eaeaf..755a407f3e7 100644 --- a/hw/vfio/container-legacy.c +++ b/hw/vfio/container-legacy.c @@ -25,7 +25,7 @@ #include "hw/vfio/vfio-device.h" #include "system/address-spaces.h" #include "system/memory.h" -#include "system/ram_addr.h" +#include "system/physmem.h" #include "qemu/error-report.h" #include "qemu/range.h" #include "system/reset.h" diff --git a/hw/vfio/container.c b/hw/vfio/container.c index 3fb19a1c8ad..9ddec300e35 100644 --- a/hw/vfio/container.c +++ b/hw/vfio/container.c @@ -20,6 +20,7 @@ #include "qemu/error-report.h" #include "hw/vfio/vfio-container.h" #include "hw/vfio/vfio-device.h" /* vfio_device_reset_handler */ +#include "system/physmem.h" #include "system/reset.h" #include "vfio-helpers.h" diff --git a/hw/vfio/listener.c b/hw/vfio/listener.c index b5cefc9395c..c6bb58f5209 100644 --- a/hw/vfio/listener.c +++ b/hw/vfio/listener.c @@ -30,7 +30,6 @@ #include "hw/vfio/pci.h" #include "system/address-spaces.h" #include "system/memory.h" -#include "system/ram_addr.h" #include "hw/hw.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" diff --git a/migration/ram.c b/migration/ram.c index db745f2a028..21b8c78fa91 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -53,6 +53,7 @@ #include "qemu/rcu_queue.h" #include "migration/colo.h" #include "system/cpu-throttle.h" +#include "system/physmem.h" #include "system/ramblock.h" #include "savevm.h" #include "qemu/iov.h" diff --git a/system/memory.c b/system/memory.c index dd045da60c0..80656c69568 100644 --- a/system/memory.c +++ b/system/memory.c @@ -25,6 +25,7 @@ #include "qemu/target-info.h" #include "qom/object.h" #include "trace.h" +#include "system/physmem.h" #include "system/ram_addr.h" #include "system/kvm.h" #include "system/runstate.h" diff --git a/system/physmem.c b/system/physmem.c index 784c2810964..b245cd14d43 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -43,6 +43,7 @@ #include "system/kvm.h" #include "system/tcg.h" #include "system/qtest.h" +#include "system/physmem.h" #include "system/ramblock.h" #include "qemu/timer.h" #include "qemu/config-file.h" diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index 077ff4b2b2c..b96c953f809 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -27,7 +27,7 @@ #include "user/cpu_loop.h" #include "user/page-protection.h" #else -#include "system/ram_addr.h" +#include "system/physmem.h" #endif #include "accel/tcg/cpu-ldst.h" #include "accel/tcg/probe.h" -- 2.51.0