2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
24 #include <sys/types.h>
37 #include "qemu-common.h"
42 #if defined(CONFIG_USER_ONLY)
46 //#define DEBUG_TB_INVALIDATE
49 //#define DEBUG_UNASSIGNED
51 /* make various TB consistency checks */
52 //#define DEBUG_TB_CHECK
53 //#define DEBUG_TLB_CHECK
55 //#define DEBUG_IOPORT
56 //#define DEBUG_SUBPAGE
58 #if !defined(CONFIG_USER_ONLY)
59 /* TB consistency checks only implemented for usermode emulation. */
63 #define SMC_BITMAP_USE_THRESHOLD 10
65 #if defined(TARGET_SPARC64)
66 #define TARGET_PHYS_ADDR_SPACE_BITS 41
67 #elif defined(TARGET_SPARC)
68 #define TARGET_PHYS_ADDR_SPACE_BITS 36
69 #elif defined(TARGET_ALPHA)
70 #define TARGET_PHYS_ADDR_SPACE_BITS 42
71 #define TARGET_VIRT_ADDR_SPACE_BITS 42
72 #elif defined(TARGET_PPC64)
73 #define TARGET_PHYS_ADDR_SPACE_BITS 42
74 #elif defined(TARGET_X86_64) && !defined(CONFIG_KQEMU)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 42
76 #elif defined(TARGET_I386) && !defined(CONFIG_KQEMU)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 36
79 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
80 #define TARGET_PHYS_ADDR_SPACE_BITS 32
83 static TranslationBlock *tbs;
84 int code_gen_max_blocks;
85 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
87 /* any access to the tbs or the page table must use this lock */
88 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
90 #if defined(__arm__) || defined(__sparc_v9__)
91 /* The prologue must be reachable with a direct jump. ARM and Sparc64
92 have limited branch ranges (possibly also PPC) so place it in a
93 section close to code segment. */
94 #define code_gen_section \
95 __attribute__((__section__(".gen_code"))) \
96 __attribute__((aligned (32)))
98 #define code_gen_section \
99 __attribute__((aligned (32)))
102 uint8_t code_gen_prologue[1024] code_gen_section;
103 static uint8_t *code_gen_buffer;
104 static unsigned long code_gen_buffer_size;
105 /* threshold to flush the translated code buffer */
106 static unsigned long code_gen_buffer_max_size;
107 uint8_t *code_gen_ptr;
109 #if !defined(CONFIG_USER_ONLY)
111 uint8_t *phys_ram_dirty;
112 static int in_migration;
114 typedef struct RAMBlock {
118 struct RAMBlock *next;
121 static RAMBlock *ram_blocks;
122 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
123 then we can no longer assume contiguous ram offsets, and external uses
124 of this variable will break. */
125 ram_addr_t last_ram_offset;
129 /* current CPU in the current thread. It is only valid inside
131 CPUState *cpu_single_env;
132 /* 0 = Do not count executed instructions.
133 1 = Precise instruction counting.
134 2 = Adaptive rate instruction counting. */
136 /* Current instruction counter. While executing translated code this may
137 include some instructions that have not yet been executed. */
140 typedef struct PageDesc {
141 /* list of TBs intersecting this ram page */
142 TranslationBlock *first_tb;
143 /* in order to optimize self modifying code, we count the number
144 of lookups we do to a given page to use a bitmap */
145 unsigned int code_write_count;
146 uint8_t *code_bitmap;
147 #if defined(CONFIG_USER_ONLY)
152 typedef struct PhysPageDesc {
153 /* offset in host memory of the page + io_index in the low bits */
154 ram_addr_t phys_offset;
155 ram_addr_t region_offset;
159 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
160 /* XXX: this is a temporary hack for alpha target.
161 * In the future, this is to be replaced by a multi-level table
162 * to actually be able to handle the complete 64 bits address space.
164 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
166 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
169 #define L1_SIZE (1 << L1_BITS)
170 #define L2_SIZE (1 << L2_BITS)
172 unsigned long qemu_real_host_page_size;
173 unsigned long qemu_host_page_bits;
174 unsigned long qemu_host_page_size;
175 unsigned long qemu_host_page_mask;
177 /* XXX: for system emulation, it could just be an array */
178 static PageDesc *l1_map[L1_SIZE];
179 static PhysPageDesc **l1_phys_map;
181 #if !defined(CONFIG_USER_ONLY)
182 static void io_mem_init(void);
184 /* io memory support */
185 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
186 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
187 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
188 static char io_mem_used[IO_MEM_NB_ENTRIES];
189 static int io_mem_watch;
194 static const char *logfilename = "qemu.log";
196 static const char *logfilename = "/tmp/qemu.log";
200 static int log_append = 0;
203 static int tlb_flush_count;
204 static int tb_flush_count;
205 static int tb_phys_invalidate_count;
207 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
208 typedef struct subpage_t {
209 target_phys_addr_t base;
210 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
211 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
212 void *opaque[TARGET_PAGE_SIZE][2][4];
213 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
217 static void map_exec(void *addr, long size)
220 VirtualProtect(addr, size,
221 PAGE_EXECUTE_READWRITE, &old_protect);
225 static void map_exec(void *addr, long size)
227 unsigned long start, end, page_size;
229 page_size = getpagesize();
230 start = (unsigned long)addr;
231 start &= ~(page_size - 1);
233 end = (unsigned long)addr + size;
234 end += page_size - 1;
235 end &= ~(page_size - 1);
237 mprotect((void *)start, end - start,
238 PROT_READ | PROT_WRITE | PROT_EXEC);
242 static void page_init(void)
244 /* NOTE: we can always suppose that qemu_host_page_size >=
248 SYSTEM_INFO system_info;
250 GetSystemInfo(&system_info);
251 qemu_real_host_page_size = system_info.dwPageSize;
254 qemu_real_host_page_size = getpagesize();
256 if (qemu_host_page_size == 0)
257 qemu_host_page_size = qemu_real_host_page_size;
258 if (qemu_host_page_size < TARGET_PAGE_SIZE)
259 qemu_host_page_size = TARGET_PAGE_SIZE;
260 qemu_host_page_bits = 0;
261 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
262 qemu_host_page_bits++;
263 qemu_host_page_mask = ~(qemu_host_page_size - 1);
264 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
265 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
267 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
269 long long startaddr, endaddr;
274 last_brk = (unsigned long)sbrk(0);
275 f = fopen("/proc/self/maps", "r");
278 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
280 startaddr = MIN(startaddr,
281 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
282 endaddr = MIN(endaddr,
283 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
284 page_set_flags(startaddr & TARGET_PAGE_MASK,
285 TARGET_PAGE_ALIGN(endaddr),
296 static inline PageDesc **page_l1_map(target_ulong index)
298 #if TARGET_LONG_BITS > 32
299 /* Host memory outside guest VM. For 32-bit targets we have already
300 excluded high addresses. */
301 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
304 return &l1_map[index >> L2_BITS];
307 static inline PageDesc *page_find_alloc(target_ulong index)
310 lp = page_l1_map(index);
316 /* allocate if not found */
317 #if defined(CONFIG_USER_ONLY)
318 size_t len = sizeof(PageDesc) * L2_SIZE;
319 /* Don't use qemu_malloc because it may recurse. */
320 p = mmap(0, len, PROT_READ | PROT_WRITE,
321 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
324 unsigned long addr = h2g(p);
325 page_set_flags(addr & TARGET_PAGE_MASK,
326 TARGET_PAGE_ALIGN(addr + len),
330 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
334 return p + (index & (L2_SIZE - 1));
337 static inline PageDesc *page_find(target_ulong index)
340 lp = page_l1_map(index);
347 return p + (index & (L2_SIZE - 1));
350 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
355 p = (void **)l1_phys_map;
356 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
358 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
359 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
361 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
364 /* allocate if not found */
367 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
368 memset(p, 0, sizeof(void *) * L1_SIZE);
372 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
376 /* allocate if not found */
379 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
381 for (i = 0; i < L2_SIZE; i++) {
382 pd[i].phys_offset = IO_MEM_UNASSIGNED;
383 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
386 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
389 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
391 return phys_page_find_alloc(index, 0);
394 #if !defined(CONFIG_USER_ONLY)
395 static void tlb_protect_code(ram_addr_t ram_addr);
396 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
398 #define mmap_lock() do { } while(0)
399 #define mmap_unlock() do { } while(0)
402 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
404 #if defined(CONFIG_USER_ONLY)
405 /* Currently it is not recommended to allocate big chunks of data in
406 user mode. It will change when a dedicated libc will be used */
407 #define USE_STATIC_CODE_GEN_BUFFER
410 #ifdef USE_STATIC_CODE_GEN_BUFFER
411 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
414 static void code_gen_alloc(unsigned long tb_size)
416 #ifdef USE_STATIC_CODE_GEN_BUFFER
417 code_gen_buffer = static_code_gen_buffer;
418 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
419 map_exec(code_gen_buffer, code_gen_buffer_size);
421 code_gen_buffer_size = tb_size;
422 if (code_gen_buffer_size == 0) {
423 #if defined(CONFIG_USER_ONLY)
424 /* in user mode, phys_ram_size is not meaningful */
425 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
427 /* XXX: needs adjustments */
428 code_gen_buffer_size = (unsigned long)(ram_size / 4);
431 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
432 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
433 /* The code gen buffer location may have constraints depending on
434 the host cpu and OS */
435 #if defined(__linux__)
440 flags = MAP_PRIVATE | MAP_ANONYMOUS;
441 #if defined(__x86_64__)
443 /* Cannot map more than that */
444 if (code_gen_buffer_size > (800 * 1024 * 1024))
445 code_gen_buffer_size = (800 * 1024 * 1024);
446 #elif defined(__sparc_v9__)
447 // Map the buffer below 2G, so we can use direct calls and branches
449 start = (void *) 0x60000000UL;
450 if (code_gen_buffer_size > (512 * 1024 * 1024))
451 code_gen_buffer_size = (512 * 1024 * 1024);
452 #elif defined(__arm__)
453 /* Map the buffer below 32M, so we can use direct calls and branches */
455 start = (void *) 0x01000000UL;
456 if (code_gen_buffer_size > 16 * 1024 * 1024)
457 code_gen_buffer_size = 16 * 1024 * 1024;
459 code_gen_buffer = mmap(start, code_gen_buffer_size,
460 PROT_WRITE | PROT_READ | PROT_EXEC,
462 if (code_gen_buffer == MAP_FAILED) {
463 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
467 #elif defined(__FreeBSD__) || defined(__DragonFly__)
471 flags = MAP_PRIVATE | MAP_ANONYMOUS;
472 #if defined(__x86_64__)
473 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
474 * 0x40000000 is free */
476 addr = (void *)0x40000000;
477 /* Cannot map more than that */
478 if (code_gen_buffer_size > (800 * 1024 * 1024))
479 code_gen_buffer_size = (800 * 1024 * 1024);
481 code_gen_buffer = mmap(addr, code_gen_buffer_size,
482 PROT_WRITE | PROT_READ | PROT_EXEC,
484 if (code_gen_buffer == MAP_FAILED) {
485 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
490 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
491 map_exec(code_gen_buffer, code_gen_buffer_size);
493 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
494 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
495 code_gen_buffer_max_size = code_gen_buffer_size -
496 code_gen_max_block_size();
497 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
498 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
501 /* Must be called before using the QEMU cpus. 'tb_size' is the size
502 (in bytes) allocated to the translation buffer. Zero means default
504 void cpu_exec_init_all(unsigned long tb_size)
507 code_gen_alloc(tb_size);
508 code_gen_ptr = code_gen_buffer;
510 #if !defined(CONFIG_USER_ONLY)
515 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
517 #define CPU_COMMON_SAVE_VERSION 1
519 static void cpu_common_save(QEMUFile *f, void *opaque)
521 CPUState *env = opaque;
523 cpu_synchronize_state(env, 0);
525 qemu_put_be32s(f, &env->halted);
526 qemu_put_be32s(f, &env->interrupt_request);
529 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
531 CPUState *env = opaque;
533 if (version_id != CPU_COMMON_SAVE_VERSION)
536 qemu_get_be32s(f, &env->halted);
537 qemu_get_be32s(f, &env->interrupt_request);
538 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
539 version_id is increased. */
540 env->interrupt_request &= ~0x01;
542 cpu_synchronize_state(env, 1);
548 void cpu_exec_init(CPUState *env)
553 #if defined(CONFIG_USER_ONLY)
556 env->next_cpu = NULL;
559 while (*penv != NULL) {
560 penv = &(*penv)->next_cpu;
563 env->cpu_index = cpu_index;
565 TAILQ_INIT(&env->breakpoints);
566 TAILQ_INIT(&env->watchpoints);
568 #if defined(CONFIG_USER_ONLY)
571 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
572 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
573 cpu_common_save, cpu_common_load, env);
574 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
575 cpu_save, cpu_load, env);
579 static inline void invalidate_page_bitmap(PageDesc *p)
581 if (p->code_bitmap) {
582 qemu_free(p->code_bitmap);
583 p->code_bitmap = NULL;
585 p->code_write_count = 0;
588 /* set to NULL all the 'first_tb' fields in all PageDescs */
589 static void page_flush_tb(void)
594 for(i = 0; i < L1_SIZE; i++) {
597 for(j = 0; j < L2_SIZE; j++) {
599 invalidate_page_bitmap(p);
606 /* flush all the translation blocks */
607 /* XXX: tb_flush is currently not thread safe */
608 void tb_flush(CPUState *env1)
611 #if defined(DEBUG_FLUSH)
612 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
613 (unsigned long)(code_gen_ptr - code_gen_buffer),
615 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
617 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
618 cpu_abort(env1, "Internal error: code buffer overflow\n");
622 for(env = first_cpu; env != NULL; env = env->next_cpu) {
623 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
626 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
629 code_gen_ptr = code_gen_buffer;
630 /* XXX: flush processor icache at this point if cache flush is
635 #ifdef DEBUG_TB_CHECK
637 static void tb_invalidate_check(target_ulong address)
639 TranslationBlock *tb;
641 address &= TARGET_PAGE_MASK;
642 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
643 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
644 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
645 address >= tb->pc + tb->size)) {
646 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
647 address, (long)tb->pc, tb->size);
653 /* verify that all the pages have correct rights for code */
654 static void tb_page_check(void)
656 TranslationBlock *tb;
657 int i, flags1, flags2;
659 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
660 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
661 flags1 = page_get_flags(tb->pc);
662 flags2 = page_get_flags(tb->pc + tb->size - 1);
663 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
664 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
665 (long)tb->pc, tb->size, flags1, flags2);
671 static void tb_jmp_check(TranslationBlock *tb)
673 TranslationBlock *tb1;
676 /* suppress any remaining jumps to this TB */
680 tb1 = (TranslationBlock *)((long)tb1 & ~3);
683 tb1 = tb1->jmp_next[n1];
685 /* check end of list */
687 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
693 /* invalidate one TB */
694 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
697 TranslationBlock *tb1;
701 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
704 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
708 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
710 TranslationBlock *tb1;
716 tb1 = (TranslationBlock *)((long)tb1 & ~3);
718 *ptb = tb1->page_next[n1];
721 ptb = &tb1->page_next[n1];
725 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
727 TranslationBlock *tb1, **ptb;
730 ptb = &tb->jmp_next[n];
733 /* find tb(n) in circular list */
737 tb1 = (TranslationBlock *)((long)tb1 & ~3);
738 if (n1 == n && tb1 == tb)
741 ptb = &tb1->jmp_first;
743 ptb = &tb1->jmp_next[n1];
746 /* now we can suppress tb(n) from the list */
747 *ptb = tb->jmp_next[n];
749 tb->jmp_next[n] = NULL;
753 /* reset the jump entry 'n' of a TB so that it is not chained to
755 static inline void tb_reset_jump(TranslationBlock *tb, int n)
757 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
760 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
765 target_phys_addr_t phys_pc;
766 TranslationBlock *tb1, *tb2;
768 /* remove the TB from the hash list */
769 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
770 h = tb_phys_hash_func(phys_pc);
771 tb_remove(&tb_phys_hash[h], tb,
772 offsetof(TranslationBlock, phys_hash_next));
774 /* remove the TB from the page list */
775 if (tb->page_addr[0] != page_addr) {
776 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
777 tb_page_remove(&p->first_tb, tb);
778 invalidate_page_bitmap(p);
780 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
781 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
782 tb_page_remove(&p->first_tb, tb);
783 invalidate_page_bitmap(p);
786 tb_invalidated_flag = 1;
788 /* remove the TB from the hash list */
789 h = tb_jmp_cache_hash_func(tb->pc);
790 for(env = first_cpu; env != NULL; env = env->next_cpu) {
791 if (env->tb_jmp_cache[h] == tb)
792 env->tb_jmp_cache[h] = NULL;
795 /* suppress this TB from the two jump lists */
796 tb_jmp_remove(tb, 0);
797 tb_jmp_remove(tb, 1);
799 /* suppress any remaining jumps to this TB */
805 tb1 = (TranslationBlock *)((long)tb1 & ~3);
806 tb2 = tb1->jmp_next[n1];
807 tb_reset_jump(tb1, n1);
808 tb1->jmp_next[n1] = NULL;
811 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
813 tb_phys_invalidate_count++;
816 static inline void set_bits(uint8_t *tab, int start, int len)
822 mask = 0xff << (start & 7);
823 if ((start & ~7) == (end & ~7)) {
825 mask &= ~(0xff << (end & 7));
830 start = (start + 8) & ~7;
832 while (start < end1) {
837 mask = ~(0xff << (end & 7));
843 static void build_page_bitmap(PageDesc *p)
845 int n, tb_start, tb_end;
846 TranslationBlock *tb;
848 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
853 tb = (TranslationBlock *)((long)tb & ~3);
854 /* NOTE: this is subtle as a TB may span two physical pages */
856 /* NOTE: tb_end may be after the end of the page, but
857 it is not a problem */
858 tb_start = tb->pc & ~TARGET_PAGE_MASK;
859 tb_end = tb_start + tb->size;
860 if (tb_end > TARGET_PAGE_SIZE)
861 tb_end = TARGET_PAGE_SIZE;
864 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
866 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
867 tb = tb->page_next[n];
871 TranslationBlock *tb_gen_code(CPUState *env,
872 target_ulong pc, target_ulong cs_base,
873 int flags, int cflags)
875 TranslationBlock *tb;
877 target_ulong phys_pc, phys_page2, virt_page2;
880 phys_pc = get_phys_addr_code(env, pc);
883 /* flush must be done */
885 /* cannot fail at this point */
887 /* Don't forget to invalidate previous TB info. */
888 tb_invalidated_flag = 1;
890 tc_ptr = code_gen_ptr;
892 tb->cs_base = cs_base;
895 cpu_gen_code(env, tb, &code_gen_size);
896 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
898 /* check next page if needed */
899 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
901 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
902 phys_page2 = get_phys_addr_code(env, virt_page2);
904 tb_link_phys(tb, phys_pc, phys_page2);
908 /* invalidate all TBs which intersect with the target physical page
909 starting in range [start;end[. NOTE: start and end must refer to
910 the same physical page. 'is_cpu_write_access' should be true if called
911 from a real cpu write access: the virtual CPU will exit the current
912 TB if code is modified inside this TB. */
913 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
914 int is_cpu_write_access)
916 TranslationBlock *tb, *tb_next, *saved_tb;
917 CPUState *env = cpu_single_env;
918 target_ulong tb_start, tb_end;
921 #ifdef TARGET_HAS_PRECISE_SMC
922 int current_tb_not_found = is_cpu_write_access;
923 TranslationBlock *current_tb = NULL;
924 int current_tb_modified = 0;
925 target_ulong current_pc = 0;
926 target_ulong current_cs_base = 0;
927 int current_flags = 0;
928 #endif /* TARGET_HAS_PRECISE_SMC */
930 p = page_find(start >> TARGET_PAGE_BITS);
933 if (!p->code_bitmap &&
934 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
935 is_cpu_write_access) {
936 /* build code bitmap */
937 build_page_bitmap(p);
940 /* we remove all the TBs in the range [start, end[ */
941 /* XXX: see if in some cases it could be faster to invalidate all the code */
945 tb = (TranslationBlock *)((long)tb & ~3);
946 tb_next = tb->page_next[n];
947 /* NOTE: this is subtle as a TB may span two physical pages */
949 /* NOTE: tb_end may be after the end of the page, but
950 it is not a problem */
951 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
952 tb_end = tb_start + tb->size;
954 tb_start = tb->page_addr[1];
955 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
957 if (!(tb_end <= start || tb_start >= end)) {
958 #ifdef TARGET_HAS_PRECISE_SMC
959 if (current_tb_not_found) {
960 current_tb_not_found = 0;
962 if (env->mem_io_pc) {
963 /* now we have a real cpu fault */
964 current_tb = tb_find_pc(env->mem_io_pc);
967 if (current_tb == tb &&
968 (current_tb->cflags & CF_COUNT_MASK) != 1) {
969 /* If we are modifying the current TB, we must stop
970 its execution. We could be more precise by checking
971 that the modification is after the current PC, but it
972 would require a specialized function to partially
973 restore the CPU state */
975 current_tb_modified = 1;
976 cpu_restore_state(current_tb, env,
977 env->mem_io_pc, NULL);
978 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base,
981 #endif /* TARGET_HAS_PRECISE_SMC */
982 /* we need to do that to handle the case where a signal
983 occurs while doing tb_phys_invalidate() */
986 saved_tb = env->current_tb;
987 env->current_tb = NULL;
989 tb_phys_invalidate(tb, -1);
991 env->current_tb = saved_tb;
992 if (env->interrupt_request && env->current_tb)
993 cpu_interrupt(env, env->interrupt_request);
998 #if !defined(CONFIG_USER_ONLY)
999 /* if no code remaining, no need to continue to use slow writes */
1001 invalidate_page_bitmap(p);
1002 if (is_cpu_write_access) {
1003 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1007 #ifdef TARGET_HAS_PRECISE_SMC
1008 if (current_tb_modified) {
1009 /* we generate a block containing just the instruction
1010 modifying the memory. It will ensure that it cannot modify
1012 env->current_tb = NULL;
1013 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1014 cpu_resume_from_signal(env, NULL);
1019 /* len must be <= 8 and start must be a multiple of len */
1020 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1026 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1027 cpu_single_env->mem_io_vaddr, len,
1028 cpu_single_env->eip,
1029 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1032 p = page_find(start >> TARGET_PAGE_BITS);
1035 if (p->code_bitmap) {
1036 offset = start & ~TARGET_PAGE_MASK;
1037 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1038 if (b & ((1 << len) - 1))
1042 tb_invalidate_phys_page_range(start, start + len, 1);
1046 #if !defined(CONFIG_SOFTMMU)
1047 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1048 unsigned long pc, void *puc)
1050 TranslationBlock *tb;
1053 #ifdef TARGET_HAS_PRECISE_SMC
1054 TranslationBlock *current_tb = NULL;
1055 CPUState *env = cpu_single_env;
1056 int current_tb_modified = 0;
1057 target_ulong current_pc = 0;
1058 target_ulong current_cs_base = 0;
1059 int current_flags = 0;
1062 addr &= TARGET_PAGE_MASK;
1063 p = page_find(addr >> TARGET_PAGE_BITS);
1067 #ifdef TARGET_HAS_PRECISE_SMC
1068 if (tb && pc != 0) {
1069 current_tb = tb_find_pc(pc);
1072 while (tb != NULL) {
1074 tb = (TranslationBlock *)((long)tb & ~3);
1075 #ifdef TARGET_HAS_PRECISE_SMC
1076 if (current_tb == tb &&
1077 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1078 /* If we are modifying the current TB, we must stop
1079 its execution. We could be more precise by checking
1080 that the modification is after the current PC, but it
1081 would require a specialized function to partially
1082 restore the CPU state */
1084 current_tb_modified = 1;
1085 cpu_restore_state(current_tb, env, pc, puc);
1086 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base,
1089 #endif /* TARGET_HAS_PRECISE_SMC */
1090 tb_phys_invalidate(tb, addr);
1091 tb = tb->page_next[n];
1094 #ifdef TARGET_HAS_PRECISE_SMC
1095 if (current_tb_modified) {
1096 /* we generate a block containing just the instruction
1097 modifying the memory. It will ensure that it cannot modify
1099 env->current_tb = NULL;
1100 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1101 cpu_resume_from_signal(env, puc);
1107 /* add the tb in the target page and protect it if necessary */
1108 static inline void tb_alloc_page(TranslationBlock *tb,
1109 unsigned int n, target_ulong page_addr)
1112 TranslationBlock *last_first_tb;
1114 tb->page_addr[n] = page_addr;
1115 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1116 tb->page_next[n] = p->first_tb;
1117 last_first_tb = p->first_tb;
1118 p->first_tb = (TranslationBlock *)((long)tb | n);
1119 invalidate_page_bitmap(p);
1121 #if defined(TARGET_HAS_SMC) || 1
1123 #if defined(CONFIG_USER_ONLY)
1124 if (p->flags & PAGE_WRITE) {
1129 /* force the host page as non writable (writes will have a
1130 page fault + mprotect overhead) */
1131 page_addr &= qemu_host_page_mask;
1133 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1134 addr += TARGET_PAGE_SIZE) {
1136 p2 = page_find (addr >> TARGET_PAGE_BITS);
1140 p2->flags &= ~PAGE_WRITE;
1141 page_get_flags(addr);
1143 mprotect(g2h(page_addr), qemu_host_page_size,
1144 (prot & PAGE_BITS) & ~PAGE_WRITE);
1145 #ifdef DEBUG_TB_INVALIDATE
1146 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1151 /* if some code is already present, then the pages are already
1152 protected. So we handle the case where only the first TB is
1153 allocated in a physical page */
1154 if (!last_first_tb) {
1155 tlb_protect_code(page_addr);
1159 #endif /* TARGET_HAS_SMC */
1162 /* Allocate a new translation block. Flush the translation buffer if
1163 too many translation blocks or too much generated code. */
1164 TranslationBlock *tb_alloc(target_ulong pc)
1166 TranslationBlock *tb;
1168 if (nb_tbs >= code_gen_max_blocks ||
1169 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1171 tb = &tbs[nb_tbs++];
1177 void tb_free(TranslationBlock *tb)
1179 /* In practice this is mostly used for single use temporary TB
1180 Ignore the hard cases and just back up if this TB happens to
1181 be the last one generated. */
1182 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1183 code_gen_ptr = tb->tc_ptr;
1188 /* add a new TB and link it to the physical page tables. phys_page2 is
1189 (-1) to indicate that only one page contains the TB. */
1190 void tb_link_phys(TranslationBlock *tb,
1191 target_ulong phys_pc, target_ulong phys_page2)
1194 TranslationBlock **ptb;
1196 /* Grab the mmap lock to stop another thread invalidating this TB
1197 before we are done. */
1199 /* add in the physical hash table */
1200 h = tb_phys_hash_func(phys_pc);
1201 ptb = &tb_phys_hash[h];
1202 tb->phys_hash_next = *ptb;
1205 /* add in the page list */
1206 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1207 if (phys_page2 != -1)
1208 tb_alloc_page(tb, 1, phys_page2);
1210 tb->page_addr[1] = -1;
1212 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1213 tb->jmp_next[0] = NULL;
1214 tb->jmp_next[1] = NULL;
1216 /* init original jump addresses */
1217 if (tb->tb_next_offset[0] != 0xffff)
1218 tb_reset_jump(tb, 0);
1219 if (tb->tb_next_offset[1] != 0xffff)
1220 tb_reset_jump(tb, 1);
1222 #ifdef DEBUG_TB_CHECK
1228 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1229 tb[1].tc_ptr. Return NULL if not found */
1230 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1232 int m_min, m_max, m;
1234 TranslationBlock *tb;
1238 if (tc_ptr < (unsigned long)code_gen_buffer ||
1239 tc_ptr >= (unsigned long)code_gen_ptr)
1241 /* binary search (cf Knuth) */
1244 while (m_min <= m_max) {
1245 m = (m_min + m_max) >> 1;
1247 v = (unsigned long)tb->tc_ptr;
1250 else if (tc_ptr < v) {
1259 static void tb_reset_jump_recursive(TranslationBlock *tb);
1261 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1263 TranslationBlock *tb1, *tb_next, **ptb;
1266 tb1 = tb->jmp_next[n];
1268 /* find head of list */
1271 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1274 tb1 = tb1->jmp_next[n1];
1276 /* we are now sure now that tb jumps to tb1 */
1279 /* remove tb from the jmp_first list */
1280 ptb = &tb_next->jmp_first;
1284 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1285 if (n1 == n && tb1 == tb)
1287 ptb = &tb1->jmp_next[n1];
1289 *ptb = tb->jmp_next[n];
1290 tb->jmp_next[n] = NULL;
1292 /* suppress the jump to next tb in generated code */
1293 tb_reset_jump(tb, n);
1295 /* suppress jumps in the tb on which we could have jumped */
1296 tb_reset_jump_recursive(tb_next);
1300 static void tb_reset_jump_recursive(TranslationBlock *tb)
1302 tb_reset_jump_recursive2(tb, 0);
1303 tb_reset_jump_recursive2(tb, 1);
1306 #if defined(TARGET_HAS_ICE)
1307 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1309 target_phys_addr_t addr;
1311 ram_addr_t ram_addr;
1314 addr = cpu_get_phys_page_debug(env, pc);
1315 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1317 pd = IO_MEM_UNASSIGNED;
1319 pd = p->phys_offset;
1321 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1322 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1326 /* Add a watchpoint. */
1327 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1328 int flags, CPUWatchpoint **watchpoint)
1330 target_ulong len_mask = ~(len - 1);
1333 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1334 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1335 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1336 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1339 wp = qemu_malloc(sizeof(*wp));
1342 wp->len_mask = len_mask;
1345 /* keep all GDB-injected watchpoints in front */
1347 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1349 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1351 tlb_flush_page(env, addr);
1358 /* Remove a specific watchpoint. */
1359 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1362 target_ulong len_mask = ~(len - 1);
1365 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1366 if (addr == wp->vaddr && len_mask == wp->len_mask
1367 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1368 cpu_watchpoint_remove_by_ref(env, wp);
1375 /* Remove a specific watchpoint by reference. */
1376 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1378 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1380 tlb_flush_page(env, watchpoint->vaddr);
1382 qemu_free(watchpoint);
1385 /* Remove all matching watchpoints. */
1386 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1388 CPUWatchpoint *wp, *next;
1390 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1391 if (wp->flags & mask)
1392 cpu_watchpoint_remove_by_ref(env, wp);
1396 /* Add a breakpoint. */
1397 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1398 CPUBreakpoint **breakpoint)
1400 #if defined(TARGET_HAS_ICE)
1403 bp = qemu_malloc(sizeof(*bp));
1408 /* keep all GDB-injected breakpoints in front */
1410 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1412 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1414 breakpoint_invalidate(env, pc);
1424 /* Remove a specific breakpoint. */
1425 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1427 #if defined(TARGET_HAS_ICE)
1430 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1431 if (bp->pc == pc && bp->flags == flags) {
1432 cpu_breakpoint_remove_by_ref(env, bp);
1442 /* Remove a specific breakpoint by reference. */
1443 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1445 #if defined(TARGET_HAS_ICE)
1446 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1448 breakpoint_invalidate(env, breakpoint->pc);
1450 qemu_free(breakpoint);
1454 /* Remove all matching breakpoints. */
1455 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1457 #if defined(TARGET_HAS_ICE)
1458 CPUBreakpoint *bp, *next;
1460 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1461 if (bp->flags & mask)
1462 cpu_breakpoint_remove_by_ref(env, bp);
1467 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1468 CPU loop after each instruction */
1469 void cpu_single_step(CPUState *env, int enabled)
1471 #if defined(TARGET_HAS_ICE)
1472 if (env->singlestep_enabled != enabled) {
1473 env->singlestep_enabled = enabled;
1475 kvm_update_guest_debug(env, 0);
1477 /* must flush all the translated code to avoid inconsistencies */
1478 /* XXX: only flush what is necessary */
1485 /* enable or disable low levels log */
1486 void cpu_set_log(int log_flags)
1488 loglevel = log_flags;
1489 if (loglevel && !logfile) {
1490 logfile = fopen(logfilename, log_append ? "a" : "w");
1492 perror(logfilename);
1495 #if !defined(CONFIG_SOFTMMU)
1496 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1498 static char logfile_buf[4096];
1499 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1502 setvbuf(logfile, NULL, _IOLBF, 0);
1506 if (!loglevel && logfile) {
1512 void cpu_set_log_filename(const char *filename)
1514 logfilename = strdup(filename);
1519 cpu_set_log(loglevel);
1522 static void cpu_unlink_tb(CPUState *env)
1524 #if defined(USE_NPTL)
1525 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1526 problem and hope the cpu will stop of its own accord. For userspace
1527 emulation this often isn't actually as bad as it sounds. Often
1528 signals are used primarily to interrupt blocking syscalls. */
1530 TranslationBlock *tb;
1531 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1533 tb = env->current_tb;
1534 /* if the cpu is currently executing code, we must unlink it and
1535 all the potentially executing TB */
1536 if (tb && !testandset(&interrupt_lock)) {
1537 env->current_tb = NULL;
1538 tb_reset_jump_recursive(tb);
1539 resetlock(&interrupt_lock);
1544 /* mask must never be zero, except for A20 change call */
1545 void cpu_interrupt(CPUState *env, int mask)
1549 old_mask = env->interrupt_request;
1550 env->interrupt_request |= mask;
1552 #ifndef CONFIG_USER_ONLY
1554 * If called from iothread context, wake the target cpu in
1557 if (!qemu_cpu_self(env)) {
1564 env->icount_decr.u16.high = 0xffff;
1565 #ifndef CONFIG_USER_ONLY
1567 && (mask & ~old_mask) != 0) {
1568 cpu_abort(env, "Raised interrupt while not in I/O function");
1576 void cpu_reset_interrupt(CPUState *env, int mask)
1578 env->interrupt_request &= ~mask;
1581 void cpu_exit(CPUState *env)
1583 env->exit_request = 1;
1587 const CPULogItem cpu_log_items[] = {
1588 { CPU_LOG_TB_OUT_ASM, "out_asm",
1589 "show generated host assembly code for each compiled TB" },
1590 { CPU_LOG_TB_IN_ASM, "in_asm",
1591 "show target assembly code for each compiled TB" },
1592 { CPU_LOG_TB_OP, "op",
1593 "show micro ops for each compiled TB" },
1594 { CPU_LOG_TB_OP_OPT, "op_opt",
1597 "before eflags optimization and "
1599 "after liveness analysis" },
1600 { CPU_LOG_INT, "int",
1601 "show interrupts/exceptions in short format" },
1602 { CPU_LOG_EXEC, "exec",
1603 "show trace before each executed TB (lots of logs)" },
1604 { CPU_LOG_TB_CPU, "cpu",
1605 "show CPU state before block translation" },
1607 { CPU_LOG_PCALL, "pcall",
1608 "show protected mode far calls/returns/exceptions" },
1609 { CPU_LOG_RESET, "cpu_reset",
1610 "show CPU state before CPU resets" },
1613 { CPU_LOG_IOPORT, "ioport",
1614 "show all i/o ports accesses" },
1619 static int cmp1(const char *s1, int n, const char *s2)
1621 if (strlen(s2) != n)
1623 return memcmp(s1, s2, n) == 0;
1626 /* takes a comma separated list of log masks. Return 0 if error. */
1627 int cpu_str_to_log_mask(const char *str)
1629 const CPULogItem *item;
1636 p1 = strchr(p, ',');
1639 if(cmp1(p,p1-p,"all")) {
1640 for(item = cpu_log_items; item->mask != 0; item++) {
1644 for(item = cpu_log_items; item->mask != 0; item++) {
1645 if (cmp1(p, p1 - p, item->name))
1659 void cpu_abort(CPUState *env, const char *fmt, ...)
1666 fprintf(stderr, "qemu: fatal: ");
1667 vfprintf(stderr, fmt, ap);
1668 fprintf(stderr, "\n");
1670 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1672 cpu_dump_state(env, stderr, fprintf, 0);
1674 if (qemu_log_enabled()) {
1675 qemu_log("qemu: fatal: ");
1676 qemu_log_vprintf(fmt, ap2);
1679 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1681 log_cpu_state(env, 0);
1691 CPUState *cpu_copy(CPUState *env)
1693 CPUState *new_env = cpu_init(env->cpu_model_str);
1694 CPUState *next_cpu = new_env->next_cpu;
1695 int cpu_index = new_env->cpu_index;
1696 #if defined(TARGET_HAS_ICE)
1701 memcpy(new_env, env, sizeof(CPUState));
1703 /* Preserve chaining and index. */
1704 new_env->next_cpu = next_cpu;
1705 new_env->cpu_index = cpu_index;
1707 /* Clone all break/watchpoints.
1708 Note: Once we support ptrace with hw-debug register access, make sure
1709 BP_CPU break/watchpoints are handled correctly on clone. */
1710 TAILQ_INIT(&env->breakpoints);
1711 TAILQ_INIT(&env->watchpoints);
1712 #if defined(TARGET_HAS_ICE)
1713 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1714 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1716 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1717 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1725 #if !defined(CONFIG_USER_ONLY)
1727 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1731 /* Discard jump cache entries for any tb which might potentially
1732 overlap the flushed page. */
1733 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1734 memset (&env->tb_jmp_cache[i], 0,
1735 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1737 i = tb_jmp_cache_hash_page(addr);
1738 memset (&env->tb_jmp_cache[i], 0,
1739 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1742 /* NOTE: if flush_global is true, also flush global entries (not
1744 void tlb_flush(CPUState *env, int flush_global)
1748 #if defined(DEBUG_TLB)
1749 printf("tlb_flush:\n");
1751 /* must reset current TB so that interrupts cannot modify the
1752 links while we are modifying them */
1753 env->current_tb = NULL;
1755 for(i = 0; i < CPU_TLB_SIZE; i++) {
1756 env->tlb_table[0][i].addr_read = -1;
1757 env->tlb_table[0][i].addr_write = -1;
1758 env->tlb_table[0][i].addr_code = -1;
1759 env->tlb_table[1][i].addr_read = -1;
1760 env->tlb_table[1][i].addr_write = -1;
1761 env->tlb_table[1][i].addr_code = -1;
1762 #if (NB_MMU_MODES >= 3)
1763 env->tlb_table[2][i].addr_read = -1;
1764 env->tlb_table[2][i].addr_write = -1;
1765 env->tlb_table[2][i].addr_code = -1;
1767 #if (NB_MMU_MODES >= 4)
1768 env->tlb_table[3][i].addr_read = -1;
1769 env->tlb_table[3][i].addr_write = -1;
1770 env->tlb_table[3][i].addr_code = -1;
1772 #if (NB_MMU_MODES >= 5)
1773 env->tlb_table[4][i].addr_read = -1;
1774 env->tlb_table[4][i].addr_write = -1;
1775 env->tlb_table[4][i].addr_code = -1;
1780 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1783 if (env->kqemu_enabled) {
1784 kqemu_flush(env, flush_global);
1790 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1792 if (addr == (tlb_entry->addr_read &
1793 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1794 addr == (tlb_entry->addr_write &
1795 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1796 addr == (tlb_entry->addr_code &
1797 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1798 tlb_entry->addr_read = -1;
1799 tlb_entry->addr_write = -1;
1800 tlb_entry->addr_code = -1;
1804 void tlb_flush_page(CPUState *env, target_ulong addr)
1808 #if defined(DEBUG_TLB)
1809 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1811 /* must reset current TB so that interrupts cannot modify the
1812 links while we are modifying them */
1813 env->current_tb = NULL;
1815 addr &= TARGET_PAGE_MASK;
1816 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1817 tlb_flush_entry(&env->tlb_table[0][i], addr);
1818 tlb_flush_entry(&env->tlb_table[1][i], addr);
1819 #if (NB_MMU_MODES >= 3)
1820 tlb_flush_entry(&env->tlb_table[2][i], addr);
1822 #if (NB_MMU_MODES >= 4)
1823 tlb_flush_entry(&env->tlb_table[3][i], addr);
1825 #if (NB_MMU_MODES >= 5)
1826 tlb_flush_entry(&env->tlb_table[4][i], addr);
1829 tlb_flush_jmp_cache(env, addr);
1832 if (env->kqemu_enabled) {
1833 kqemu_flush_page(env, addr);
1838 /* update the TLBs so that writes to code in the virtual page 'addr'
1840 static void tlb_protect_code(ram_addr_t ram_addr)
1842 cpu_physical_memory_reset_dirty(ram_addr,
1843 ram_addr + TARGET_PAGE_SIZE,
1847 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1848 tested for self modifying code */
1849 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1852 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1855 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1856 unsigned long start, unsigned long length)
1859 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1860 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1861 if ((addr - start) < length) {
1862 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1867 /* Note: start and end must be within the same ram block. */
1868 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1872 unsigned long length, start1;
1876 start &= TARGET_PAGE_MASK;
1877 end = TARGET_PAGE_ALIGN(end);
1879 length = end - start;
1882 len = length >> TARGET_PAGE_BITS;
1884 /* XXX: should not depend on cpu context */
1886 if (env->kqemu_enabled) {
1889 for(i = 0; i < len; i++) {
1890 kqemu_set_notdirty(env, addr);
1891 addr += TARGET_PAGE_SIZE;
1895 mask = ~dirty_flags;
1896 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1897 for(i = 0; i < len; i++)
1900 /* we modify the TLB cache so that the dirty bit will be set again
1901 when accessing the range */
1902 start1 = (unsigned long)qemu_get_ram_ptr(start);
1903 /* Chek that we don't span multiple blocks - this breaks the
1904 address comparisons below. */
1905 if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
1906 != (end - 1) - start) {
1910 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1911 for(i = 0; i < CPU_TLB_SIZE; i++)
1912 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1913 for(i = 0; i < CPU_TLB_SIZE; i++)
1914 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1915 #if (NB_MMU_MODES >= 3)
1916 for(i = 0; i < CPU_TLB_SIZE; i++)
1917 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1919 #if (NB_MMU_MODES >= 4)
1920 for(i = 0; i < CPU_TLB_SIZE; i++)
1921 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1923 #if (NB_MMU_MODES >= 5)
1924 for(i = 0; i < CPU_TLB_SIZE; i++)
1925 tlb_reset_dirty_range(&env->tlb_table[4][i], start1, length);
1930 int cpu_physical_memory_set_dirty_tracking(int enable)
1932 in_migration = enable;
1933 if (kvm_enabled()) {
1934 return kvm_set_migration_log(enable);
1939 int cpu_physical_memory_get_dirty_tracking(void)
1941 return in_migration;
1944 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
1945 target_phys_addr_t end_addr)
1950 ret = kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1954 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1956 ram_addr_t ram_addr;
1959 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1960 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
1961 + tlb_entry->addend);
1962 ram_addr = qemu_ram_addr_from_host(p);
1963 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1964 tlb_entry->addr_write |= TLB_NOTDIRTY;
1969 /* update the TLB according to the current state of the dirty bits */
1970 void cpu_tlb_update_dirty(CPUState *env)
1973 for(i = 0; i < CPU_TLB_SIZE; i++)
1974 tlb_update_dirty(&env->tlb_table[0][i]);
1975 for(i = 0; i < CPU_TLB_SIZE; i++)
1976 tlb_update_dirty(&env->tlb_table[1][i]);
1977 #if (NB_MMU_MODES >= 3)
1978 for(i = 0; i < CPU_TLB_SIZE; i++)
1979 tlb_update_dirty(&env->tlb_table[2][i]);
1981 #if (NB_MMU_MODES >= 4)
1982 for(i = 0; i < CPU_TLB_SIZE; i++)
1983 tlb_update_dirty(&env->tlb_table[3][i]);
1985 #if (NB_MMU_MODES >= 5)
1986 for(i = 0; i < CPU_TLB_SIZE; i++)
1987 tlb_update_dirty(&env->tlb_table[4][i]);
1991 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1993 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1994 tlb_entry->addr_write = vaddr;
1997 /* update the TLB corresponding to virtual page vaddr
1998 so that it is no longer dirty */
1999 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2003 vaddr &= TARGET_PAGE_MASK;
2004 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2005 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
2006 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
2007 #if (NB_MMU_MODES >= 3)
2008 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
2010 #if (NB_MMU_MODES >= 4)
2011 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
2013 #if (NB_MMU_MODES >= 5)
2014 tlb_set_dirty1(&env->tlb_table[4][i], vaddr);
2018 /* add a new TLB entry. At most one entry for a given virtual address
2019 is permitted. Return 0 if OK or 2 if the page could not be mapped
2020 (can only happen in non SOFTMMU mode for I/O pages or pages
2021 conflicting with the host address space). */
2022 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2023 target_phys_addr_t paddr, int prot,
2024 int mmu_idx, int is_softmmu)
2029 target_ulong address;
2030 target_ulong code_address;
2031 target_phys_addr_t addend;
2035 target_phys_addr_t iotlb;
2037 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2039 pd = IO_MEM_UNASSIGNED;
2041 pd = p->phys_offset;
2043 #if defined(DEBUG_TLB)
2044 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2045 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2050 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2051 /* IO memory case (romd handled later) */
2052 address |= TLB_MMIO;
2054 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2055 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2057 iotlb = pd & TARGET_PAGE_MASK;
2058 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2059 iotlb |= IO_MEM_NOTDIRTY;
2061 iotlb |= IO_MEM_ROM;
2063 /* IO handlers are currently passed a physical address.
2064 It would be nice to pass an offset from the base address
2065 of that region. This would avoid having to special case RAM,
2066 and avoid full address decoding in every device.
2067 We can't use the high bits of pd for this because
2068 IO_MEM_ROMD uses these as a ram address. */
2069 iotlb = (pd & ~TARGET_PAGE_MASK);
2071 iotlb += p->region_offset;
2077 code_address = address;
2078 /* Make accesses to pages with watchpoints go via the
2079 watchpoint trap routines. */
2080 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2081 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2082 iotlb = io_mem_watch + paddr;
2083 /* TODO: The memory case can be optimized by not trapping
2084 reads of pages with a write breakpoint. */
2085 address |= TLB_MMIO;
2089 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2090 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2091 te = &env->tlb_table[mmu_idx][index];
2092 te->addend = addend - vaddr;
2093 if (prot & PAGE_READ) {
2094 te->addr_read = address;
2099 if (prot & PAGE_EXEC) {
2100 te->addr_code = code_address;
2104 if (prot & PAGE_WRITE) {
2105 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2106 (pd & IO_MEM_ROMD)) {
2107 /* Write access calls the I/O callback. */
2108 te->addr_write = address | TLB_MMIO;
2109 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2110 !cpu_physical_memory_is_dirty(pd)) {
2111 te->addr_write = address | TLB_NOTDIRTY;
2113 te->addr_write = address;
2116 te->addr_write = -1;
2123 void tlb_flush(CPUState *env, int flush_global)
2127 void tlb_flush_page(CPUState *env, target_ulong addr)
2131 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2132 target_phys_addr_t paddr, int prot,
2133 int mmu_idx, int is_softmmu)
2139 * Walks guest process memory "regions" one by one
2140 * and calls callback function 'fn' for each region.
2142 int walk_memory_regions(void *priv,
2143 int (*fn)(void *, unsigned long, unsigned long, unsigned long))
2145 unsigned long start, end;
2147 int i, j, prot, prot1;
2153 for (i = 0; i <= L1_SIZE; i++) {
2154 p = (i < L1_SIZE) ? l1_map[i] : NULL;
2155 for (j = 0; j < L2_SIZE; j++) {
2156 prot1 = (p == NULL) ? 0 : p[j].flags;
2158 * "region" is one continuous chunk of memory
2159 * that has same protection flags set.
2161 if (prot1 != prot) {
2162 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2164 rc = (*fn)(priv, start, end, prot);
2165 /* callback can stop iteration by returning != 0 */
2182 static int dump_region(void *priv, unsigned long start,
2183 unsigned long end, unsigned long prot)
2185 FILE *f = (FILE *)priv;
2187 (void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2188 start, end, end - start,
2189 ((prot & PAGE_READ) ? 'r' : '-'),
2190 ((prot & PAGE_WRITE) ? 'w' : '-'),
2191 ((prot & PAGE_EXEC) ? 'x' : '-'));
2196 /* dump memory mappings */
2197 void page_dump(FILE *f)
2199 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2200 "start", "end", "size", "prot");
2201 walk_memory_regions(f, dump_region);
2204 int page_get_flags(target_ulong address)
2208 p = page_find(address >> TARGET_PAGE_BITS);
2214 /* modify the flags of a page and invalidate the code if
2215 necessary. The flag PAGE_WRITE_ORG is positioned automatically
2216 depending on PAGE_WRITE */
2217 void page_set_flags(target_ulong start, target_ulong end, int flags)
2222 /* mmap_lock should already be held. */
2223 start = start & TARGET_PAGE_MASK;
2224 end = TARGET_PAGE_ALIGN(end);
2225 if (flags & PAGE_WRITE)
2226 flags |= PAGE_WRITE_ORG;
2227 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2228 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2229 /* We may be called for host regions that are outside guest
2233 /* if the write protection is set, then we invalidate the code
2235 if (!(p->flags & PAGE_WRITE) &&
2236 (flags & PAGE_WRITE) &&
2238 tb_invalidate_phys_page(addr, 0, NULL);
2244 int page_check_range(target_ulong start, target_ulong len, int flags)
2250 if (start + len < start)
2251 /* we've wrapped around */
2254 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2255 start = start & TARGET_PAGE_MASK;
2257 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2258 p = page_find(addr >> TARGET_PAGE_BITS);
2261 if( !(p->flags & PAGE_VALID) )
2264 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2266 if (flags & PAGE_WRITE) {
2267 if (!(p->flags & PAGE_WRITE_ORG))
2269 /* unprotect the page if it was put read-only because it
2270 contains translated code */
2271 if (!(p->flags & PAGE_WRITE)) {
2272 if (!page_unprotect(addr, 0, NULL))
2281 /* called from signal handler: invalidate the code and unprotect the
2282 page. Return TRUE if the fault was successfully handled. */
2283 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2285 unsigned int page_index, prot, pindex;
2287 target_ulong host_start, host_end, addr;
2289 /* Technically this isn't safe inside a signal handler. However we
2290 know this only ever happens in a synchronous SEGV handler, so in
2291 practice it seems to be ok. */
2294 host_start = address & qemu_host_page_mask;
2295 page_index = host_start >> TARGET_PAGE_BITS;
2296 p1 = page_find(page_index);
2301 host_end = host_start + qemu_host_page_size;
2304 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2308 /* if the page was really writable, then we change its
2309 protection back to writable */
2310 if (prot & PAGE_WRITE_ORG) {
2311 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2312 if (!(p1[pindex].flags & PAGE_WRITE)) {
2313 mprotect((void *)g2h(host_start), qemu_host_page_size,
2314 (prot & PAGE_BITS) | PAGE_WRITE);
2315 p1[pindex].flags |= PAGE_WRITE;
2316 /* and since the content will be modified, we must invalidate
2317 the corresponding translated code. */
2318 tb_invalidate_phys_page(address, pc, puc);
2319 #ifdef DEBUG_TB_CHECK
2320 tb_invalidate_check(address);
2330 static inline void tlb_set_dirty(CPUState *env,
2331 unsigned long addr, target_ulong vaddr)
2334 #endif /* defined(CONFIG_USER_ONLY) */
2336 #if !defined(CONFIG_USER_ONLY)
2338 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2339 ram_addr_t memory, ram_addr_t region_offset);
2340 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2341 ram_addr_t orig_memory, ram_addr_t region_offset);
2342 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2345 if (addr > start_addr) \
2348 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2349 if (start_addr2 > 0) \
2353 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2354 end_addr2 = TARGET_PAGE_SIZE - 1; \
2356 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2357 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2362 /* register physical memory. 'size' must be a multiple of the target
2363 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2364 io memory page. The address used when calling the IO function is
2365 the offset from the start of the region, plus region_offset. Both
2366 start_addr and region_offset are rounded down to a page boundary
2367 before calculating this offset. This should not be a problem unless
2368 the low bits of start_addr and region_offset differ. */
2369 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2371 ram_addr_t phys_offset,
2372 ram_addr_t region_offset)
2374 target_phys_addr_t addr, end_addr;
2377 ram_addr_t orig_size = size;
2381 /* XXX: should not depend on cpu context */
2383 if (env->kqemu_enabled) {
2384 kqemu_set_phys_mem(start_addr, size, phys_offset);
2388 kvm_set_phys_mem(start_addr, size, phys_offset);
2390 if (phys_offset == IO_MEM_UNASSIGNED) {
2391 region_offset = start_addr;
2393 region_offset &= TARGET_PAGE_MASK;
2394 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2395 end_addr = start_addr + (target_phys_addr_t)size;
2396 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2397 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2398 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2399 ram_addr_t orig_memory = p->phys_offset;
2400 target_phys_addr_t start_addr2, end_addr2;
2401 int need_subpage = 0;
2403 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2405 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2406 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2407 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2408 &p->phys_offset, orig_memory,
2411 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2414 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2416 p->region_offset = 0;
2418 p->phys_offset = phys_offset;
2419 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2420 (phys_offset & IO_MEM_ROMD))
2421 phys_offset += TARGET_PAGE_SIZE;
2424 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2425 p->phys_offset = phys_offset;
2426 p->region_offset = region_offset;
2427 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2428 (phys_offset & IO_MEM_ROMD)) {
2429 phys_offset += TARGET_PAGE_SIZE;
2431 target_phys_addr_t start_addr2, end_addr2;
2432 int need_subpage = 0;
2434 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2435 end_addr2, need_subpage);
2437 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2438 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2439 &p->phys_offset, IO_MEM_UNASSIGNED,
2440 addr & TARGET_PAGE_MASK);
2441 subpage_register(subpage, start_addr2, end_addr2,
2442 phys_offset, region_offset);
2443 p->region_offset = 0;
2447 region_offset += TARGET_PAGE_SIZE;
2450 /* since each CPU stores ram addresses in its TLB cache, we must
2451 reset the modified entries */
2453 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2458 /* XXX: temporary until new memory mapping API */
2459 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2463 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2465 return IO_MEM_UNASSIGNED;
2466 return p->phys_offset;
2469 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2472 kvm_coalesce_mmio_region(addr, size);
2475 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2478 kvm_uncoalesce_mmio_region(addr, size);
2482 /* XXX: better than nothing */
2483 static ram_addr_t kqemu_ram_alloc(ram_addr_t size)
2486 if ((last_ram_offset + size) > kqemu_phys_ram_size) {
2487 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2488 (uint64_t)size, (uint64_t)kqemu_phys_ram_size);
2491 addr = last_ram_offset;
2492 last_ram_offset = TARGET_PAGE_ALIGN(last_ram_offset + size);
2497 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2499 RAMBlock *new_block;
2502 if (kqemu_phys_ram_base) {
2503 return kqemu_ram_alloc(size);
2507 size = TARGET_PAGE_ALIGN(size);
2508 new_block = qemu_malloc(sizeof(*new_block));
2510 new_block->host = qemu_vmalloc(size);
2511 new_block->offset = last_ram_offset;
2512 new_block->length = size;
2514 new_block->next = ram_blocks;
2515 ram_blocks = new_block;
2517 phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2518 (last_ram_offset + size) >> TARGET_PAGE_BITS);
2519 memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2520 0xff, size >> TARGET_PAGE_BITS);
2522 last_ram_offset += size;
2525 kvm_setup_guest_memory(new_block->host, size);
2527 return new_block->offset;
2530 void qemu_ram_free(ram_addr_t addr)
2532 /* TODO: implement this. */
2535 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2536 With the exception of the softmmu code in this file, this should
2537 only be used for local memory (e.g. video ram) that the device owns,
2538 and knows it isn't going to access beyond the end of the block.
2540 It should not be used for general purpose DMA.
2541 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2543 void *qemu_get_ram_ptr(ram_addr_t addr)
2550 if (kqemu_phys_ram_base) {
2551 return kqemu_phys_ram_base + addr;
2556 prevp = &ram_blocks;
2558 while (block && (block->offset > addr
2559 || block->offset + block->length <= addr)) {
2561 prevp = &prev->next;
2563 block = block->next;
2566 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2569 /* Move this entry to to start of the list. */
2571 prev->next = block->next;
2572 block->next = *prevp;
2575 return block->host + (addr - block->offset);
2578 /* Some of the softmmu routines need to translate from a host pointer
2579 (typically a TLB entry) back to a ram offset. */
2580 ram_addr_t qemu_ram_addr_from_host(void *ptr)
2585 uint8_t *host = ptr;
2588 if (kqemu_phys_ram_base) {
2589 return host - kqemu_phys_ram_base;
2594 prevp = &ram_blocks;
2596 while (block && (block->host > host
2597 || block->host + block->length <= host)) {
2599 prevp = &prev->next;
2601 block = block->next;
2604 fprintf(stderr, "Bad ram pointer %p\n", ptr);
2607 return block->offset + (host - block->host);
2610 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2612 #ifdef DEBUG_UNASSIGNED
2613 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2615 #if defined(TARGET_SPARC)
2616 do_unassigned_access(addr, 0, 0, 0, 1);
2621 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2623 #ifdef DEBUG_UNASSIGNED
2624 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2626 #if defined(TARGET_SPARC)
2627 do_unassigned_access(addr, 0, 0, 0, 2);
2632 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2634 #ifdef DEBUG_UNASSIGNED
2635 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2637 #if defined(TARGET_SPARC)
2638 do_unassigned_access(addr, 0, 0, 0, 4);
2643 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2645 #ifdef DEBUG_UNASSIGNED
2646 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2648 #if defined(TARGET_SPARC)
2649 do_unassigned_access(addr, 1, 0, 0, 1);
2653 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2655 #ifdef DEBUG_UNASSIGNED
2656 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2658 #if defined(TARGET_SPARC)
2659 do_unassigned_access(addr, 1, 0, 0, 2);
2663 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2665 #ifdef DEBUG_UNASSIGNED
2666 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2668 #if defined(TARGET_SPARC)
2669 do_unassigned_access(addr, 1, 0, 0, 4);
2673 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2674 unassigned_mem_readb,
2675 unassigned_mem_readw,
2676 unassigned_mem_readl,
2679 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2680 unassigned_mem_writeb,
2681 unassigned_mem_writew,
2682 unassigned_mem_writel,
2685 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2689 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2690 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2691 #if !defined(CONFIG_USER_ONLY)
2692 tb_invalidate_phys_page_fast(ram_addr, 1);
2693 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2696 stb_p(qemu_get_ram_ptr(ram_addr), val);
2698 if (cpu_single_env->kqemu_enabled &&
2699 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2700 kqemu_modify_page(cpu_single_env, ram_addr);
2702 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2703 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2704 /* we remove the notdirty callback only if the code has been
2706 if (dirty_flags == 0xff)
2707 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2710 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2714 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2715 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2716 #if !defined(CONFIG_USER_ONLY)
2717 tb_invalidate_phys_page_fast(ram_addr, 2);
2718 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2721 stw_p(qemu_get_ram_ptr(ram_addr), val);
2723 if (cpu_single_env->kqemu_enabled &&
2724 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2725 kqemu_modify_page(cpu_single_env, ram_addr);
2727 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2728 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2729 /* we remove the notdirty callback only if the code has been
2731 if (dirty_flags == 0xff)
2732 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2735 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2739 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2740 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2741 #if !defined(CONFIG_USER_ONLY)
2742 tb_invalidate_phys_page_fast(ram_addr, 4);
2743 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2746 stl_p(qemu_get_ram_ptr(ram_addr), val);
2748 if (cpu_single_env->kqemu_enabled &&
2749 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2750 kqemu_modify_page(cpu_single_env, ram_addr);
2752 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2753 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2754 /* we remove the notdirty callback only if the code has been
2756 if (dirty_flags == 0xff)
2757 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2760 static CPUReadMemoryFunc *error_mem_read[3] = {
2761 NULL, /* never used */
2762 NULL, /* never used */
2763 NULL, /* never used */
2766 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2767 notdirty_mem_writeb,
2768 notdirty_mem_writew,
2769 notdirty_mem_writel,
2772 /* Generate a debug exception if a watchpoint has been hit. */
2773 static void check_watchpoint(int offset, int len_mask, int flags)
2775 CPUState *env = cpu_single_env;
2776 target_ulong pc, cs_base;
2777 TranslationBlock *tb;
2782 if (env->watchpoint_hit) {
2783 /* We re-entered the check after replacing the TB. Now raise
2784 * the debug interrupt so that is will trigger after the
2785 * current instruction. */
2786 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2789 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2790 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2791 if ((vaddr == (wp->vaddr & len_mask) ||
2792 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2793 wp->flags |= BP_WATCHPOINT_HIT;
2794 if (!env->watchpoint_hit) {
2795 env->watchpoint_hit = wp;
2796 tb = tb_find_pc(env->mem_io_pc);
2798 cpu_abort(env, "check_watchpoint: could not find TB for "
2799 "pc=%p", (void *)env->mem_io_pc);
2801 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2802 tb_phys_invalidate(tb, -1);
2803 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2804 env->exception_index = EXCP_DEBUG;
2806 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2807 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2809 cpu_resume_from_signal(env, NULL);
2812 wp->flags &= ~BP_WATCHPOINT_HIT;
2817 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2818 so these check for a hit then pass through to the normal out-of-line
2820 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2822 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2823 return ldub_phys(addr);
2826 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2828 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2829 return lduw_phys(addr);
2832 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2834 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2835 return ldl_phys(addr);
2838 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2841 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2842 stb_phys(addr, val);
2845 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2848 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2849 stw_phys(addr, val);
2852 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2855 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2856 stl_phys(addr, val);
2859 static CPUReadMemoryFunc *watch_mem_read[3] = {
2865 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2871 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2877 idx = SUBPAGE_IDX(addr);
2878 #if defined(DEBUG_SUBPAGE)
2879 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2880 mmio, len, addr, idx);
2882 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2883 addr + mmio->region_offset[idx][0][len]);
2888 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2889 uint32_t value, unsigned int len)
2893 idx = SUBPAGE_IDX(addr);
2894 #if defined(DEBUG_SUBPAGE)
2895 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2896 mmio, len, addr, idx, value);
2898 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2899 addr + mmio->region_offset[idx][1][len],
2903 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2905 #if defined(DEBUG_SUBPAGE)
2906 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2909 return subpage_readlen(opaque, addr, 0);
2912 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2915 #if defined(DEBUG_SUBPAGE)
2916 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2918 subpage_writelen(opaque, addr, value, 0);
2921 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2923 #if defined(DEBUG_SUBPAGE)
2924 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2927 return subpage_readlen(opaque, addr, 1);
2930 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2933 #if defined(DEBUG_SUBPAGE)
2934 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2936 subpage_writelen(opaque, addr, value, 1);
2939 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2941 #if defined(DEBUG_SUBPAGE)
2942 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2945 return subpage_readlen(opaque, addr, 2);
2948 static void subpage_writel (void *opaque,
2949 target_phys_addr_t addr, uint32_t value)
2951 #if defined(DEBUG_SUBPAGE)
2952 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2954 subpage_writelen(opaque, addr, value, 2);
2957 static CPUReadMemoryFunc *subpage_read[] = {
2963 static CPUWriteMemoryFunc *subpage_write[] = {
2969 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2970 ram_addr_t memory, ram_addr_t region_offset)
2975 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2977 idx = SUBPAGE_IDX(start);
2978 eidx = SUBPAGE_IDX(end);
2979 #if defined(DEBUG_SUBPAGE)
2980 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2981 mmio, start, end, idx, eidx, memory);
2983 memory >>= IO_MEM_SHIFT;
2984 for (; idx <= eidx; idx++) {
2985 for (i = 0; i < 4; i++) {
2986 if (io_mem_read[memory][i]) {
2987 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2988 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2989 mmio->region_offset[idx][0][i] = region_offset;
2991 if (io_mem_write[memory][i]) {
2992 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2993 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2994 mmio->region_offset[idx][1][i] = region_offset;
3002 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3003 ram_addr_t orig_memory, ram_addr_t region_offset)
3008 mmio = qemu_mallocz(sizeof(subpage_t));
3011 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
3012 #if defined(DEBUG_SUBPAGE)
3013 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3014 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3016 *phys = subpage_memory | IO_MEM_SUBPAGE;
3017 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
3023 static int get_free_io_mem_idx(void)
3027 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3028 if (!io_mem_used[i]) {
3032 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3036 static void io_mem_init(void)
3040 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
3041 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
3042 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
3046 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
3047 watch_mem_write, NULL);
3049 if (kqemu_phys_ram_base) {
3050 /* alloc dirty bits array */
3051 phys_ram_dirty = qemu_vmalloc(kqemu_phys_ram_size >> TARGET_PAGE_BITS);
3052 memset(phys_ram_dirty, 0xff, kqemu_phys_ram_size >> TARGET_PAGE_BITS);
3057 /* mem_read and mem_write are arrays of functions containing the
3058 function to access byte (index 0), word (index 1) and dword (index
3059 2). Functions can be omitted with a NULL function pointer.
3060 If io_index is non zero, the corresponding io zone is
3061 modified. If it is zero, a new io zone is allocated. The return
3062 value can be used with cpu_register_physical_memory(). (-1) is
3063 returned if error. */
3064 int cpu_register_io_memory(int io_index,
3065 CPUReadMemoryFunc **mem_read,
3066 CPUWriteMemoryFunc **mem_write,
3069 int i, subwidth = 0;
3071 if (io_index <= 0) {
3072 io_index = get_free_io_mem_idx();
3076 if (io_index >= IO_MEM_NB_ENTRIES)
3080 for(i = 0;i < 3; i++) {
3081 if (!mem_read[i] || !mem_write[i])
3082 subwidth = IO_MEM_SUBWIDTH;
3083 io_mem_read[io_index][i] = mem_read[i];
3084 io_mem_write[io_index][i] = mem_write[i];
3086 io_mem_opaque[io_index] = opaque;
3087 return (io_index << IO_MEM_SHIFT) | subwidth;
3090 void cpu_unregister_io_memory(int io_table_address)
3093 int io_index = io_table_address >> IO_MEM_SHIFT;
3095 for (i=0;i < 3; i++) {
3096 io_mem_read[io_index][i] = unassigned_mem_read[i];
3097 io_mem_write[io_index][i] = unassigned_mem_write[i];
3099 io_mem_opaque[io_index] = NULL;
3100 io_mem_used[io_index] = 0;
3103 #endif /* !defined(CONFIG_USER_ONLY) */
3105 /* physical memory access (slow version, mainly for debug) */
3106 #if defined(CONFIG_USER_ONLY)
3107 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3108 int len, int is_write)
3115 page = addr & TARGET_PAGE_MASK;
3116 l = (page + TARGET_PAGE_SIZE) - addr;
3119 flags = page_get_flags(page);
3120 if (!(flags & PAGE_VALID))
3123 if (!(flags & PAGE_WRITE))
3125 /* XXX: this code should not depend on lock_user */
3126 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3127 /* FIXME - should this return an error rather than just fail? */
3130 unlock_user(p, addr, l);
3132 if (!(flags & PAGE_READ))
3134 /* XXX: this code should not depend on lock_user */
3135 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3136 /* FIXME - should this return an error rather than just fail? */
3139 unlock_user(p, addr, 0);
3148 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3149 int len, int is_write)
3154 target_phys_addr_t page;
3159 page = addr & TARGET_PAGE_MASK;
3160 l = (page + TARGET_PAGE_SIZE) - addr;
3163 p = phys_page_find(page >> TARGET_PAGE_BITS);
3165 pd = IO_MEM_UNASSIGNED;
3167 pd = p->phys_offset;
3171 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3172 target_phys_addr_t addr1 = addr;
3173 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3175 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3176 /* XXX: could force cpu_single_env to NULL to avoid
3178 if (l >= 4 && ((addr1 & 3) == 0)) {
3179 /* 32 bit write access */
3181 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3183 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3184 /* 16 bit write access */
3186 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3189 /* 8 bit write access */
3191 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3195 unsigned long addr1;
3196 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3198 ptr = qemu_get_ram_ptr(addr1);
3199 memcpy(ptr, buf, l);
3200 if (!cpu_physical_memory_is_dirty(addr1)) {
3201 /* invalidate code */
3202 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3204 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3205 (0xff & ~CODE_DIRTY_FLAG);
3209 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3210 !(pd & IO_MEM_ROMD)) {
3211 target_phys_addr_t addr1 = addr;
3213 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3215 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3216 if (l >= 4 && ((addr1 & 3) == 0)) {
3217 /* 32 bit read access */
3218 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3221 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3222 /* 16 bit read access */
3223 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3227 /* 8 bit read access */
3228 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3234 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3235 (addr & ~TARGET_PAGE_MASK);
3236 memcpy(buf, ptr, l);
3245 /* used for ROM loading : can write in RAM and ROM */
3246 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3247 const uint8_t *buf, int len)
3251 target_phys_addr_t page;
3256 page = addr & TARGET_PAGE_MASK;
3257 l = (page + TARGET_PAGE_SIZE) - addr;
3260 p = phys_page_find(page >> TARGET_PAGE_BITS);
3262 pd = IO_MEM_UNASSIGNED;
3264 pd = p->phys_offset;
3267 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3268 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3269 !(pd & IO_MEM_ROMD)) {
3272 unsigned long addr1;
3273 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3275 ptr = qemu_get_ram_ptr(addr1);
3276 memcpy(ptr, buf, l);
3286 target_phys_addr_t addr;
3287 target_phys_addr_t len;
3290 static BounceBuffer bounce;
3292 typedef struct MapClient {
3294 void (*callback)(void *opaque);
3295 LIST_ENTRY(MapClient) link;
3298 static LIST_HEAD(map_client_list, MapClient) map_client_list
3299 = LIST_HEAD_INITIALIZER(map_client_list);
3301 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3303 MapClient *client = qemu_malloc(sizeof(*client));
3305 client->opaque = opaque;
3306 client->callback = callback;
3307 LIST_INSERT_HEAD(&map_client_list, client, link);
3311 void cpu_unregister_map_client(void *_client)
3313 MapClient *client = (MapClient *)_client;
3315 LIST_REMOVE(client, link);
3318 static void cpu_notify_map_clients(void)
3322 while (!LIST_EMPTY(&map_client_list)) {
3323 client = LIST_FIRST(&map_client_list);
3324 client->callback(client->opaque);
3325 LIST_REMOVE(client, link);
3329 /* Map a physical memory region into a host virtual address.
3330 * May map a subset of the requested range, given by and returned in *plen.
3331 * May return NULL if resources needed to perform the mapping are exhausted.
3332 * Use only for reads OR writes - not for read-modify-write operations.
3333 * Use cpu_register_map_client() to know when retrying the map operation is
3334 * likely to succeed.
3336 void *cpu_physical_memory_map(target_phys_addr_t addr,
3337 target_phys_addr_t *plen,
3340 target_phys_addr_t len = *plen;
3341 target_phys_addr_t done = 0;
3343 uint8_t *ret = NULL;
3345 target_phys_addr_t page;
3348 unsigned long addr1;
3351 page = addr & TARGET_PAGE_MASK;
3352 l = (page + TARGET_PAGE_SIZE) - addr;
3355 p = phys_page_find(page >> TARGET_PAGE_BITS);
3357 pd = IO_MEM_UNASSIGNED;
3359 pd = p->phys_offset;
3362 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3363 if (done || bounce.buffer) {
3366 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3370 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3372 ptr = bounce.buffer;
3374 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3375 ptr = qemu_get_ram_ptr(addr1);
3379 } else if (ret + done != ptr) {
3391 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3392 * Will also mark the memory as dirty if is_write == 1. access_len gives
3393 * the amount of memory that was actually read or written by the caller.
3395 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3396 int is_write, target_phys_addr_t access_len)
3398 if (buffer != bounce.buffer) {
3400 ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3401 while (access_len) {
3403 l = TARGET_PAGE_SIZE;
3406 if (!cpu_physical_memory_is_dirty(addr1)) {
3407 /* invalidate code */
3408 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3410 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3411 (0xff & ~CODE_DIRTY_FLAG);
3420 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3422 qemu_free(bounce.buffer);
3423 bounce.buffer = NULL;
3424 cpu_notify_map_clients();
3427 /* warning: addr must be aligned */
3428 uint32_t ldl_phys(target_phys_addr_t addr)
3436 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3438 pd = IO_MEM_UNASSIGNED;
3440 pd = p->phys_offset;
3443 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3444 !(pd & IO_MEM_ROMD)) {
3446 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3448 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3449 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3452 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3453 (addr & ~TARGET_PAGE_MASK);
3459 /* warning: addr must be aligned */
3460 uint64_t ldq_phys(target_phys_addr_t addr)
3468 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3470 pd = IO_MEM_UNASSIGNED;
3472 pd = p->phys_offset;
3475 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3476 !(pd & IO_MEM_ROMD)) {
3478 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3480 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3481 #ifdef TARGET_WORDS_BIGENDIAN
3482 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3483 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3485 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3486 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3490 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3491 (addr & ~TARGET_PAGE_MASK);
3498 uint32_t ldub_phys(target_phys_addr_t addr)
3501 cpu_physical_memory_read(addr, &val, 1);
3506 uint32_t lduw_phys(target_phys_addr_t addr)
3509 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3510 return tswap16(val);
3513 /* warning: addr must be aligned. The ram page is not masked as dirty
3514 and the code inside is not invalidated. It is useful if the dirty
3515 bits are used to track modified PTEs */
3516 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3523 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3525 pd = IO_MEM_UNASSIGNED;
3527 pd = p->phys_offset;
3530 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3531 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3533 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3534 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3536 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3537 ptr = qemu_get_ram_ptr(addr1);
3540 if (unlikely(in_migration)) {
3541 if (!cpu_physical_memory_is_dirty(addr1)) {
3542 /* invalidate code */
3543 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3545 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3546 (0xff & ~CODE_DIRTY_FLAG);
3552 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3559 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3561 pd = IO_MEM_UNASSIGNED;
3563 pd = p->phys_offset;
3566 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3567 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3569 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3570 #ifdef TARGET_WORDS_BIGENDIAN
3571 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3572 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3574 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3575 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3578 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3579 (addr & ~TARGET_PAGE_MASK);
3584 /* warning: addr must be aligned */
3585 void stl_phys(target_phys_addr_t addr, uint32_t val)
3592 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3594 pd = IO_MEM_UNASSIGNED;
3596 pd = p->phys_offset;
3599 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3600 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3602 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3603 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3605 unsigned long addr1;
3606 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3608 ptr = qemu_get_ram_ptr(addr1);
3610 if (!cpu_physical_memory_is_dirty(addr1)) {
3611 /* invalidate code */
3612 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3614 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3615 (0xff & ~CODE_DIRTY_FLAG);
3621 void stb_phys(target_phys_addr_t addr, uint32_t val)
3624 cpu_physical_memory_write(addr, &v, 1);
3628 void stw_phys(target_phys_addr_t addr, uint32_t val)
3630 uint16_t v = tswap16(val);
3631 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3635 void stq_phys(target_phys_addr_t addr, uint64_t val)
3638 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3643 /* virtual memory access for debug (includes writing to ROM) */
3644 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3645 uint8_t *buf, int len, int is_write)
3648 target_phys_addr_t phys_addr;
3652 page = addr & TARGET_PAGE_MASK;
3653 phys_addr = cpu_get_phys_page_debug(env, page);
3654 /* if no physical page mapped, return an error */
3655 if (phys_addr == -1)
3657 l = (page + TARGET_PAGE_SIZE) - addr;
3660 phys_addr += (addr & ~TARGET_PAGE_MASK);
3661 #if !defined(CONFIG_USER_ONLY)
3663 cpu_physical_memory_write_rom(phys_addr, buf, l);
3666 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3674 /* in deterministic execution mode, instructions doing device I/Os
3675 must be at the end of the TB */
3676 void cpu_io_recompile(CPUState *env, void *retaddr)
3678 TranslationBlock *tb;
3680 target_ulong pc, cs_base;
3683 tb = tb_find_pc((unsigned long)retaddr);
3685 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3688 n = env->icount_decr.u16.low + tb->icount;
3689 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3690 /* Calculate how many instructions had been executed before the fault
3692 n = n - env->icount_decr.u16.low;
3693 /* Generate a new TB ending on the I/O insn. */
3695 /* On MIPS and SH, delay slot instructions can only be restarted if
3696 they were already the first instruction in the TB. If this is not
3697 the first instruction in a TB then re-execute the preceding
3699 #if defined(TARGET_MIPS)
3700 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3701 env->active_tc.PC -= 4;
3702 env->icount_decr.u16.low++;
3703 env->hflags &= ~MIPS_HFLAG_BMASK;
3705 #elif defined(TARGET_SH4)
3706 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3709 env->icount_decr.u16.low++;
3710 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3713 /* This should never happen. */
3714 if (n > CF_COUNT_MASK)
3715 cpu_abort(env, "TB too big during recompile");
3717 cflags = n | CF_LAST_IO;
3719 cs_base = tb->cs_base;
3721 tb_phys_invalidate(tb, -1);
3722 /* FIXME: In theory this could raise an exception. In practice
3723 we have already translated the block once so it's probably ok. */
3724 tb_gen_code(env, pc, cs_base, flags, cflags);
3725 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3726 the first in the TB) then we end up generating a whole new TB and
3727 repeating the fault, which is horribly inefficient.
3728 Better would be to execute just this insn uncached, or generate a
3730 cpu_resume_from_signal(env, NULL);
3733 void dump_exec_info(FILE *f,
3734 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3736 int i, target_code_size, max_target_code_size;
3737 int direct_jmp_count, direct_jmp2_count, cross_page;
3738 TranslationBlock *tb;
3740 target_code_size = 0;
3741 max_target_code_size = 0;
3743 direct_jmp_count = 0;
3744 direct_jmp2_count = 0;
3745 for(i = 0; i < nb_tbs; i++) {
3747 target_code_size += tb->size;
3748 if (tb->size > max_target_code_size)
3749 max_target_code_size = tb->size;
3750 if (tb->page_addr[1] != -1)
3752 if (tb->tb_next_offset[0] != 0xffff) {
3754 if (tb->tb_next_offset[1] != 0xffff) {
3755 direct_jmp2_count++;
3759 /* XXX: avoid using doubles ? */
3760 cpu_fprintf(f, "Translation buffer state:\n");
3761 cpu_fprintf(f, "gen code size %ld/%ld\n",
3762 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3763 cpu_fprintf(f, "TB count %d/%d\n",
3764 nb_tbs, code_gen_max_blocks);
3765 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3766 nb_tbs ? target_code_size / nb_tbs : 0,
3767 max_target_code_size);
3768 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3769 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3770 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3771 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3773 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3774 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3776 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3778 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3779 cpu_fprintf(f, "\nStatistics:\n");
3780 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3781 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3782 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3783 tcg_dump_info(f, cpu_fprintf);
3786 #if !defined(CONFIG_USER_ONLY)
3788 #define MMUSUFFIX _cmmu
3789 #define GETPC() NULL
3790 #define env cpu_single_env
3791 #define SOFTMMU_CODE_ACCESS
3794 #include "softmmu_template.h"
3797 #include "softmmu_template.h"
3800 #include "softmmu_template.h"
3803 #include "softmmu_template.h"