arch/um/kernel/tlb.c

   1 /*
   2  * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com)
   3  * Licensed under the GPL
   4  */
   5
   6 #include "linux/mm.h"
   7 #include "asm/page.h"
   8 #include "asm/pgalloc.h"
   9 #include "asm/tlbflush.h"
  10 #include "choose-mode.h"
  11 #include "mode_kern.h"
  12 #include "as-layout.h"
  13 #include "user_util.h"
  14 #include "tlb.h"
  15 #include "mem.h"
  16 #include "mem_user.h"
  17 #include "os.h"
  18
  19 static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
  20                     int r, int w, int x, struct host_vm_op *ops, int *index,
  21                     int last_filled, union mm_context *mmu, void **flush,
  22                     int (*do_ops)(union mm_context *, struct host_vm_op *,
  23                                   int, int, void **))
  24 {
  25         __u64 offset;
  26         struct host_vm_op *last;
  27         int fd, ret = 0;
  28
  29         fd = phys_mapping(phys, &offset);
  30         if(*index != -1){
  31                 last = &ops[*index];
  32                 if((last->type == MMAP) &&
  33                    (last->u.mmap.addr + last->u.mmap.len == virt) &&
  34                    (last->u.mmap.r == r) && (last->u.mmap.w == w) &&
  35                    (last->u.mmap.x == x) && (last->u.mmap.fd == fd) &&
  36                    (last->u.mmap.offset + last->u.mmap.len == offset)){
  37                         last->u.mmap.len += len;
  38                         return 0;
  39                 }
  40         }
  41
  42         if(*index == last_filled){
  43                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
  44                 *index = -1;
  45         }
  46
  47         ops[++*index] = ((struct host_vm_op) { .type    = MMAP,
  48                                                 .u = { .mmap = {
  49                                                        .addr    = virt,
  50                                                        .len     = len,
  51                                                        .r       = r,
  52                                                        .w       = w,
  53                                                        .x       = x,
  54                                                        .fd      = fd,
  55                                                        .offset  = offset }
  56                            } });
  57         return ret;
  58 }
  59
  60 static int add_munmap(unsigned long addr, unsigned long len,
  61                       struct host_vm_op *ops, int *index, int last_filled,
  62                       union mm_context *mmu, void **flush,
  63                       int (*do_ops)(union mm_context *, struct host_vm_op *,
  64                                     int, int, void **))
  65 {
  66         struct host_vm_op *last;
  67         int ret = 0;
  68
  69         if(*index != -1){
  70                 last = &ops[*index];
  71                 if((last->type == MUNMAP) &&
  72                    (last->u.munmap.addr + last->u.mmap.len == addr)){
  73                         last->u.munmap.len += len;
  74                         return 0;
  75                 }
  76         }
  77
  78         if(*index == last_filled){
  79                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
  80                 *index = -1;
  81         }
  82
  83         ops[++*index] = ((struct host_vm_op) { .type    = MUNMAP,
  84                                                .u = { .munmap = {
  85                                                         .addr   = addr,
  86                                                         .len    = len } } });
  87         return ret;
  88 }
  89
  90 static int add_mprotect(unsigned long addr, unsigned long len, int r, int w,
  91                         int x, struct host_vm_op *ops, int *index,
  92                         int last_filled, union mm_context *mmu, void **flush,
  93                         int (*do_ops)(union mm_context *, struct host_vm_op *,
  94                                       int, int, void **))
  95 {
  96         struct host_vm_op *last;
  97         int ret = 0;
  98
  99         if(*index != -1){
 100                 last = &ops[*index];
 101                 if((last->type == MPROTECT) &&
 102                    (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
 103                    (last->u.mprotect.r == r) && (last->u.mprotect.w == w) &&
 104                    (last->u.mprotect.x == x)){
 105                         last->u.mprotect.len += len;
 106                         return 0;
 107                 }
 108         }
 109
 110         if(*index == last_filled){
 111                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
 112                 *index = -1;
 113         }
 114
 115         ops[++*index] = ((struct host_vm_op) { .type    = MPROTECT,
 116                                                .u = { .mprotect = {
 117                                                        .addr    = addr,
 118                                                        .len     = len,
 119                                                        .r       = r,
 120                                                        .w       = w,
 121                                                        .x       = x } } });
 122         return ret;
 123 }
 124
 125 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
 126
 127 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
 128                       unsigned long end_addr, int force,
 129                       int (*do_ops)(union mm_context *, struct host_vm_op *,
 130                                     int, int, void **))
 131 {
 132         pgd_t *npgd;
 133         pud_t *npud;
 134         pmd_t *npmd;
 135         pte_t *npte;
 136         union mm_context *mmu = &mm->context;
 137         unsigned long addr, end;
 138         int r, w, x;
 139         struct host_vm_op ops[1];
 140         void *flush = NULL;
 141         int op_index = -1, last_op = ARRAY_SIZE(ops) - 1;
 142         int ret = 0;
 143
 144         if(mm == NULL)
 145                 return;
 146
 147         ops[0].type = NONE;
 148         for(addr = start_addr; addr < end_addr && !ret;){
 149                 npgd = pgd_offset(mm, addr);
 150                 if(!pgd_present(*npgd)){
 151                         end = ADD_ROUND(addr, PGDIR_SIZE);
 152                         if(end > end_addr)
 153                                 end = end_addr;
 154                         if(force || pgd_newpage(*npgd)){
 155                                 ret = add_munmap(addr, end - addr, ops,
 156                                                  &op_index, last_op, mmu,
 157                                                  &flush, do_ops);
 158                                 pgd_mkuptodate(*npgd);
 159                         }
 160                         addr = end;
 161                         continue;
 162                 }
 163
 164                 npud = pud_offset(npgd, addr);
 165                 if(!pud_present(*npud)){
 166                         end = ADD_ROUND(addr, PUD_SIZE);
 167                         if(end > end_addr)
 168                                 end = end_addr;
 169                         if(force || pud_newpage(*npud)){
 170                                 ret = add_munmap(addr, end - addr, ops,
 171                                                  &op_index, last_op, mmu,
 172                                                  &flush, do_ops);
 173                                 pud_mkuptodate(*npud);
 174                         }
 175                         addr = end;
 176                         continue;
 177                 }
 178
 179                 npmd = pmd_offset(npud, addr);
 180                 if(!pmd_present(*npmd)){
 181                         end = ADD_ROUND(addr, PMD_SIZE);
 182                         if(end > end_addr)
 183                                 end = end_addr;
 184                         if(force || pmd_newpage(*npmd)){
 185                                 ret = add_munmap(addr, end - addr, ops,
 186                                                  &op_index, last_op, mmu,
 187                                                  &flush, do_ops);
 188                                 pmd_mkuptodate(*npmd);
 189                         }
 190                         addr = end;
 191                         continue;
 192                 }
 193
 194                 npte = pte_offset_kernel(npmd, addr);
 195                 r = pte_read(*npte);
 196                 w = pte_write(*npte);
 197                 x = pte_exec(*npte);
 198                 if (!pte_young(*npte)) {
 199                         r = 0;
 200                         w = 0;
 201                 } else if (!pte_dirty(*npte)) {
 202                         w = 0;
 203                 }
 204                 if(force || pte_newpage(*npte)){
 205                         if(pte_present(*npte))
 206                                 ret = add_mmap(addr,
 207                                                pte_val(*npte) & PAGE_MASK,
 208                                                PAGE_SIZE, r, w, x, ops,
 209                                                &op_index, last_op, mmu,
 210                                                &flush, do_ops);
 211                         else ret = add_munmap(addr, PAGE_SIZE, ops,
 212                                               &op_index, last_op, mmu,
 213                                               &flush, do_ops);
 214                 }
 215                 else if(pte_newprot(*npte))
 216                         ret = add_mprotect(addr, PAGE_SIZE, r, w, x, ops,
 217                                            &op_index, last_op, mmu,
 218                                            &flush, do_ops);
 219
 220                 *npte = pte_mkuptodate(*npte);
 221                 addr += PAGE_SIZE;
 222         }
 223         if(!ret)
 224                 ret = (*do_ops)(mmu, ops, op_index, 1, &flush);
 225
 226 /* This is not an else because ret is modified above */
 227         if(ret) {
 228                 printk("fix_range_common: failed, killing current process\n");
 229                 force_sig(SIGKILL, current);
 230         }
 231 }
 232
 233 int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
 234 {
 235         struct mm_struct *mm;
 236         pgd_t *pgd;
 237         pud_t *pud;
 238         pmd_t *pmd;
 239         pte_t *pte;
 240         unsigned long addr, last;
 241         int updated = 0, err;
 242
 243         mm = &init_mm;
 244         for(addr = start; addr < end;){
 245                 pgd = pgd_offset(mm, addr);
 246                 if(!pgd_present(*pgd)){
 247                         last = ADD_ROUND(addr, PGDIR_SIZE);
 248                         if(last > end)
 249                                 last = end;
 250                         if(pgd_newpage(*pgd)){
 251                                 updated = 1;
 252                                 err = os_unmap_memory((void *) addr,
 253                                                       last - addr);
 254                                 if(err < 0)
 255                                         panic("munmap failed, errno = %d\n",
 256                                               -err);
 257                         }
 258                         addr = last;
 259                         continue;
 260                 }
 261
 262                 pud = pud_offset(pgd, addr);
 263                 if(!pud_present(*pud)){
 264                         last = ADD_ROUND(addr, PUD_SIZE);
 265                         if(last > end)
 266                                 last = end;
 267                         if(pud_newpage(*pud)){
 268                                 updated = 1;
 269                                 err = os_unmap_memory((void *) addr,
 270                                                       last - addr);
 271                                 if(err < 0)
 272                                         panic("munmap failed, errno = %d\n",
 273                                               -err);
 274                         }
 275                         addr = last;
 276                         continue;
 277                 }
 278
 279                 pmd = pmd_offset(pud, addr);
 280                 if(!pmd_present(*pmd)){
 281                         last = ADD_ROUND(addr, PMD_SIZE);
 282                         if(last > end)
 283                                 last = end;
 284                         if(pmd_newpage(*pmd)){
 285                                 updated = 1;
 286                                 err = os_unmap_memory((void *) addr,
 287                                                       last - addr);
 288                                 if(err < 0)
 289                                         panic("munmap failed, errno = %d\n",
 290                                               -err);
 291                         }
 292                         addr = last;
 293                         continue;
 294                 }
 295
 296                 pte = pte_offset_kernel(pmd, addr);
 297                 if(!pte_present(*pte) || pte_newpage(*pte)){
 298                         updated = 1;
 299                         err = os_unmap_memory((void *) addr,
 300                                               PAGE_SIZE);
 301                         if(err < 0)
 302                                 panic("munmap failed, errno = %d\n",
 303                                       -err);
 304                         if(pte_present(*pte))
 305                                 map_memory(addr,
 306                                            pte_val(*pte) & PAGE_MASK,
 307                                            PAGE_SIZE, 1, 1, 1);
 308                 }
 309                 else if(pte_newprot(*pte)){
 310                         updated = 1;
 311                         os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
 312                 }
 313                 addr += PAGE_SIZE;
 314         }
 315         return(updated);
 316 }
 317
 318 pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
 319 {
 320         return(pgd_offset(mm, address));
 321 }
 322
 323 pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
 324 {
 325         return(pud_offset(pgd, address));
 326 }
 327
 328 pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
 329 {
 330         return(pmd_offset(pud, address));
 331 }
 332
 333 pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
 334 {
 335         return(pte_offset_kernel(pmd, address));
 336 }
 337
 338 pte_t *addr_pte(struct task_struct *task, unsigned long addr)
 339 {
 340         pgd_t *pgd = pgd_offset(task->mm, addr);
 341         pud_t *pud = pud_offset(pgd, addr);
 342         pmd_t *pmd = pmd_offset(pud, addr);
 343
 344         return(pte_offset_map(pmd, addr));
 345 }
 346
 347 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
 348 {
 349         address &= PAGE_MASK;
 350         flush_tlb_range(vma, address, address + PAGE_SIZE);
 351 }
 352
 353 void flush_tlb_all(void)
 354 {
 355         flush_tlb_mm(current->mm);
 356 }
 357
 358 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 359 {
 360         CHOOSE_MODE_PROC(flush_tlb_kernel_range_tt,
 361                          flush_tlb_kernel_range_common, start, end);
 362 }
 363
 364 void flush_tlb_kernel_vm(void)
 365 {
 366         CHOOSE_MODE(flush_tlb_kernel_vm_tt(),
 367                     flush_tlb_kernel_range_common(start_vm, end_vm));
 368 }
 369
 370 void __flush_tlb_one(unsigned long addr)
 371 {
 372         CHOOSE_MODE_PROC(__flush_tlb_one_tt, __flush_tlb_one_skas, addr);
 373 }
 374
 375 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 376                      unsigned long end)
 377 {
 378         CHOOSE_MODE_PROC(flush_tlb_range_tt, flush_tlb_range_skas, vma, start,
 379                          end);
 380 }
 381
 382 void flush_tlb_mm(struct mm_struct *mm)
 383 {
 384         CHOOSE_MODE_PROC(flush_tlb_mm_tt, flush_tlb_mm_skas, mm);
 385 }
 386
 387 void force_flush_all(void)
 388 {
 389         CHOOSE_MODE(force_flush_all_tt(), force_flush_all_skas());
 390 }
 391