mm/vmalloc.c

   1 /*
   2  *  linux/mm/vmalloc.c
   3  *
   4  *  Copyright (C) 1993  Linus Torvalds
   5  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
   6  *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
   7  */
   8
   9 #include <linux/config.h>
  10 #include <linux/slab.h>
  11 #include <linux/vmalloc.h>
  12 #include <linux/spinlock.h>
  13 #include <linux/highmem.h>
  14 #include <linux/smp_lock.h>
  15
  16 #include <asm/uaccess.h>
  17 #include <asm/pgalloc.h>
  18
  19 rwlock_t vmlist_lock = RW_LOCK_UNLOCKED;
  20 struct vm_struct * vmlist;
  21
  22 static inline void free_area_pte(pmd_t * pmd, unsigned long address, unsigned long size)
  23 {
  24         pte_t * pte;
  25         unsigned long end;
  26
  27         if (pmd_none(*pmd))
  28                 return;
  29         if (pmd_bad(*pmd)) {
  30                 pmd_ERROR(*pmd);
  31                 pmd_clear(pmd);
  32                 return;
  33         }
  34         pte = pte_offset(pmd, address);
  35         address &= ~PMD_MASK;
  36         end = address + size;
  37         if (end > PMD_SIZE)
  38                 end = PMD_SIZE;
  39         do {
  40                 pte_t page;
  41                 page = ptep_get_and_clear(pte);
  42                 address += PAGE_SIZE;
  43                 pte++;
  44                 if (pte_none(page))
  45                         continue;
  46                 if (pte_present(page)) {
  47                         struct page *ptpage = pte_page(page);
  48                         if (VALID_PAGE(ptpage) && (!PageReserved(ptpage)))
  49                                 __free_page(ptpage);
  50                         continue;
  51                 }
  52                 printk(KERN_CRIT "Whee.. Swapped out page in kernel page table\n");
  53         } while (address < end);
  54 }
  55
  56 static inline void free_area_pmd(pgd_t * dir, unsigned long address, unsigned long size)
  57 {
  58         pmd_t * pmd;
  59         unsigned long end;
  60
  61         if (pgd_none(*dir))
  62                 return;
  63         if (pgd_bad(*dir)) {
  64                 pgd_ERROR(*dir);
  65                 pgd_clear(dir);
  66                 return;
  67         }
  68         pmd = pmd_offset(dir, address);
  69         address &= ~PGDIR_MASK;
  70         end = address + size;
  71         if (end > PGDIR_SIZE)
  72                 end = PGDIR_SIZE;
  73         do {
  74                 free_area_pte(pmd, address, end - address);
  75                 address = (address + PMD_SIZE) & PMD_MASK;
  76                 pmd++;
  77         } while (address < end);
  78 }
  79
  80 void vmfree_area_pages(unsigned long address, unsigned long size)
  81 {
  82         pgd_t * dir;
  83         unsigned long end = address + size;
  84
  85         dir = pgd_offset_k(address);
  86         flush_cache_all();
  87         do {
  88                 free_area_pmd(dir, address, end - address);
  89                 address = (address + PGDIR_SIZE) & PGDIR_MASK;
  90                 dir++;
  91         } while (address && (address < end));
  92         flush_tlb_all();
  93 }
  94
  95 static inline int alloc_area_pte (pte_t * pte, unsigned long address,
  96                         unsigned long size, int gfp_mask, pgprot_t prot)
  97 {
  98         unsigned long end;
  99
 100         address &= ~PMD_MASK;
 101         end = address + size;
 102         if (end > PMD_SIZE)
 103                 end = PMD_SIZE;
 104         do {
 105                 struct page * page;
 106                 spin_unlock(&init_mm.page_table_lock);
 107                 page = alloc_page(gfp_mask);
 108                 spin_lock(&init_mm.page_table_lock);
 109                 if (!pte_none(*pte))
 110                         printk(KERN_ERR "alloc_area_pte: page already exists\n");
 111                 if (!page)
 112                         return -ENOMEM;
 113                 set_pte(pte, mk_pte(page, prot));
 114                 address += PAGE_SIZE;
 115                 pte++;
 116         } while (address < end);
 117         return 0;
 118 }
 119
 120 static inline int alloc_area_pmd(pmd_t * pmd, unsigned long address, unsigned long size, int gfp_mask, pgprot_t prot)
 121 {
 122         unsigned long end;
 123
 124         address &= ~PGDIR_MASK;
 125         end = address + size;
 126         if (end > PGDIR_SIZE)
 127                 end = PGDIR_SIZE;
 128         do {
 129                 pte_t * pte = pte_alloc(&init_mm, pmd, address);
 130                 if (!pte)
 131                         return -ENOMEM;
 132                 if (alloc_area_pte(pte, address, end - address, gfp_mask, prot))
 133                         return -ENOMEM;
 134                 address = (address + PMD_SIZE) & PMD_MASK;
 135                 pmd++;
 136         } while (address < end);
 137         return 0;
 138 }
 139
 140 inline int vmalloc_area_pages (unsigned long address, unsigned long size,
 141                                int gfp_mask, pgprot_t prot)
 142 {
 143         pgd_t * dir;
 144         unsigned long end = address + size;
 145         int ret;
 146
 147         dir = pgd_offset_k(address);
 148         spin_lock(&init_mm.page_table_lock);
 149         do {
 150                 pmd_t *pmd;
 151
 152                 pmd = pmd_alloc(&init_mm, dir, address);
 153                 ret = -ENOMEM;
 154                 if (!pmd)
 155                         break;
 156
 157                 ret = -ENOMEM;
 158                 if (alloc_area_pmd(pmd, address, end - address, gfp_mask, prot))
 159                         break;
 160
 161                 address = (address + PGDIR_SIZE) & PGDIR_MASK;
 162                 dir++;
 163
 164                 ret = 0;
 165         } while (address && (address < end));
 166         spin_unlock(&init_mm.page_table_lock);
 167         flush_cache_all();
 168         return ret;
 169 }
 170
 171 struct vm_struct * get_vm_area(unsigned long size, unsigned long flags)
 172 {
 173         unsigned long addr;
 174         struct vm_struct **p, *tmp, *area;
 175
 176         area = (struct vm_struct *) kmalloc(sizeof(*area), GFP_KERNEL);
 177         if (!area)
 178                 return NULL;
 179
 180         size += PAGE_SIZE;
 181         if (!size) {
 182                 kfree (area);
 183                 return NULL;
 184         }
 185
 186         addr = VMALLOC_START;
 187         write_lock(&vmlist_lock);
 188         for (p = &vmlist; (tmp = *p) ; p = &tmp->next) {
 189                 if ((size + addr) < addr)
 190                         goto out;
 191                 if (size + addr <= (unsigned long) tmp->addr)
 192                         break;
 193                 addr = tmp->size + (unsigned long) tmp->addr;
 194                 if (addr > VMALLOC_END-size)
 195                         goto out;
 196         }
 197         area->flags = flags;
 198         area->addr = (void *)addr;
 199         area->size = size;
 200         area->next = *p;
 201         *p = area;
 202         write_unlock(&vmlist_lock);
 203         return area;
 204
 205 out:
 206         write_unlock(&vmlist_lock);
 207         kfree(area);
 208         return NULL;
 209 }
 210
 211 void vfree(void * addr)
 212 {
 213         struct vm_struct **p, *tmp;
 214
 215         if (!addr)
 216                 return;
 217         if ((PAGE_SIZE-1) & (unsigned long) addr) {
 218                 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
 219                 return;
 220         }
 221         write_lock(&vmlist_lock);
 222         for (p = &vmlist ; (tmp = *p) ; p = &tmp->next) {
 223                 if (tmp->addr == addr) {
 224                         *p = tmp->next;
 225                         vmfree_area_pages(VMALLOC_VMADDR(tmp->addr), tmp->size);
 226                         write_unlock(&vmlist_lock);
 227                         kfree(tmp);
 228                         return;
 229                 }
 230         }
 231         write_unlock(&vmlist_lock);
 232         printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", addr);
 233 }
 234
 235 void * __vmalloc (unsigned long size, int gfp_mask, pgprot_t prot)
 236 {
 237         void * addr;
 238         struct vm_struct *area;
 239
 240         size = PAGE_ALIGN(size);
 241         if (!size || (size >> PAGE_SHIFT) > num_physpages) {
 242                 BUG();
 243                 return NULL;
 244         }
 245         area = get_vm_area(size, VM_ALLOC);
 246         if (!area)
 247                 return NULL;
 248         addr = area->addr;
 249         if (vmalloc_area_pages(VMALLOC_VMADDR(addr), size, gfp_mask, prot)) {
 250                 vfree(addr);
 251                 return NULL;
 252         }
 253         return addr;
 254 }
 255
 256 long vread(char *buf, char *addr, unsigned long count)
 257 {
 258         struct vm_struct *tmp;
 259         char *vaddr, *buf_start = buf;
 260         unsigned long n;
 261
 262         /* Don't allow overflow */
 263         if ((unsigned long) addr + count < count)
 264                 count = -(unsigned long) addr;
 265
 266         read_lock(&vmlist_lock);
 267         for (tmp = vmlist; tmp; tmp = tmp->next) {
 268                 vaddr = (char *) tmp->addr;
 269                 if (addr >= vaddr + tmp->size - PAGE_SIZE)
 270                         continue;
 271                 while (addr < vaddr) {
 272                         if (count == 0)
 273                                 goto finished;
 274                         *buf = '\0';
 275                         buf++;
 276                         addr++;
 277                         count--;
 278                 }
 279                 n = vaddr + tmp->size - PAGE_SIZE - addr;
 280                 do {
 281                         if (count == 0)
 282                                 goto finished;
 283                         *buf = *addr;
 284                         buf++;
 285                         addr++;
 286                         count--;
 287                 } while (--n > 0);
 288         }
 289 finished:
 290         read_unlock(&vmlist_lock);
 291         return buf - buf_start;
 292 }
 293
 294 long vwrite(char *buf, char *addr, unsigned long count)
 295 {
 296         struct vm_struct *tmp;
 297         char *vaddr, *buf_start = buf;
 298         unsigned long n;
 299
 300         /* Don't allow overflow */
 301         if ((unsigned long) addr + count < count)
 302                 count = -(unsigned long) addr;
 303
 304         read_lock(&vmlist_lock);
 305         for (tmp = vmlist; tmp; tmp = tmp->next) {
 306                 vaddr = (char *) tmp->addr;
 307                 if (addr >= vaddr + tmp->size - PAGE_SIZE)
 308                         continue;
 309                 while (addr < vaddr) {
 310                         if (count == 0)
 311                                 goto finished;
 312                         buf++;
 313                         addr++;
 314                         count--;
 315                 }
 316                 n = vaddr + tmp->size - PAGE_SIZE - addr;
 317                 do {
 318                         if (count == 0)
 319                                 goto finished;
 320                         *addr = *buf;
 321                         buf++;
 322                         addr++;
 323                         count--;
 324                 } while (--n > 0);
 325         }
 326 finished:
 327         read_unlock(&vmlist_lock);
 328         return buf - buf_start;
 329 }