Auto merge with /home/aegl/GIT/linus

[powerpc.git] / arch / sparc64 / kernel / pci_iommu.c

diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c
index f009b1b..2803bc7 100644 (file)
--- a/arch/sparc64/kernel/pci_iommu.c
+++ b/arch/sparc64/kernel/pci_iommu.c
@@ -196,6 +196,34 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long
        return NULL;
 }
 
+static int iommu_alloc_ctx(struct pci_iommu *iommu)
+{
+       int lowest = iommu->ctx_lowest_free;
+       int sz = IOMMU_NUM_CTXS - lowest;
+       int n = find_next_zero_bit(iommu->ctx_bitmap, sz, lowest);
+
+       if (unlikely(n == sz)) {
+               n = find_next_zero_bit(iommu->ctx_bitmap, lowest, 1);
+               if (unlikely(n == lowest)) {
+                       printk(KERN_WARNING "IOMMU: Ran out of contexts.\n");
+                       n = 0;
+               }
+       }
+       if (n)
+               __set_bit(n, iommu->ctx_bitmap);
+
+       return n;
+}
+
+static inline void iommu_free_ctx(struct pci_iommu *iommu, int ctx)
+{
+       if (likely(ctx)) {
+               __clear_bit(ctx, iommu->ctx_bitmap);
+               if (ctx < iommu->ctx_lowest_free)
+                       iommu->ctx_lowest_free = ctx;
+       }
+}
+
 /* Allocate and map kernel buffer of size SIZE using consistent mode
  * DMA for PCI device PDEV.  Return non-NULL cpu-side address if
  * successful and set *DMA_ADDRP to the PCI side dma address.
@@ -236,7 +264,7 @@ void *pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_ad
        npages = size >> IO_PAGE_SHIFT;
        ctx = 0;
        if (iommu->iommu_ctxflush)
-               ctx = iommu->iommu_cur_ctx++;
+               ctx = iommu_alloc_ctx(iommu);
        first_page = __pa(first_page);
        while (npages--) {
                iopte_val(*iopte) = (IOPTE_CONSISTENT(ctx) |
@@ -317,6 +345,8 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
                }
        }
 
+       iommu_free_ctx(iommu, ctx);
+
        spin_unlock_irqrestore(&iommu->lock, flags);
 
        order = get_order(size);
@@ -360,7 +390,7 @@ dma_addr_t pci_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direct
        base_paddr = __pa(oaddr & IO_PAGE_MASK);
        ctx = 0;
        if (iommu->iommu_ctxflush)
-               ctx = iommu->iommu_cur_ctx++;
+               ctx = iommu_alloc_ctx(iommu);
        if (strbuf->strbuf_enabled)
                iopte_protection = IOPTE_STREAMING(ctx);
        else
@@ -380,46 +410,62 @@ bad:
        return PCI_DMA_ERROR_CODE;
 }
 
-static void pci_strbuf_flush(struct pci_strbuf *strbuf, struct pci_iommu *iommu, u32 vaddr, unsigned long ctx, unsigned long npages)
+static void pci_strbuf_flush(struct pci_strbuf *strbuf, struct pci_iommu *iommu, u32 vaddr, unsigned long ctx, unsigned long npages, int direction)
 {
        int limit;
 
-       PCI_STC_FLUSHFLAG_INIT(strbuf);
        if (strbuf->strbuf_ctxflush &&
            iommu->iommu_ctxflush) {
                unsigned long matchreg, flushreg;
+               u64 val;
 
                flushreg = strbuf->strbuf_ctxflush;
                matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
 
-               limit = 10000;
-               do {
-                       pci_iommu_write(flushreg, ctx);
-                       udelay(10);
-                       limit--;
-                       if (!limit)
-                               break;
-               } while(((long)pci_iommu_read(matchreg)) < 0L);
-               if (!limit)
+               pci_iommu_write(flushreg, ctx);
+               val = pci_iommu_read(matchreg);
+               val &= 0xffff;
+               if (!val)
+                       goto do_flush_sync;
+
+               while (val) {
+                       if (val & 0x1)
+                               pci_iommu_write(flushreg, ctx);
+                       val >>= 1;
+               }
+               val = pci_iommu_read(matchreg);
+               if (unlikely(val)) {
                        printk(KERN_WARNING "pci_strbuf_flush: ctx flush "
-                              "timeout vaddr[%08x] ctx[%lx]\n",
-                              vaddr, ctx);
+                              "timeout matchreg[%lx] ctx[%lx]\n",
+                              val, ctx);
+                       goto do_page_flush;
+               }
        } else {
                unsigned long i;
 
+       do_page_flush:
                for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE)
                        pci_iommu_write(strbuf->strbuf_pflush, vaddr);
        }
 
+do_flush_sync:
+       /* If the device could not have possibly put dirty data into
+        * the streaming cache, no flush-flag synchronization needs
+        * to be performed.
+        */
+       if (direction == PCI_DMA_TODEVICE)
+               return;
+
+       PCI_STC_FLUSHFLAG_INIT(strbuf);
        pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
        (void) pci_iommu_read(iommu->write_complete_reg);
 
-       limit = 10000;
+       limit = 100000;
        while (!PCI_STC_FLUSHFLAG_SET(strbuf)) {
                limit--;
                if (!limit)
                        break;
-               udelay(10);
+               udelay(1);
                membar("#LoadLoad");
        }
        if (!limit)
@@ -464,7 +510,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
 
        /* Step 1: Kick data out of streaming buffers if necessary. */
        if (strbuf->strbuf_enabled)
-               pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages);
+               pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
 
        /* Step 2: Clear out first TSB entry. */
        iopte_make_dummy(iommu, base);
@@ -472,6 +518,8 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
        free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
                               npages, ctx);
 
+       iommu_free_ctx(iommu, ctx);
+
        spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
@@ -611,7 +659,7 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int
        /* Step 4: Choose a context if necessary. */
        ctx = 0;
        if (iommu->iommu_ctxflush)
-               ctx = iommu->iommu_cur_ctx++;
+               ctx = iommu_alloc_ctx(iommu);
 
        /* Step 5: Create the mappings. */
        if (strbuf->strbuf_enabled)
@@ -676,7 +724,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
 
        /* Step 1: Kick data out of streaming buffers if necessary. */
        if (strbuf->strbuf_enabled)
-               pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages);
+               pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
 
        /* Step 2: Clear out first TSB entry. */
        iopte_make_dummy(iommu, base);
@@ -684,6 +732,8 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
        free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
                               npages, ctx);
 
+       iommu_free_ctx(iommu, ctx);
+
        spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
@@ -722,7 +772,7 @@ void pci_dma_sync_single_for_cpu(struct pci_dev *pdev, dma_addr_t bus_addr, size
        }
 
        /* Step 2: Kick data out of streaming buffers. */
-       pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages);
+       pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
 
        spin_unlock_irqrestore(&iommu->lock, flags);
 }
@@ -766,7 +816,7 @@ void pci_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist *sglist, i
        i--;
        npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length)
                  - bus_addr) >> IO_PAGE_SHIFT;
-       pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages);
+       pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
 
        spin_unlock_irqrestore(&iommu->lock, flags);
 }