*
* On SYSIO, using an 8K page size we have 1GB of SBUS
* DMA space mapped. We divide this space into equally
- * sized clusters. Currently we allow clusters up to a
- * size of 1MB. If anything begins to generate DMA
- * mapping requests larger than this we will need to
- * increase things a bit.
+ * sized clusters. We allocate a DMA mapping from the
+ * cluster that matches the order of the allocation, or
+ * if the order is greater than the number of clusters,
+ * we try to allocate from the last cluster.
*/
#define NCLUSTERS 8UL
#define STRBUF_TAG_VALID 0x02UL
-static void strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
+static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages, int direction)
{
- iommu->strbuf_flushflag = 0UL;
- while (npages--)
- upa_writeq(base + (npages << IO_PAGE_SHIFT),
+ unsigned long n;
+ int limit;
+
+ n = npages;
+ while (n--)
+ upa_writeq(base + (n << IO_PAGE_SHIFT),
iommu->strbuf_regs + STRBUF_PFLUSH);
+ /* If the device could not have possibly put dirty data into
+ * the streaming cache, no flush-flag synchronization needs
+ * to be performed.
+ */
+ if (direction == SBUS_DMA_TODEVICE)
+ return;
+
+ iommu->strbuf_flushflag = 0UL;
+
/* Whoopee cushion! */
upa_writeq(__pa(&iommu->strbuf_flushflag),
iommu->strbuf_regs + STRBUF_FSYNC);
upa_readq(iommu->sbus_control_reg);
- while (iommu->strbuf_flushflag == 0UL)
- membar("#LoadLoad");
+
+ limit = 100000;
+ while (iommu->strbuf_flushflag == 0UL) {
+ limit--;
+ if (!limit)
+ break;
+ udelay(1);
+ rmb();
+ }
+ if (!limit)
+ printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout "
+ "vaddr[%08x] npages[%ld]\n",
+ base, npages);
}
static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages)
{
- iopte_t *iopte, *limit, *first;
- unsigned long cnum, ent, flush_point;
+ iopte_t *iopte, *limit, *first, *cluster;
+ unsigned long cnum, ent, nent, flush_point, found;
cnum = 0;
+ nent = 1;
while ((1UL << cnum) < npages)
cnum++;
+ if(cnum >= NCLUSTERS) {
+ nent = 1UL << (cnum - NCLUSTERS);
+ cnum = NCLUSTERS - 1;
+ }
iopte = iommu->page_table + (cnum * CLUSTER_NPAGES);
if (cnum == 0)
flush_point = iommu->alloc_info[cnum].flush;
first = iopte;
+ cluster = NULL;
+ found = 0;
for (;;) {
if (iopte_val(*iopte) == 0UL) {
- if ((iopte + (1 << cnum)) >= limit)
- ent = 0;
- else
- ent = ent + 1;
- iommu->alloc_info[cnum].next = ent;
- if (ent == flush_point)
- __iommu_flushall(iommu);
- break;
+ found++;
+ if (!cluster)
+ cluster = iopte;
+ } else {
+ /* Used cluster in the way */
+ cluster = NULL;
+ found = 0;
}
+
+ if (found == nent)
+ break;
+
iopte += (1 << cnum);
ent++;
if (iopte >= limit) {
iopte = (iommu->page_table + (cnum * CLUSTER_NPAGES));
ent = 0;
+
+ /* Multiple cluster allocations must not wrap */
+ cluster = NULL;
+ found = 0;
}
if (ent == flush_point)
__iommu_flushall(iommu);
goto bad;
}
+ /* ent/iopte points to the last cluster entry we're going to use,
+ * so save our place for the next allocation.
+ */
+ if ((iopte + (1 << cnum)) >= limit)
+ ent = 0;
+ else
+ ent = ent + 1;
+ iommu->alloc_info[cnum].next = ent;
+ if (ent == flush_point)
+ __iommu_flushall(iommu);
+
/* I've got your streaming cluster right here buddy boy... */
- return iopte;
+ return cluster;
bad:
printk(KERN_EMERG "sbus: alloc_streaming_cluster of npages(%ld) failed!\n",
static void free_streaming_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages)
{
- unsigned long cnum, ent;
+ unsigned long cnum, ent, nent;
iopte_t *iopte;
cnum = 0;
+ nent = 1;
while ((1UL << cnum) < npages)
cnum++;
+ if(cnum >= NCLUSTERS) {
+ nent = 1UL << (cnum - NCLUSTERS);
+ cnum = NCLUSTERS - 1;
+ }
ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum);
iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT);
- iopte_val(*iopte) = 0UL;
+ do {
+ iopte_val(*iopte) = 0UL;
+ iopte += 1 << cnum;
+ } while(--nent);
/* If the global flush might not have caught this entry,
* adjust the flush point such that we will flush before
order = get_order(size);
if (order >= 10)
return NULL;
- first_page = __get_free_pages(GFP_KERNEL, order);
+ first_page = __get_free_pages(GFP_KERNEL|__GFP_COMP, order);
if (first_page == 0UL)
return NULL;
memset((char *)first_page, 0, PAGE_SIZE << order);
spin_lock_irqsave(&iommu->lock, flags);
free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT);
- strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT);
+ sbus_strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT, direction);
spin_unlock_irqrestore(&iommu->lock, flags);
}
iommu = sdev->bus->iommu;
spin_lock_irqsave(&iommu->lock, flags);
free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT);
- strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT);
+ sbus_strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT, direction);
spin_unlock_irqrestore(&iommu->lock, flags);
}
size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK));
spin_lock_irqsave(&iommu->lock, flags);
- strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT);
+ sbus_strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT, direction);
spin_unlock_irqrestore(&iommu->lock, flags);
}
size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base;
spin_lock_irqsave(&iommu->lock, flags);
- strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT);
+ sbus_strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT, direction);
spin_unlock_irqrestore(&iommu->lock, flags);
}
#undef bogon
-#define NUM_SYSIO_OFFSETS (sizeof(sysio_irq_offsets) / sizeof(sysio_irq_offsets[0]))
+#define NUM_SYSIO_OFFSETS ARRAY_SIZE(sysio_irq_offsets)
/* Convert Interrupt Mapping register pointer to associated
* Interrupt Clear register pointer, SYSIO specific version.
/* Error interrupt handling. */
#define SYSIO_UE_AFSR 0x0030UL
#define SYSIO_UE_AFAR 0x0038UL
-#define SYSIO_UEAFSR_PPIO 0x8000000000000000 /* Primary PIO is cause */
-#define SYSIO_UEAFSR_PDRD 0x4000000000000000 /* Primary DVMA read is cause */
-#define SYSIO_UEAFSR_PDWR 0x2000000000000000 /* Primary DVMA write is cause */
-#define SYSIO_UEAFSR_SPIO 0x1000000000000000 /* Secondary PIO is cause */
-#define SYSIO_UEAFSR_SDRD 0x0800000000000000 /* Secondary DVMA read is cause */
-#define SYSIO_UEAFSR_SDWR 0x0400000000000000 /* Secondary DVMA write is cause*/
-#define SYSIO_UEAFSR_RESV1 0x03ff000000000000 /* Reserved */
-#define SYSIO_UEAFSR_DOFF 0x0000e00000000000 /* Doubleword Offset */
-#define SYSIO_UEAFSR_SIZE 0x00001c0000000000 /* Bad transfer size is 2**SIZE */
-#define SYSIO_UEAFSR_MID 0x000003e000000000 /* UPA MID causing the fault */
-#define SYSIO_UEAFSR_RESV2 0x0000001fffffffff /* Reserved */
+#define SYSIO_UEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */
+#define SYSIO_UEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */
+#define SYSIO_UEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */
+#define SYSIO_UEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO is cause */
+#define SYSIO_UEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */
+#define SYSIO_UEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/
+#define SYSIO_UEAFSR_RESV1 0x03ff000000000000UL /* Reserved */
+#define SYSIO_UEAFSR_DOFF 0x0000e00000000000UL /* Doubleword Offset */
+#define SYSIO_UEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */
+#define SYSIO_UEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */
+#define SYSIO_UEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */
static irqreturn_t sysio_ue_handler(int irq, void *dev_id, struct pt_regs *regs)
{
struct sbus_bus *sbus = dev_id;
#define SYSIO_CE_AFSR 0x0040UL
#define SYSIO_CE_AFAR 0x0048UL
-#define SYSIO_CEAFSR_PPIO 0x8000000000000000 /* Primary PIO is cause */
-#define SYSIO_CEAFSR_PDRD 0x4000000000000000 /* Primary DVMA read is cause */
-#define SYSIO_CEAFSR_PDWR 0x2000000000000000 /* Primary DVMA write is cause */
-#define SYSIO_CEAFSR_SPIO 0x1000000000000000 /* Secondary PIO is cause */
-#define SYSIO_CEAFSR_SDRD 0x0800000000000000 /* Secondary DVMA read is cause */
-#define SYSIO_CEAFSR_SDWR 0x0400000000000000 /* Secondary DVMA write is cause*/
-#define SYSIO_CEAFSR_RESV1 0x0300000000000000 /* Reserved */
-#define SYSIO_CEAFSR_ESYND 0x00ff000000000000 /* Syndrome Bits */
-#define SYSIO_CEAFSR_DOFF 0x0000e00000000000 /* Double Offset */
-#define SYSIO_CEAFSR_SIZE 0x00001c0000000000 /* Bad transfer size is 2**SIZE */
-#define SYSIO_CEAFSR_MID 0x000003e000000000 /* UPA MID causing the fault */
-#define SYSIO_CEAFSR_RESV2 0x0000001fffffffff /* Reserved */
+#define SYSIO_CEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */
+#define SYSIO_CEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */
+#define SYSIO_CEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */
+#define SYSIO_CEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO cause */
+#define SYSIO_CEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */
+#define SYSIO_CEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/
+#define SYSIO_CEAFSR_RESV1 0x0300000000000000UL /* Reserved */
+#define SYSIO_CEAFSR_ESYND 0x00ff000000000000UL /* Syndrome Bits */
+#define SYSIO_CEAFSR_DOFF 0x0000e00000000000UL /* Double Offset */
+#define SYSIO_CEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */
+#define SYSIO_CEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */
+#define SYSIO_CEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */
static irqreturn_t sysio_ce_handler(int irq, void *dev_id, struct pt_regs *regs)
{
struct sbus_bus *sbus = dev_id;
#define SYSIO_SBUS_AFSR 0x2010UL
#define SYSIO_SBUS_AFAR 0x2018UL
-#define SYSIO_SBAFSR_PLE 0x8000000000000000 /* Primary Late PIO Error */
-#define SYSIO_SBAFSR_PTO 0x4000000000000000 /* Primary SBUS Timeout */
-#define SYSIO_SBAFSR_PBERR 0x2000000000000000 /* Primary SBUS Error ACK */
-#define SYSIO_SBAFSR_SLE 0x1000000000000000 /* Secondary Late PIO Error */
-#define SYSIO_SBAFSR_STO 0x0800000000000000 /* Secondary SBUS Timeout */
-#define SYSIO_SBAFSR_SBERR 0x0400000000000000 /* Secondary SBUS Error ACK */
-#define SYSIO_SBAFSR_RESV1 0x03ff000000000000 /* Reserved */
-#define SYSIO_SBAFSR_RD 0x0000800000000000 /* Primary was late PIO read */
-#define SYSIO_SBAFSR_RESV2 0x0000600000000000 /* Reserved */
-#define SYSIO_SBAFSR_SIZE 0x00001c0000000000 /* Size of transfer */
-#define SYSIO_SBAFSR_MID 0x000003e000000000 /* MID causing the error */
-#define SYSIO_SBAFSR_RESV3 0x0000001fffffffff /* Reserved */
+#define SYSIO_SBAFSR_PLE 0x8000000000000000UL /* Primary Late PIO Error */
+#define SYSIO_SBAFSR_PTO 0x4000000000000000UL /* Primary SBUS Timeout */
+#define SYSIO_SBAFSR_PBERR 0x2000000000000000UL /* Primary SBUS Error ACK */
+#define SYSIO_SBAFSR_SLE 0x1000000000000000UL /* Secondary Late PIO Error */
+#define SYSIO_SBAFSR_STO 0x0800000000000000UL /* Secondary SBUS Timeout */
+#define SYSIO_SBAFSR_SBERR 0x0400000000000000UL /* Secondary SBUS Error ACK */
+#define SYSIO_SBAFSR_RESV1 0x03ff000000000000UL /* Reserved */
+#define SYSIO_SBAFSR_RD 0x0000800000000000UL /* Primary was late PIO read */
+#define SYSIO_SBAFSR_RESV2 0x0000600000000000UL /* Reserved */
+#define SYSIO_SBAFSR_SIZE 0x00001c0000000000UL /* Size of transfer */
+#define SYSIO_SBAFSR_MID 0x000003e000000000UL /* MID causing the error */
+#define SYSIO_SBAFSR_RESV3 0x0000001fffffffffUL /* Reserved */
static irqreturn_t sysio_sbus_error_handler(int irq, void *dev_id, struct pt_regs *regs)
{
struct sbus_bus *sbus = dev_id;
}
#define ECC_CONTROL 0x0020UL
-#define SYSIO_ECNTRL_ECCEN 0x8000000000000000 /* Enable ECC Checking */
-#define SYSIO_ECNTRL_UEEN 0x4000000000000000 /* Enable UE Interrupts */
-#define SYSIO_ECNTRL_CEEN 0x2000000000000000 /* Enable CE Interrupts */
+#define SYSIO_ECNTRL_ECCEN 0x8000000000000000UL /* Enable ECC Checking */
+#define SYSIO_ECNTRL_UEEN 0x4000000000000000UL /* Enable UE Interrupts */
+#define SYSIO_ECNTRL_CEEN 0x2000000000000000UL /* Enable CE Interrupts */
#define SYSIO_UE_INO 0x34
#define SYSIO_CE_INO 0x35