* Copyright 2002 Andi Kleen, SuSE Labs.
*/
-#include <linux/config.h>
#include <linux/types.h>
#include <linux/ctype.h>
#include <linux/agp_backend.h>
#include <linux/module.h>
#include <linux/topology.h>
#include <linux/interrupt.h>
+#include <linux/bitops.h>
#include <asm/atomic.h>
#include <asm/io.h>
#include <asm/mtrr.h>
-#include <asm/bitops.h>
#include <asm/pgtable.h>
#include <asm/proto.h>
#include <asm/cacheflush.h>
#include <asm/kdebug.h>
-
-#ifdef CONFIG_PREEMPT
-#define preempt_atomic() in_atomic()
-#else
-#define preempt_atomic() 1
-#endif
-
-dma_addr_t bad_dma_address;
+#include <asm/swiotlb.h>
+#include <asm/dma.h>
+#include <asm/k8.h>
unsigned long iommu_bus_base; /* GART remapping area (physical) */
static unsigned long iommu_size; /* size of remapping area bytes */
u32 *iommu_gatt_base; /* Remapping table */
-int no_iommu;
-static int no_agp;
-#ifdef CONFIG_IOMMU_DEBUG
-int panic_on_overflow = 1;
-int force_iommu = 1;
-#else
-int panic_on_overflow = 0;
-int force_iommu = 0;
-#endif
-int iommu_merge = 0;
-int iommu_sac_force = 0;
-
/* If this is disabled the IOMMU will use an optimized flushing strategy
of only flushing when an mapping is reused. With it true the GART is flushed
for every mapping. Problem is that doing the lazy flush seems to trigger
also seen with Qlogic at least). */
int iommu_fullflush = 1;
-#define MAX_NB 8
-
/* Allocation bitmap for the remapping area */
-static spinlock_t iommu_bitmap_lock = SPIN_LOCK_UNLOCKED;
+static DEFINE_SPINLOCK(iommu_bitmap_lock);
static unsigned long *iommu_gart_bitmap; /* guarded by iommu_bitmap_lock */
+static u32 gart_unmapped_entry;
+
#define GPTE_VALID 1
#define GPTE_COHERENT 2
#define GPTE_ENCODE(x) \
#define to_pages(addr,size) \
(round_up(((addr) & ~PAGE_MASK) + (size), PAGE_SIZE) >> PAGE_SHIFT)
-#define for_all_nb(dev) \
- dev = NULL; \
- while ((dev = pci_find_device(PCI_VENDOR_ID_AMD, 0x1103, dev))!=NULL)\
- if (dev->bus->number == 0 && \
- (PCI_SLOT(dev->devfn) >= 24) && (PCI_SLOT(dev->devfn) <= 31))
-
-static struct pci_dev *northbridges[MAX_NB];
-static u32 northbridge_flush_word[MAX_NB];
-
#define EMERGENCY_PAGES 32 /* = 128KB */
#ifdef CONFIG_AGP
static unsigned long next_bit; /* protected by iommu_bitmap_lock */
static int need_flush; /* global flush state. set for each gart wrap */
-static dma_addr_t pci_map_area(struct pci_dev *dev, unsigned long phys_mem,
- size_t size, int dir);
static unsigned long alloc_iommu(int size)
{
offset = find_next_zero_string(iommu_gart_bitmap,next_bit,iommu_pages,size);
if (offset == -1) {
need_flush = 1;
- offset = find_next_zero_string(iommu_gart_bitmap,0,next_bit,size);
+ offset = find_next_zero_string(iommu_gart_bitmap,0,iommu_pages,size);
}
if (offset != -1) {
set_bit_string(iommu_gart_bitmap, offset, size);
static void free_iommu(unsigned long offset, int size)
{
unsigned long flags;
- if (size == 1) {
- clear_bit(offset, iommu_gart_bitmap);
- return;
- }
spin_lock_irqsave(&iommu_bitmap_lock, flags);
__clear_bit_string(iommu_gart_bitmap, offset, size);
spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
/*
* Use global flush state to avoid races with multiple flushers.
*/
-static void flush_gart(struct pci_dev *dev)
+static void flush_gart(void)
{
unsigned long flags;
- int bus = dev ? dev->bus->number : -1;
- cpumask_const_t bus_cpumask = pcibus_to_cpumask(bus);
- int flushed = 0;
- int i;
-
spin_lock_irqsave(&iommu_bitmap_lock, flags);
- if (need_flush) {
- for (i = 0; i < MAX_NB; i++) {
- u32 w;
- if (!northbridges[i])
- continue;
- if (bus >= 0 && !(cpu_isset_const(i, bus_cpumask)))
- continue;
- pci_write_config_dword(northbridges[i], 0x9c,
- northbridge_flush_word[i] | 1);
- /* Make sure the hardware actually executed the flush. */
- do {
- pci_read_config_dword(northbridges[i], 0x9c, &w);
- } while (w & 1);
- flushed++;
- }
- if (!flushed)
- printk("nothing to flush? %d\n", bus);
+ if (need_flush) {
+ k8_flush_garts();
need_flush = 0;
}
spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
}
-/*
- * Allocate memory for a consistent mapping.
- */
-void *pci_alloc_consistent(struct pci_dev *hwdev, size_t size,
- dma_addr_t *dma_handle)
-{
- void *memory;
- int gfp = preempt_atomic() ? GFP_ATOMIC : GFP_KERNEL;
- unsigned long dma_mask = 0;
- u64 bus;
-
- if (hwdev)
- dma_mask = hwdev->dev.coherent_dma_mask;
- if (dma_mask == 0)
- dma_mask = 0xffffffff;
-
- /* Kludge to make it bug-to-bug compatible with i386. i386
- uses the normal dma_mask for alloc_consistent. */
- if (hwdev)
- dma_mask &= hwdev->dma_mask;
-
- again:
- memory = (void *)__get_free_pages(gfp, get_order(size));
- if (memory == NULL)
- return NULL;
-
- {
- int high, mmu;
- bus = virt_to_bus(memory);
- high = (bus + size) >= dma_mask;
- mmu = high;
- if (force_iommu && !(gfp & GFP_DMA))
- mmu = 1;
- if (no_iommu || dma_mask < 0xffffffffUL) {
- if (high) {
- if (!(gfp & GFP_DMA)) {
- gfp |= GFP_DMA;
- goto again;
- }
- goto free;
- }
- mmu = 0;
- }
- memset(memory, 0, size);
- if (!mmu) {
- *dma_handle = virt_to_bus(memory);
- return memory;
- }
- }
-
- *dma_handle = pci_map_area(hwdev, bus, size, PCI_DMA_BIDIRECTIONAL);
- if (*dma_handle == bad_dma_address)
- goto error;
- flush_gart(hwdev);
- return memory;
-
-error:
- if (panic_on_overflow)
- panic("pci_alloc_consistent: overflow %lu bytes\n", size);
-free:
- free_pages((unsigned long)memory, get_order(size));
- return NULL;
-}
-
-/*
- * Unmap consistent memory.
- * The caller must ensure that the device has finished accessing the mapping.
- */
-void pci_free_consistent(struct pci_dev *hwdev, size_t size,
- void *vaddr, dma_addr_t bus)
-{
- pci_unmap_single(hwdev, bus, size, 0);
- free_pages((unsigned long)vaddr, get_order(size));
-}
-
#ifdef CONFIG_IOMMU_LEAK
#define SET_LEAK(x) if (iommu_leak_tab) \
iommu_leak_tab[x] = __builtin_return_address(0);
#define CLEAR_LEAK(x) if (iommu_leak_tab) \
- iommu_leak_tab[x] = 0;
+ iommu_leak_tab[x] = NULL;
/* Debugging aid for drivers that don't free their IOMMU tables */
static void **iommu_leak_tab;
#define CLEAR_LEAK(x)
#endif
-static void iommu_full(struct pci_dev *dev, size_t size, int dir)
+static void iommu_full(struct device *dev, size_t size, int dir)
{
/*
* Ran out of IOMMU space for this operation. This is very bad.
*/
printk(KERN_ERR
- "PCI-DMA: Out of IOMMU space for %lu bytes at device %s[%s]\n",
- size, dev ? pci_pretty_name(dev) : "", dev ? dev->slot_name : "?");
+ "PCI-DMA: Out of IOMMU space for %lu bytes at device %s\n",
+ size, dev->bus_id);
if (size > PAGE_SIZE*EMERGENCY_PAGES) {
if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL)
- panic("PCI-DMA: Memory will be corrupted\n");
+ panic("PCI-DMA: Memory would be corrupted\n");
if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL)
- panic("PCI-DMA: Random memory will be DMAed\n");
+ panic(KERN_ERR "PCI-DMA: Random memory would be DMAed\n");
}
#ifdef CONFIG_IOMMU_LEAK
#endif
}
-static inline int need_iommu(struct pci_dev *dev, unsigned long addr, size_t size)
+static inline int need_iommu(struct device *dev, unsigned long addr, size_t size)
{
- u64 mask = dev ? dev->dma_mask : 0xffffffff;
+ u64 mask = *dev->dma_mask;
int high = addr + size >= mask;
int mmu = high;
if (force_iommu)
mmu = 1;
- if (no_iommu) {
- if (high)
- panic("PCI-DMA: high address but no IOMMU.\n");
- mmu = 0;
- }
return mmu;
}
-static inline int nonforced_iommu(struct pci_dev *dev, unsigned long addr, size_t size)
+static inline int nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
{
- u64 mask = dev ? dev->dma_mask : 0xffffffff;
+ u64 mask = *dev->dma_mask;
int high = addr + size >= mask;
int mmu = high;
- if (no_iommu) {
- if (high)
- panic("PCI-DMA: high address but no IOMMU.\n");
- mmu = 0;
- }
return mmu;
}
/* Map a single continuous physical area into the IOMMU.
* Caller needs to check if the iommu is needed and flush.
*/
-static dma_addr_t pci_map_area(struct pci_dev *dev, unsigned long phys_mem,
+static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
size_t size, int dir)
{
unsigned long npages = to_pages(phys_mem, size);
if (!nonforced_iommu(dev, phys_mem, size))
return phys_mem;
if (panic_on_overflow)
- panic("pci_map_area overflow %lu bytes\n", size);
+ panic("dma_map_area overflow %lu bytes\n", size);
iommu_full(dev, size, dir);
return bad_dma_address;
}
return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK);
}
+static dma_addr_t gart_map_simple(struct device *dev, char *buf,
+ size_t size, int dir)
+{
+ dma_addr_t map = dma_map_area(dev, virt_to_bus(buf), size, dir);
+ flush_gart();
+ return map;
+}
+
/* Map a single area into the IOMMU */
-dma_addr_t pci_map_single(struct pci_dev *dev, void *addr, size_t size, int dir)
-{
+dma_addr_t gart_map_single(struct device *dev, void *addr, size_t size, int dir)
+{
unsigned long phys_mem, bus;
- BUG_ON(dir == PCI_DMA_NONE);
-
-#ifdef CONFIG_SWIOTLB
- if (swiotlb)
- return swiotlb_map_single(&dev->dev,addr,size,dir);
-#endif
+ if (!dev)
+ dev = &fallback_dev;
phys_mem = virt_to_phys(addr);
if (!need_iommu(dev, phys_mem, size))
return phys_mem;
- bus = pci_map_area(dev, phys_mem, size, dir);
- flush_gart(dev);
+ bus = gart_map_simple(dev, addr, size, dir);
return bus;
-}
+}
+
+/*
+ * Free a DMA mapping.
+ */
+void gart_unmap_single(struct device *dev, dma_addr_t dma_addr,
+ size_t size, int direction)
+{
+ unsigned long iommu_page;
+ int npages;
+ int i;
+
+ if (dma_addr < iommu_bus_base + EMERGENCY_PAGES*PAGE_SIZE ||
+ dma_addr >= iommu_bus_base + iommu_size)
+ return;
+ iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT;
+ npages = to_pages(dma_addr, size);
+ for (i = 0; i < npages; i++) {
+ iommu_gatt_base[iommu_page + i] = gart_unmapped_entry;
+ CLEAR_LEAK(iommu_page + i);
+ }
+ free_iommu(iommu_page, npages);
+}
+
+/*
+ * Wrapper for pci_unmap_single working with scatterlists.
+ */
+void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
+{
+ int i;
-/* Fallback for pci_map_sg in case of overflow */
-static int pci_map_sg_nonforce(struct pci_dev *dev, struct scatterlist *sg,
+ for (i = 0; i < nents; i++) {
+ struct scatterlist *s = &sg[i];
+ if (!s->dma_length || !s->length)
+ break;
+ gart_unmap_single(dev, s->dma_address, s->dma_length, dir);
+ }
+}
+
+/* Fallback for dma_map_sg in case of overflow */
+static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
int nents, int dir)
{
int i;
#ifdef CONFIG_IOMMU_DEBUG
- printk(KERN_DEBUG "pci_map_sg overflow\n");
+ printk(KERN_DEBUG "dma_map_sg overflow\n");
#endif
for (i = 0; i < nents; i++ ) {
struct scatterlist *s = &sg[i];
unsigned long addr = page_to_phys(s->page) + s->offset;
if (nonforced_iommu(dev, addr, s->length)) {
- addr = pci_map_area(dev, addr, s->length, dir);
+ addr = dma_map_area(dev, addr, s->length, dir);
if (addr == bad_dma_address) {
if (i > 0)
- pci_unmap_sg(dev, sg, i, dir);
+ gart_unmap_sg(dev, sg, i, dir);
nents = 0;
sg[0].dma_length = 0;
break;
s->dma_address = addr;
s->dma_length = s->length;
}
- flush_gart(dev);
+ flush_gart();
return nents;
}
/* Map multiple scatterlist entries continuous into the first. */
-static int __pci_map_cont(struct scatterlist *sg, int start, int stopat,
+static int __dma_map_cont(struct scatterlist *sg, int start, int stopat,
struct scatterlist *sout, unsigned long pages)
{
unsigned long iommu_start = alloc_iommu(pages);
SET_LEAK(iommu_page);
addr += PAGE_SIZE;
iommu_page++;
- }
+ }
}
BUG_ON(iommu_page - iommu_start != pages);
return 0;
}
-static inline int pci_map_cont(struct scatterlist *sg, int start, int stopat,
+static inline int dma_map_cont(struct scatterlist *sg, int start, int stopat,
struct scatterlist *sout,
unsigned long pages, int need)
{
sout->dma_length = sg[start].length;
return 0;
}
- return __pci_map_cont(sg, start, stopat, sout, pages);
+ return __dma_map_cont(sg, start, stopat, sout, pages);
}
/*
* DMA map all entries in a scatterlist.
* Merge chunks that have page aligned sizes into a continuous mapping.
- */
-int pci_map_sg(struct pci_dev *dev, struct scatterlist *sg, int nents, int dir)
+ */
+int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
{
int i;
int out;
unsigned long pages = 0;
int need = 0, nextneed;
- BUG_ON(dir == PCI_DMA_NONE);
if (nents == 0)
return 0;
-#ifdef CONFIG_SWIOTLB
- if (swiotlb)
- return swiotlb_map_sg(&dev->dev,sg,nents,dir);
-#endif
+ if (!dev)
+ dev = &fallback_dev;
out = 0;
start = 0;
boundary and the new one doesn't have an offset. */
if (!iommu_merge || !nextneed || !need || s->offset ||
(ps->offset + ps->length) % PAGE_SIZE) {
- if (pci_map_cont(sg, start, i, sg+out, pages,
+ if (dma_map_cont(sg, start, i, sg+out, pages,
need) < 0)
goto error;
out++;
pages = 0;
start = i;
}
- }
+ }
need = nextneed;
pages += to_pages(s->offset, s->length);
}
- if (pci_map_cont(sg, start, i, sg+out, pages, need) < 0)
+ if (dma_map_cont(sg, start, i, sg+out, pages, need) < 0)
goto error;
out++;
- flush_gart(dev);
+ flush_gart();
if (out < nents)
sg[out].dma_length = 0;
return out;
error:
- flush_gart(NULL);
- pci_unmap_sg(dev, sg, nents, dir);
- /* When it was forced try again unforced */
- if (force_iommu)
- return pci_map_sg_nonforce(dev, sg, nents, dir);
+ flush_gart();
+ gart_unmap_sg(dev, sg, nents, dir);
+ /* When it was forced or merged try again in a dumb way */
+ if (force_iommu || iommu_merge) {
+ out = dma_map_sg_nonforce(dev, sg, nents, dir);
+ if (out > 0)
+ return out;
+ }
if (panic_on_overflow)
- panic("pci_map_sg: overflow on %lu pages\n", pages);
+ panic("dma_map_sg: overflow on %lu pages\n", pages);
iommu_full(dev, pages << PAGE_SHIFT, dir);
for (i = 0; i < nents; i++)
sg[i].dma_address = bad_dma_address;
return 0;
}
-/*
- * Free a PCI mapping.
- */
-void pci_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr,
- size_t size, int direction)
-{
- unsigned long iommu_page;
- int npages;
- int i;
-
-#ifdef CONFIG_SWIOTLB
- if (swiotlb) {
- swiotlb_unmap_single(&hwdev->dev,dma_addr,size,direction);
- return;
- }
-#endif
-
- if (dma_addr < iommu_bus_base + EMERGENCY_PAGES*PAGE_SIZE ||
- dma_addr >= iommu_bus_base + iommu_size)
- return;
- iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT;
- npages = to_pages(dma_addr, size);
- for (i = 0; i < npages; i++) {
- iommu_gatt_base[iommu_page + i] = 0;
- CLEAR_LEAK(iommu_page + i);
- }
- free_iommu(iommu_page, npages);
-}
-
-/*
- * Wrapper for pci_unmap_single working with scatterlists.
- */
-void pci_unmap_sg(struct pci_dev *dev, struct scatterlist *sg, int nents,
- int dir)
-{
- int i;
- for (i = 0; i < nents; i++) {
- struct scatterlist *s = &sg[i];
- if (!s->dma_length || !s->length)
- break;
- pci_unmap_single(dev, s->dma_address, s->dma_length, dir);
- }
-}
-
-int pci_dma_supported(struct pci_dev *dev, u64 mask)
-{
- /* Copied from i386. Doesn't make much sense, because it will
- only work for pci_alloc_consistent.
- The caller just has to use GFP_DMA in this case. */
- if (mask < 0x00ffffff)
- return 0;
-
- /* Tell the device to use SAC when IOMMU force is on.
- This allows the driver to use cheaper accesses in some cases.
-
- Problem with this is that if we overflow the IOMMU area
- and return DAC as fallback address the device may not handle it correctly.
-
- As a special case some controllers have a 39bit address mode
- that is as efficient as 32bit (aic79xx). Don't force SAC for these.
- Assume all masks <= 40 bits are of this type. Normally this doesn't
- make any difference, but gives more gentle handling of IOMMU overflow. */
- if (iommu_sac_force && (mask >= 0xffffffffffULL)) {
- printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->slot_name,mask);
- return 0;
- }
-
- return 1;
-}
-
-EXPORT_SYMBOL(pci_unmap_sg);
-EXPORT_SYMBOL(pci_map_sg);
-EXPORT_SYMBOL(pci_map_single);
-EXPORT_SYMBOL(pci_unmap_single);
-EXPORT_SYMBOL(pci_dma_supported);
-EXPORT_SYMBOL(no_iommu);
-EXPORT_SYMBOL(force_iommu);
-EXPORT_SYMBOL(bad_dma_address);
-EXPORT_SYMBOL(iommu_merge);
+static int no_agp;
static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size)
{
void *gatt;
unsigned aper_base, new_aper_base;
unsigned aper_size, gatt_size, new_aper_size;
-
+ int i;
+
+ printk(KERN_INFO "PCI-DMA: Disabling AGP.\n");
aper_size = aper_base = info->aper_size = 0;
- for_all_nb(dev) {
+ dev = NULL;
+ for (i = 0; i < num_k8_northbridges; i++) {
+ dev = k8_northbridges[i];
new_aper_base = read_aperture(dev, &new_aper_size);
if (!new_aper_base)
goto nommu;
panic("Cannot allocate GATT table");
memset(gatt, 0, gatt_size);
agp_gatt_table = gatt;
-
- for_all_nb(dev) {
+
+ for (i = 0; i < num_k8_northbridges; i++) {
u32 ctl;
u32 gatt_reg;
+ dev = k8_northbridges[i];
gatt_reg = __pa(gatt) >> 12;
gatt_reg <<= 4;
pci_write_config_dword(dev, 0x98, gatt_reg);
pci_write_config_dword(dev, 0x90, ctl);
}
- flush_gart(NULL);
+ flush_gart();
printk("PCI-DMA: aperture base @ %x size %u KB\n",aper_base, aper_size>>10);
return 0;
nommu:
/* Should not happen anymore */
printk(KERN_ERR "PCI-DMA: More than 4GB of RAM and no IOMMU\n"
- KERN_ERR "PCI-DMA: 32bit PCI IO may malfunction.");
+ KERN_ERR "PCI-DMA: 32bit PCI IO may malfunction.\n");
return -1;
}
extern int agp_amd64_init(void);
-static int __init pci_iommu_init(void)
+static struct dma_mapping_ops gart_dma_ops = {
+ .mapping_error = NULL,
+ .map_single = gart_map_single,
+ .map_simple = gart_map_simple,
+ .unmap_single = gart_unmap_single,
+ .sync_single_for_cpu = NULL,
+ .sync_single_for_device = NULL,
+ .sync_single_range_for_cpu = NULL,
+ .sync_single_range_for_device = NULL,
+ .sync_sg_for_cpu = NULL,
+ .sync_sg_for_device = NULL,
+ .map_sg = gart_map_sg,
+ .unmap_sg = gart_unmap_sg,
+};
+
+void __init gart_iommu_init(void)
{
struct agp_kern_info info;
unsigned long aper_size;
unsigned long iommu_start;
- struct pci_dev *dev;
-
+ unsigned long scratch;
+ long i;
+
+ if (cache_k8_northbridges() < 0 || num_k8_northbridges == 0) {
+ printk(KERN_INFO "PCI-GART: No AMD northbridge found.\n");
+ return;
+ }
#ifndef CONFIG_AGP_AMD64
no_agp = 1;
/* Add other K8 AGP bridge drivers here */
no_agp = no_agp ||
(agp_amd64_init() < 0) ||
- (agp_copy_info(&info) < 0);
+ (agp_copy_info(agp_bridge, &info) < 0);
#endif
- if (swiotlb) {
- no_iommu = 1;
- printk(KERN_INFO "PCI-DMA: Using software bounce buffering for IO (SWIOTLB)\n");
- return -1;
- }
-
- if (no_iommu || (!force_iommu && end_pfn < 0xffffffff>>PAGE_SHIFT) ||
- !iommu_aperture) {
- printk(KERN_INFO "PCI-DMA: Disabling IOMMU.\n");
- no_iommu = 1;
- return -1;
- }
+ if (swiotlb)
+ return;
- if (no_agp) {
- int err = -1;
- printk(KERN_INFO "PCI-DMA: Disabling AGP.\n");
- no_agp = 1;
- if (force_iommu || end_pfn >= 0xffffffff>>PAGE_SHIFT)
- err = init_k8_gatt(&info);
- if (err < 0) {
- printk(KERN_INFO "PCI-DMA: Disabling IOMMU.\n");
- no_iommu = 1;
- return -1;
+ /* Did we detect a different HW IOMMU? */
+ if (iommu_detected && !iommu_aperture)
+ return;
+
+ if (no_iommu ||
+ (!force_iommu && end_pfn <= MAX_DMA32_PFN) ||
+ !iommu_aperture ||
+ (no_agp && init_k8_gatt(&info) < 0)) {
+ if (end_pfn > MAX_DMA32_PFN) {
+ printk(KERN_ERR "WARNING more than 4GB of memory "
+ "but GART IOMMU not available.\n"
+ KERN_ERR "WARNING 32bit PCI may malfunction.\n");
}
- }
-
+ return;
+ }
+
+ printk(KERN_INFO "PCI-DMA: using GART IOMMU.\n");
aper_size = info.aper_size * 1024 * 1024;
iommu_size = check_iommu_size(info.aper_base, aper_size);
iommu_pages = iommu_size >> PAGE_SHIFT;
*/
clear_kernel_mapping((unsigned long)__va(iommu_bus_base), iommu_size);
- for_all_nb(dev) {
- u32 flag;
- int cpu = PCI_SLOT(dev->devfn) - 24;
- if (cpu >= MAX_NB)
- continue;
- northbridges[cpu] = dev;
- pci_read_config_dword(dev, 0x9c, &flag); /* cache flush word */
- northbridge_flush_word[cpu] = flag;
- }
-
- flush_gart(NULL);
-
- return 0;
+ /*
+ * Try to workaround a bug (thanks to BenH)
+ * Set unmapped entries to a scratch page instead of 0.
+ * Any prefetches that hit unmapped entries won't get an bus abort
+ * then.
+ */
+ scratch = get_zeroed_page(GFP_KERNEL);
+ if (!scratch)
+ panic("Cannot allocate iommu scratch page");
+ gart_unmapped_entry = GPTE_ENCODE(__pa(scratch));
+ for (i = EMERGENCY_PAGES; i < iommu_pages; i++)
+ iommu_gatt_base[i] = gart_unmapped_entry;
+
+ flush_gart();
+ dma_ops = &gart_dma_ops;
}
-/* Must execute after PCI subsystem */
-fs_initcall(pci_iommu_init);
-
-/* iommu=[size][,noagp][,off][,force][,noforce][,leak][,memaper[=order]][,merge]
- [,forcesac][,fullflush][,nomerge]
- size set size of iommu (in bytes)
- noagp don't initialize the AGP driver and use full aperture.
- off don't use the IOMMU
- leak turn on simple iommu leak tracing (only when CONFIG_IOMMU_LEAK is on)
- memaper[=order] allocate an own aperture over RAM with size 32MB^order.
- noforce don't force IOMMU usage. Default.
- force Force IOMMU.
- merge Do SG merging. Implies force (experimental)
- nomerge Don't do SG merging.
- forcesac For SAC mode for masks <40bits (experimental)
- fullflush Flush IOMMU on each allocation (default)
- nofullflush Don't use IOMMU fullflush
- allowed overwrite iommu off workarounds for specific chipsets.
- soft Use software bounce buffering (default for Intel machines)
-*/
-__init int iommu_setup(char *opt)
-{
- int arg;
- char *p = opt;
-
- for (;;) {
- if (!memcmp(p,"noagp", 5))
- no_agp = 1;
- if (!memcmp(p,"off", 3))
- no_iommu = 1;
- if (!memcmp(p,"force", 5)) {
- force_iommu = 1;
- iommu_aperture_allowed = 1;
- }
- if (!memcmp(p,"allowed",7))
- iommu_aperture_allowed = 1;
- if (!memcmp(p,"noforce", 7)) {
- iommu_merge = 0;
- force_iommu = 0;
- }
- if (!memcmp(p, "memaper", 7)) {
- fallback_aper_force = 1;
- p += 7;
- if (*p == '=' && get_option(&p, &arg))
- fallback_aper_order = arg;
- }
- if (!memcmp(p, "panic", 5))
- panic_on_overflow = 1;
- if (!memcmp(p, "nopanic", 7))
- panic_on_overflow = 0;
- if (!memcmp(p, "merge", 5)) {
- iommu_merge = 1;
- force_iommu = 1;
- }
- if (!memcmp(p, "nomerge", 7))
- iommu_merge = 0;
- if (!memcmp(p, "forcesac", 8))
- iommu_sac_force = 1;
- if (!memcmp(p, "fullflush", 9))
- iommu_fullflush = 1;
- if (!memcmp(p, "nofullflush", 11))
- iommu_fullflush = 0;
- if (!memcmp(p, "soft", 4))
- swiotlb = 1;
+void gart_parse_options(char *p)
+{
+ int arg;
+
#ifdef CONFIG_IOMMU_LEAK
- if (!memcmp(p,"leak", 4)) {
- leak_trace = 1;
- p += 4;
- if (*p == '=') ++p;
- if (isdigit(*p) && get_option(&p, &arg))
- iommu_leak_pages = arg;
- } else
+ if (!strncmp(p,"leak",4)) {
+ leak_trace = 1;
+ p += 4;
+ if (*p == '=') ++p;
+ if (isdigit(*p) && get_option(&p, &arg))
+ iommu_leak_pages = arg;
+ }
#endif
- if (isdigit(*p) && get_option(&p, &arg))
- iommu_size = arg;
- do {
- if (*p == ' ' || *p == 0)
- return 0;
- } while (*p++ != ',');
- }
- return 1;
-}
+ if (isdigit(*p) && get_option(&p, &arg))
+ iommu_size = arg;
+ if (!strncmp(p, "fullflush",8))
+ iommu_fullflush = 1;
+ if (!strncmp(p, "nofullflush",11))
+ iommu_fullflush = 0;
+ if (!strncmp(p,"noagp",5))
+ no_agp = 1;
+ if (!strncmp(p, "noaperture",10))
+ fix_aperture = 0;
+ /* duplicated from pci-dma.c */
+ if (!strncmp(p,"force",5))
+ iommu_aperture_allowed = 1;
+ if (!strncmp(p,"allowed",7))
+ iommu_aperture_allowed = 1;
+ if (!strncmp(p, "memaper", 7)) {
+ fallback_aper_force = 1;
+ p += 7;
+ if (*p == '=') {
+ ++p;
+ if (get_option(&p, &arg))
+ fallback_aper_order = arg;
+ }
+ }
+}
git://git.onelab.eu / linux-2.6.git / blobdiff