/*
- * Copyright (c) 2006 QLogic, Inc. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
#include <asm/pgtable.h>
#include "ipath_kernel.h"
+#include "ips_common.h"
#include "ipath_layer.h"
-#include "ipath_common.h"
static int ipath_open(struct inode *, struct file *);
static int ipath_close(struct inode *, struct file *);
* on to yet another method of dealing with this
*/
kinfo->spi_rcvhdr_base = (u64) pd->port_rcvhdrq_phys;
- kinfo->spi_rcvhdr_tailaddr = (u64)pd->port_rcvhdrqtailaddr_phys;
kinfo->spi_rcv_egrbufs = (u64) pd->port_rcvegr_phys;
kinfo->spi_pioavailaddr = (u64) dd->ipath_pioavailregs_phys;
kinfo->spi_status = (u64) kinfo->spi_pioavailaddr +
u16 lkey = key & 0x7FFF;
int ret;
- if (lkey == (IPATH_DEFAULT_P_KEY & 0x7FFF)) {
+ if (lkey == (IPS_DEFAULT_P_KEY & 0x7FFF)) {
/* nothing to do; this key always valid */
ret = 0;
goto bail;
unsigned e, egrcnt, alloced, egrperchunk, chunk, egrsize, egroff;
size_t size;
int ret;
- gfp_t gfp_flags;
-
- /*
- * GFP_USER, but without GFP_FS, so buffer cache can be
- * coalesced (we hope); otherwise, even at order 4,
- * heavy filesystem activity makes these fail, and we can
- * use compound pages.
- */
- gfp_flags = __GFP_WAIT | __GFP_IO | __GFP_COMP;
egrcnt = dd->ipath_rcvegrcnt;
/* TID number offset for this port */
* memory pressure (creating large files and then copying them over
* NFS while doing lots of MPI jobs), we hit some allocation
* failures, even though we can sleep... (2.6.10) Still get
- * failures at 64K. 32K is the lowest we can go without wasting
- * additional memory.
+ * failures at 64K. 32K is the lowest we can go without waiting
+ * more memory again. It seems likely that the coalescing in
+ * free_pages, etc. still has issues (as it has had previously
+ * during 2.6.x development).
*/
size = 0x8000;
alloced = ALIGN(egrsize * egrcnt, size);
goto bail_rcvegrbuf;
}
for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
+ /*
+ * GFP_USER, but without GFP_FS, so buffer cache can be
+ * coalesced (we hope); otherwise, even at order 4,
+ * heavy filesystem activity makes these fail
+ */
+ gfp_t gfp_flags = __GFP_WAIT | __GFP_IO | __GFP_COMP;
pd->port_rcvegrbuf[e] = dma_alloc_coherent(
&dd->pcidev->dev, size, &pd->port_rcvegrbuf_phys[e],
bail_rcvegrbuf_phys:
for (e = 0; e < pd->port_rcvegrbuf_chunks &&
- pd->port_rcvegrbuf[e]; e++) {
+ pd->port_rcvegrbuf[e]; e++)
dma_free_coherent(&dd->pcidev->dev, size,
pd->port_rcvegrbuf[e],
pd->port_rcvegrbuf_phys[e]);
- }
vfree(pd->port_rcvegrbuf_phys);
pd->port_rcvegrbuf_phys = NULL;
bail_rcvegrbuf:
{
int ret = 0;
struct ipath_devdata *dd = pd->port_dd;
+ u64 physaddr, uaddr, off, atmp;
+ struct page *pagep;
u32 head32;
+ u64 head;
/* for now, if major version is different, bail */
if ((uinfo->spu_userversion >> 16) != IPATH_USER_SWMAJOR) {
/* for now we do nothing with rcvhdrcnt: uinfo->spu_rcvhdrcnt */
+ /* set up for the rcvhdr Q tail register writeback to user memory */
+ if (!uinfo->spu_rcvhdraddr ||
+ !access_ok(VERIFY_WRITE, (u64 __user *) (unsigned long)
+ uinfo->spu_rcvhdraddr, sizeof(u64))) {
+ ipath_dbg("Port %d rcvhdrtail addr %llx not valid\n",
+ pd->port_port,
+ (unsigned long long) uinfo->spu_rcvhdraddr);
+ ret = -EINVAL;
+ goto done;
+ }
+
+ off = offset_in_page(uinfo->spu_rcvhdraddr);
+ uaddr = PAGE_MASK & (unsigned long) uinfo->spu_rcvhdraddr;
+ ret = ipath_get_user_pages_nocopy(uaddr, &pagep);
+ if (ret) {
+ dev_info(&dd->pcidev->dev, "Failed to lookup and lock "
+ "address %llx for rcvhdrtail: errno %d\n",
+ (unsigned long long) uinfo->spu_rcvhdraddr, -ret);
+ goto done;
+ }
+ ipath_stats.sps_pagelocks++;
+ pd->port_rcvhdrtail_uaddr = uaddr;
+ pd->port_rcvhdrtail_pagep = pagep;
+ pd->port_rcvhdrtail_kvaddr =
+ page_address(pagep);
+ pd->port_rcvhdrtail_kvaddr += off;
+ physaddr = page_to_phys(pagep) + off;
+ ipath_cdbg(VERBOSE, "port %d user addr %llx hdrtailaddr, %llx "
+ "physical (off=%llx)\n",
+ pd->port_port,
+ (unsigned long long) uinfo->spu_rcvhdraddr,
+ (unsigned long long) physaddr, (unsigned long long) off);
+ ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdrtailaddr,
+ pd->port_port, physaddr);
+ atmp = ipath_read_kreg64_port(dd,
+ dd->ipath_kregs->kr_rcvhdrtailaddr,
+ pd->port_port);
+ if (physaddr != atmp) {
+ ipath_dev_err(dd,
+ "Catastrophic software error, "
+ "RcvHdrTailAddr%u written as %llx, "
+ "read back as %llx\n", pd->port_port,
+ (unsigned long long) physaddr,
+ (unsigned long long) atmp);
+ ret = -EINVAL;
+ goto done;
+ }
+
/* for right now, kernel piobufs are at end, so port 1 is at 0 */
pd->port_piobufs = dd->ipath_piobufbase +
dd->ipath_pbufsport * (pd->port_port -
ret = ipath_create_user_egr(pd);
if (ret)
goto done;
+ /* enable receives now */
+ /* atomically set enable bit for this port */
+ set_bit(INFINIPATH_R_PORTENABLE_SHIFT + pd->port_port,
+ &dd->ipath_rcvctrl);
/*
- * set the eager head register for this port to the current values
+ * set the head registers for this port to the current values
* of the tail pointers, since we don't know if they were
* updated on last use of the port.
*/
+ head32 = ipath_read_ureg32(dd, ur_rcvhdrtail, pd->port_port);
+ head = (u64) head32;
+ ipath_write_ureg(dd, ur_rcvhdrhead, head, pd->port_port);
head32 = ipath_read_ureg32(dd, ur_rcvegrindextail, pd->port_port);
ipath_write_ureg(dd, ur_rcvegrindexhead, head32, pd->port_port);
dd->ipath_lastegrheads[pd->port_port] = -1;
dd->ipath_lastrcvhdrqtails[pd->port_port] = -1;
- ipath_cdbg(VERBOSE, "Wrote port%d egrhead %x from tail regs\n",
- pd->port_port, head32);
+ ipath_cdbg(VERBOSE, "Wrote port%d head %llx, egrhead %x from "
+ "tail regs\n", pd->port_port,
+ (unsigned long long) head, head32);
pd->port_tidcursor = 0; /* start at beginning after open */
/*
* now enable the port; the tail registers will be written to memory
* transition from 0 to 1, so clear it first, then set it as part of
* enabling the port. This will (very briefly) affect any other
* open ports, but it shouldn't be long enough to be an issue.
- * We explictly set the in-memory copy to 0 beforehand, so we don't
- * have to wait to be sure the DMA update has happened.
*/
- *pd->port_rcvhdrtail_kvaddr = 0ULL;
- set_bit(INFINIPATH_R_PORTENABLE_SHIFT + pd->port_port,
- &dd->ipath_rcvctrl);
ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
dd->ipath_rcvctrl & ~INFINIPATH_R_TAILUPD);
ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
dd->ipath_rcvctrl);
-done:
- return ret;
-}
-
-
-/* common code for the mappings on dma_alloc_coherent mem */
-static int ipath_mmap_mem(struct vm_area_struct *vma,
- struct ipath_portdata *pd, unsigned len,
- int write_ok, dma_addr_t addr, char *what)
-{
- struct ipath_devdata *dd = pd->port_dd;
- unsigned pfn = (unsigned long)addr >> PAGE_SHIFT;
- int ret;
-
- if ((vma->vm_end - vma->vm_start) > len) {
- dev_info(&dd->pcidev->dev,
- "FAIL on %s: len %lx > %x\n", what,
- vma->vm_end - vma->vm_start, len);
- ret = -EFAULT;
- goto bail;
- }
-
- if (!write_ok) {
- if (vma->vm_flags & VM_WRITE) {
- dev_info(&dd->pcidev->dev,
- "%s must be mapped readonly\n", what);
- ret = -EPERM;
- goto bail;
- }
- /* don't allow them to later change with mprotect */
- vma->vm_flags &= ~VM_MAYWRITE;
- }
-
- ret = remap_pfn_range(vma, vma->vm_start, pfn,
- len, vma->vm_page_prot);
- if (ret)
- dev_info(&dd->pcidev->dev,
- "%s port%u mmap of %lx, %x bytes r%c failed: %d\n",
- what, pd->port_port, (unsigned long)addr, len,
- write_ok?'w':'o', ret);
- else
- ipath_cdbg(VERBOSE, "%s port%u mmaped %lx, %x bytes r%c\n",
- what, pd->port_port, (unsigned long)addr, len,
- write_ok?'w':'o');
-bail:
+done:
return ret;
}
unsigned long phys;
int ret;
- /*
- * This is real hardware, so use io_remap. This is the mechanism
- * for the user process to update the head registers for their port
- * in the chip.
- */
+ /* it's the real hardware, so io_remap works */
+
if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
dev_info(&dd->pcidev->dev, "FAIL mmap userreg: reqlen "
"%lx > PAGE\n", vma->vm_end - vma->vm_start);
int ret;
/*
- * When we map the PIO buffers in the chip, we want to map them as
- * writeonly, no read possible. This prevents access to previous
- * process data, and catches users who might try to read the i/o
- * space due to a bug.
+ * When we map the PIO buffers, we want to map them as writeonly, no
+ * read possible.
*/
+
if ((vma->vm_end - vma->vm_start) >
(dd->ipath_pbufsport * dd->ipath_palign)) {
dev_info(&dd->pcidev->dev, "FAIL mmap piobufs: "
}
phys = dd->ipath_physaddr + pd->port_piobufs;
-
/*
- * Don't mark this as non-cached, or we don't get the
+ * Do *NOT* mark this as non-cached (PWT bit), or we don't get the
* write combining behavior we want on the PIO buffers!
+ * vma->vm_page_prot =
+ * pgprot_noncached(vma->vm_page_prot);
*/
if (vma->vm_flags & VM_READ) {
}
/* don't allow them to later change to readable with mprotect */
- vma->vm_flags &= ~VM_MAYREAD;
+
+ vma->vm_flags &= ~VM_MAYWRITE;
vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
ret = io_remap_pfn_range(vma, vma->vm_start, phys >> PAGE_SHIFT,
dma_addr_t *phys;
int ret;
+ if (!pd->port_rcvegrbuf) {
+ ret = -EFAULT;
+ goto bail;
+ }
+
size = pd->port_rcvegrbuf_size;
total_size = pd->port_rcvegrbuf_chunks * size;
if ((vma->vm_end - vma->vm_start) > total_size) {
ret = -EPERM;
goto bail;
}
- /* don't allow them to later change to writeable with mprotect */
- vma->vm_flags &= ~VM_MAYWRITE;
start = vma->vm_start;
phys = pd->port_rcvegrbuf_phys;
+ /* don't allow them to later change to writeable with mprotect */
+ vma->vm_flags &= ~VM_MAYWRITE;
+
for (i = 0; i < pd->port_rcvegrbuf_chunks; i++, start += size) {
ret = remap_pfn_range(vma, start, phys[i] >> PAGE_SHIFT,
size, vma->vm_page_prot);
return ret;
}
+static int mmap_rcvhdrq(struct vm_area_struct *vma,
+ struct ipath_portdata *pd)
+{
+ struct ipath_devdata *dd = pd->port_dd;
+ size_t total_size;
+ int ret;
+
+ /*
+ * kmalloc'ed memory, physically contiguous; this is from
+ * spi_rcvhdr_base; we allow user to map read-write so they can
+ * write hdrq entries to allow protocol code to directly poll
+ * whether a hdrq entry has been written.
+ */
+ total_size = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
+ sizeof(u32), PAGE_SIZE);
+ if ((vma->vm_end - vma->vm_start) > total_size) {
+ dev_info(&dd->pcidev->dev,
+ "FAIL on rcvhdrq: reqlen %lx > actual %lx\n",
+ vma->vm_end - vma->vm_start,
+ (unsigned long) total_size);
+ ret = -EFAULT;
+ goto bail;
+ }
+
+ ret = remap_pfn_range(vma, vma->vm_start,
+ pd->port_rcvhdrq_phys >> PAGE_SHIFT,
+ vma->vm_end - vma->vm_start,
+ vma->vm_page_prot);
+bail:
+ return ret;
+}
+
+static int mmap_pioavailregs(struct vm_area_struct *vma,
+ struct ipath_portdata *pd)
+{
+ struct ipath_devdata *dd = pd->port_dd;
+ int ret;
+
+ /*
+ * when we map the PIO bufferavail registers, we want to map them as
+ * readonly, no write possible.
+ *
+ * kmalloc'ed memory, physically contiguous, one page only, readonly
+ */
+
+ if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
+ dev_info(&dd->pcidev->dev, "FAIL on pioavailregs_dma: "
+ "reqlen %lx > actual %lx\n",
+ vma->vm_end - vma->vm_start,
+ (unsigned long) PAGE_SIZE);
+ ret = -EFAULT;
+ goto bail;
+ }
+
+ if (vma->vm_flags & VM_WRITE) {
+ dev_info(&dd->pcidev->dev,
+ "Can't map pioavailregs as writable (flags=%lx)\n",
+ vma->vm_flags);
+ ret = -EPERM;
+ goto bail;
+ }
+
+ /* don't allow them to later change with mprotect */
+ vma->vm_flags &= ~VM_MAYWRITE;
+
+ ret = remap_pfn_range(vma, vma->vm_start,
+ dd->ipath_pioavailregs_phys >> PAGE_SHIFT,
+ PAGE_SIZE, vma->vm_page_prot);
+bail:
+ return ret;
+}
+
/**
* ipath_mmap - mmap various structures into user space
* @fp: the file pointer
pd = port_fp(fp);
dd = pd->port_dd;
-
/*
* This is the ipath_do_user_init() code, mapping the shared buffers
* into the user process. The address referred to by vm_pgoff is the
pgaddr = vma->vm_pgoff << PAGE_SHIFT;
/*
- * Must fit in 40 bits for our hardware; some checked elsewhere,
- * but we'll be paranoid. Check for 0 is mostly in case one of the
- * allocations failed, but user called mmap anyway. We want to catch
- * that before it can match.
+ * note that ureg does *NOT* have the kregvirt as part of it, to be
+ * sure that for 32 bit programs, we don't end up trying to map a >
+ * 44 address. Has to match ipath_get_base_info() code that sets
+ * __spi_uregbase
*/
- if (!pgaddr || pgaddr >= (1ULL<<40)) {
- ipath_dev_err(dd, "Bad phys addr %llx, start %lx, end %lx\n",
- (unsigned long long)pgaddr, vma->vm_start, vma->vm_end);
- return -EINVAL;
- }
- /* just the offset of the port user registers, not physical addr */
ureg = dd->ipath_uregbase + dd->ipath_palign * pd->port_port;
ipath_cdbg(MM, "ushare: pgaddr %llx vm_start=%lx, vmlen %lx\n",
(unsigned long long) pgaddr, vma->vm_start,
vma->vm_end - vma->vm_start);
- if (vma->vm_start & (PAGE_SIZE-1)) {
- ipath_dev_err(dd,
- "vm_start not aligned: %lx, end=%lx phys %lx\n",
- vma->vm_start, vma->vm_end, (unsigned long)pgaddr);
- ret = -EINVAL;
- }
- else if (pgaddr == ureg)
+ if (pgaddr == ureg)
ret = mmap_ureg(vma, dd, ureg);
else if (pgaddr == pd->port_piobufs)
ret = mmap_piobufs(vma, dd, pd);
else if (pgaddr == (u64) pd->port_rcvegr_phys)
ret = mmap_rcvegrbufs(vma, pd);
- else if (pgaddr == (u64) pd->port_rcvhdrq_phys) {
- /*
- * The rcvhdrq itself; readonly except on HT-400 (so have
- * to allow writable mapping), multiple pages, contiguous
- * from an i/o perspective.
- */
- unsigned total_size =
- ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize
- * sizeof(u32), PAGE_SIZE);
- ret = ipath_mmap_mem(vma, pd, total_size, 1,
- pd->port_rcvhdrq_phys,
- "rcvhdrq");
- }
- else if (pgaddr == (u64)pd->port_rcvhdrqtailaddr_phys)
- /* in-memory copy of rcvhdrq tail register */
- ret = ipath_mmap_mem(vma, pd, PAGE_SIZE, 0,
- pd->port_rcvhdrqtailaddr_phys,
- "rcvhdrq tail");
+ else if (pgaddr == (u64) pd->port_rcvhdrq_phys)
+ ret = mmap_rcvhdrq(vma, pd);
else if (pgaddr == dd->ipath_pioavailregs_phys)
- /* in-memory copy of pioavail registers */
- ret = ipath_mmap_mem(vma, pd, PAGE_SIZE, 0,
- dd->ipath_pioavailregs_phys,
- "pioavail registers");
+ ret = mmap_pioavailregs(vma, pd);
else
ret = -EINVAL;
static int ipath_open(struct inode *in, struct file *fp)
{
- int ret, user_minor;
+ int ret, minor;
mutex_lock(&ipath_mutex);
- user_minor = iminor(in) - IPATH_USER_MINOR_BASE;
+ minor = iminor(in);
ipath_cdbg(VERBOSE, "open on dev %lx (minor %d)\n",
- (long)in->i_rdev, user_minor);
+ (long)in->i_rdev, minor);
- if (user_minor)
- ret = find_free_port(user_minor - 1, fp);
+ if (minor)
+ ret = find_free_port(minor - 1, fp);
else
ret = find_best_unit(fp);
}
if (dd->ipath_kregbase) {
- int i;
- /* atomically clear receive enable port. */
- clear_bit(INFINIPATH_R_PORTENABLE_SHIFT + port,
- &dd->ipath_rcvctrl);
- ipath_write_kreg( dd, dd->ipath_kregs->kr_rcvctrl,
- dd->ipath_rcvctrl);
- /* and read back from chip to be sure that nothing
- * else is in flight when we do the rest */
- (void)ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
+ if (pd->port_rcvhdrtail_uaddr) {
+ pd->port_rcvhdrtail_uaddr = 0;
+ pd->port_rcvhdrtail_kvaddr = NULL;
+ ipath_release_user_pages_on_close(
+ &pd->port_rcvhdrtail_pagep, 1);
+ pd->port_rcvhdrtail_pagep = NULL;
+ ipath_stats.sps_pageunlocks++;
+ }
+ ipath_write_kreg_port(
+ dd, dd->ipath_kregs->kr_rcvhdrtailaddr,
+ port, 0ULL);
+ ipath_write_kreg_port(
+ dd, dd->ipath_kregs->kr_rcvhdraddr,
+ pd->port_port, 0);
/* clean up the pkeys for this port user */
ipath_clean_part_key(pd, dd);
+ if (port < dd->ipath_cfgports) {
+ int i = dd->ipath_pbufsport * (port - 1);
+ ipath_disarm_piobufs(dd, i, dd->ipath_pbufsport);
- /*
- * be paranoid, and never write 0's to these, just use an
- * unused part of the port 0 tail page. Of course,
- * rcvhdraddr points to a large chunk of memory, so this
- * could still trash things, but at least it won't trash
- * page 0, and by disabling the port, it should stop "soon",
- * even if a packet or two is in already in flight after we
- * disabled the port.
- */
- ipath_write_kreg_port(dd,
- dd->ipath_kregs->kr_rcvhdrtailaddr, port,
- dd->ipath_dummy_hdrq_phys);
- ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
- pd->port_port, dd->ipath_dummy_hdrq_phys);
-
- i = dd->ipath_pbufsport * (port - 1);
- ipath_disarm_piobufs(dd, i, dd->ipath_pbufsport);
-
- if (dd->ipath_pageshadow)
- unlock_expected_tids(pd);
- ipath_stats.sps_ports--;
- ipath_cdbg(PROC, "%s[%u] closed port %u:%u\n",
- pd->port_comm, pd->port_pid,
- dd->ipath_unit, port);
-
- dd->ipath_f_clear_tids(dd, pd->port_port);
+ /* atomically clear receive enable port. */
+ clear_bit(INFINIPATH_R_PORTENABLE_SHIFT + port,
+ &dd->ipath_rcvctrl);
+ ipath_write_kreg(
+ dd,
+ dd->ipath_kregs->kr_rcvctrl,
+ dd->ipath_rcvctrl);
+
+ if (dd->ipath_pageshadow)
+ unlock_expected_tids(pd);
+ ipath_stats.sps_ports--;
+ ipath_cdbg(PROC, "%s[%u] closed port %u:%u\n",
+ pd->port_comm, pd->port_pid,
+ dd->ipath_unit, port);
+ }
}
pd->port_cnt = 0;
pd->port_pid = 0;
- dd->ipath_pd[pd->port_port] = NULL; /* before releasing mutex */
+ dd->ipath_f_clear_tids(dd, pd->port_port);
+
+ ipath_free_pddata(dd, pd->port_port, 0);
+
mutex_unlock(&ipath_mutex);
- ipath_free_pddata(dd, pd); /* after releasing the mutex */
return ret;
}
"error %d\n", -ret);
goto bail;
}
+ ret = ipath_diag_init();
+ if (ret < 0) {
+ ipath_dev_err(dd, "Unable to set up diag support: "
+ "error %d\n", -ret);
+ goto bail_sma;
+ }
+
ret = init_cdev(0, "ipath", &ipath_file_ops, &wildcard_cdev,
&wildcard_class_dev);
if (ret < 0) {
ipath_dev_err(dd, "Could not create wildcard "
"minor: error %d\n", -ret);
- goto bail_sma;
+ goto bail_diag;
}
atomic_set(&user_setup, 1);
snprintf(name, sizeof(name), "ipath%d", dd->ipath_unit);
ret = init_cdev(dd->ipath_unit + 1, name, &ipath_file_ops,
- &dd->user_cdev, &dd->user_class_dev);
+ &dd->cdev, &dd->class_dev);
if (ret < 0)
ipath_dev_err(dd, "Could not create user minor %d, %s\n",
dd->ipath_unit + 1, name);
goto bail;
+bail_diag:
+ ipath_diag_cleanup();
bail_sma:
user_cleanup();
bail:
return ret;
}
-void ipath_user_remove(struct ipath_devdata *dd)
+void ipath_user_del(struct ipath_devdata *dd)
{
- cleanup_cdev(&dd->user_cdev, &dd->user_class_dev);
+ cleanup_cdev(&dd->cdev, &dd->class_dev);
if (atomic_dec_return(&user_count) == 0) {
if (atomic_read(&user_setup) == 0)
goto bail;
cleanup_cdev(&wildcard_cdev, &wildcard_class_dev);
+ ipath_diag_cleanup();
user_cleanup();
atomic_set(&user_setup, 0);
bail:
return;
}
-