/*
* Public API and common code for RelayFS.
*
- * Please see Documentation/filesystems/relayfs.txt for API description.
- *
- * Copyright (C) 2002, 2003 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
- * Copyright (C) 1999, 2000, 2001, 2002 - Karim Yaghmour (karim@opersys.com)
+ * See Documentation/filesystems/relayfs.txt for an overview of relayfs.
+ *
+ * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
+ * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com)
*
* This file is released under the GPL.
*/
-#include <linux/init.h>
#include <linux/errno.h>
#include <linux/stddef.h>
#include <linux/slab.h>
#include <linux/module.h>
-#include <linux/sched.h>
#include <linux/string.h>
-#include <linux/time.h>
-#include <linux/page-flags.h>
-#include <linux/vmalloc.h>
-#include <linux/mm.h>
-#include <linux/mman.h>
-#include <linux/delay.h>
-
-#include <asm/io.h>
-#include <asm/current.h>
-#include <asm/uaccess.h>
-#include <asm/bitops.h>
-#include <asm/pgtable.h>
-#include <asm/relay.h>
-#include <asm/hardirq.h>
-
-#include "relay_lockless.h"
-#include "relay_locking.h"
-#include "resize.h"
-
-/* Relay channel table, indexed by channel id */
-static struct rchan * rchan_table[RELAY_MAX_CHANNELS];
-static rwlock_t rchan_table_lock = RW_LOCK_UNLOCKED;
-
-/* Relay operation structs, one per scheme */
-static struct relay_ops lockless_ops = {
- .reserve = lockless_reserve,
- .commit = lockless_commit,
- .get_offset = lockless_get_offset,
- .finalize = lockless_finalize,
- .reset = lockless_reset,
- .reset_index = lockless_reset_index
-};
-
-static struct relay_ops locking_ops = {
- .reserve = locking_reserve,
- .commit = locking_commit,
- .get_offset = locking_get_offset,
- .finalize = locking_finalize,
- .reset = locking_reset,
- .reset_index = locking_reset_index
-};
-
-/*
- * Low-level relayfs kernel API. These functions should not normally be
- * used by clients. See high-level kernel API below.
- */
-
-/**
- * rchan_get - get channel associated with id, incrementing refcount
- * @rchan_id: the channel id
- *
- * Returns channel if successful, NULL otherwise.
- */
-struct rchan *
-rchan_get(int rchan_id)
-{
- struct rchan *rchan;
-
- if ((rchan_id < 0) || (rchan_id >= RELAY_MAX_CHANNELS))
- return NULL;
-
- read_lock(&rchan_table_lock);
- rchan = rchan_table[rchan_id];
- if (rchan)
- atomic_inc(&rchan->refcount);
- read_unlock(&rchan_table_lock);
-
- return rchan;
-}
-
-/**
- * clear_readers - clear non-VFS readers
- * @rchan: the channel
- *
- * Clear the channel pointers of all non-VFS readers open on the channel.
- */
-static inline void
-clear_readers(struct rchan *rchan)
-{
- struct list_head *p;
- struct rchan_reader *reader;
-
- read_lock(&rchan->open_readers_lock);
- list_for_each(p, &rchan->open_readers) {
- reader = list_entry(p, struct rchan_reader, list);
- if (!reader->vfs_reader)
- reader->rchan = NULL;
- }
- read_unlock(&rchan->open_readers_lock);
-}
-
-/**
- * rchan_alloc_id - reserve a channel id and store associated channel
- * @rchan: the channel
- *
- * Returns channel id if successful, -1 otherwise.
- */
-static inline int
-rchan_alloc_id(struct rchan *rchan)
-{
- int i;
- int rchan_id = -1;
-
- if (rchan == NULL)
- return -1;
-
- write_lock(&rchan_table_lock);
- for (i = 0; i < RELAY_MAX_CHANNELS; i++) {
- if (rchan_table[i] == NULL) {
- rchan_table[i] = rchan;
- rchan_id = rchan->id = i;
- break;
- }
- }
- if (rchan_id != -1)
- atomic_inc(&rchan->refcount);
- write_unlock(&rchan_table_lock);
-
- return rchan_id;
-}
-
-/**
- * rchan_free_id - revoke a channel id and remove associated channel
- * @rchan_id: the channel id
- */
-static inline void
-rchan_free_id(int rchan_id)
-{
- struct rchan *rchan;
-
- if ((rchan_id < 0) || (rchan_id >= RELAY_MAX_CHANNELS))
- return;
-
- write_lock(&rchan_table_lock);
- rchan = rchan_table[rchan_id];
- rchan_table[rchan_id] = NULL;
- write_unlock(&rchan_table_lock);
-}
-
-/**
- * rchan_destroy_buf - destroy the current channel buffer
- * @rchan: the channel
- */
-static inline void
-rchan_destroy_buf(struct rchan *rchan)
-{
- if (rchan->buf && !rchan->init_buf)
- free_rchan_buf(rchan->buf,
- rchan->buf_page_array,
- rchan->buf_page_count);
-}
+#include <linux/relayfs_fs.h>
+#include "relay.h"
+#include "buffers.h"
/**
- * relay_release - perform end-of-buffer processing for last buffer
- * @rchan: the channel
+ * relay_buf_empty - boolean, is the channel buffer empty?
+ * @buf: channel buffer
*
- * Returns 0 if successful, negative otherwise.
- *
- * Releases the channel buffer, destroys the channel, and removes the
- * relay file from the relayfs filesystem. Should only be called from
- * rchan_put(). If we're here, it means by definition refcount is 0.
- */
-static int
-relay_release(struct rchan *rchan)
-{
- if (rchan == NULL)
- return -EBADF;
-
- rchan_destroy_buf(rchan);
- rchan_free_id(rchan->id);
- relayfs_remove_file(rchan->dentry);
- clear_readers(rchan);
- kfree(rchan);
-
- return 0;
-}
-
-/**
- * rchan_get - decrement channel refcount, releasing it if 0
- * @rchan: the channel
- *
- * If the refcount reaches 0, the channel will be destroyed.
- */
-void
-rchan_put(struct rchan *rchan)
-{
- if (atomic_dec_and_test(&rchan->refcount))
- relay_release(rchan);
-}
-
-/**
- * relay_reserve - reserve a slot in the channel buffer
- * @rchan: the channel
- * @len: the length of the slot to reserve
- * @td: the time delta between buffer start and current write, or TSC
- * @err: receives the result flags
- * @interrupting: 1 if interrupting previous, used only in locking scheme
- *
- * Returns pointer to the beginning of the reserved slot, NULL if error.
- *
- * The errcode value contains the result flags and is an ORed combination
- * of the following:
- *
- * RELAY_BUFFER_SWITCH_NONE - no buffer switch occurred
- * RELAY_EVENT_DISCARD_NONE - event should not be discarded
- * RELAY_BUFFER_SWITCH - buffer switch occurred
- * RELAY_EVENT_DISCARD - event should be discarded (all buffers are full)
- * RELAY_EVENT_TOO_LONG - event won't fit into even an empty buffer
- *
- * buffer_start and buffer_end callbacks are triggered at this point
- * if applicable.
- */
-char *
-relay_reserve(struct rchan *rchan,
- u32 len,
- struct timeval *ts,
- u32 *td,
- int *err,
- int *interrupting)
-{
- if (rchan == NULL)
- return NULL;
-
- *interrupting = 0;
-
- return rchan->relay_ops->reserve(rchan, len, ts, td, err, interrupting);
-}
-
-
-/**
- * wakeup_readers - wake up VFS readers waiting on a channel
- * @private: the channel
- *
- * This is the work function used to defer reader waking. The
- * reason waking is deferred is that calling directly from commit
- * causes problems if you're writing from say the scheduler.
- */
-static void
-wakeup_readers(void *private)
-{
- struct rchan *rchan = (struct rchan *)private;
-
- wake_up_interruptible(&rchan->read_wait);
-}
-
-
-/**
- * relay_commit - commit a reserved slot in the buffer
- * @rchan: the channel
- * @from: commit the length starting here
- * @len: length committed
- * @interrupting: 1 if interrupting previous, used only in locking scheme
- *
- * After the write into the reserved buffer has been complted, this
- * function must be called in order for the relay to determine whether
- * buffers are complete and to wake up VFS readers.
- *
- * delivery callback is triggered at this point if applicable.
- */
-void
-relay_commit(struct rchan *rchan,
- char *from,
- u32 len,
- int reserve_code,
- int interrupting)
-{
- int deliver;
-
- if (rchan == NULL)
- return;
-
- deliver = packet_delivery(rchan) ||
- (reserve_code & RELAY_BUFFER_SWITCH);
-
- rchan->relay_ops->commit(rchan, from, len, deliver, interrupting);
-
- /* The params are always the same, so no worry about re-queuing */
- if (deliver && waitqueue_active(&rchan->read_wait)) {
- PREPARE_WORK(&rchan->wake_readers, wakeup_readers, rchan);
- schedule_delayed_work(&rchan->wake_readers, 1);
- }
-}
-
-/**
- * relay_get_offset - get current and max channel buffer offsets
- * @rchan: the channel
- * @max_offset: maximum channel offset
- *
- * Returns the current and maximum channel buffer offsets.
- */
-u32
-relay_get_offset(struct rchan *rchan, u32 *max_offset)
-{
- return rchan->relay_ops->get_offset(rchan, max_offset);
-}
-
-/**
- * reset_index - try once to reset the current channel index
- * @rchan: the channel
- * @old_index: the index read before reset
- *
- * Attempts to reset the channel index to 0. It tries once, and
- * if it fails, returns negative, 0 otherwise.
- */
-int
-reset_index(struct rchan *rchan, u32 old_index)
-{
- return rchan->relay_ops->reset_index(rchan, old_index);
-}
-
-/*
- * close() vm_op implementation for relayfs file mapping.
- */
-static void
-relay_file_mmap_close(struct vm_area_struct *vma)
-{
- struct file *filp = vma->vm_file;
- struct rchan_reader *reader;
- struct rchan *rchan;
-
- reader = (struct rchan_reader *)filp->private_data;
- rchan = reader->rchan;
-
- atomic_dec(&rchan->mapped);
-
- rchan->callbacks->fileop_notify(reader->rchan->id, filp,
- RELAY_FILE_UNMAP);
-}
-
-/*
- * vm_ops for relay file mappings.
+ * Returns 1 if the buffer is empty, 0 otherwise.
*/
-static struct vm_operations_struct relay_file_mmap_ops = {
- .close = relay_file_mmap_close
-};
-
-/* \begin{Code inspired from BTTV driver} */
-static inline unsigned long
-kvirt_to_pa(unsigned long adr)
+int relay_buf_empty(struct rchan_buf *buf)
{
- unsigned long kva, ret;
-
- kva = (unsigned long) page_address(vmalloc_to_page((void *) adr));
- kva |= adr & (PAGE_SIZE - 1);
- ret = __pa(kva);
- return ret;
-}
-
-static int
-relay_mmap_region(struct vm_area_struct *vma,
- const char *adr,
- const char *start_pos,
- unsigned long size)
-{
- unsigned long start = (unsigned long) adr;
- unsigned long page, pos;
-
- pos = (unsigned long) start_pos;
-
- while (size > 0) {
- page = kvirt_to_pa(pos);
- if (remap_page_range(vma, start, page, PAGE_SIZE, PAGE_SHARED))
- return -EAGAIN;
- start += PAGE_SIZE;
- pos += PAGE_SIZE;
- size -= PAGE_SIZE;
- }
-
- return 0;
+ return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;
}
-/* \end{Code inspired from BTTV driver} */
/**
- * relay_mmap_buffer: - mmap buffer to process address space
- * @rchan_id: relay channel id
- * @vma: vm_area_struct describing memory to be mapped
+ * relay_buf_full - boolean, is the channel buffer full?
+ * @buf: channel buffer
*
- * Returns:
- * 0 if ok
- * -EAGAIN, when remap failed
- * -EINVAL, invalid requested length
- *
- * Caller should already have grabbed mmap_sem.
+ * Returns 1 if the buffer is full, 0 otherwise.
*/
-int
-__relay_mmap_buffer(struct rchan *rchan,
- struct vm_area_struct *vma)
+int relay_buf_full(struct rchan_buf *buf)
{
- int err = 0;
- unsigned long length = vma->vm_end - vma->vm_start;
- struct file *filp = vma->vm_file;
-
- if (rchan == NULL) {
- err = -EBADF;
- goto exit;
- }
-
- if (rchan->init_buf) {
- err = -EPERM;
- goto exit;
- }
-
- if (length != (unsigned long)rchan->alloc_size) {
- err = -EINVAL;
- goto exit;
- }
-
- err = relay_mmap_region(vma,
- (char *)vma->vm_start,
- rchan->buf,
- rchan->alloc_size);
-
- if (err == 0) {
- vma->vm_ops = &relay_file_mmap_ops;
- err = rchan->callbacks->fileop_notify(rchan->id, filp,
- RELAY_FILE_MAP);
- if (err == 0)
- atomic_inc(&rchan->mapped);
- }
-exit:
- return err;
+ size_t ready = buf->subbufs_produced - buf->subbufs_consumed;
+ return (ready >= buf->chan->n_subbufs) ? 1 : 0;
}
/*
- * High-level relayfs kernel API. See Documentation/filesystems/relafys.txt.
+ * High-level relayfs kernel API and associated functions.
*/
/*
*/
/*
- * buffer_end() default callback. Does nothing.
- */
-static int
-buffer_end_default_callback(int rchan_id,
- char *current_write_pos,
- char *end_of_buffer,
- struct timeval end_time,
- u32 end_tsc,
- int using_tsc)
-{
- return 0;
-}
-
-/*
- * buffer_start() default callback. Does nothing.
+ * subbuf_start() default callback. Does nothing.
*/
-static int
-buffer_start_default_callback(int rchan_id,
- char *current_write_pos,
- u32 buffer_id,
- struct timeval start_time,
- u32 start_tsc,
- int using_tsc)
+static int subbuf_start_default_callback (struct rchan_buf *buf,
+ void *subbuf,
+ void *prev_subbuf,
+ size_t prev_padding)
{
- return 0;
-}
+ if (relay_buf_full(buf))
+ return 0;
-/*
- * deliver() default callback. Does nothing.
- */
-static void
-deliver_default_callback(int rchan_id, char *from, u32 len)
-{
+ return 1;
}
/*
- * user_deliver() default callback. Does nothing.
+ * buf_mapped() default callback. Does nothing.
*/
-static void
-user_deliver_default_callback(int rchan_id, char *from, u32 len)
+static void buf_mapped_default_callback(struct rchan_buf *buf,
+ struct file *filp)
{
}
/*
- * needs_resize() default callback. Does nothing.
+ * buf_unmapped() default callback. Does nothing.
*/
-static void
-needs_resize_default_callback(int rchan_id,
- int resize_type,
- u32 suggested_buf_size,
- u32 suggested_n_bufs)
+static void buf_unmapped_default_callback(struct rchan_buf *buf,
+ struct file *filp)
{
}
/*
- * fileop_notify() default callback. Does nothing.
+ * create_buf_file_create() default callback. Creates file to represent buf.
*/
-static int
-fileop_notify_default_callback(int rchan_id,
- struct file *filp,
- enum relay_fileop fileop)
+static struct dentry *create_buf_file_default_callback(const char *filename,
+ struct dentry *parent,
+ int mode,
+ struct rchan_buf *buf,
+ int *is_global)
{
- return 0;
+ return relayfs_create_file(filename, parent, mode,
+ &relay_file_operations, buf);
}
/*
- * ioctl() default callback. Does nothing.
+ * remove_buf_file() default callback. Removes file representing relay buffer.
*/
-static int
-ioctl_default_callback(int rchan_id,
- unsigned int cmd,
- unsigned long arg)
+static int remove_buf_file_default_callback(struct dentry *dentry)
{
- return 0;
+ return relayfs_remove(dentry);
}
/* relay channel default callbacks */
static struct rchan_callbacks default_channel_callbacks = {
- .buffer_start = buffer_start_default_callback,
- .buffer_end = buffer_end_default_callback,
- .deliver = deliver_default_callback,
- .user_deliver = user_deliver_default_callback,
- .needs_resize = needs_resize_default_callback,
- .fileop_notify = fileop_notify_default_callback,
- .ioctl = ioctl_default_callback,
+ .subbuf_start = subbuf_start_default_callback,
+ .buf_mapped = buf_mapped_default_callback,
+ .buf_unmapped = buf_unmapped_default_callback,
+ .create_buf_file = create_buf_file_default_callback,
+ .remove_buf_file = remove_buf_file_default_callback,
};
/**
- * check_attribute_flags - check sanity of channel attributes
- * @flags: channel attributes
- * @resizeable: 1 if true
+ * wakeup_readers - wake up readers waiting on a channel
+ * @private: the channel buffer
*
- * Returns 0 if successful, negative otherwise.
+ * This is the work function used to defer reader waking. The
+ * reason waking is deferred is that calling directly from write
+ * causes problems if you're writing from say the scheduler.
*/
-static int
-check_attribute_flags(u32 *attribute_flags, int resizeable)
+static void wakeup_readers(void *private)
{
- u32 flags = *attribute_flags;
-
- if (!(flags & RELAY_DELIVERY_BULK) && !(flags & RELAY_DELIVERY_PACKET))
- return -EINVAL; /* Delivery mode must be specified */
-
- if (!(flags & RELAY_USAGE_SMP) && !(flags & RELAY_USAGE_GLOBAL))
- return -EINVAL; /* Usage must be specified */
-
- if (resizeable) { /* Resizeable can never be continuous */
- *attribute_flags &= ~RELAY_MODE_CONTINUOUS;
- *attribute_flags |= RELAY_MODE_NO_OVERWRITE;
- }
-
- if ((flags & RELAY_MODE_CONTINUOUS) &&
- (flags & RELAY_MODE_NO_OVERWRITE))
- return -EINVAL; /* Can't have it both ways */
-
- if (!(flags & RELAY_MODE_CONTINUOUS) &&
- !(flags & RELAY_MODE_NO_OVERWRITE))
- *attribute_flags |= RELAY_MODE_CONTINUOUS; /* Default to continuous */
-
- if (!(flags & RELAY_SCHEME_ANY))
- return -EINVAL; /* One or both must be specified */
- else if (flags & RELAY_SCHEME_LOCKLESS) {
- if (have_cmpxchg())
- *attribute_flags &= ~RELAY_SCHEME_LOCKING;
- else if (flags & RELAY_SCHEME_LOCKING)
- *attribute_flags &= ~RELAY_SCHEME_LOCKLESS;
- else
- return -EINVAL; /* Locking scheme not an alternative */
- }
-
- if (!(flags & RELAY_TIMESTAMP_ANY))
- return -EINVAL; /* One or both must be specified */
- else if (flags & RELAY_TIMESTAMP_TSC) {
- if (have_tsc())
- *attribute_flags &= ~RELAY_TIMESTAMP_GETTIMEOFDAY;
- else if (flags & RELAY_TIMESTAMP_GETTIMEOFDAY)
- *attribute_flags &= ~RELAY_TIMESTAMP_TSC;
- else
- return -EINVAL; /* gettimeofday not an alternative */
- }
-
- return 0;
+ struct rchan_buf *buf = private;
+ wake_up_interruptible(&buf->read_wait);
}
-/*
- * High-level API functions.
- */
-
/**
- * __relay_reset - internal reset function
- * @rchan: the channel
- * @init: 1 if this is a first-time channel initialization
+ * __relay_reset - reset a channel buffer
+ * @buf: the channel buffer
+ * @init: 1 if this is a first-time initialization
*
* See relay_reset for description of effect.
*/
-void
-__relay_reset(struct rchan *rchan, int init)
+static inline void __relay_reset(struct rchan_buf *buf, unsigned int init)
{
- int i;
-
+ size_t i;
+
if (init) {
- rchan->version = RELAYFS_CHANNEL_VERSION;
- init_MUTEX(&rchan->resize_sem);
- init_waitqueue_head(&rchan->read_wait);
- init_waitqueue_head(&rchan->write_wait);
- atomic_set(&rchan->refcount, 0);
- INIT_LIST_HEAD(&rchan->open_readers);
- rchan->open_readers_lock = RW_LOCK_UNLOCKED;
+ init_waitqueue_head(&buf->read_wait);
+ kref_init(&buf->kref);
+ INIT_WORK(&buf->wake_readers, NULL, NULL);
+ } else {
+ cancel_delayed_work(&buf->wake_readers);
+ flush_scheduled_work();
}
-
- rchan->buf_id = rchan->buf_idx = 0;
- atomic_set(&rchan->suspended, 0);
- atomic_set(&rchan->mapped, 0);
- rchan->half_switch = 0;
- rchan->bufs_produced = 0;
- rchan->bufs_consumed = 0;
- rchan->bytes_consumed = 0;
- rchan->initialized = 0;
- rchan->finalized = 0;
- rchan->resize_min = rchan->resize_max = 0;
- rchan->resizing = 0;
- rchan->replace_buffer = 0;
- rchan->resize_buf = NULL;
- rchan->resize_buf_size = 0;
- rchan->resize_alloc_size = 0;
- rchan->resize_n_bufs = 0;
- rchan->resize_err = 0;
- rchan->resize_failures = 0;
- rchan->resize_order = 0;
- rchan->expand_page_array = NULL;
- rchan->expand_page_count = 0;
- rchan->shrink_page_array = NULL;
- rchan->shrink_page_count = 0;
- rchan->resize_page_array = NULL;
- rchan->resize_page_count = 0;
- rchan->old_buf_page_array = NULL;
- rchan->expand_buf_id = 0;
+ buf->subbufs_produced = 0;
+ buf->subbufs_consumed = 0;
+ buf->bytes_consumed = 0;
+ buf->finalized = 0;
+ buf->data = buf->start;
+ buf->offset = 0;
- INIT_WORK(&rchan->wake_readers, NULL, NULL);
- INIT_WORK(&rchan->wake_writers, NULL, NULL);
+ for (i = 0; i < buf->chan->n_subbufs; i++)
+ buf->padding[i] = 0;
- for (i = 0; i < RELAY_MAX_BUFS; i++)
- rchan->unused_bytes[i] = 0;
-
- rchan->relay_ops->reset(rchan, init);
+ buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);
}
/**
* relay_reset - reset the channel
- * @rchan: the channel
+ * @chan: the channel
*
- * Returns 0 if successful, negative if not.
+ * This has the effect of erasing all data from all channel buffers
+ * and restarting the channel in its initial state. The buffers
+ * are not freed, so any mappings are still in effect.
*
- * This has the effect of erasing all data from the buffer and
- * restarting the channel in its initial state. The buffer itself
- * is not freed, so any mappings are still in effect.
- *
- * NOTE: Care should be taken that the channnel isn't actually
+ * NOTE: Care should be taken that the channel isn't actually
* being used by anything when this call is made.
*/
-int
-relay_reset(int rchan_id)
+void relay_reset(struct rchan *chan)
{
- struct rchan *rchan;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- __relay_reset(rchan, 0);
- update_readers_consumed(rchan, 0, 0);
+ unsigned int i;
+ struct rchan_buf *prev = NULL;
- rchan_put(rchan);
-
- return 0;
-}
-
-/**
- * check_init_buf - check the sanity of init_buf, if present
- * @init_buf: the initbuf
- * @init_buf_size: the total initbuf size
- * @bufsize: the channel's sub-buffer size
- * @nbufs: the number of sub-buffers in the channel
- *
- * Returns 0 if ok, negative otherwise.
- */
-static int
-check_init_buf(char *init_buf, u32 init_buf_size, u32 bufsize, u32 nbufs)
-{
- int err = 0;
-
- if (init_buf && nbufs == 1) /* 1 sub-buffer makes no sense */
- err = -EINVAL;
-
- if (init_buf && (bufsize * nbufs != init_buf_size))
- err = -EINVAL;
-
- return err;
-}
-
-/**
- * rchan_create_buf - allocate the initial channel buffer
- * @rchan: the channel
- * @size_alloc: the total size of the channel buffer
- *
- * Returns 0 if successful, negative otherwise.
- */
-static inline int
-rchan_create_buf(struct rchan *rchan, int size_alloc)
-{
- struct page **page_array;
- int page_count;
-
- if ((rchan->buf = (char *)alloc_rchan_buf(size_alloc, &page_array, &page_count)) == NULL) {
- rchan->buf_page_array = NULL;
- rchan->buf_page_count = 0;
- return -ENOMEM;
- }
-
- rchan->buf_page_array = page_array;
- rchan->buf_page_count = page_count;
-
- return 0;
-}
-
-/**
- * rchan_create - allocate and initialize a channel, including buffer
- * @chanpath: path specifying the relayfs channel file to create
- * @bufsize: the size of the sub-buffers within the channel buffer
- * @nbufs: the number of sub-buffers within the channel buffer
- * @rchan_flags: flags specifying buffer attributes
- * @err: err code
- *
- * Returns channel if successful, NULL otherwise, err receives errcode.
- *
- * Allocates a struct rchan representing a relay channel, according
- * to the attributes passed in via rchan_flags. Does some basic sanity
- * checking but doesn't try to do anything smart. In particular, the
- * number of buffers must be a power of 2, and if the lockless scheme
- * is being used, the sub-buffer size must also be a power of 2. The
- * locking scheme can use buffers of any size.
- */
-static struct rchan *
-rchan_create(const char *chanpath,
- int bufsize,
- int nbufs,
- u32 rchan_flags,
- char *init_buf,
- u32 init_buf_size,
- int *err)
-{
- int size_alloc;
- struct rchan *rchan = NULL;
-
- *err = 0;
-
- rchan = (struct rchan *)kmalloc(sizeof(struct rchan), GFP_KERNEL);
- if (rchan == NULL) {
- *err = -ENOMEM;
- return NULL;
- }
- rchan->buf = rchan->init_buf = NULL;
-
- *err = check_init_buf(init_buf, init_buf_size, bufsize, nbufs);
- if (*err)
- goto exit;
-
- if (nbufs == 1 && bufsize) {
- rchan->n_bufs = nbufs;
- rchan->buf_size = bufsize;
- size_alloc = bufsize;
- goto alloc;
- }
-
- if (bufsize <= 0 ||
- (rchan_flags & RELAY_SCHEME_LOCKLESS && hweight32(bufsize) != 1) ||
- hweight32(nbufs) != 1 ||
- nbufs < RELAY_MIN_BUFS ||
- nbufs > RELAY_MAX_BUFS) {
- *err = -EINVAL;
- goto exit;
- }
-
- size_alloc = FIX_SIZE(bufsize * nbufs);
- if (size_alloc > RELAY_MAX_BUF_SIZE) {
- *err = -EINVAL;
- goto exit;
- }
- rchan->n_bufs = nbufs;
- rchan->buf_size = bufsize;
-
- if (rchan_flags & RELAY_SCHEME_LOCKLESS) {
- offset_bits(rchan) = ffs(bufsize) - 1;
- offset_mask(rchan) = RELAY_BUF_OFFSET_MASK(offset_bits(rchan));
- bufno_bits(rchan) = ffs(nbufs) - 1;
- }
-alloc:
- if (rchan_alloc_id(rchan) == -1) {
- *err = -ENOMEM;
- goto exit;
- }
-
- if (init_buf == NULL) {
- *err = rchan_create_buf(rchan, size_alloc);
- if (*err) {
- rchan_free_id(rchan->id);
- goto exit;
- }
- } else
- rchan->buf = rchan->init_buf = init_buf;
-
- rchan->alloc_size = size_alloc;
-
- if (rchan_flags & RELAY_SCHEME_LOCKLESS)
- rchan->relay_ops = &lockless_ops;
- else
- rchan->relay_ops = &locking_ops;
-
-exit:
- if (*err) {
- kfree(rchan);
- rchan = NULL;
- }
-
- return rchan;
-}
-
-
-static char tmpname[NAME_MAX];
+ if (!chan)
+ return;
-/**
- * rchan_create_dir - create directory for file
- * @chanpath: path to file, including filename
- * @residual: filename remaining after parse
- * @topdir: the directory filename should be created in
- *
- * Returns 0 if successful, negative otherwise.
- *
- * Inspired by xlate_proc_name() in procfs. Given a file path which
- * includes the filename, creates any and all directories necessary
- * to create the file.
- */
-static int
-rchan_create_dir(const char * chanpath,
- const char **residual,
- struct dentry **topdir)
-{
- const char *cp = chanpath, *next;
- struct dentry *parent = NULL;
- int len, err = 0;
-
- while (1) {
- next = strchr(cp, '/');
- if (!next)
+ for (i = 0; i < NR_CPUS; i++) {
+ if (!chan->buf[i] || chan->buf[i] == prev)
break;
-
- len = next - cp;
-
- strncpy(tmpname, cp, len);
- tmpname[len] = '\0';
- err = relayfs_create_dir(tmpname, parent, &parent);
- if (err && (err != -EEXIST))
- return err;
- cp += len + 1;
+ __relay_reset(chan->buf[i], 0);
+ prev = chan->buf[i];
}
-
- *residual = cp;
- *topdir = parent;
-
- return err;
-}
-
-/**
- * rchan_create_file - create file, including parent directories
- * @chanpath: path to file, including filename
- * @dentry: result dentry
- * @data: data to associate with the file
- *
- * Returns 0 if successful, negative otherwise.
- */
-static int
-rchan_create_file(const char * chanpath,
- struct dentry **dentry,
- struct rchan * data,
- int mode)
-{
- int err;
- const char * fname;
- struct dentry *topdir;
-
- err = rchan_create_dir(chanpath, &fname, &topdir);
- if (err && (err != -EEXIST))
- return err;
-
- err = relayfs_create_file(fname, topdir, dentry, (void *)data, mode);
-
- return err;
}
/**
- * relay_open - create a new file/channel buffer in relayfs
- * @chanpath: name of file to create, including path
- * @bufsize: size of sub-buffers
- * @nbufs: number of sub-buffers
- * @flags: channel attributes
- * @callbacks: client callback functions
- * @start_reserve: number of bytes to reserve at start of each sub-buffer
- * @end_reserve: number of bytes to reserve at end of each sub-buffer
- * @rchan_start_reserve: additional reserve at start of first sub-buffer
- * @resize_min: minimum total buffer size, if set
- * @resize_max: maximum total buffer size, if set
- * @mode: the perms to be given to the relayfs file, 0 to accept defaults
- * @init_buf: initial memory buffer to start out with, NULL if N/A
- * @init_buf_size: initial memory buffer size to start out with, 0 if N/A
- *
- * Returns channel id if successful, negative otherwise.
+ * relay_open_buf - create a new channel buffer in relayfs
*
- * Creates a relay channel using the sizes and attributes specified.
- * The default permissions, used if mode == 0 are S_IRUSR | S_IWUSR. See
- * Documentation/filesystems/relayfs.txt for details.
+ * Internal - used by relay_open().
*/
-int
-relay_open(const char *chanpath,
- int bufsize,
- int nbufs,
- u32 flags,
- struct rchan_callbacks *channel_callbacks,
- u32 start_reserve,
- u32 end_reserve,
- u32 rchan_start_reserve,
- u32 resize_min,
- u32 resize_max,
- int mode,
- char *init_buf,
- u32 init_buf_size)
+static struct rchan_buf *relay_open_buf(struct rchan *chan,
+ const char *filename,
+ struct dentry *parent,
+ int *is_global)
{
- int err;
- struct rchan *rchan;
+ struct rchan_buf *buf;
struct dentry *dentry;
- struct rchan_callbacks *callbacks = NULL;
- if (chanpath == NULL)
- return -EINVAL;
+ if (*is_global)
+ return chan->buf[0];
- if (nbufs != 1) {
- err = check_attribute_flags(&flags, resize_min ? 1 : 0);
- if (err)
- return err;
- }
-
- rchan = rchan_create(chanpath, bufsize, nbufs, flags, init_buf, init_buf_size, &err);
-
- if (err < 0)
- return err;
+ buf = relay_create_buf(chan);
+ if (!buf)
+ return NULL;
/* Create file in fs */
- if ((err = rchan_create_file(chanpath, &dentry, rchan, mode)) < 0) {
- rchan_destroy_buf(rchan);
- rchan_free_id(rchan->id);
- kfree(rchan);
- return err;
- }
-
- rchan->dentry = dentry;
-
- if (channel_callbacks == NULL)
- callbacks = &default_channel_callbacks;
- else
- callbacks = channel_callbacks;
-
- if (callbacks->buffer_end == NULL)
- callbacks->buffer_end = buffer_end_default_callback;
- if (callbacks->buffer_start == NULL)
- callbacks->buffer_start = buffer_start_default_callback;
- if (callbacks->deliver == NULL)
- callbacks->deliver = deliver_default_callback;
- if (callbacks->user_deliver == NULL)
- callbacks->user_deliver = user_deliver_default_callback;
- if (callbacks->needs_resize == NULL)
- callbacks->needs_resize = needs_resize_default_callback;
- if (callbacks->fileop_notify == NULL)
- callbacks->fileop_notify = fileop_notify_default_callback;
- if (callbacks->ioctl == NULL)
- callbacks->ioctl = ioctl_default_callback;
- rchan->callbacks = callbacks;
-
- /* Just to let the client know the sizes used */
- rchan->callbacks->needs_resize(rchan->id,
- RELAY_RESIZE_REPLACED,
- rchan->buf_size,
- rchan->n_bufs);
-
- rchan->flags = flags;
- rchan->start_reserve = start_reserve;
- rchan->end_reserve = end_reserve;
- rchan->rchan_start_reserve = rchan_start_reserve;
-
- __relay_reset(rchan, 1);
-
- if (resize_min > 0 && resize_max > 0 &&
- resize_max < RELAY_MAX_TOTAL_BUF_SIZE) {
- rchan->resize_min = resize_min;
- rchan->resize_max = resize_max;
- init_shrink_timer(rchan);
- }
+ dentry = chan->cb->create_buf_file(filename, parent, S_IRUSR,
+ buf, is_global);
+ if (!dentry) {
+ relay_destroy_buf(buf);
+ return NULL;
+ }
- rchan_get(rchan->id);
+ buf->dentry = dentry;
+ __relay_reset(buf, 1);
- return rchan->id;
+ return buf;
}
/**
- * relay_discard_init_buf - alloc channel buffer and copy init_buf into it
- * @rchan_id: the channel id
- *
- * Returns 0 if successful, negative otherwise.
+ * relay_close_buf - close a channel buffer
+ * @buf: channel buffer
*
- * NOTE: May sleep. Should also be called only when the channel isn't
- * actively being written into.
+ * Marks the buffer finalized and restores the default callbacks.
+ * The channel buffer and channel buffer data structure are then freed
+ * automatically when the last reference is given up.
*/
-int
-relay_discard_init_buf(int rchan_id)
+static inline void relay_close_buf(struct rchan_buf *buf)
{
- struct rchan *rchan;
- int err = 0;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- if (rchan->init_buf == NULL) {
- err = -EINVAL;
- goto out;
- }
-
- err = rchan_create_buf(rchan, rchan->alloc_size);
- if (err)
- goto out;
-
- memcpy(rchan->buf, rchan->init_buf, rchan->n_bufs * rchan->buf_size);
- rchan->init_buf = NULL;
-out:
- rchan_put(rchan);
-
- return err;
+ buf->finalized = 1;
+ buf->chan->cb = &default_channel_callbacks;
+ cancel_delayed_work(&buf->wake_readers);
+ flush_scheduled_work();
+ kref_put(&buf->kref, relay_remove_buf);
}
-/**
- * relay_finalize - perform end-of-buffer processing for last buffer
- * @rchan_id: the channel id
- * @releasing: true if called when releasing file
- *
- * Returns 0 if successful, negative otherwise.
- */
-static int
-relay_finalize(int rchan_id)
+static inline void setup_callbacks(struct rchan *chan,
+ struct rchan_callbacks *cb)
{
- struct rchan *rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- if (rchan->finalized == 0) {
- rchan->relay_ops->finalize(rchan);
- rchan->finalized = 1;
- }
-
- if (waitqueue_active(&rchan->read_wait)) {
- PREPARE_WORK(&rchan->wake_readers, wakeup_readers, rchan);
- schedule_delayed_work(&rchan->wake_readers, 1);
+ if (!cb) {
+ chan->cb = &default_channel_callbacks;
+ return;
}
- rchan_put(rchan);
-
- return 0;
+ if (!cb->subbuf_start)
+ cb->subbuf_start = subbuf_start_default_callback;
+ if (!cb->buf_mapped)
+ cb->buf_mapped = buf_mapped_default_callback;
+ if (!cb->buf_unmapped)
+ cb->buf_unmapped = buf_unmapped_default_callback;
+ if (!cb->create_buf_file)
+ cb->create_buf_file = create_buf_file_default_callback;
+ if (!cb->remove_buf_file)
+ cb->remove_buf_file = remove_buf_file_default_callback;
+ chan->cb = cb;
}
/**
- * restore_callbacks - restore default channel callbacks
- * @rchan: the channel
+ * relay_open - create a new relayfs channel
+ * @base_filename: base name of files to create
+ * @parent: dentry of parent directory, NULL for root directory
+ * @subbuf_size: size of sub-buffers
+ * @n_subbufs: number of sub-buffers
+ * @cb: client callback functions
*
- * Restore callbacks to the default versions.
- */
-static inline void
-restore_callbacks(struct rchan *rchan)
-{
- if (rchan->callbacks != &default_channel_callbacks)
- rchan->callbacks = &default_channel_callbacks;
-}
-
-/**
- * relay_close - close the channel
- * @rchan_id: relay channel id
- *
- * Finalizes the last sub-buffer and marks the channel as finalized.
- * The channel buffer and channel data structure are then freed
- * automatically when the last reference to the channel is given up.
- */
-int
-relay_close(int rchan_id)
-{
- int err;
- struct rchan *rchan;
-
- if ((rchan_id < 0) || (rchan_id >= RELAY_MAX_CHANNELS))
- return -EBADF;
-
- err = relay_finalize(rchan_id);
-
- if (!err) {
- read_lock(&rchan_table_lock);
- rchan = rchan_table[rchan_id];
- read_unlock(&rchan_table_lock);
-
- if (rchan) {
- restore_callbacks(rchan);
- if (rchan->resize_min)
- del_timer(&rchan->shrink_timer);
- rchan_put(rchan);
- }
- }
-
- return err;
-}
-
-/**
- * relay_write - reserve a slot in the channel and write data into it
- * @rchan_id: relay channel id
- * @data_ptr: data to be written into reserved slot
- * @count: number of bytes to write
- * @td_offset: optional offset where time delta should be written
- * @wrote_pos: optional ptr returning buf pos written to, ignored if NULL
+ * Returns channel pointer if successful, NULL otherwise.
*
- * Returns the number of bytes written, 0 or negative on failure.
- *
- * Reserves space in the channel and writes count bytes of data_ptr
- * to it. Automatically performs any necessary locking, depending
- * on the scheme and SMP usage in effect (no locking is done for the
- * lockless scheme regardless of usage).
- *
- * If td_offset is >= 0, the internal time delta calculated when
- * slot was reserved will be written at that offset.
- *
- * If wrote_pos is non-NULL, it will receive the location the data
- * was written to, which may be needed for some applications but is not
- * normally interesting.
- */
-int
-relay_write(int rchan_id,
- const void *data_ptr,
- size_t count,
- int td_offset,
- void **wrote_pos)
-{
- unsigned long flags;
- char *reserved, *write_pos;
- int bytes_written = 0;
- int reserve_code, interrupting;
- struct timeval ts;
- u32 td;
- struct rchan *rchan;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- relay_lock_channel(rchan, flags); /* nop for lockless */
-
- write_pos = reserved = relay_reserve(rchan, count, &ts, &td,
- &reserve_code, &interrupting);
-
- if (reserved != NULL) {
- relay_write_direct(write_pos, data_ptr, count);
- if ((td_offset >= 0) && (td_offset < count - sizeof(td)))
- *((u32 *)(reserved + td_offset)) = td;
- bytes_written = count;
- } else if (reserve_code == RELAY_WRITE_TOO_LONG)
- bytes_written = -EINVAL;
-
- if (bytes_written > 0)
- relay_commit(rchan, reserved, bytes_written, reserve_code, interrupting);
-
- relay_unlock_channel(rchan, flags); /* nop for lockless */
-
- rchan_put(rchan);
-
- if (wrote_pos)
- *wrote_pos = reserved;
-
- return bytes_written;
-}
-
-/**
- * wakeup_writers - wake up VFS writers waiting on a channel
- * @private: the channel
- *
- * This is the work function used to defer writer waking. The
- * reason waking is deferred is that calling directly from
- * buffers_consumed causes problems if you're writing from say
- * the scheduler.
- */
-static void
-wakeup_writers(void *private)
-{
- struct rchan *rchan = (struct rchan *)private;
-
- wake_up_interruptible(&rchan->write_wait);
-}
-
-
-/**
- * __relay_buffers_consumed - internal version of relay_buffers_consumed
- * @rchan: the relay channel
- * @bufs_consumed: number of buffers to add to current count for channel
- *
- * Internal - updates the channel's consumed buffer count.
+ * Creates a channel buffer for each cpu using the sizes and
+ * attributes specified. The created channel buffer files
+ * will be named base_filename0...base_filenameN-1. File
+ * permissions will be S_IRUSR.
*/
-static void
-__relay_buffers_consumed(struct rchan *rchan, u32 bufs_consumed)
+struct rchan *relay_open(const char *base_filename,
+ struct dentry *parent,
+ size_t subbuf_size,
+ size_t n_subbufs,
+ struct rchan_callbacks *cb)
{
- rchan->bufs_consumed += bufs_consumed;
-
- if (rchan->bufs_consumed > rchan->bufs_produced)
- rchan->bufs_consumed = rchan->bufs_produced;
-
- atomic_set(&rchan->suspended, 0);
-
- PREPARE_WORK(&rchan->wake_writers, wakeup_writers, rchan);
- schedule_delayed_work(&rchan->wake_writers, 1);
-}
+ unsigned int i;
+ struct rchan *chan;
+ char *tmpname;
+ int is_global = 0;
-/**
- * __reader_buffers_consumed - update reader/channel consumed buffer count
- * @reader: channel reader
- * @bufs_consumed: number of buffers to add to current count for channel
- *
- * Internal - updates the reader's consumed buffer count. If the reader's
- * resulting total is greater than the channel's, update the channel's.
-*/
-static void
-__reader_buffers_consumed(struct rchan_reader *reader, u32 bufs_consumed)
-{
- reader->bufs_consumed += bufs_consumed;
-
- if (reader->bufs_consumed > reader->rchan->bufs_consumed)
- __relay_buffers_consumed(reader->rchan, bufs_consumed);
-}
+ if (!base_filename)
+ return NULL;
-/**
- * relay_buffers_consumed - add to the # buffers consumed for the channel
- * @reader: channel reader
- * @bufs_consumed: number of buffers to add to current count for channel
- *
- * Adds to the channel's consumed buffer count. buffers_consumed should
- * be the number of buffers newly consumed, not the total number consumed.
- *
- * NOTE: kernel clients don't need to call this function if the reader
- * is auto-consuming or the channel is MODE_CONTINUOUS.
- */
-void
-relay_buffers_consumed(struct rchan_reader *reader, u32 bufs_consumed)
-{
- if (reader && reader->rchan)
- __reader_buffers_consumed(reader, bufs_consumed);
-}
+ if (!(subbuf_size && n_subbufs))
+ return NULL;
-/**
- * __relay_bytes_consumed - internal version of relay_bytes_consumed
- * @rchan: the relay channel
- * @bytes_consumed: number of bytes to add to current count for channel
- * @read_offset: where the bytes were consumed from
- *
- * Internal - updates the channel's consumed count.
-*/
-static void
-__relay_bytes_consumed(struct rchan *rchan, u32 bytes_consumed, u32 read_offset)
-{
- u32 consuming_idx;
- u32 unused;
+ chan = kcalloc(1, sizeof(struct rchan), GFP_KERNEL);
+ if (!chan)
+ return NULL;
- consuming_idx = read_offset / rchan->buf_size;
+ chan->version = RELAYFS_CHANNEL_VERSION;
+ chan->n_subbufs = n_subbufs;
+ chan->subbuf_size = subbuf_size;
+ chan->alloc_size = FIX_SIZE(subbuf_size * n_subbufs);
+ setup_callbacks(chan, cb);
+ kref_init(&chan->kref);
- if (consuming_idx >= rchan->n_bufs)
- consuming_idx = rchan->n_bufs - 1;
- rchan->bytes_consumed += bytes_consumed;
+ tmpname = kmalloc(NAME_MAX + 1, GFP_KERNEL);
+ if (!tmpname)
+ goto free_chan;
- unused = rchan->unused_bytes[consuming_idx];
-
- if (rchan->bytes_consumed + unused >= rchan->buf_size) {
- __relay_buffers_consumed(rchan, 1);
- rchan->bytes_consumed = 0;
+ for_each_online_cpu(i) {
+ sprintf(tmpname, "%s%d", base_filename, i);
+ chan->buf[i] = relay_open_buf(chan, tmpname, parent,
+ &is_global);
+ chan->buf[i]->cpu = i;
+ if (!chan->buf[i])
+ goto free_bufs;
}
-}
-
-/**
- * __reader_bytes_consumed - update reader/channel consumed count
- * @reader: channel reader
- * @bytes_consumed: number of bytes to add to current count for channel
- * @read_offset: where the bytes were consumed from
- *
- * Internal - updates the reader's consumed count. If the reader's
- * resulting total is greater than the channel's, update the channel's.
-*/
-static void
-__reader_bytes_consumed(struct rchan_reader *reader, u32 bytes_consumed, u32 read_offset)
-{
- u32 consuming_idx;
- u32 unused;
-
- consuming_idx = read_offset / reader->rchan->buf_size;
- if (consuming_idx >= reader->rchan->n_bufs)
- consuming_idx = reader->rchan->n_bufs - 1;
+ kfree(tmpname);
+ return chan;
- reader->bytes_consumed += bytes_consumed;
-
- unused = reader->rchan->unused_bytes[consuming_idx];
-
- if (reader->bytes_consumed + unused >= reader->rchan->buf_size) {
- reader->bufs_consumed++;
- reader->bytes_consumed = 0;
+free_bufs:
+ for (i = 0; i < NR_CPUS; i++) {
+ if (!chan->buf[i])
+ break;
+ relay_close_buf(chan->buf[i]);
+ if (is_global)
+ break;
}
+ kfree(tmpname);
- if ((reader->bufs_consumed > reader->rchan->bufs_consumed) ||
- ((reader->bufs_consumed == reader->rchan->bufs_consumed) &&
- (reader->bytes_consumed > reader->rchan->bytes_consumed)))
- __relay_bytes_consumed(reader->rchan, bytes_consumed, read_offset);
-}
-
-/**
- * relay_bytes_consumed - add to the # bytes consumed for the channel
- * @reader: channel reader
- * @bytes_consumed: number of bytes to add to current count for channel
- * @read_offset: where the bytes were consumed from
- *
- * Adds to the channel's consumed count. bytes_consumed should be the
- * number of bytes actually read e.g. return value of relay_read() and
- * the read_offset should be the actual offset the bytes were read from
- * e.g. the actual_read_offset set by relay_read(). See
- * Documentation/filesystems/relayfs.txt for more details.
- *
- * NOTE: kernel clients don't need to call this function if the reader
- * is auto-consuming or the channel is MODE_CONTINUOUS.
- */
-void
-relay_bytes_consumed(struct rchan_reader *reader, u32 bytes_consumed, u32 read_offset)
-{
- if (reader && reader->rchan)
- __reader_bytes_consumed(reader, bytes_consumed, read_offset);
-}
-
-/**
- * update_readers_consumed - apply offset change to reader
- * @rchan: the channel
- *
- * Apply the consumed counts to all readers open on the channel.
- */
-void
-update_readers_consumed(struct rchan *rchan, u32 bufs_consumed, u32 bytes_consumed)
-{
- struct list_head *p;
- struct rchan_reader *reader;
-
- read_lock(&rchan->open_readers_lock);
- list_for_each(p, &rchan->open_readers) {
- reader = list_entry(p, struct rchan_reader, list);
- reader->bufs_consumed = bufs_consumed;
- reader->bytes_consumed = bytes_consumed;
- if (reader->vfs_reader)
- reader->pos.file->f_pos = 0;
- else
- reader->pos.f_pos = 0;
- reader->offset_changed = 1;
- }
- read_unlock(&rchan->open_readers_lock);
+free_chan:
+ kref_put(&chan->kref, relay_destroy_channel);
+ return NULL;
}
/**
- * do_read - utility function to do the actual read to user
- * @rchan: the channel
- * @buf: user buf to read into, NULL if just getting info
- * @count: bytes requested
- * @read_offset: offset into channel
- * @new_offset: new offset into channel after read
- * @actual_read_offset: read offset actually used
+ * relay_switch_subbuf - switch to a new sub-buffer
+ * @buf: channel buffer
+ * @length: size of current event
*
- * Returns the number of bytes read, 0 if none.
- */
-static ssize_t
-do_read(struct rchan *rchan, char *buf, size_t count, u32 read_offset, u32 *new_offset, u32 *actual_read_offset)
-{
- u32 read_bufno, cur_bufno;
- u32 avail_offset, cur_idx, max_offset, buf_end_offset;
- u32 avail_count, buf_size;
- int unused_bytes = 0;
- size_t read_count = 0;
- u32 last_buf_byte_offset;
-
- *actual_read_offset = read_offset;
-
- buf_size = rchan->buf_size;
- if (unlikely(!buf_size)) BUG();
-
- read_bufno = read_offset / buf_size;
- if (unlikely(read_bufno >= RELAY_MAX_BUFS)) BUG();
- unused_bytes = rchan->unused_bytes[read_bufno];
-
- avail_offset = cur_idx = relay_get_offset(rchan, &max_offset);
-
- if (cur_idx == read_offset) {
- if (atomic_read(&rchan->suspended) == 1) {
- read_offset += 1;
- if (read_offset >= max_offset)
- read_offset = 0;
- *actual_read_offset = read_offset;
- } else {
- *new_offset = read_offset;
- return 0;
- }
- } else {
- last_buf_byte_offset = (read_bufno + 1) * buf_size - 1;
- if (read_offset == last_buf_byte_offset) {
- if (unused_bytes != 1) {
- read_offset += 1;
- if (read_offset >= max_offset)
- read_offset = 0;
- *actual_read_offset = read_offset;
- }
- }
- }
-
- read_bufno = read_offset / buf_size;
- if (unlikely(read_bufno >= RELAY_MAX_BUFS)) BUG();
- unused_bytes = rchan->unused_bytes[read_bufno];
-
- cur_bufno = cur_idx / buf_size;
+ * Returns either the length passed in or 0 if full.
- buf_end_offset = (read_bufno + 1) * buf_size - unused_bytes;
- if (avail_offset > buf_end_offset)
- avail_offset = buf_end_offset;
- else if (avail_offset < read_offset)
- avail_offset = buf_end_offset;
- avail_count = avail_offset - read_offset;
- read_count = avail_count >= count ? count : avail_count;
-
- if (read_count && buf != NULL)
- if (copy_to_user(buf, rchan->buf + read_offset, read_count))
- return -EFAULT;
-
- if (read_bufno == cur_bufno)
- if (read_count && (read_offset + read_count >= buf_end_offset) && (read_offset + read_count <= cur_idx)) {
- *new_offset = cur_idx;
- return read_count;
- }
-
- if (read_offset + read_count + unused_bytes > max_offset)
- *new_offset = 0;
- else if (read_offset + read_count >= buf_end_offset)
- *new_offset = read_offset + read_count + unused_bytes;
- else
- *new_offset = read_offset + read_count;
-
- return read_count;
-}
-
-/**
- * __relay_read - read bytes from channel, relative to current reader pos
- * @reader: channel reader
- * @buf: user buf to read into, NULL if just getting info
- * @count: bytes requested
- * @read_offset: offset into channel
- * @new_offset: new offset into channel after read
- * @actual_read_offset: read offset actually used
- * @wait: if non-zero, wait for something to read
- *
- * Internal - see relay_read() for details.
- *
- * Returns the number of bytes read, 0 if none, negative on failure.
+ * Performs sub-buffer-switch tasks such as invoking callbacks,
+ * updating padding counts, waking up readers, etc.
*/
-static ssize_t
-__relay_read(struct rchan_reader *reader, char *buf, size_t count, u32 read_offset, u32 *new_offset, u32 *actual_read_offset, int wait)
+size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)
{
- int err = 0;
- size_t read_count = 0;
- struct rchan *rchan = reader->rchan;
-
- if (!wait && !rchan->initialized)
- return -EAGAIN;
-
- if (using_lockless(rchan))
- read_offset &= idx_mask(rchan);
-
- if (read_offset >= rchan->n_bufs * rchan->buf_size) {
- *new_offset = 0;
- if (!wait)
- return -EAGAIN;
- else
- return -EINTR;
- }
-
- if (buf != NULL && wait) {
- err = wait_event_interruptible(rchan->read_wait,
- ((rchan->finalized == 1) ||
- (atomic_read(&rchan->suspended) == 1) ||
- (relay_get_offset(rchan, NULL) != read_offset)));
+ void *old, *new;
+ size_t old_subbuf, new_subbuf;
- if (rchan->finalized)
- return 0;
+ if (unlikely(length > buf->chan->subbuf_size))
+ goto toobig;
- if (reader->offset_changed) {
- reader->offset_changed = 0;
- return -EINTR;
+ if (buf->offset != buf->chan->subbuf_size + 1) {
+ buf->prev_padding = buf->chan->subbuf_size - buf->offset;
+ old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
+ buf->padding[old_subbuf] = buf->prev_padding;
+ buf->subbufs_produced++;
+ if (waitqueue_active(&buf->read_wait)) {
+ PREPARE_WORK(&buf->wake_readers, wakeup_readers, buf);
+ schedule_delayed_work(&buf->wake_readers, 1);
}
-
- if (err)
- return err;
}
- read_count = do_read(rchan, buf, count, read_offset, new_offset, actual_read_offset);
-
- if (read_count < 0)
- err = read_count;
-
- if (err)
- return err;
- else
- return read_count;
-}
-
-/**
- * relay_read - read bytes from channel, relative to current reader pos
- * @reader: channel reader
- * @buf: user buf to read into, NULL if just getting info
- * @count: bytes requested
- * @wait: if non-zero, wait for something to read
- * @actual_read_offset: set read offset actually used, must not be NULL
- *
- * Reads count bytes from the channel, or as much as is available within
- * the sub-buffer currently being read. The read offset that will be
- * read from is the position contained within the reader object. If the
- * wait flag is set, buf is non-NULL, and there is nothing available,
- * it will wait until there is. If the wait flag is 0 and there is
- * nothing available, -EAGAIN is returned. If buf is NULL, the value
- * returned is the number of bytes that would have been read.
- * actual_read_offset is the value that should be passed as the read
- * offset to relay_bytes_consumed, needed only if the reader is not
- * auto-consuming and the channel is MODE_NO_OVERWRITE, but in any case,
- * it must not be NULL. See Documentation/filesystems/relayfs.txt for
- * more details.
- */
-ssize_t
-relay_read(struct rchan_reader *reader, char *buf, size_t count, int wait, u32 *actual_read_offset)
-{
- u32 new_offset;
- u32 read_offset;
- ssize_t read_count;
-
- if (reader == NULL || reader->rchan == NULL)
- return -EBADF;
-
- if (actual_read_offset == NULL)
- return -EINVAL;
-
- if (reader->vfs_reader)
- read_offset = (u32)(reader->pos.file->f_pos);
- else
- read_offset = reader->pos.f_pos;
- *actual_read_offset = read_offset;
-
- read_count = __relay_read(reader, buf, count, read_offset,
- &new_offset, actual_read_offset, wait);
-
- if (read_count < 0)
- return read_count;
-
- if (reader->vfs_reader)
- reader->pos.file->f_pos = new_offset;
- else
- reader->pos.f_pos = new_offset;
-
- if (reader->auto_consume && ((read_count) || (new_offset != read_offset)))
- __reader_bytes_consumed(reader, read_count, *actual_read_offset);
-
- if (read_count == 0 && !wait)
- return -EAGAIN;
-
- return read_count;
-}
-
-/**
- * relay_bytes_avail - number of bytes available in current sub-buffer
- * @reader: channel reader
- *
- * Returns the number of bytes available relative to the reader's
- * current read position within the corresponding sub-buffer, 0 if
- * there is nothing available. See Documentation/filesystems/relayfs.txt
- * for more details.
- */
-ssize_t
-relay_bytes_avail(struct rchan_reader *reader)
-{
- u32 f_pos;
- u32 new_offset;
- u32 actual_read_offset;
- ssize_t bytes_read;
-
- if (reader == NULL || reader->rchan == NULL)
- return -EBADF;
-
- if (reader->vfs_reader)
- f_pos = (u32)reader->pos.file->f_pos;
- else
- f_pos = reader->pos.f_pos;
- new_offset = f_pos;
-
- bytes_read = __relay_read(reader, NULL, reader->rchan->buf_size,
- f_pos, &new_offset, &actual_read_offset, 0);
-
- if ((new_offset != f_pos) &&
- ((bytes_read == -EINTR) || (bytes_read == 0)))
- bytes_read = -EAGAIN;
- else if ((bytes_read < 0) && (bytes_read != -EAGAIN))
- bytes_read = 0;
-
- return bytes_read;
-}
-
-/**
- * rchan_empty - boolean, is the channel empty wrt reader?
- * @reader: channel reader
- *
- * Returns 1 if the channel is empty, 0 otherwise.
- */
-int
-rchan_empty(struct rchan_reader *reader)
-{
- ssize_t avail_count;
- u32 buffers_ready;
- struct rchan *rchan = reader->rchan;
- u32 cur_idx, curbuf_bytes;
- int mapped;
-
- if (atomic_read(&rchan->suspended) == 1)
+ old = buf->data;
+ new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
+ new = buf->start + new_subbuf * buf->chan->subbuf_size;
+ buf->offset = 0;
+ if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {
+ buf->offset = buf->chan->subbuf_size + 1;
return 0;
-
- mapped = atomic_read(&rchan->mapped);
-
- if (mapped && bulk_delivery(rchan)) {
- buffers_ready = rchan->bufs_produced - rchan->bufs_consumed;
- return buffers_ready ? 0 : 1;
}
+ buf->data = new;
+ buf->padding[new_subbuf] = 0;
- if (mapped && packet_delivery(rchan)) {
- buffers_ready = rchan->bufs_produced - rchan->bufs_consumed;
- if (buffers_ready)
- return 0;
- else {
- cur_idx = relay_get_offset(rchan, NULL);
- curbuf_bytes = cur_idx % rchan->buf_size;
- return curbuf_bytes == rchan->bytes_consumed ? 1 : 0;
- }
- }
-
- avail_count = relay_bytes_avail(reader);
-
- return avail_count ? 0 : 1;
-}
-
-/**
- * rchan_full - boolean, is the channel full wrt consuming reader?
- * @reader: channel reader
- *
- * Returns 1 if the channel is full, 0 otherwise.
- */
-int
-rchan_full(struct rchan_reader *reader)
-{
- u32 buffers_ready;
- struct rchan *rchan = reader->rchan;
-
- if (mode_continuous(rchan))
- return 0;
-
- buffers_ready = rchan->bufs_produced - rchan->bufs_consumed;
-
- return buffers_ready > reader->rchan->n_bufs - 1 ? 1 : 0;
-}
+ if (unlikely(length + buf->offset > buf->chan->subbuf_size))
+ goto toobig;
-/**
- * relay_info - get status and other information about a relay channel
- * @rchan_id: relay channel id
- * @rchan_info: pointer to the rchan_info struct to be filled in
- *
- * Fills in an rchan_info struct with channel status and attribute
- * information. See Documentation/filesystems/relayfs.txt for details.
- *
- * Returns 0 if successful, negative otherwise.
- */
-int
-relay_info(int rchan_id, struct rchan_info *rchan_info)
-{
- int i;
- struct rchan *rchan;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- rchan_info->flags = rchan->flags;
- rchan_info->buf_size = rchan->buf_size;
- rchan_info->buf_addr = rchan->buf;
- rchan_info->alloc_size = rchan->alloc_size;
- rchan_info->n_bufs = rchan->n_bufs;
- rchan_info->cur_idx = relay_get_offset(rchan, NULL);
- rchan_info->bufs_produced = rchan->bufs_produced;
- rchan_info->bufs_consumed = rchan->bufs_consumed;
- rchan_info->buf_id = rchan->buf_id;
-
- for (i = 0; i < rchan->n_bufs; i++) {
- rchan_info->unused_bytes[i] = rchan->unused_bytes[i];
- if (using_lockless(rchan))
- rchan_info->buffer_complete[i] = (atomic_read(&fill_count(rchan, i)) == rchan->buf_size);
- else
- rchan_info->buffer_complete[i] = 0;
- }
-
- rchan_put(rchan);
+ return length;
+toobig:
+ buf->chan->last_toobig = length;
return 0;
}
/**
- * __add_rchan_reader - creates and adds a reader to a channel
- * @rchan: relay channel
- * @filp: the file associated with rchan, if applicable
- * @auto_consume: boolean, whether reader's reads automatically consume
- * @map_reader: boolean, whether reader's reading via a channel mapping
+ * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count
+ * @chan: the channel
+ * @cpu: the cpu associated with the channel buffer to update
+ * @subbufs_consumed: number of sub-buffers to add to current buf's count
*
- * Returns a pointer to the reader object create, NULL if unsuccessful
+ * Adds to the channel buffer's consumed sub-buffer count.
+ * subbufs_consumed should be the number of sub-buffers newly consumed,
+ * not the total consumed.
*
- * Creates and initializes an rchan_reader object for reading the channel.
- * If filp is non-NULL, the reader is a VFS reader, otherwise not.
- *
- * If the reader is a map reader, it isn't considered a VFS reader for
- * our purposes. Also, map_readers can't be auto-consuming.
+ * NOTE: kernel clients don't need to call this function if the channel
+ * mode is 'overwrite'.
*/
-struct rchan_reader *
-__add_rchan_reader(struct rchan *rchan, struct file *filp, int auto_consume, int map_reader)
+void relay_subbufs_consumed(struct rchan *chan,
+ unsigned int cpu,
+ size_t subbufs_consumed)
{
- struct rchan_reader *reader;
- u32 will_read;
-
- reader = kmalloc(sizeof(struct rchan_reader), GFP_KERNEL);
+ struct rchan_buf *buf;
- if (reader) {
- write_lock(&rchan->open_readers_lock);
- reader->rchan = rchan;
- if (filp) {
- reader->vfs_reader = 1;
- reader->pos.file = filp;
- } else {
- reader->vfs_reader = 0;
- reader->pos.f_pos = 0;
- }
- reader->map_reader = map_reader;
- reader->auto_consume = auto_consume;
+ if (!chan)
+ return;
- if (!map_reader) {
- will_read = rchan->bufs_produced % rchan->n_bufs;
- if (!will_read && atomic_read(&rchan->suspended))
- will_read = rchan->n_bufs;
- reader->bufs_consumed = rchan->bufs_produced - will_read;
- rchan->bufs_consumed = reader->bufs_consumed;
- rchan->bytes_consumed = reader->bytes_consumed = 0;
- reader->offset_changed = 0;
- }
-
- list_add(&reader->list, &rchan->open_readers);
- write_unlock(&rchan->open_readers_lock);
- }
+ if (cpu >= NR_CPUS || !chan->buf[cpu])
+ return;
- return reader;
+ buf = chan->buf[cpu];
+ buf->subbufs_consumed += subbufs_consumed;
+ if (buf->subbufs_consumed > buf->subbufs_produced)
+ buf->subbufs_consumed = buf->subbufs_produced;
}
/**
- * add_rchan_reader - create a reader for a channel
- * @rchan_id: relay channel handle
- * @auto_consume: boolean, whether reader's reads automatically consume
- *
- * Returns a pointer to the reader object created, NULL if unsuccessful
+ * relay_destroy_channel - free the channel struct
*
- * Creates and initializes an rchan_reader object for reading the channel.
- * This function is useful only for non-VFS readers.
+ * Should only be called from kref_put().
*/
-struct rchan_reader *
-add_rchan_reader(int rchan_id, int auto_consume)
+void relay_destroy_channel(struct kref *kref)
{
- struct rchan *rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return NULL;
-
- return __add_rchan_reader(rchan, NULL, auto_consume, 0);
+ struct rchan *chan = container_of(kref, struct rchan, kref);
+ kfree(chan);
}
/**
- * add_map_reader - create a map reader for a channel
- * @rchan_id: relay channel handle
- *
- * Returns a pointer to the reader object created, NULL if unsuccessful
+ * relay_close - close the channel
+ * @chan: the channel
*
- * Creates and initializes an rchan_reader object for reading the channel.
- * This function is useful only for map readers.
+ * Closes all channel buffers and frees the channel.
*/
-struct rchan_reader *
-add_map_reader(int rchan_id)
+void relay_close(struct rchan *chan)
{
- struct rchan *rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return NULL;
+ unsigned int i;
+ struct rchan_buf *prev = NULL;
- return __add_rchan_reader(rchan, NULL, 0, 1);
-}
+ if (!chan)
+ return;
-/**
- * __remove_rchan_reader - destroy a channel reader
- * @reader: channel reader
- *
- * Internal - removes reader from the open readers list, and frees it.
- */
-void
-__remove_rchan_reader(struct rchan_reader *reader)
-{
- struct list_head *p;
- struct rchan_reader *found_reader = NULL;
-
- write_lock(&reader->rchan->open_readers_lock);
- list_for_each(p, &reader->rchan->open_readers) {
- found_reader = list_entry(p, struct rchan_reader, list);
- if (found_reader == reader) {
- list_del(&found_reader->list);
+ for (i = 0; i < NR_CPUS; i++) {
+ if (!chan->buf[i] || chan->buf[i] == prev)
break;
- }
+ relay_close_buf(chan->buf[i]);
+ prev = chan->buf[i];
}
- write_unlock(&reader->rchan->open_readers_lock);
- if (found_reader)
- kfree(found_reader);
-}
-
-/**
- * remove_rchan_reader - destroy a channel reader
- * @reader: channel reader
- *
- * Finds and removes the given reader from the channel. This function
- * is useful only for non-VFS readers.
- *
- * Returns 0 if successful, negative otherwise.
- */
-int
-remove_rchan_reader(struct rchan_reader *reader)
-{
- int err = 0;
-
- if (reader) {
- rchan_put(reader->rchan);
- __remove_rchan_reader(reader);
- } else
- err = -EINVAL;
+ if (chan->last_toobig)
+ printk(KERN_WARNING "relayfs: one or more items not logged "
+ "[item size (%Zd) > sub-buffer size (%Zd)]\n",
+ chan->last_toobig, chan->subbuf_size);
- return err;
+ kref_put(&chan->kref, relay_destroy_channel);
}
/**
- * remove_map_reader - destroy a map reader
- * @reader: channel reader
+ * relay_flush - close the channel
+ * @chan: the channel
*
- * Finds and removes the given map reader from the channel. This function
- * is useful only for map readers.
- *
- * Returns 0 if successful, negative otherwise.
+ * Flushes all channel buffers i.e. forces buffer switch.
*/
-int
-remove_map_reader(struct rchan_reader *reader)
+void relay_flush(struct rchan *chan)
{
- return remove_rchan_reader(reader);
-}
+ unsigned int i;
+ struct rchan_buf *prev = NULL;
-EXPORT_SYMBOL(relay_open);
-EXPORT_SYMBOL(relay_close);
-EXPORT_SYMBOL(relay_reset);
-EXPORT_SYMBOL(relay_reserve);
-EXPORT_SYMBOL(relay_commit);
-EXPORT_SYMBOL(relay_read);
-EXPORT_SYMBOL(relay_write);
-EXPORT_SYMBOL(relay_bytes_avail);
-EXPORT_SYMBOL(relay_buffers_consumed);
-EXPORT_SYMBOL(relay_bytes_consumed);
-EXPORT_SYMBOL(relay_info);
-EXPORT_SYMBOL(relay_discard_init_buf);
+ if (!chan)
+ return;
+ for (i = 0; i < NR_CPUS; i++) {
+ if (!chan->buf[i] || chan->buf[i] == prev)
+ break;
+ relay_switch_subbuf(chan->buf[i], 0);
+ prev = chan->buf[i];
+ }
+}
+EXPORT_SYMBOL_GPL(relay_open);
+EXPORT_SYMBOL_GPL(relay_close);
+EXPORT_SYMBOL_GPL(relay_flush);
+EXPORT_SYMBOL_GPL(relay_reset);
+EXPORT_SYMBOL_GPL(relay_subbufs_consumed);
+EXPORT_SYMBOL_GPL(relay_switch_subbuf);
+EXPORT_SYMBOL_GPL(relay_buf_full);
git://git.onelab.eu / linux-2.6.git / blobdiff