+++ /dev/null
-#
-# relayfs Makefile
-#
-
-obj-$(CONFIG_RELAYFS_FS) += relayfs.o
-
-relayfs-y := relay.o relay_lockless.o relay_locking.o inode.o resize.o
-relayfs-$(CONFIG_KLOG_CHANNEL) += klog.o
+++ /dev/null
-/*
- * VFS-related code for RelayFS, a high-speed data relay filesystem.
- *
- * Copyright (C) 2003 - Tom Zanussi <zanussi@us.ibm.com>, IBM Corp
- * Copyright (C) 2003 - Karim Yaghmour <karim@opersys.com>
- *
- * Based on ramfs, Copyright (C) 2002 - Linus Torvalds
- *
- * This file is released under the GPL.
- */
-
-#include <linux/module.h>
-#include <linux/fs.h>
-#include <linux/mount.h>
-#include <linux/pagemap.h>
-#include <linux/highmem.h>
-#include <linux/init.h>
-#include <linux/string.h>
-#include <linux/smp_lock.h>
-#include <linux/backing-dev.h>
-#include <linux/namei.h>
-#include <linux/poll.h>
-#include <asm/uaccess.h>
-#include <asm/relay.h>
-
-#define RELAYFS_MAGIC 0x26F82121
-
-static struct super_operations relayfs_ops;
-static struct address_space_operations relayfs_aops;
-static struct inode_operations relayfs_file_inode_operations;
-static struct file_operations relayfs_file_operations;
-static struct inode_operations relayfs_dir_inode_operations;
-
-static struct vfsmount * relayfs_mount;
-static int relayfs_mount_count;
-
-static struct backing_dev_info relayfs_backing_dev_info = {
- .ra_pages = 0, /* No readahead */
- .memory_backed = 1, /* Does not contribute to dirty memory */
-};
-
-static struct inode *
-relayfs_get_inode(struct super_block *sb, int mode, dev_t dev)
-{
- struct inode * inode;
-
- inode = new_inode(sb);
-
- if (inode) {
- inode->i_mode = mode;
- inode->i_uid = current->fsuid;
- inode->i_gid = current->fsgid;
- inode->i_blksize = PAGE_CACHE_SIZE;
- inode->i_blocks = 0;
- inode->i_mapping->a_ops = &relayfs_aops;
- inode->i_mapping->backing_dev_info = &relayfs_backing_dev_info;
- inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
- switch (mode & S_IFMT) {
- default:
- init_special_inode(inode, mode, dev);
- break;
- case S_IFREG:
- inode->i_op = &relayfs_file_inode_operations;
- inode->i_fop = &relayfs_file_operations;
- break;
- case S_IFDIR:
- inode->i_op = &relayfs_dir_inode_operations;
- inode->i_fop = &simple_dir_operations;
-
- /* directory inodes start off with i_nlink == 2 (for "." entry) */
- inode->i_nlink++;
- break;
- case S_IFLNK:
- inode->i_op = &page_symlink_inode_operations;
- break;
- }
- }
- return inode;
-}
-
-/*
- * File creation. Allocate an inode, and we're done..
- */
-/* SMP-safe */
-static int
-relayfs_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
-{
- struct inode * inode;
- int error = -ENOSPC;
-
- inode = relayfs_get_inode(dir->i_sb, mode, dev);
-
- if (inode) {
- d_instantiate(dentry, inode);
- dget(dentry); /* Extra count - pin the dentry in core */
- error = 0;
- }
- return error;
-}
-
-static int
-relayfs_mkdir(struct inode * dir, struct dentry * dentry, int mode)
-{
- int retval;
-
- retval = relayfs_mknod(dir, dentry, mode | S_IFDIR, 0);
-
- if (!retval)
- dir->i_nlink++;
- return retval;
-}
-
-static int
-relayfs_create(struct inode *dir, struct dentry *dentry, int mode, struct nameidata *nd)
-{
- return relayfs_mknod(dir, dentry, mode | S_IFREG, 0);
-}
-
-static int
-relayfs_symlink(struct inode * dir, struct dentry *dentry, const char * symname)
-{
- struct inode *inode;
- int error = -ENOSPC;
-
- inode = relayfs_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
-
- if (inode) {
- int l = strlen(symname)+1;
- error = page_symlink(inode, symname, l);
- if (!error) {
- d_instantiate(dentry, inode);
- dget(dentry);
- } else
- iput(inode);
- }
- return error;
-}
-
-/**
- * relayfs_create_entry - create a relayfs directory or file
- * @name: the name of the file to create
- * @parent: parent directory
- * @dentry: result dentry
- * @entry_type: type of file to create (S_IFREG, S_IFDIR)
- * @mode: mode
- * @data: data to associate with the file
- *
- * Creates a file or directory with the specifed permissions.
- */
-static int
-relayfs_create_entry(const char * name, struct dentry * parent, struct dentry **dentry, int entry_type, int mode, void * data)
-{
- struct qstr qname;
- struct dentry * d;
-
- int error = 0;
-
- error = simple_pin_fs("relayfs", &relayfs_mount, &relayfs_mount_count);
- if (error) {
- printk(KERN_ERR "Couldn't mount relayfs: errcode %d\n", error);
- return error;
- }
-
- qname.name = name;
- qname.len = strlen(name);
- qname.hash = full_name_hash(name, qname.len);
-
- if (parent == NULL)
- if (relayfs_mount && relayfs_mount->mnt_sb)
- parent = relayfs_mount->mnt_sb->s_root;
-
- if (parent == NULL) {
- simple_release_fs(&relayfs_mount, &relayfs_mount_count);
- return -EINVAL;
- }
-
- parent = dget(parent);
- down(&parent->d_inode->i_sem);
- d = lookup_hash(&qname, parent);
- if (IS_ERR(d)) {
- error = PTR_ERR(d);
- goto release_mount;
- }
-
- if (d->d_inode) {
- error = -EEXIST;
- goto release_mount;
- }
-
- if (entry_type == S_IFREG)
- error = relayfs_create(parent->d_inode, d, entry_type | mode, NULL);
- else
- error = relayfs_mkdir(parent->d_inode, d, entry_type | mode);
- if (error)
- goto release_mount;
-
- if ((entry_type == S_IFREG) && data) {
- d->d_inode->u.generic_ip = data;
- goto exit; /* don't release mount for regular files */
- }
-
-release_mount:
- simple_release_fs(&relayfs_mount, &relayfs_mount_count);
-exit:
- *dentry = d;
- up(&parent->d_inode->i_sem);
- dput(parent);
-
- return error;
-}
-
-/**
- * relayfs_create_file - create a file in the relay filesystem
- * @name: the name of the file to create
- * @parent: parent directory
- * @dentry: result dentry
- * @data: data to associate with the file
- * @mode: mode, if not specied the default perms are used
- *
- * The file will be created user rw on behalf of current user.
- */
-int
-relayfs_create_file(const char * name, struct dentry * parent, struct dentry **dentry, void * data, int mode)
-{
- if (!mode)
- mode = S_IRUSR | S_IWUSR;
-
- return relayfs_create_entry(name, parent, dentry, S_IFREG,
- mode, data);
-}
-
-/**
- * relayfs_create_dir - create a directory in the relay filesystem
- * @name: the name of the directory to create
- * @parent: parent directory
- * @dentry: result dentry
- *
- * The directory will be created world rwx on behalf of current user.
- */
-int
-relayfs_create_dir(const char * name, struct dentry * parent, struct dentry **dentry)
-{
- return relayfs_create_entry(name, parent, dentry, S_IFDIR,
- S_IRWXU | S_IRUGO | S_IXUGO, NULL);
-}
-
-/**
- * relayfs_remove_file - remove a file in the relay filesystem
- * @dentry: file dentry
- *
- * Remove a file previously created by relayfs_create_file.
- */
-int
-relayfs_remove_file(struct dentry *dentry)
-{
- struct dentry *parent;
- int is_reg;
-
- parent = dentry->d_parent;
- if (parent == NULL)
- return -EINVAL;
-
- is_reg = S_ISREG(dentry->d_inode->i_mode);
-
- parent = dget(parent);
- down(&parent->d_inode->i_sem);
- if (dentry->d_inode) {
- simple_unlink(parent->d_inode, dentry);
- d_delete(dentry);
- }
- dput(dentry);
- up(&parent->d_inode->i_sem);
- dput(parent);
-
- if(is_reg)
- simple_release_fs(&relayfs_mount, &relayfs_mount_count);
-
- return 0;
-}
-
-/**
- * relayfs_open - open file op for relayfs files
- * @inode: the inode
- * @filp: the file
- *
- * Associates the channel with the file, and increments the
- * channel refcount. Reads will be 'auto-consuming'.
- */
-int
-relayfs_open(struct inode *inode, struct file *filp)
-{
- struct rchan *rchan;
- struct rchan_reader *reader;
- int retval = 0;
-
- if (inode->u.generic_ip) {
- rchan = (struct rchan *)inode->u.generic_ip;
- if (rchan == NULL)
- return -EACCES;
- reader = __add_rchan_reader(rchan, filp, 1, 0);
- if (reader == NULL)
- return -ENOMEM;
- filp->private_data = reader;
- retval = rchan->callbacks->fileop_notify(rchan->id, filp,
- RELAY_FILE_OPEN);
- if (retval == 0)
- /* Inc relay channel refcount for file */
- rchan_get(rchan->id);
- else {
- __remove_rchan_reader(reader);
- retval = -EPERM;
- }
- }
-
- return retval;
-}
-
-/**
- * relayfs_mmap - mmap file op for relayfs files
- * @filp: the file
- * @vma: the vma describing what to map
- *
- * Calls upon relay_mmap_buffer to map the file into user space.
- */
-int
-relayfs_mmap(struct file *filp, struct vm_area_struct *vma)
-{
- struct rchan *rchan;
-
- rchan = ((struct rchan_reader *)filp->private_data)->rchan;
-
- return __relay_mmap_buffer(rchan, vma);
-}
-
-/**
- * relayfs_file_read - read file op for relayfs files
- * @filp: the file
- * @buf: user buf to read into
- * @count: bytes requested
- * @offset: offset into file
- *
- * Reads count bytes from the channel, or as much as is available within
- * the sub-buffer currently being read. Reads are 'auto-consuming'.
- * See relay_read() for details.
- *
- * Returns bytes read on success, 0 or -EAGAIN if nothing available,
- * negative otherwise.
- */
-ssize_t
-relayfs_file_read(struct file *filp, char * buf, size_t count, loff_t *offset)
-{
- size_t read_count;
- struct rchan_reader *reader;
- u32 dummy; /* all VFS readers are auto-consuming */
-
- if (offset != &filp->f_pos) /* pread, seeking not supported */
- return -ESPIPE;
-
- if (count == 0)
- return 0;
-
- reader = (struct rchan_reader *)filp->private_data;
- read_count = relay_read(reader, buf, count,
- filp->f_flags & (O_NDELAY | O_NONBLOCK) ? 0 : 1, &dummy);
-
- return read_count;
-}
-
-/**
- * relayfs_file_write - write file op for relayfs files
- * @filp: the file
- * @buf: user buf to write from
- * @count: bytes to write
- * @offset: offset into file
- *
- * Reserves a slot in the relay buffer and writes count bytes
- * into it. The current limit for a single write is 2 pages
- * worth. The user_deliver() channel callback will be invoked on
- *
- * Returns bytes written on success, 0 or -EAGAIN if nothing available,
- * negative otherwise.
- */
-ssize_t
-relayfs_file_write(struct file *filp, const char *buf, size_t count, loff_t *offset)
-{
- int write_count;
- char * write_buf;
- struct rchan *rchan;
- int err = 0;
- void *wrote_pos;
- struct rchan_reader *reader;
-
- reader = (struct rchan_reader *)filp->private_data;
- if (reader == NULL)
- return -EPERM;
-
- rchan = reader->rchan;
- if (rchan == NULL)
- return -EPERM;
-
- if (count == 0)
- return 0;
-
- /* Change this if need to write more than 2 pages at once */
- if (count > 2 * PAGE_SIZE)
- return -EINVAL;
-
- write_buf = (char *)__get_free_pages(GFP_KERNEL, 1);
- if (write_buf == NULL)
- return -ENOMEM;
-
- if (copy_from_user(write_buf, buf, count))
- return -EFAULT;
-
- if (filp->f_flags & (O_NDELAY | O_NONBLOCK)) {
- write_count = relay_write(rchan->id, write_buf, count, -1, &wrote_pos);
- if (write_count == 0)
- return -EAGAIN;
- } else {
- err = wait_event_interruptible(rchan->write_wait,
- (write_count = relay_write(rchan->id, write_buf, count, -1, &wrote_pos)));
- if (err)
- return err;
- }
-
- free_pages((unsigned long)write_buf, 1);
-
- rchan->callbacks->user_deliver(rchan->id, wrote_pos, write_count);
-
- return write_count;
-}
-
-/**
- * relayfs_ioctl - ioctl file op for relayfs files
- * @inode: the inode
- * @filp: the file
- * @cmd: the command
- * @arg: command arg
- *
- * Passes the specified cmd/arg to the kernel client. arg may be a
- * pointer to user-space data, in which case the kernel client is
- * responsible for copying the data to/from user space appropriately.
- * The kernel client is also responsible for returning a meaningful
- * return value for ioctl calls.
- *
- * Returns result of relay channel callback, -EPERM if unsuccessful.
- */
-int
-relayfs_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
-{
- struct rchan *rchan;
- struct rchan_reader *reader;
-
- reader = (struct rchan_reader *)filp->private_data;
- if (reader == NULL)
- return -EPERM;
-
- rchan = reader->rchan;
- if (rchan == NULL)
- return -EPERM;
-
- return rchan->callbacks->ioctl(rchan->id, cmd, arg);
-}
-
-/**
- * relayfs_poll - poll file op for relayfs files
- * @filp: the file
- * @wait: poll table
- *
- * Poll implemention.
- */
-static unsigned int
-relayfs_poll(struct file *filp, poll_table *wait)
-{
- struct rchan_reader *reader;
- unsigned int mask = 0;
-
- reader = (struct rchan_reader *)filp->private_data;
-
- if (reader->rchan->finalized)
- return POLLERR;
-
- if (filp->f_mode & FMODE_READ) {
- poll_wait(filp, &reader->rchan->read_wait, wait);
- if (!rchan_empty(reader))
- mask |= POLLIN | POLLRDNORM;
- }
-
- if (filp->f_mode & FMODE_WRITE) {
- poll_wait(filp, &reader->rchan->write_wait, wait);
- if (!rchan_full(reader))
- mask |= POLLOUT | POLLWRNORM;
- }
-
- return mask;
-}
-
-/**
- * relayfs_release - release file op for relayfs files
- * @inode: the inode
- * @filp: the file
- *
- * Decrements the channel refcount, as the filesystem is
- * no longer using it.
- */
-int
-relayfs_release(struct inode *inode, struct file *filp)
-{
- struct rchan_reader *reader;
- struct rchan *rchan;
-
- reader = (struct rchan_reader *)filp->private_data;
- if (reader == NULL || reader->rchan == NULL)
- return 0;
- rchan = reader->rchan;
-
- rchan->callbacks->fileop_notify(reader->rchan->id, filp,
- RELAY_FILE_CLOSE);
- __remove_rchan_reader(reader);
- /* The channel is no longer in use as far as this file is concerned */
- rchan_put(rchan);
-
- return 0;
-}
-
-static struct address_space_operations relayfs_aops = {
- .readpage = simple_readpage,
- .prepare_write = simple_prepare_write,
- .commit_write = simple_commit_write
-};
-
-static struct file_operations relayfs_file_operations = {
- .open = relayfs_open,
- .read = relayfs_file_read,
- .write = relayfs_file_write,
- .ioctl = relayfs_ioctl,
- .poll = relayfs_poll,
- .mmap = relayfs_mmap,
- .fsync = simple_sync_file,
- .release = relayfs_release,
-};
-
-static struct inode_operations relayfs_file_inode_operations = {
- .getattr = simple_getattr,
-};
-
-static struct inode_operations relayfs_dir_inode_operations = {
- .create = relayfs_create,
- .lookup = simple_lookup,
- .link = simple_link,
- .unlink = simple_unlink,
- .symlink = relayfs_symlink,
- .mkdir = relayfs_mkdir,
- .rmdir = simple_rmdir,
- .mknod = relayfs_mknod,
- .rename = simple_rename,
-};
-
-static struct super_operations relayfs_ops = {
- .statfs = simple_statfs,
- .drop_inode = generic_delete_inode,
-};
-
-static int
-relayfs_fill_super(struct super_block * sb, void * data, int silent)
-{
- struct inode * inode;
- struct dentry * root;
-
- sb->s_blocksize = PAGE_CACHE_SIZE;
- sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
- sb->s_magic = RELAYFS_MAGIC;
- sb->s_op = &relayfs_ops;
- inode = relayfs_get_inode(sb, S_IFDIR | 0755, 0);
-
- if (!inode)
- return -ENOMEM;
-
- root = d_alloc_root(inode);
- if (!root) {
- iput(inode);
- return -ENOMEM;
- }
- sb->s_root = root;
-
- return 0;
-}
-
-static struct super_block *
-relayfs_get_sb(struct file_system_type *fs_type,
- int flags, const char *dev_name, void *data)
-{
- return get_sb_single(fs_type, flags, data, relayfs_fill_super);
-}
-
-static struct file_system_type relayfs_fs_type = {
- .owner = THIS_MODULE,
- .name = "relayfs",
- .get_sb = relayfs_get_sb,
- .kill_sb = kill_litter_super,
-};
-
-static int __init
-init_relayfs_fs(void)
-{
- int err = register_filesystem(&relayfs_fs_type);
-#ifdef CONFIG_KLOG_CHANNEL
- if (!err)
- create_klog_channel();
-#endif
- return err;
-}
-
-static void __exit
-exit_relayfs_fs(void)
-{
-#ifdef CONFIG_KLOG_CHANNEL
- remove_klog_channel();
-#endif
- unregister_filesystem(&relayfs_fs_type);
-}
-
-module_init(init_relayfs_fs)
-module_exit(exit_relayfs_fs)
-
-MODULE_AUTHOR("Tom Zanussi <zanussi@us.ibm.com> and Karim Yaghmour <karim@opersys.com>");
-MODULE_DESCRIPTION("Relay Filesystem");
-MODULE_LICENSE("GPL");
-
+++ /dev/null
-/*
- * KLOG Generic Logging facility built upon the relayfs infrastructure
- *
- * Authors: Hubertus Franke (frankeh@us.ibm.com)
- * Tom Zanussi (zanussi@us.ibm.com)
- *
- * Please direct all questions/comments to zanussi@us.ibm.com
- *
- * Copyright (C) 2003, IBM Corp
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/kernel.h>
-#include <linux/smp_lock.h>
-#include <linux/console.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/config.h>
-#include <linux/delay.h>
-#include <linux/smp.h>
-#include <linux/sysctl.h>
-#include <linux/relayfs_fs.h>
-#include <linux/klog.h>
-
-/* klog channel id */
-static int klog_channel = -1;
-
-/* maximum size of klog formatting buffer beyond which truncation will occur */
-#define KLOG_BUF_SIZE (512)
-/* per-cpu klog formatting buffer */
-static char buf[NR_CPUS][KLOG_BUF_SIZE];
-
-/*
- * klog_enabled determines whether klog()/klog_raw() actually do write
- * to the klog channel at any given time. If klog_enabled == 1 they do,
- * otherwise they don't. Settable using sysctl fs.relayfs.klog_enabled.
- */
-#ifdef CONFIG_KLOG_CHANNEL_AUTOENABLE
-static int klog_enabled = 1;
-#else
-static int klog_enabled = 0;
-#endif
-
-/**
- * klog - write a formatted string into the klog channel
- * @fmt: format string
- *
- * Returns number of bytes written, negative number on failure.
- */
-int klog(const char *fmt, ...)
-{
- va_list args;
- int len, err;
- char *cbuf;
- unsigned long flags;
-
- if (!klog_enabled || klog_channel < 0)
- return 0;
-
- local_irq_save(flags);
- cbuf = buf[smp_processor_id()];
-
- va_start(args, fmt);
- len = vsnprintf(cbuf, KLOG_BUF_SIZE, fmt, args);
- va_end(args);
-
- err = relay_write(klog_channel, cbuf, len, -1, NULL);
- local_irq_restore(flags);
-
- return err;
-}
-
-/**
- * klog_raw - directly write into the klog channel
- * @buf: buffer containing data to write
- * @len: # bytes to write
- *
- * Returns number of bytes written, negative number on failure.
- */
-int klog_raw(const char *buf,int len)
-{
- int err = 0;
-
- if (klog_enabled && klog_channel >= 0)
- err = relay_write(klog_channel, buf, len, -1, NULL);
-
- return err;
-}
-
-/**
- * relayfs sysctl data
- *
- * Only sys/fs/relayfs/klog_enabled for now.
- */
-#define CTL_ENABLE_KLOG 100
-#define CTL_RELAYFS 100
-
-static struct ctl_table_header *relayfs_ctl_table_header;
-
-static struct ctl_table relayfs_table[] =
-{
- {
- .ctl_name = CTL_ENABLE_KLOG,
- .procname = "klog_enabled",
- .data = &klog_enabled,
- .maxlen = sizeof(int),
- .mode = 0644,
- .proc_handler = &proc_dointvec,
- },
- {
- 0
- }
-};
-
-static struct ctl_table relayfs_dir_table[] =
-{
- {
- .ctl_name = CTL_RELAYFS,
- .procname = "relayfs",
- .data = NULL,
- .maxlen = 0,
- .mode = 0555,
- .child = relayfs_table,
- },
- {
- 0
- }
-};
-
-static struct ctl_table relayfs_root_table[] =
-{
- {
- .ctl_name = CTL_FS,
- .procname = "fs",
- .data = NULL,
- .maxlen = 0,
- .mode = 0555,
- .child = relayfs_dir_table,
- },
- {
- 0
- }
-};
-
-/**
- * create_klog_channel - creates channel /mnt/relay/klog
- *
- * Returns channel id on success, negative otherwise.
- */
-int
-create_klog_channel(void)
-{
- u32 bufsize, nbufs;
- u32 channel_flags;
-
- channel_flags = RELAY_DELIVERY_PACKET | RELAY_USAGE_GLOBAL;
- channel_flags |= RELAY_SCHEME_ANY | RELAY_TIMESTAMP_ANY;
-
- bufsize = 1 << (CONFIG_KLOG_CHANNEL_SHIFT - 2);
- nbufs = 4;
-
- klog_channel = relay_open("klog",
- bufsize,
- nbufs,
- channel_flags,
- NULL,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- NULL,
- 0);
-
- if (klog_channel < 0)
- printk("klog channel creation failed, errcode: %d\n", klog_channel);
- else {
- printk("klog channel created (%u bytes)\n", 1 << CONFIG_KLOG_CHANNEL_SHIFT);
- relayfs_ctl_table_header = register_sysctl_table(relayfs_root_table, 1);
- }
-
- return klog_channel;
-}
-
-/**
- * remove_klog_channel - destroys channel /mnt/relay/klog
- *
- * Returns 0, negative otherwise.
- */
-int
-remove_klog_channel(void)
-{
- if (relayfs_ctl_table_header)
- unregister_sysctl_table(relayfs_ctl_table_header);
-
- return relay_close(klog_channel);
-}
-
-EXPORT_SYMBOL(klog);
-EXPORT_SYMBOL(klog_raw);
-
+++ /dev/null
-/*
- * Public API and common code for RelayFS.
- *
- * Please see Documentation/filesystems/relayfs.txt for API description.
- *
- * Copyright (C) 2002, 2003 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
- * Copyright (C) 1999, 2000, 2001, 2002 - Karim Yaghmour (karim@opersys.com)
- *
- * This file is released under the GPL.
- */
-
-#include <linux/init.h>
-#include <linux/errno.h>
-#include <linux/stddef.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-#include <linux/sched.h>
-#include <linux/string.h>
-#include <linux/time.h>
-#include <linux/page-flags.h>
-#include <linux/vmalloc.h>
-#include <linux/mm.h>
-#include <linux/mman.h>
-#include <linux/delay.h>
-
-#include <asm/io.h>
-#include <asm/current.h>
-#include <asm/uaccess.h>
-#include <asm/bitops.h>
-#include <asm/pgtable.h>
-#include <asm/relay.h>
-#include <asm/hardirq.h>
-
-#include "relay_lockless.h"
-#include "relay_locking.h"
-#include "resize.h"
-
-/* Relay channel table, indexed by channel id */
-static struct rchan * rchan_table[RELAY_MAX_CHANNELS];
-static rwlock_t rchan_table_lock = RW_LOCK_UNLOCKED;
-
-/* Relay operation structs, one per scheme */
-static struct relay_ops lockless_ops = {
- .reserve = lockless_reserve,
- .commit = lockless_commit,
- .get_offset = lockless_get_offset,
- .finalize = lockless_finalize,
- .reset = lockless_reset,
- .reset_index = lockless_reset_index
-};
-
-static struct relay_ops locking_ops = {
- .reserve = locking_reserve,
- .commit = locking_commit,
- .get_offset = locking_get_offset,
- .finalize = locking_finalize,
- .reset = locking_reset,
- .reset_index = locking_reset_index
-};
-
-/*
- * Low-level relayfs kernel API. These functions should not normally be
- * used by clients. See high-level kernel API below.
- */
-
-/**
- * rchan_get - get channel associated with id, incrementing refcount
- * @rchan_id: the channel id
- *
- * Returns channel if successful, NULL otherwise.
- */
-struct rchan *
-rchan_get(int rchan_id)
-{
- struct rchan *rchan;
-
- if ((rchan_id < 0) || (rchan_id >= RELAY_MAX_CHANNELS))
- return NULL;
-
- read_lock(&rchan_table_lock);
- rchan = rchan_table[rchan_id];
- if (rchan)
- atomic_inc(&rchan->refcount);
- read_unlock(&rchan_table_lock);
-
- return rchan;
-}
-
-/**
- * clear_readers - clear non-VFS readers
- * @rchan: the channel
- *
- * Clear the channel pointers of all non-VFS readers open on the channel.
- */
-static inline void
-clear_readers(struct rchan *rchan)
-{
- struct list_head *p;
- struct rchan_reader *reader;
-
- read_lock(&rchan->open_readers_lock);
- list_for_each(p, &rchan->open_readers) {
- reader = list_entry(p, struct rchan_reader, list);
- if (!reader->vfs_reader)
- reader->rchan = NULL;
- }
- read_unlock(&rchan->open_readers_lock);
-}
-
-/**
- * rchan_alloc_id - reserve a channel id and store associated channel
- * @rchan: the channel
- *
- * Returns channel id if successful, -1 otherwise.
- */
-static inline int
-rchan_alloc_id(struct rchan *rchan)
-{
- int i;
- int rchan_id = -1;
-
- if (rchan == NULL)
- return -1;
-
- write_lock(&rchan_table_lock);
- for (i = 0; i < RELAY_MAX_CHANNELS; i++) {
- if (rchan_table[i] == NULL) {
- rchan_table[i] = rchan;
- rchan_id = rchan->id = i;
- break;
- }
- }
- if (rchan_id != -1)
- atomic_inc(&rchan->refcount);
- write_unlock(&rchan_table_lock);
-
- return rchan_id;
-}
-
-/**
- * rchan_free_id - revoke a channel id and remove associated channel
- * @rchan_id: the channel id
- */
-static inline void
-rchan_free_id(int rchan_id)
-{
- struct rchan *rchan;
-
- if ((rchan_id < 0) || (rchan_id >= RELAY_MAX_CHANNELS))
- return;
-
- write_lock(&rchan_table_lock);
- rchan = rchan_table[rchan_id];
- rchan_table[rchan_id] = NULL;
- write_unlock(&rchan_table_lock);
-}
-
-/**
- * rchan_destroy_buf - destroy the current channel buffer
- * @rchan: the channel
- */
-static inline void
-rchan_destroy_buf(struct rchan *rchan)
-{
- if (rchan->buf && !rchan->init_buf)
- free_rchan_buf(rchan->buf,
- rchan->buf_page_array,
- rchan->buf_page_count);
-}
-
-/**
- * relay_release - perform end-of-buffer processing for last buffer
- * @rchan: the channel
- *
- * Returns 0 if successful, negative otherwise.
- *
- * Releases the channel buffer, destroys the channel, and removes the
- * relay file from the relayfs filesystem. Should only be called from
- * rchan_put(). If we're here, it means by definition refcount is 0.
- */
-static int
-relay_release(struct rchan *rchan)
-{
- if (rchan == NULL)
- return -EBADF;
-
- rchan_destroy_buf(rchan);
- rchan_free_id(rchan->id);
- relayfs_remove_file(rchan->dentry);
- clear_readers(rchan);
- kfree(rchan);
-
- return 0;
-}
-
-/**
- * rchan_get - decrement channel refcount, releasing it if 0
- * @rchan: the channel
- *
- * If the refcount reaches 0, the channel will be destroyed.
- */
-void
-rchan_put(struct rchan *rchan)
-{
- if (atomic_dec_and_test(&rchan->refcount))
- relay_release(rchan);
-}
-
-/**
- * relay_reserve - reserve a slot in the channel buffer
- * @rchan: the channel
- * @len: the length of the slot to reserve
- * @td: the time delta between buffer start and current write, or TSC
- * @err: receives the result flags
- * @interrupting: 1 if interrupting previous, used only in locking scheme
- *
- * Returns pointer to the beginning of the reserved slot, NULL if error.
- *
- * The errcode value contains the result flags and is an ORed combination
- * of the following:
- *
- * RELAY_BUFFER_SWITCH_NONE - no buffer switch occurred
- * RELAY_EVENT_DISCARD_NONE - event should not be discarded
- * RELAY_BUFFER_SWITCH - buffer switch occurred
- * RELAY_EVENT_DISCARD - event should be discarded (all buffers are full)
- * RELAY_EVENT_TOO_LONG - event won't fit into even an empty buffer
- *
- * buffer_start and buffer_end callbacks are triggered at this point
- * if applicable.
- */
-char *
-relay_reserve(struct rchan *rchan,
- u32 len,
- struct timeval *ts,
- u32 *td,
- int *err,
- int *interrupting)
-{
- if (rchan == NULL)
- return NULL;
-
- *interrupting = 0;
-
- return rchan->relay_ops->reserve(rchan, len, ts, td, err, interrupting);
-}
-
-
-/**
- * wakeup_readers - wake up VFS readers waiting on a channel
- * @private: the channel
- *
- * This is the work function used to defer reader waking. The
- * reason waking is deferred is that calling directly from commit
- * causes problems if you're writing from say the scheduler.
- */
-static void
-wakeup_readers(void *private)
-{
- struct rchan *rchan = (struct rchan *)private;
-
- wake_up_interruptible(&rchan->read_wait);
-}
-
-
-/**
- * relay_commit - commit a reserved slot in the buffer
- * @rchan: the channel
- * @from: commit the length starting here
- * @len: length committed
- * @interrupting: 1 if interrupting previous, used only in locking scheme
- *
- * After the write into the reserved buffer has been complted, this
- * function must be called in order for the relay to determine whether
- * buffers are complete and to wake up VFS readers.
- *
- * delivery callback is triggered at this point if applicable.
- */
-void
-relay_commit(struct rchan *rchan,
- char *from,
- u32 len,
- int reserve_code,
- int interrupting)
-{
- int deliver;
-
- if (rchan == NULL)
- return;
-
- deliver = packet_delivery(rchan) ||
- (reserve_code & RELAY_BUFFER_SWITCH);
-
- rchan->relay_ops->commit(rchan, from, len, deliver, interrupting);
-
- /* The params are always the same, so no worry about re-queuing */
- if (deliver && waitqueue_active(&rchan->read_wait)) {
- PREPARE_WORK(&rchan->wake_readers, wakeup_readers, rchan);
- schedule_delayed_work(&rchan->wake_readers, 1);
- }
-}
-
-/**
- * relay_get_offset - get current and max channel buffer offsets
- * @rchan: the channel
- * @max_offset: maximum channel offset
- *
- * Returns the current and maximum channel buffer offsets.
- */
-u32
-relay_get_offset(struct rchan *rchan, u32 *max_offset)
-{
- return rchan->relay_ops->get_offset(rchan, max_offset);
-}
-
-/**
- * reset_index - try once to reset the current channel index
- * @rchan: the channel
- * @old_index: the index read before reset
- *
- * Attempts to reset the channel index to 0. It tries once, and
- * if it fails, returns negative, 0 otherwise.
- */
-int
-reset_index(struct rchan *rchan, u32 old_index)
-{
- return rchan->relay_ops->reset_index(rchan, old_index);
-}
-
-/*
- * close() vm_op implementation for relayfs file mapping.
- */
-static void
-relay_file_mmap_close(struct vm_area_struct *vma)
-{
- struct file *filp = vma->vm_file;
- struct rchan_reader *reader;
- struct rchan *rchan;
-
- reader = (struct rchan_reader *)filp->private_data;
- rchan = reader->rchan;
-
- atomic_dec(&rchan->mapped);
-
- rchan->callbacks->fileop_notify(reader->rchan->id, filp,
- RELAY_FILE_UNMAP);
-}
-
-/*
- * vm_ops for relay file mappings.
- */
-static struct vm_operations_struct relay_file_mmap_ops = {
- .close = relay_file_mmap_close
-};
-
-/* \begin{Code inspired from BTTV driver} */
-static inline unsigned long
-kvirt_to_pa(unsigned long adr)
-{
- unsigned long kva, ret;
-
- kva = (unsigned long) page_address(vmalloc_to_page((void *) adr));
- kva |= adr & (PAGE_SIZE - 1);
- ret = __pa(kva);
- return ret;
-}
-
-static int
-relay_mmap_region(struct vm_area_struct *vma,
- const char *adr,
- const char *start_pos,
- unsigned long size)
-{
- unsigned long start = (unsigned long) adr;
- unsigned long page, pos;
-
- pos = (unsigned long) start_pos;
-
- while (size > 0) {
- page = kvirt_to_pa(pos);
- if (remap_page_range(vma, start, page, PAGE_SIZE, PAGE_SHARED))
- return -EAGAIN;
- start += PAGE_SIZE;
- pos += PAGE_SIZE;
- size -= PAGE_SIZE;
- }
-
- return 0;
-}
-/* \end{Code inspired from BTTV driver} */
-
-/**
- * relay_mmap_buffer: - mmap buffer to process address space
- * @rchan_id: relay channel id
- * @vma: vm_area_struct describing memory to be mapped
- *
- * Returns:
- * 0 if ok
- * -EAGAIN, when remap failed
- * -EINVAL, invalid requested length
- *
- * Caller should already have grabbed mmap_sem.
- */
-int
-__relay_mmap_buffer(struct rchan *rchan,
- struct vm_area_struct *vma)
-{
- int err = 0;
- unsigned long length = vma->vm_end - vma->vm_start;
- struct file *filp = vma->vm_file;
-
- if (rchan == NULL) {
- err = -EBADF;
- goto exit;
- }
-
- if (rchan->init_buf) {
- err = -EPERM;
- goto exit;
- }
-
- if (length != (unsigned long)rchan->alloc_size) {
- err = -EINVAL;
- goto exit;
- }
-
- err = relay_mmap_region(vma,
- (char *)vma->vm_start,
- rchan->buf,
- rchan->alloc_size);
-
- if (err == 0) {
- vma->vm_ops = &relay_file_mmap_ops;
- err = rchan->callbacks->fileop_notify(rchan->id, filp,
- RELAY_FILE_MAP);
- if (err == 0)
- atomic_inc(&rchan->mapped);
- }
-exit:
- return err;
-}
-
-/*
- * High-level relayfs kernel API. See Documentation/filesystems/relafys.txt.
- */
-
-/*
- * rchan_callback implementations defining default channel behavior. Used
- * in place of corresponding NULL values in client callback struct.
- */
-
-/*
- * buffer_end() default callback. Does nothing.
- */
-static int
-buffer_end_default_callback(int rchan_id,
- char *current_write_pos,
- char *end_of_buffer,
- struct timeval end_time,
- u32 end_tsc,
- int using_tsc)
-{
- return 0;
-}
-
-/*
- * buffer_start() default callback. Does nothing.
- */
-static int
-buffer_start_default_callback(int rchan_id,
- char *current_write_pos,
- u32 buffer_id,
- struct timeval start_time,
- u32 start_tsc,
- int using_tsc)
-{
- return 0;
-}
-
-/*
- * deliver() default callback. Does nothing.
- */
-static void
-deliver_default_callback(int rchan_id, char *from, u32 len)
-{
-}
-
-/*
- * user_deliver() default callback. Does nothing.
- */
-static void
-user_deliver_default_callback(int rchan_id, char *from, u32 len)
-{
-}
-
-/*
- * needs_resize() default callback. Does nothing.
- */
-static void
-needs_resize_default_callback(int rchan_id,
- int resize_type,
- u32 suggested_buf_size,
- u32 suggested_n_bufs)
-{
-}
-
-/*
- * fileop_notify() default callback. Does nothing.
- */
-static int
-fileop_notify_default_callback(int rchan_id,
- struct file *filp,
- enum relay_fileop fileop)
-{
- return 0;
-}
-
-/*
- * ioctl() default callback. Does nothing.
- */
-static int
-ioctl_default_callback(int rchan_id,
- unsigned int cmd,
- unsigned long arg)
-{
- return 0;
-}
-
-/* relay channel default callbacks */
-static struct rchan_callbacks default_channel_callbacks = {
- .buffer_start = buffer_start_default_callback,
- .buffer_end = buffer_end_default_callback,
- .deliver = deliver_default_callback,
- .user_deliver = user_deliver_default_callback,
- .needs_resize = needs_resize_default_callback,
- .fileop_notify = fileop_notify_default_callback,
- .ioctl = ioctl_default_callback,
-};
-
-/**
- * check_attribute_flags - check sanity of channel attributes
- * @flags: channel attributes
- * @resizeable: 1 if true
- *
- * Returns 0 if successful, negative otherwise.
- */
-static int
-check_attribute_flags(u32 *attribute_flags, int resizeable)
-{
- u32 flags = *attribute_flags;
-
- if (!(flags & RELAY_DELIVERY_BULK) && !(flags & RELAY_DELIVERY_PACKET))
- return -EINVAL; /* Delivery mode must be specified */
-
- if (!(flags & RELAY_USAGE_SMP) && !(flags & RELAY_USAGE_GLOBAL))
- return -EINVAL; /* Usage must be specified */
-
- if (resizeable) { /* Resizeable can never be continuous */
- *attribute_flags &= ~RELAY_MODE_CONTINUOUS;
- *attribute_flags |= RELAY_MODE_NO_OVERWRITE;
- }
-
- if ((flags & RELAY_MODE_CONTINUOUS) &&
- (flags & RELAY_MODE_NO_OVERWRITE))
- return -EINVAL; /* Can't have it both ways */
-
- if (!(flags & RELAY_MODE_CONTINUOUS) &&
- !(flags & RELAY_MODE_NO_OVERWRITE))
- *attribute_flags |= RELAY_MODE_CONTINUOUS; /* Default to continuous */
-
- if (!(flags & RELAY_SCHEME_ANY))
- return -EINVAL; /* One or both must be specified */
- else if (flags & RELAY_SCHEME_LOCKLESS) {
- if (have_cmpxchg())
- *attribute_flags &= ~RELAY_SCHEME_LOCKING;
- else if (flags & RELAY_SCHEME_LOCKING)
- *attribute_flags &= ~RELAY_SCHEME_LOCKLESS;
- else
- return -EINVAL; /* Locking scheme not an alternative */
- }
-
- if (!(flags & RELAY_TIMESTAMP_ANY))
- return -EINVAL; /* One or both must be specified */
- else if (flags & RELAY_TIMESTAMP_TSC) {
- if (have_tsc())
- *attribute_flags &= ~RELAY_TIMESTAMP_GETTIMEOFDAY;
- else if (flags & RELAY_TIMESTAMP_GETTIMEOFDAY)
- *attribute_flags &= ~RELAY_TIMESTAMP_TSC;
- else
- return -EINVAL; /* gettimeofday not an alternative */
- }
-
- return 0;
-}
-
-/*
- * High-level API functions.
- */
-
-/**
- * __relay_reset - internal reset function
- * @rchan: the channel
- * @init: 1 if this is a first-time channel initialization
- *
- * See relay_reset for description of effect.
- */
-void
-__relay_reset(struct rchan *rchan, int init)
-{
- int i;
-
- if (init) {
- rchan->version = RELAYFS_CHANNEL_VERSION;
- init_MUTEX(&rchan->resize_sem);
- init_waitqueue_head(&rchan->read_wait);
- init_waitqueue_head(&rchan->write_wait);
- atomic_set(&rchan->refcount, 0);
- INIT_LIST_HEAD(&rchan->open_readers);
- rchan->open_readers_lock = RW_LOCK_UNLOCKED;
- }
-
- rchan->buf_id = rchan->buf_idx = 0;
- atomic_set(&rchan->suspended, 0);
- atomic_set(&rchan->mapped, 0);
- rchan->half_switch = 0;
- rchan->bufs_produced = 0;
- rchan->bufs_consumed = 0;
- rchan->bytes_consumed = 0;
- rchan->initialized = 0;
- rchan->finalized = 0;
- rchan->resize_min = rchan->resize_max = 0;
- rchan->resizing = 0;
- rchan->replace_buffer = 0;
- rchan->resize_buf = NULL;
- rchan->resize_buf_size = 0;
- rchan->resize_alloc_size = 0;
- rchan->resize_n_bufs = 0;
- rchan->resize_err = 0;
- rchan->resize_failures = 0;
- rchan->resize_order = 0;
-
- rchan->expand_page_array = NULL;
- rchan->expand_page_count = 0;
- rchan->shrink_page_array = NULL;
- rchan->shrink_page_count = 0;
- rchan->resize_page_array = NULL;
- rchan->resize_page_count = 0;
- rchan->old_buf_page_array = NULL;
- rchan->expand_buf_id = 0;
-
- INIT_WORK(&rchan->wake_readers, NULL, NULL);
- INIT_WORK(&rchan->wake_writers, NULL, NULL);
-
- for (i = 0; i < RELAY_MAX_BUFS; i++)
- rchan->unused_bytes[i] = 0;
-
- rchan->relay_ops->reset(rchan, init);
-}
-
-/**
- * relay_reset - reset the channel
- * @rchan: the channel
- *
- * Returns 0 if successful, negative if not.
- *
- * This has the effect of erasing all data from the buffer and
- * restarting the channel in its initial state. The buffer itself
- * is not freed, so any mappings are still in effect.
- *
- * NOTE: Care should be taken that the channnel isn't actually
- * being used by anything when this call is made.
- */
-int
-relay_reset(int rchan_id)
-{
- struct rchan *rchan;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- __relay_reset(rchan, 0);
- update_readers_consumed(rchan, 0, 0);
-
- rchan_put(rchan);
-
- return 0;
-}
-
-/**
- * check_init_buf - check the sanity of init_buf, if present
- * @init_buf: the initbuf
- * @init_buf_size: the total initbuf size
- * @bufsize: the channel's sub-buffer size
- * @nbufs: the number of sub-buffers in the channel
- *
- * Returns 0 if ok, negative otherwise.
- */
-static int
-check_init_buf(char *init_buf, u32 init_buf_size, u32 bufsize, u32 nbufs)
-{
- int err = 0;
-
- if (init_buf && nbufs == 1) /* 1 sub-buffer makes no sense */
- err = -EINVAL;
-
- if (init_buf && (bufsize * nbufs != init_buf_size))
- err = -EINVAL;
-
- return err;
-}
-
-/**
- * rchan_create_buf - allocate the initial channel buffer
- * @rchan: the channel
- * @size_alloc: the total size of the channel buffer
- *
- * Returns 0 if successful, negative otherwise.
- */
-static inline int
-rchan_create_buf(struct rchan *rchan, int size_alloc)
-{
- struct page **page_array;
- int page_count;
-
- if ((rchan->buf = (char *)alloc_rchan_buf(size_alloc, &page_array, &page_count)) == NULL) {
- rchan->buf_page_array = NULL;
- rchan->buf_page_count = 0;
- return -ENOMEM;
- }
-
- rchan->buf_page_array = page_array;
- rchan->buf_page_count = page_count;
-
- return 0;
-}
-
-/**
- * rchan_create - allocate and initialize a channel, including buffer
- * @chanpath: path specifying the relayfs channel file to create
- * @bufsize: the size of the sub-buffers within the channel buffer
- * @nbufs: the number of sub-buffers within the channel buffer
- * @rchan_flags: flags specifying buffer attributes
- * @err: err code
- *
- * Returns channel if successful, NULL otherwise, err receives errcode.
- *
- * Allocates a struct rchan representing a relay channel, according
- * to the attributes passed in via rchan_flags. Does some basic sanity
- * checking but doesn't try to do anything smart. In particular, the
- * number of buffers must be a power of 2, and if the lockless scheme
- * is being used, the sub-buffer size must also be a power of 2. The
- * locking scheme can use buffers of any size.
- */
-static struct rchan *
-rchan_create(const char *chanpath,
- int bufsize,
- int nbufs,
- u32 rchan_flags,
- char *init_buf,
- u32 init_buf_size,
- int *err)
-{
- int size_alloc;
- struct rchan *rchan = NULL;
-
- *err = 0;
-
- rchan = (struct rchan *)kmalloc(sizeof(struct rchan), GFP_KERNEL);
- if (rchan == NULL) {
- *err = -ENOMEM;
- return NULL;
- }
- rchan->buf = rchan->init_buf = NULL;
-
- *err = check_init_buf(init_buf, init_buf_size, bufsize, nbufs);
- if (*err)
- goto exit;
-
- if (nbufs == 1 && bufsize) {
- rchan->n_bufs = nbufs;
- rchan->buf_size = bufsize;
- size_alloc = bufsize;
- goto alloc;
- }
-
- if (bufsize <= 0 ||
- (rchan_flags & RELAY_SCHEME_LOCKLESS && hweight32(bufsize) != 1) ||
- hweight32(nbufs) != 1 ||
- nbufs < RELAY_MIN_BUFS ||
- nbufs > RELAY_MAX_BUFS) {
- *err = -EINVAL;
- goto exit;
- }
-
- size_alloc = FIX_SIZE(bufsize * nbufs);
- if (size_alloc > RELAY_MAX_BUF_SIZE) {
- *err = -EINVAL;
- goto exit;
- }
- rchan->n_bufs = nbufs;
- rchan->buf_size = bufsize;
-
- if (rchan_flags & RELAY_SCHEME_LOCKLESS) {
- offset_bits(rchan) = ffs(bufsize) - 1;
- offset_mask(rchan) = RELAY_BUF_OFFSET_MASK(offset_bits(rchan));
- bufno_bits(rchan) = ffs(nbufs) - 1;
- }
-alloc:
- if (rchan_alloc_id(rchan) == -1) {
- *err = -ENOMEM;
- goto exit;
- }
-
- if (init_buf == NULL) {
- *err = rchan_create_buf(rchan, size_alloc);
- if (*err) {
- rchan_free_id(rchan->id);
- goto exit;
- }
- } else
- rchan->buf = rchan->init_buf = init_buf;
-
- rchan->alloc_size = size_alloc;
-
- if (rchan_flags & RELAY_SCHEME_LOCKLESS)
- rchan->relay_ops = &lockless_ops;
- else
- rchan->relay_ops = &locking_ops;
-
-exit:
- if (*err) {
- kfree(rchan);
- rchan = NULL;
- }
-
- return rchan;
-}
-
-
-static char tmpname[NAME_MAX];
-
-/**
- * rchan_create_dir - create directory for file
- * @chanpath: path to file, including filename
- * @residual: filename remaining after parse
- * @topdir: the directory filename should be created in
- *
- * Returns 0 if successful, negative otherwise.
- *
- * Inspired by xlate_proc_name() in procfs. Given a file path which
- * includes the filename, creates any and all directories necessary
- * to create the file.
- */
-static int
-rchan_create_dir(const char * chanpath,
- const char **residual,
- struct dentry **topdir)
-{
- const char *cp = chanpath, *next;
- struct dentry *parent = NULL;
- int len, err = 0;
-
- while (1) {
- next = strchr(cp, '/');
- if (!next)
- break;
-
- len = next - cp;
-
- strncpy(tmpname, cp, len);
- tmpname[len] = '\0';
- err = relayfs_create_dir(tmpname, parent, &parent);
- if (err && (err != -EEXIST))
- return err;
- cp += len + 1;
- }
-
- *residual = cp;
- *topdir = parent;
-
- return err;
-}
-
-/**
- * rchan_create_file - create file, including parent directories
- * @chanpath: path to file, including filename
- * @dentry: result dentry
- * @data: data to associate with the file
- *
- * Returns 0 if successful, negative otherwise.
- */
-static int
-rchan_create_file(const char * chanpath,
- struct dentry **dentry,
- struct rchan * data,
- int mode)
-{
- int err;
- const char * fname;
- struct dentry *topdir;
-
- err = rchan_create_dir(chanpath, &fname, &topdir);
- if (err && (err != -EEXIST))
- return err;
-
- err = relayfs_create_file(fname, topdir, dentry, (void *)data, mode);
-
- return err;
-}
-
-/**
- * relay_open - create a new file/channel buffer in relayfs
- * @chanpath: name of file to create, including path
- * @bufsize: size of sub-buffers
- * @nbufs: number of sub-buffers
- * @flags: channel attributes
- * @callbacks: client callback functions
- * @start_reserve: number of bytes to reserve at start of each sub-buffer
- * @end_reserve: number of bytes to reserve at end of each sub-buffer
- * @rchan_start_reserve: additional reserve at start of first sub-buffer
- * @resize_min: minimum total buffer size, if set
- * @resize_max: maximum total buffer size, if set
- * @mode: the perms to be given to the relayfs file, 0 to accept defaults
- * @init_buf: initial memory buffer to start out with, NULL if N/A
- * @init_buf_size: initial memory buffer size to start out with, 0 if N/A
- *
- * Returns channel id if successful, negative otherwise.
- *
- * Creates a relay channel using the sizes and attributes specified.
- * The default permissions, used if mode == 0 are S_IRUSR | S_IWUSR. See
- * Documentation/filesystems/relayfs.txt for details.
- */
-int
-relay_open(const char *chanpath,
- int bufsize,
- int nbufs,
- u32 flags,
- struct rchan_callbacks *channel_callbacks,
- u32 start_reserve,
- u32 end_reserve,
- u32 rchan_start_reserve,
- u32 resize_min,
- u32 resize_max,
- int mode,
- char *init_buf,
- u32 init_buf_size)
-{
- int err;
- struct rchan *rchan;
- struct dentry *dentry;
- struct rchan_callbacks *callbacks = NULL;
-
- if (chanpath == NULL)
- return -EINVAL;
-
- if (nbufs != 1) {
- err = check_attribute_flags(&flags, resize_min ? 1 : 0);
- if (err)
- return err;
- }
-
- rchan = rchan_create(chanpath, bufsize, nbufs, flags, init_buf, init_buf_size, &err);
-
- if (err < 0)
- return err;
-
- /* Create file in fs */
- if ((err = rchan_create_file(chanpath, &dentry, rchan, mode)) < 0) {
- rchan_destroy_buf(rchan);
- rchan_free_id(rchan->id);
- kfree(rchan);
- return err;
- }
-
- rchan->dentry = dentry;
-
- if (channel_callbacks == NULL)
- callbacks = &default_channel_callbacks;
- else
- callbacks = channel_callbacks;
-
- if (callbacks->buffer_end == NULL)
- callbacks->buffer_end = buffer_end_default_callback;
- if (callbacks->buffer_start == NULL)
- callbacks->buffer_start = buffer_start_default_callback;
- if (callbacks->deliver == NULL)
- callbacks->deliver = deliver_default_callback;
- if (callbacks->user_deliver == NULL)
- callbacks->user_deliver = user_deliver_default_callback;
- if (callbacks->needs_resize == NULL)
- callbacks->needs_resize = needs_resize_default_callback;
- if (callbacks->fileop_notify == NULL)
- callbacks->fileop_notify = fileop_notify_default_callback;
- if (callbacks->ioctl == NULL)
- callbacks->ioctl = ioctl_default_callback;
- rchan->callbacks = callbacks;
-
- /* Just to let the client know the sizes used */
- rchan->callbacks->needs_resize(rchan->id,
- RELAY_RESIZE_REPLACED,
- rchan->buf_size,
- rchan->n_bufs);
-
- rchan->flags = flags;
- rchan->start_reserve = start_reserve;
- rchan->end_reserve = end_reserve;
- rchan->rchan_start_reserve = rchan_start_reserve;
-
- __relay_reset(rchan, 1);
-
- if (resize_min > 0 && resize_max > 0 &&
- resize_max < RELAY_MAX_TOTAL_BUF_SIZE) {
- rchan->resize_min = resize_min;
- rchan->resize_max = resize_max;
- init_shrink_timer(rchan);
- }
-
- rchan_get(rchan->id);
-
- return rchan->id;
-}
-
-/**
- * relay_discard_init_buf - alloc channel buffer and copy init_buf into it
- * @rchan_id: the channel id
- *
- * Returns 0 if successful, negative otherwise.
- *
- * NOTE: May sleep. Should also be called only when the channel isn't
- * actively being written into.
- */
-int
-relay_discard_init_buf(int rchan_id)
-{
- struct rchan *rchan;
- int err = 0;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- if (rchan->init_buf == NULL) {
- err = -EINVAL;
- goto out;
- }
-
- err = rchan_create_buf(rchan, rchan->alloc_size);
- if (err)
- goto out;
-
- memcpy(rchan->buf, rchan->init_buf, rchan->n_bufs * rchan->buf_size);
- rchan->init_buf = NULL;
-out:
- rchan_put(rchan);
-
- return err;
-}
-
-/**
- * relay_finalize - perform end-of-buffer processing for last buffer
- * @rchan_id: the channel id
- * @releasing: true if called when releasing file
- *
- * Returns 0 if successful, negative otherwise.
- */
-static int
-relay_finalize(int rchan_id)
-{
- struct rchan *rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- if (rchan->finalized == 0) {
- rchan->relay_ops->finalize(rchan);
- rchan->finalized = 1;
- }
-
- if (waitqueue_active(&rchan->read_wait)) {
- PREPARE_WORK(&rchan->wake_readers, wakeup_readers, rchan);
- schedule_delayed_work(&rchan->wake_readers, 1);
- }
-
- rchan_put(rchan);
-
- return 0;
-}
-
-/**
- * restore_callbacks - restore default channel callbacks
- * @rchan: the channel
- *
- * Restore callbacks to the default versions.
- */
-static inline void
-restore_callbacks(struct rchan *rchan)
-{
- if (rchan->callbacks != &default_channel_callbacks)
- rchan->callbacks = &default_channel_callbacks;
-}
-
-/**
- * relay_close - close the channel
- * @rchan_id: relay channel id
- *
- * Finalizes the last sub-buffer and marks the channel as finalized.
- * The channel buffer and channel data structure are then freed
- * automatically when the last reference to the channel is given up.
- */
-int
-relay_close(int rchan_id)
-{
- int err;
- struct rchan *rchan;
-
- if ((rchan_id < 0) || (rchan_id >= RELAY_MAX_CHANNELS))
- return -EBADF;
-
- err = relay_finalize(rchan_id);
-
- if (!err) {
- read_lock(&rchan_table_lock);
- rchan = rchan_table[rchan_id];
- read_unlock(&rchan_table_lock);
-
- if (rchan) {
- restore_callbacks(rchan);
- if (rchan->resize_min)
- del_timer(&rchan->shrink_timer);
- rchan_put(rchan);
- }
- }
-
- return err;
-}
-
-/**
- * relay_write - reserve a slot in the channel and write data into it
- * @rchan_id: relay channel id
- * @data_ptr: data to be written into reserved slot
- * @count: number of bytes to write
- * @td_offset: optional offset where time delta should be written
- * @wrote_pos: optional ptr returning buf pos written to, ignored if NULL
- *
- * Returns the number of bytes written, 0 or negative on failure.
- *
- * Reserves space in the channel and writes count bytes of data_ptr
- * to it. Automatically performs any necessary locking, depending
- * on the scheme and SMP usage in effect (no locking is done for the
- * lockless scheme regardless of usage).
- *
- * If td_offset is >= 0, the internal time delta calculated when
- * slot was reserved will be written at that offset.
- *
- * If wrote_pos is non-NULL, it will receive the location the data
- * was written to, which may be needed for some applications but is not
- * normally interesting.
- */
-int
-relay_write(int rchan_id,
- const void *data_ptr,
- size_t count,
- int td_offset,
- void **wrote_pos)
-{
- unsigned long flags;
- char *reserved, *write_pos;
- int bytes_written = 0;
- int reserve_code, interrupting;
- struct timeval ts;
- u32 td;
- struct rchan *rchan;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- relay_lock_channel(rchan, flags); /* nop for lockless */
-
- write_pos = reserved = relay_reserve(rchan, count, &ts, &td,
- &reserve_code, &interrupting);
-
- if (reserved != NULL) {
- relay_write_direct(write_pos, data_ptr, count);
- if ((td_offset >= 0) && (td_offset < count - sizeof(td)))
- *((u32 *)(reserved + td_offset)) = td;
- bytes_written = count;
- } else if (reserve_code == RELAY_WRITE_TOO_LONG)
- bytes_written = -EINVAL;
-
- if (bytes_written > 0)
- relay_commit(rchan, reserved, bytes_written, reserve_code, interrupting);
-
- relay_unlock_channel(rchan, flags); /* nop for lockless */
-
- rchan_put(rchan);
-
- if (wrote_pos)
- *wrote_pos = reserved;
-
- return bytes_written;
-}
-
-/**
- * wakeup_writers - wake up VFS writers waiting on a channel
- * @private: the channel
- *
- * This is the work function used to defer writer waking. The
- * reason waking is deferred is that calling directly from
- * buffers_consumed causes problems if you're writing from say
- * the scheduler.
- */
-static void
-wakeup_writers(void *private)
-{
- struct rchan *rchan = (struct rchan *)private;
-
- wake_up_interruptible(&rchan->write_wait);
-}
-
-
-/**
- * __relay_buffers_consumed - internal version of relay_buffers_consumed
- * @rchan: the relay channel
- * @bufs_consumed: number of buffers to add to current count for channel
- *
- * Internal - updates the channel's consumed buffer count.
- */
-static void
-__relay_buffers_consumed(struct rchan *rchan, u32 bufs_consumed)
-{
- rchan->bufs_consumed += bufs_consumed;
-
- if (rchan->bufs_consumed > rchan->bufs_produced)
- rchan->bufs_consumed = rchan->bufs_produced;
-
- atomic_set(&rchan->suspended, 0);
-
- PREPARE_WORK(&rchan->wake_writers, wakeup_writers, rchan);
- schedule_delayed_work(&rchan->wake_writers, 1);
-}
-
-/**
- * __reader_buffers_consumed - update reader/channel consumed buffer count
- * @reader: channel reader
- * @bufs_consumed: number of buffers to add to current count for channel
- *
- * Internal - updates the reader's consumed buffer count. If the reader's
- * resulting total is greater than the channel's, update the channel's.
-*/
-static void
-__reader_buffers_consumed(struct rchan_reader *reader, u32 bufs_consumed)
-{
- reader->bufs_consumed += bufs_consumed;
-
- if (reader->bufs_consumed > reader->rchan->bufs_consumed)
- __relay_buffers_consumed(reader->rchan, bufs_consumed);
-}
-
-/**
- * relay_buffers_consumed - add to the # buffers consumed for the channel
- * @reader: channel reader
- * @bufs_consumed: number of buffers to add to current count for channel
- *
- * Adds to the channel's consumed buffer count. buffers_consumed should
- * be the number of buffers newly consumed, not the total number consumed.
- *
- * NOTE: kernel clients don't need to call this function if the reader
- * is auto-consuming or the channel is MODE_CONTINUOUS.
- */
-void
-relay_buffers_consumed(struct rchan_reader *reader, u32 bufs_consumed)
-{
- if (reader && reader->rchan)
- __reader_buffers_consumed(reader, bufs_consumed);
-}
-
-/**
- * __relay_bytes_consumed - internal version of relay_bytes_consumed
- * @rchan: the relay channel
- * @bytes_consumed: number of bytes to add to current count for channel
- * @read_offset: where the bytes were consumed from
- *
- * Internal - updates the channel's consumed count.
-*/
-static void
-__relay_bytes_consumed(struct rchan *rchan, u32 bytes_consumed, u32 read_offset)
-{
- u32 consuming_idx;
- u32 unused;
-
- consuming_idx = read_offset / rchan->buf_size;
-
- if (consuming_idx >= rchan->n_bufs)
- consuming_idx = rchan->n_bufs - 1;
- rchan->bytes_consumed += bytes_consumed;
-
- unused = rchan->unused_bytes[consuming_idx];
-
- if (rchan->bytes_consumed + unused >= rchan->buf_size) {
- __relay_buffers_consumed(rchan, 1);
- rchan->bytes_consumed = 0;
- }
-}
-
-/**
- * __reader_bytes_consumed - update reader/channel consumed count
- * @reader: channel reader
- * @bytes_consumed: number of bytes to add to current count for channel
- * @read_offset: where the bytes were consumed from
- *
- * Internal - updates the reader's consumed count. If the reader's
- * resulting total is greater than the channel's, update the channel's.
-*/
-static void
-__reader_bytes_consumed(struct rchan_reader *reader, u32 bytes_consumed, u32 read_offset)
-{
- u32 consuming_idx;
- u32 unused;
-
- consuming_idx = read_offset / reader->rchan->buf_size;
-
- if (consuming_idx >= reader->rchan->n_bufs)
- consuming_idx = reader->rchan->n_bufs - 1;
-
- reader->bytes_consumed += bytes_consumed;
-
- unused = reader->rchan->unused_bytes[consuming_idx];
-
- if (reader->bytes_consumed + unused >= reader->rchan->buf_size) {
- reader->bufs_consumed++;
- reader->bytes_consumed = 0;
- }
-
- if ((reader->bufs_consumed > reader->rchan->bufs_consumed) ||
- ((reader->bufs_consumed == reader->rchan->bufs_consumed) &&
- (reader->bytes_consumed > reader->rchan->bytes_consumed)))
- __relay_bytes_consumed(reader->rchan, bytes_consumed, read_offset);
-}
-
-/**
- * relay_bytes_consumed - add to the # bytes consumed for the channel
- * @reader: channel reader
- * @bytes_consumed: number of bytes to add to current count for channel
- * @read_offset: where the bytes were consumed from
- *
- * Adds to the channel's consumed count. bytes_consumed should be the
- * number of bytes actually read e.g. return value of relay_read() and
- * the read_offset should be the actual offset the bytes were read from
- * e.g. the actual_read_offset set by relay_read(). See
- * Documentation/filesystems/relayfs.txt for more details.
- *
- * NOTE: kernel clients don't need to call this function if the reader
- * is auto-consuming or the channel is MODE_CONTINUOUS.
- */
-void
-relay_bytes_consumed(struct rchan_reader *reader, u32 bytes_consumed, u32 read_offset)
-{
- if (reader && reader->rchan)
- __reader_bytes_consumed(reader, bytes_consumed, read_offset);
-}
-
-/**
- * update_readers_consumed - apply offset change to reader
- * @rchan: the channel
- *
- * Apply the consumed counts to all readers open on the channel.
- */
-void
-update_readers_consumed(struct rchan *rchan, u32 bufs_consumed, u32 bytes_consumed)
-{
- struct list_head *p;
- struct rchan_reader *reader;
-
- read_lock(&rchan->open_readers_lock);
- list_for_each(p, &rchan->open_readers) {
- reader = list_entry(p, struct rchan_reader, list);
- reader->bufs_consumed = bufs_consumed;
- reader->bytes_consumed = bytes_consumed;
- if (reader->vfs_reader)
- reader->pos.file->f_pos = 0;
- else
- reader->pos.f_pos = 0;
- reader->offset_changed = 1;
- }
- read_unlock(&rchan->open_readers_lock);
-}
-
-/**
- * do_read - utility function to do the actual read to user
- * @rchan: the channel
- * @buf: user buf to read into, NULL if just getting info
- * @count: bytes requested
- * @read_offset: offset into channel
- * @new_offset: new offset into channel after read
- * @actual_read_offset: read offset actually used
- *
- * Returns the number of bytes read, 0 if none.
- */
-static ssize_t
-do_read(struct rchan *rchan, char *buf, size_t count, u32 read_offset, u32 *new_offset, u32 *actual_read_offset)
-{
- u32 read_bufno, cur_bufno;
- u32 avail_offset, cur_idx, max_offset, buf_end_offset;
- u32 avail_count, buf_size;
- int unused_bytes = 0;
- size_t read_count = 0;
- u32 last_buf_byte_offset;
-
- *actual_read_offset = read_offset;
-
- buf_size = rchan->buf_size;
- if (unlikely(!buf_size)) BUG();
-
- read_bufno = read_offset / buf_size;
- if (unlikely(read_bufno >= RELAY_MAX_BUFS)) BUG();
- unused_bytes = rchan->unused_bytes[read_bufno];
-
- avail_offset = cur_idx = relay_get_offset(rchan, &max_offset);
-
- if (cur_idx == read_offset) {
- if (atomic_read(&rchan->suspended) == 1) {
- read_offset += 1;
- if (read_offset >= max_offset)
- read_offset = 0;
- *actual_read_offset = read_offset;
- } else {
- *new_offset = read_offset;
- return 0;
- }
- } else {
- last_buf_byte_offset = (read_bufno + 1) * buf_size - 1;
- if (read_offset == last_buf_byte_offset) {
- if (unused_bytes != 1) {
- read_offset += 1;
- if (read_offset >= max_offset)
- read_offset = 0;
- *actual_read_offset = read_offset;
- }
- }
- }
-
- read_bufno = read_offset / buf_size;
- if (unlikely(read_bufno >= RELAY_MAX_BUFS)) BUG();
- unused_bytes = rchan->unused_bytes[read_bufno];
-
- cur_bufno = cur_idx / buf_size;
-
- buf_end_offset = (read_bufno + 1) * buf_size - unused_bytes;
- if (avail_offset > buf_end_offset)
- avail_offset = buf_end_offset;
- else if (avail_offset < read_offset)
- avail_offset = buf_end_offset;
- avail_count = avail_offset - read_offset;
- read_count = avail_count >= count ? count : avail_count;
-
- if (read_count && buf != NULL)
- if (copy_to_user(buf, rchan->buf + read_offset, read_count))
- return -EFAULT;
-
- if (read_bufno == cur_bufno)
- if (read_count && (read_offset + read_count >= buf_end_offset) && (read_offset + read_count <= cur_idx)) {
- *new_offset = cur_idx;
- return read_count;
- }
-
- if (read_offset + read_count + unused_bytes > max_offset)
- *new_offset = 0;
- else if (read_offset + read_count >= buf_end_offset)
- *new_offset = read_offset + read_count + unused_bytes;
- else
- *new_offset = read_offset + read_count;
-
- return read_count;
-}
-
-/**
- * __relay_read - read bytes from channel, relative to current reader pos
- * @reader: channel reader
- * @buf: user buf to read into, NULL if just getting info
- * @count: bytes requested
- * @read_offset: offset into channel
- * @new_offset: new offset into channel after read
- * @actual_read_offset: read offset actually used
- * @wait: if non-zero, wait for something to read
- *
- * Internal - see relay_read() for details.
- *
- * Returns the number of bytes read, 0 if none, negative on failure.
- */
-static ssize_t
-__relay_read(struct rchan_reader *reader, char *buf, size_t count, u32 read_offset, u32 *new_offset, u32 *actual_read_offset, int wait)
-{
- int err = 0;
- size_t read_count = 0;
- struct rchan *rchan = reader->rchan;
-
- if (!wait && !rchan->initialized)
- return -EAGAIN;
-
- if (using_lockless(rchan))
- read_offset &= idx_mask(rchan);
-
- if (read_offset >= rchan->n_bufs * rchan->buf_size) {
- *new_offset = 0;
- if (!wait)
- return -EAGAIN;
- else
- return -EINTR;
- }
-
- if (buf != NULL && wait) {
- err = wait_event_interruptible(rchan->read_wait,
- ((rchan->finalized == 1) ||
- (atomic_read(&rchan->suspended) == 1) ||
- (relay_get_offset(rchan, NULL) != read_offset)));
-
- if (rchan->finalized)
- return 0;
-
- if (reader->offset_changed) {
- reader->offset_changed = 0;
- return -EINTR;
- }
-
- if (err)
- return err;
- }
-
- read_count = do_read(rchan, buf, count, read_offset, new_offset, actual_read_offset);
-
- if (read_count < 0)
- err = read_count;
-
- if (err)
- return err;
- else
- return read_count;
-}
-
-/**
- * relay_read - read bytes from channel, relative to current reader pos
- * @reader: channel reader
- * @buf: user buf to read into, NULL if just getting info
- * @count: bytes requested
- * @wait: if non-zero, wait for something to read
- * @actual_read_offset: set read offset actually used, must not be NULL
- *
- * Reads count bytes from the channel, or as much as is available within
- * the sub-buffer currently being read. The read offset that will be
- * read from is the position contained within the reader object. If the
- * wait flag is set, buf is non-NULL, and there is nothing available,
- * it will wait until there is. If the wait flag is 0 and there is
- * nothing available, -EAGAIN is returned. If buf is NULL, the value
- * returned is the number of bytes that would have been read.
- * actual_read_offset is the value that should be passed as the read
- * offset to relay_bytes_consumed, needed only if the reader is not
- * auto-consuming and the channel is MODE_NO_OVERWRITE, but in any case,
- * it must not be NULL. See Documentation/filesystems/relayfs.txt for
- * more details.
- */
-ssize_t
-relay_read(struct rchan_reader *reader, char *buf, size_t count, int wait, u32 *actual_read_offset)
-{
- u32 new_offset;
- u32 read_offset;
- ssize_t read_count;
-
- if (reader == NULL || reader->rchan == NULL)
- return -EBADF;
-
- if (actual_read_offset == NULL)
- return -EINVAL;
-
- if (reader->vfs_reader)
- read_offset = (u32)(reader->pos.file->f_pos);
- else
- read_offset = reader->pos.f_pos;
- *actual_read_offset = read_offset;
-
- read_count = __relay_read(reader, buf, count, read_offset,
- &new_offset, actual_read_offset, wait);
-
- if (read_count < 0)
- return read_count;
-
- if (reader->vfs_reader)
- reader->pos.file->f_pos = new_offset;
- else
- reader->pos.f_pos = new_offset;
-
- if (reader->auto_consume && ((read_count) || (new_offset != read_offset)))
- __reader_bytes_consumed(reader, read_count, *actual_read_offset);
-
- if (read_count == 0 && !wait)
- return -EAGAIN;
-
- return read_count;
-}
-
-/**
- * relay_bytes_avail - number of bytes available in current sub-buffer
- * @reader: channel reader
- *
- * Returns the number of bytes available relative to the reader's
- * current read position within the corresponding sub-buffer, 0 if
- * there is nothing available. See Documentation/filesystems/relayfs.txt
- * for more details.
- */
-ssize_t
-relay_bytes_avail(struct rchan_reader *reader)
-{
- u32 f_pos;
- u32 new_offset;
- u32 actual_read_offset;
- ssize_t bytes_read;
-
- if (reader == NULL || reader->rchan == NULL)
- return -EBADF;
-
- if (reader->vfs_reader)
- f_pos = (u32)reader->pos.file->f_pos;
- else
- f_pos = reader->pos.f_pos;
- new_offset = f_pos;
-
- bytes_read = __relay_read(reader, NULL, reader->rchan->buf_size,
- f_pos, &new_offset, &actual_read_offset, 0);
-
- if ((new_offset != f_pos) &&
- ((bytes_read == -EINTR) || (bytes_read == 0)))
- bytes_read = -EAGAIN;
- else if ((bytes_read < 0) && (bytes_read != -EAGAIN))
- bytes_read = 0;
-
- return bytes_read;
-}
-
-/**
- * rchan_empty - boolean, is the channel empty wrt reader?
- * @reader: channel reader
- *
- * Returns 1 if the channel is empty, 0 otherwise.
- */
-int
-rchan_empty(struct rchan_reader *reader)
-{
- ssize_t avail_count;
- u32 buffers_ready;
- struct rchan *rchan = reader->rchan;
- u32 cur_idx, curbuf_bytes;
- int mapped;
-
- if (atomic_read(&rchan->suspended) == 1)
- return 0;
-
- mapped = atomic_read(&rchan->mapped);
-
- if (mapped && bulk_delivery(rchan)) {
- buffers_ready = rchan->bufs_produced - rchan->bufs_consumed;
- return buffers_ready ? 0 : 1;
- }
-
- if (mapped && packet_delivery(rchan)) {
- buffers_ready = rchan->bufs_produced - rchan->bufs_consumed;
- if (buffers_ready)
- return 0;
- else {
- cur_idx = relay_get_offset(rchan, NULL);
- curbuf_bytes = cur_idx % rchan->buf_size;
- return curbuf_bytes == rchan->bytes_consumed ? 1 : 0;
- }
- }
-
- avail_count = relay_bytes_avail(reader);
-
- return avail_count ? 0 : 1;
-}
-
-/**
- * rchan_full - boolean, is the channel full wrt consuming reader?
- * @reader: channel reader
- *
- * Returns 1 if the channel is full, 0 otherwise.
- */
-int
-rchan_full(struct rchan_reader *reader)
-{
- u32 buffers_ready;
- struct rchan *rchan = reader->rchan;
-
- if (mode_continuous(rchan))
- return 0;
-
- buffers_ready = rchan->bufs_produced - rchan->bufs_consumed;
-
- return buffers_ready > reader->rchan->n_bufs - 1 ? 1 : 0;
-}
-
-/**
- * relay_info - get status and other information about a relay channel
- * @rchan_id: relay channel id
- * @rchan_info: pointer to the rchan_info struct to be filled in
- *
- * Fills in an rchan_info struct with channel status and attribute
- * information. See Documentation/filesystems/relayfs.txt for details.
- *
- * Returns 0 if successful, negative otherwise.
- */
-int
-relay_info(int rchan_id, struct rchan_info *rchan_info)
-{
- int i;
- struct rchan *rchan;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- rchan_info->flags = rchan->flags;
- rchan_info->buf_size = rchan->buf_size;
- rchan_info->buf_addr = rchan->buf;
- rchan_info->alloc_size = rchan->alloc_size;
- rchan_info->n_bufs = rchan->n_bufs;
- rchan_info->cur_idx = relay_get_offset(rchan, NULL);
- rchan_info->bufs_produced = rchan->bufs_produced;
- rchan_info->bufs_consumed = rchan->bufs_consumed;
- rchan_info->buf_id = rchan->buf_id;
-
- for (i = 0; i < rchan->n_bufs; i++) {
- rchan_info->unused_bytes[i] = rchan->unused_bytes[i];
- if (using_lockless(rchan))
- rchan_info->buffer_complete[i] = (atomic_read(&fill_count(rchan, i)) == rchan->buf_size);
- else
- rchan_info->buffer_complete[i] = 0;
- }
-
- rchan_put(rchan);
-
- return 0;
-}
-
-/**
- * __add_rchan_reader - creates and adds a reader to a channel
- * @rchan: relay channel
- * @filp: the file associated with rchan, if applicable
- * @auto_consume: boolean, whether reader's reads automatically consume
- * @map_reader: boolean, whether reader's reading via a channel mapping
- *
- * Returns a pointer to the reader object create, NULL if unsuccessful
- *
- * Creates and initializes an rchan_reader object for reading the channel.
- * If filp is non-NULL, the reader is a VFS reader, otherwise not.
- *
- * If the reader is a map reader, it isn't considered a VFS reader for
- * our purposes. Also, map_readers can't be auto-consuming.
- */
-struct rchan_reader *
-__add_rchan_reader(struct rchan *rchan, struct file *filp, int auto_consume, int map_reader)
-{
- struct rchan_reader *reader;
- u32 will_read;
-
- reader = kmalloc(sizeof(struct rchan_reader), GFP_KERNEL);
-
- if (reader) {
- write_lock(&rchan->open_readers_lock);
- reader->rchan = rchan;
- if (filp) {
- reader->vfs_reader = 1;
- reader->pos.file = filp;
- } else {
- reader->vfs_reader = 0;
- reader->pos.f_pos = 0;
- }
- reader->map_reader = map_reader;
- reader->auto_consume = auto_consume;
-
- if (!map_reader) {
- will_read = rchan->bufs_produced % rchan->n_bufs;
- if (!will_read && atomic_read(&rchan->suspended))
- will_read = rchan->n_bufs;
- reader->bufs_consumed = rchan->bufs_produced - will_read;
- rchan->bufs_consumed = reader->bufs_consumed;
- rchan->bytes_consumed = reader->bytes_consumed = 0;
- reader->offset_changed = 0;
- }
-
- list_add(&reader->list, &rchan->open_readers);
- write_unlock(&rchan->open_readers_lock);
- }
-
- return reader;
-}
-
-/**
- * add_rchan_reader - create a reader for a channel
- * @rchan_id: relay channel handle
- * @auto_consume: boolean, whether reader's reads automatically consume
- *
- * Returns a pointer to the reader object created, NULL if unsuccessful
- *
- * Creates and initializes an rchan_reader object for reading the channel.
- * This function is useful only for non-VFS readers.
- */
-struct rchan_reader *
-add_rchan_reader(int rchan_id, int auto_consume)
-{
- struct rchan *rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return NULL;
-
- return __add_rchan_reader(rchan, NULL, auto_consume, 0);
-}
-
-/**
- * add_map_reader - create a map reader for a channel
- * @rchan_id: relay channel handle
- *
- * Returns a pointer to the reader object created, NULL if unsuccessful
- *
- * Creates and initializes an rchan_reader object for reading the channel.
- * This function is useful only for map readers.
- */
-struct rchan_reader *
-add_map_reader(int rchan_id)
-{
- struct rchan *rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return NULL;
-
- return __add_rchan_reader(rchan, NULL, 0, 1);
-}
-
-/**
- * __remove_rchan_reader - destroy a channel reader
- * @reader: channel reader
- *
- * Internal - removes reader from the open readers list, and frees it.
- */
-void
-__remove_rchan_reader(struct rchan_reader *reader)
-{
- struct list_head *p;
- struct rchan_reader *found_reader = NULL;
-
- write_lock(&reader->rchan->open_readers_lock);
- list_for_each(p, &reader->rchan->open_readers) {
- found_reader = list_entry(p, struct rchan_reader, list);
- if (found_reader == reader) {
- list_del(&found_reader->list);
- break;
- }
- }
- write_unlock(&reader->rchan->open_readers_lock);
-
- if (found_reader)
- kfree(found_reader);
-}
-
-/**
- * remove_rchan_reader - destroy a channel reader
- * @reader: channel reader
- *
- * Finds and removes the given reader from the channel. This function
- * is useful only for non-VFS readers.
- *
- * Returns 0 if successful, negative otherwise.
- */
-int
-remove_rchan_reader(struct rchan_reader *reader)
-{
- int err = 0;
-
- if (reader) {
- rchan_put(reader->rchan);
- __remove_rchan_reader(reader);
- } else
- err = -EINVAL;
-
- return err;
-}
-
-/**
- * remove_map_reader - destroy a map reader
- * @reader: channel reader
- *
- * Finds and removes the given map reader from the channel. This function
- * is useful only for map readers.
- *
- * Returns 0 if successful, negative otherwise.
- */
-int
-remove_map_reader(struct rchan_reader *reader)
-{
- return remove_rchan_reader(reader);
-}
-
-EXPORT_SYMBOL(relay_open);
-EXPORT_SYMBOL(relay_close);
-EXPORT_SYMBOL(relay_reset);
-EXPORT_SYMBOL(relay_reserve);
-EXPORT_SYMBOL(relay_commit);
-EXPORT_SYMBOL(relay_read);
-EXPORT_SYMBOL(relay_write);
-EXPORT_SYMBOL(relay_bytes_avail);
-EXPORT_SYMBOL(relay_buffers_consumed);
-EXPORT_SYMBOL(relay_bytes_consumed);
-EXPORT_SYMBOL(relay_info);
-EXPORT_SYMBOL(relay_discard_init_buf);
-
-
+++ /dev/null
-/*
- * RelayFS locking scheme implementation.
- *
- * Copyright (C) 1999, 2000, 2001, 2002 - Karim Yaghmour (karim@opersys.com)
- * Copyright (C) 2002, 2003 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
- *
- * This file is released under the GPL.
- */
-
-#include <asm/relay.h>
-#include "relay_locking.h"
-#include "resize.h"
-
-/**
- * switch_buffers - switches between read and write buffers.
- * @cur_time: current time.
- * @cur_tsc: the TSC associated with current_time, if applicable
- * @rchan: the channel
- * @finalizing: if true, don't start a new buffer
- * @resetting: if true,
- *
- * This should be called from with interrupts disabled.
- */
-static void
-switch_buffers(struct timeval cur_time,
- u32 cur_tsc,
- struct rchan *rchan,
- int finalizing,
- int resetting,
- int finalize_buffer_only)
-{
- char *chan_buf_end;
- int bytes_written;
-
- if (!rchan->half_switch) {
- bytes_written = rchan->callbacks->buffer_end(rchan->id,
- cur_write_pos(rchan), write_buf_end(rchan),
- cur_time, cur_tsc, using_tsc(rchan));
- if (bytes_written == 0)
- rchan->unused_bytes[rchan->buf_idx % rchan->n_bufs] =
- write_buf_end(rchan) - cur_write_pos(rchan);
- }
-
- if (finalize_buffer_only) {
- rchan->bufs_produced++;
- return;
- }
-
- chan_buf_end = rchan->buf + rchan->n_bufs * rchan->buf_size;
- if((write_buf(rchan) + rchan->buf_size >= chan_buf_end) || resetting)
- write_buf(rchan) = rchan->buf;
- else
- write_buf(rchan) += rchan->buf_size;
- write_buf_end(rchan) = write_buf(rchan) + rchan->buf_size;
- write_limit(rchan) = write_buf_end(rchan) - rchan->end_reserve;
- cur_write_pos(rchan) = write_buf(rchan);
-
- rchan->buf_start_time = cur_time;
- rchan->buf_start_tsc = cur_tsc;
-
- if (resetting)
- rchan->buf_idx = 0;
- else
- rchan->buf_idx++;
- rchan->buf_id++;
-
- if (!packet_delivery(rchan))
- rchan->unused_bytes[rchan->buf_idx % rchan->n_bufs] = 0;
-
- if (resetting) {
- rchan->bufs_produced = rchan->bufs_produced + rchan->n_bufs;
- rchan->bufs_produced -= rchan->bufs_produced % rchan->n_bufs;
- rchan->bufs_consumed = rchan->bufs_produced;
- rchan->bytes_consumed = 0;
- update_readers_consumed(rchan, rchan->bufs_consumed, rchan->bytes_consumed);
- } else if (!rchan->half_switch)
- rchan->bufs_produced++;
-
- rchan->half_switch = 0;
-
- if (!finalizing) {
- bytes_written = rchan->callbacks->buffer_start(rchan->id, cur_write_pos(rchan), rchan->buf_id, cur_time, cur_tsc, using_tsc(rchan));
- cur_write_pos(rchan) += bytes_written;
- }
-}
-
-/**
- * locking_reserve - reserve a slot in the buffer for an event.
- * @rchan: the channel
- * @slot_len: the length of the slot to reserve
- * @ts: variable that will receive the time the slot was reserved
- * @tsc: the timestamp counter associated with time
- * @err: receives the result flags
- * @interrupting: if this write is interrupting another, set to non-zero
- *
- * Returns pointer to the beginning of the reserved slot, NULL if error.
- *
- * The err value contains the result flags and is an ORed combination
- * of the following:
- *
- * RELAY_BUFFER_SWITCH_NONE - no buffer switch occurred
- * RELAY_EVENT_DISCARD_NONE - event should not be discarded
- * RELAY_BUFFER_SWITCH - buffer switch occurred
- * RELAY_EVENT_DISCARD - event should be discarded (all buffers are full)
- * RELAY_EVENT_TOO_LONG - event won't fit into even an empty buffer
- */
-inline char *
-locking_reserve(struct rchan *rchan,
- u32 slot_len,
- struct timeval *ts,
- u32 *tsc,
- int *err,
- int *interrupting)
-{
- u32 buffers_ready;
- int bytes_written;
-
- *err = RELAY_BUFFER_SWITCH_NONE;
-
- if (slot_len >= rchan->buf_size) {
- *err = RELAY_WRITE_DISCARD | RELAY_WRITE_TOO_LONG;
- return NULL;
- }
-
- if (rchan->initialized == 0) {
- rchan->initialized = 1;
- get_timestamp(&rchan->buf_start_time,
- &rchan->buf_start_tsc, rchan);
- rchan->unused_bytes[0] = 0;
- bytes_written = rchan->callbacks->buffer_start(
- rchan->id, cur_write_pos(rchan),
- rchan->buf_id, rchan->buf_start_time,
- rchan->buf_start_tsc, using_tsc(rchan));
- cur_write_pos(rchan) += bytes_written;
- *tsc = get_time_delta(ts, rchan);
- return cur_write_pos(rchan);
- }
-
- *tsc = get_time_delta(ts, rchan);
-
- if (in_progress_event_size(rchan)) {
- interrupted_pos(rchan) = cur_write_pos(rchan);
- cur_write_pos(rchan) = in_progress_event_pos(rchan)
- + in_progress_event_size(rchan)
- + interrupting_size(rchan);
- *interrupting = 1;
- } else {
- in_progress_event_pos(rchan) = cur_write_pos(rchan);
- in_progress_event_size(rchan) = slot_len;
- interrupting_size(rchan) = 0;
- }
-
- if (cur_write_pos(rchan) + slot_len > write_limit(rchan)) {
- if (atomic_read(&rchan->suspended) == 1) {
- in_progress_event_pos(rchan) = NULL;
- in_progress_event_size(rchan) = 0;
- interrupting_size(rchan) = 0;
- *err = RELAY_WRITE_DISCARD;
- return NULL;
- }
-
- buffers_ready = rchan->bufs_produced - rchan->bufs_consumed;
- if (buffers_ready == rchan->n_bufs - 1) {
- if (!mode_continuous(rchan)) {
- atomic_set(&rchan->suspended, 1);
- in_progress_event_pos(rchan) = NULL;
- in_progress_event_size(rchan) = 0;
- interrupting_size(rchan) = 0;
- get_timestamp(ts, tsc, rchan);
- switch_buffers(*ts, *tsc, rchan, 0, 0, 1);
- recalc_time_delta(ts, tsc, rchan);
- rchan->half_switch = 1;
-
- cur_write_pos(rchan) = write_buf_end(rchan) - 1;
- *err = RELAY_BUFFER_SWITCH | RELAY_WRITE_DISCARD;
- return NULL;
- }
- }
-
- get_timestamp(ts, tsc, rchan);
- switch_buffers(*ts, *tsc, rchan, 0, 0, 0);
- recalc_time_delta(ts, tsc, rchan);
- *err = RELAY_BUFFER_SWITCH;
- }
-
- return cur_write_pos(rchan);
-}
-
-/**
- * locking_commit - commit a reserved slot in the buffer
- * @rchan: the channel
- * @from: commit the length starting here
- * @len: length committed
- * @deliver: length committed
- * @interrupting: not used
- *
- * Commits len bytes and calls deliver callback if applicable.
- */
-inline void
-locking_commit(struct rchan *rchan,
- char *from,
- u32 len,
- int deliver,
- int interrupting)
-{
- cur_write_pos(rchan) += len;
-
- if (interrupting) {
- cur_write_pos(rchan) = interrupted_pos(rchan);
- interrupting_size(rchan) += len;
- } else {
- in_progress_event_size(rchan) = 0;
- if (interrupting_size(rchan)) {
- cur_write_pos(rchan) += interrupting_size(rchan);
- interrupting_size(rchan) = 0;
- }
- }
-
- if (deliver) {
- if (bulk_delivery(rchan)) {
- u32 cur_idx = cur_write_pos(rchan) - rchan->buf;
- u32 cur_bufno = cur_idx / rchan->buf_size;
- from = rchan->buf + cur_bufno * rchan->buf_size;
- len = cur_idx - cur_bufno * rchan->buf_size;
- }
- rchan->callbacks->deliver(rchan->id, from, len);
- expand_check(rchan);
- }
-}
-
-/**
- * locking_finalize: - finalize last buffer at end of channel use
- * @rchan: the channel
- */
-inline void
-locking_finalize(struct rchan *rchan)
-{
- unsigned long int flags;
- struct timeval time;
- u32 tsc;
-
- local_irq_save(flags);
- get_timestamp(&time, &tsc, rchan);
- switch_buffers(time, tsc, rchan, 1, 0, 0);
- local_irq_restore(flags);
-}
-
-/**
- * locking_get_offset - get current and max 'file' offsets for VFS
- * @rchan: the channel
- * @max_offset: maximum channel offset
- *
- * Returns the current and maximum buffer offsets in VFS terms.
- */
-u32
-locking_get_offset(struct rchan *rchan,
- u32 *max_offset)
-{
- if (max_offset)
- *max_offset = rchan->buf_size * rchan->n_bufs - 1;
-
- return cur_write_pos(rchan) - rchan->buf;
-}
-
-/**
- * locking_reset - reset the channel
- * @rchan: the channel
- * @init: 1 if this is a first-time channel initialization
- */
-void locking_reset(struct rchan *rchan, int init)
-{
- if (init)
- channel_lock(rchan) = SPIN_LOCK_UNLOCKED;
- write_buf(rchan) = rchan->buf;
- write_buf_end(rchan) = write_buf(rchan) + rchan->buf_size;
- cur_write_pos(rchan) = write_buf(rchan);
- write_limit(rchan) = write_buf_end(rchan) - rchan->end_reserve;
- in_progress_event_pos(rchan) = NULL;
- in_progress_event_size(rchan) = 0;
- interrupted_pos(rchan) = NULL;
- interrupting_size(rchan) = 0;
-}
-
-/**
- * locking_reset_index - atomically set channel index to the beginning
- * @rchan: the channel
- *
- * If this fails, it means that something else just logged something
- * and therefore we probably no longer want to do this. It's up to the
- * caller anyway...
- *
- * Returns 0 if the index was successfully set, negative otherwise
- */
-int
-locking_reset_index(struct rchan *rchan, u32 old_idx)
-{
- unsigned long flags;
- struct timeval time;
- u32 tsc;
- u32 cur_idx;
-
- relay_lock_channel(rchan, flags);
- cur_idx = locking_get_offset(rchan, NULL);
- if (cur_idx != old_idx) {
- relay_unlock_channel(rchan, flags);
- return -1;
- }
-
- get_timestamp(&time, &tsc, rchan);
- switch_buffers(time, tsc, rchan, 0, 1, 0);
-
- relay_unlock_channel(rchan, flags);
-
- return 0;
-}
-
-
-
-
-
-
-
+++ /dev/null
-#ifndef _RELAY_LOCKING_H
-#define _RELAY_LOCKING_H
-
-extern char *
-locking_reserve(struct rchan *rchan,
- u32 slot_len,
- struct timeval *time_stamp,
- u32 *tsc,
- int *err,
- int *interrupting);
-
-extern void
-locking_commit(struct rchan *rchan,
- char *from,
- u32 len,
- int deliver,
- int interrupting);
-
-extern void
-locking_resume(struct rchan *rchan);
-
-extern void
-locking_finalize(struct rchan *rchan);
-
-extern u32
-locking_get_offset(struct rchan *rchan, u32 *max_offset);
-
-extern void
-locking_reset(struct rchan *rchan, int init);
-
-extern int
-locking_reset_index(struct rchan *rchan, u32 old_idx);
-
-#endif /* _RELAY_LOCKING_H */
+++ /dev/null
-/*
- * RelayFS lockless scheme implementation.
- *
- * Copyright (C) 1999, 2000, 2001, 2002 - Karim Yaghmour (karim@opersys.com)
- * Copyright (C) 2002, 2003 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
- * Copyright (C) 2002, 2003 - Bob Wisniewski (bob@watson.ibm.com), IBM Corp
- *
- * This file is released under the GPL.
- */
-
-#include <asm/relay.h>
-#include "relay_lockless.h"
-#include "resize.h"
-
-/**
- * compare_and_store_volatile - self-explicit
- * @ptr: ptr to the word that will receive the new value
- * @oval: the value we think is currently in *ptr
- * @nval: the value *ptr will get if we were right
- */
-inline int
-compare_and_store_volatile(volatile u32 *ptr,
- u32 oval,
- u32 nval)
-{
- u32 prev;
-
- barrier();
- prev = cmpxchg(ptr, oval, nval);
- barrier();
-
- return (prev == oval);
-}
-
-/**
- * atomic_set_volatile - atomically set the value in ptr to nval.
- * @ptr: ptr to the word that will receive the new value
- * @nval: the new value
- */
-inline void
-atomic_set_volatile(atomic_t *ptr,
- u32 nval)
-{
- barrier();
- atomic_set(ptr, (int)nval);
- barrier();
-}
-
-/**
- * atomic_add_volatile - atomically add val to the value at ptr.
- * @ptr: ptr to the word that will receive the addition
- * @val: the value to add to *ptr
- */
-inline void
-atomic_add_volatile(atomic_t *ptr, u32 val)
-{
- barrier();
- atomic_add((int)val, ptr);
- barrier();
-}
-
-/**
- * atomic_sub_volatile - atomically substract val from the value at ptr.
- * @ptr: ptr to the word that will receive the subtraction
- * @val: the value to subtract from *ptr
- */
-inline void
-atomic_sub_volatile(atomic_t *ptr, s32 val)
-{
- barrier();
- atomic_sub((int)val, ptr);
- barrier();
-}
-
-/**
- * lockless_commit - commit a reserved slot in the buffer
- * @rchan: the channel
- * @from: commit the length starting here
- * @len: length committed
- * @deliver: length committed
- * @interrupting: not used
- *
- * Commits len bytes and calls deliver callback if applicable.
- */
-inline void
-lockless_commit(struct rchan *rchan,
- char *from,
- u32 len,
- int deliver,
- int interrupting)
-{
- u32 bufno, idx;
-
- idx = from - rchan->buf;
-
- if (len > 0) {
- bufno = RELAY_BUFNO_GET(idx, offset_bits(rchan));
- atomic_add_volatile(&fill_count(rchan, bufno), len);
- }
-
- if (deliver) {
- u32 mask = offset_mask(rchan);
- if (bulk_delivery(rchan)) {
- from = rchan->buf + RELAY_BUF_OFFSET_CLEAR(idx, mask);
- len += RELAY_BUF_OFFSET_GET(idx, mask);
- }
- rchan->callbacks->deliver(rchan->id, from, len);
- expand_check(rchan);
- }
-}
-
-/**
- * get_buffer_end - get the address of the end of buffer
- * @rchan: the channel
- * @buf_idx: index into channel corresponding to address
- */
-static inline char *
-get_buffer_end(struct rchan *rchan, u32 buf_idx)
-{
- return rchan->buf
- + RELAY_BUF_OFFSET_CLEAR(buf_idx, offset_mask(rchan))
- + RELAY_BUF_SIZE(offset_bits(rchan));
-}
-
-
-/**
- * finalize_buffer - utility function consolidating end-of-buffer tasks.
- * @rchan: the channel
- * @end_idx: index into buffer to write the end-buffer event at
- * @size_lost: number of unused bytes at the end of the buffer
- * @time_stamp: the time of the end-buffer event
- * @tsc: the timestamp counter associated with time
- * @resetting: are we resetting the channel?
- *
- * This function must be called with local irqs disabled.
- */
-static inline void
-finalize_buffer(struct rchan *rchan,
- u32 end_idx,
- u32 size_lost,
- struct timeval *time_stamp,
- u32 *tsc,
- int resetting)
-{
- char* cur_write_pos;
- char* write_buf_end;
- u32 bufno;
- int bytes_written;
-
- cur_write_pos = rchan->buf + end_idx;
- write_buf_end = get_buffer_end(rchan, end_idx - 1);
-
- bytes_written = rchan->callbacks->buffer_end(rchan->id, cur_write_pos,
- write_buf_end, *time_stamp, *tsc, using_tsc(rchan));
- if (bytes_written == 0)
- rchan->unused_bytes[rchan->buf_idx % rchan->n_bufs] = size_lost;
-
- bufno = RELAY_BUFNO_GET(end_idx, offset_bits(rchan));
- atomic_add_volatile(&fill_count(rchan, bufno), size_lost);
- if (resetting) {
- rchan->bufs_produced = rchan->bufs_produced + rchan->n_bufs;
- rchan->bufs_produced -= rchan->bufs_produced % rchan->n_bufs;
- rchan->bufs_consumed = rchan->bufs_produced;
- rchan->bytes_consumed = 0;
- update_readers_consumed(rchan, rchan->bufs_consumed, rchan->bytes_consumed);
- } else
- rchan->bufs_produced++;
-}
-
-/**
- * lockless_finalize: - finalize last buffer at end of channel use
- * @rchan: the channel
- */
-inline void
-lockless_finalize(struct rchan *rchan)
-{
- u32 event_end_idx;
- u32 size_lost;
- unsigned long int flags;
- struct timeval time;
- u32 tsc;
-
- event_end_idx = RELAY_BUF_OFFSET_GET(idx(rchan), offset_mask(rchan));
- size_lost = RELAY_BUF_SIZE(offset_bits(rchan)) - event_end_idx;
-
- local_irq_save(flags);
- get_timestamp(&time, &tsc, rchan);
- finalize_buffer(rchan, idx(rchan) & idx_mask(rchan), size_lost,
- &time, &tsc, 0);
- local_irq_restore(flags);
-}
-
-/**
- * discard_check: - determine whether a write should be discarded
- * @rchan: the channel
- * @old_idx: index into buffer where check for space should begin
- * @write_len: the length of the write to check
- * @time_stamp: the time of the end-buffer event
- * @tsc: the timestamp counter associated with time
- *
- * The return value contains the result flags and is an ORed combination
- * of the following:
- *
- * RELAY_WRITE_DISCARD_NONE - write should not be discarded
- * RELAY_BUFFER_SWITCH - buffer switch occurred
- * RELAY_WRITE_DISCARD - write should be discarded (all buffers are full)
- * RELAY_WRITE_TOO_LONG - write won't fit into even an empty buffer
- */
-static inline int
-discard_check(struct rchan *rchan,
- u32 old_idx,
- u32 write_len,
- struct timeval *time_stamp,
- u32 *tsc)
-{
- u32 buffers_ready;
- u32 offset_mask = offset_mask(rchan);
- u8 offset_bits = offset_bits(rchan);
- u32 idx_mask = idx_mask(rchan);
- u32 size_lost;
- unsigned long int flags;
-
- if (write_len > RELAY_BUF_SIZE(offset_bits))
- return RELAY_WRITE_DISCARD | RELAY_WRITE_TOO_LONG;
-
- if (mode_continuous(rchan))
- return RELAY_WRITE_DISCARD_NONE;
-
- local_irq_save(flags);
- if (atomic_read(&rchan->suspended) == 1) {
- local_irq_restore(flags);
- return RELAY_WRITE_DISCARD;
- }
- if (rchan->half_switch) {
- local_irq_restore(flags);
- return RELAY_WRITE_DISCARD_NONE;
- }
- buffers_ready = rchan->bufs_produced - rchan->bufs_consumed;
- if (buffers_ready == rchan->n_bufs - 1) {
- atomic_set(&rchan->suspended, 1);
- size_lost = RELAY_BUF_SIZE(offset_bits)
- - RELAY_BUF_OFFSET_GET(old_idx, offset_mask);
- finalize_buffer(rchan, old_idx & idx_mask, size_lost,
- time_stamp, tsc, 0);
- rchan->half_switch = 1;
- idx(rchan) = RELAY_BUF_OFFSET_CLEAR((old_idx & idx_mask), offset_mask(rchan)) + RELAY_BUF_SIZE(offset_bits) - 1;
- local_irq_restore(flags);
-
- return RELAY_BUFFER_SWITCH | RELAY_WRITE_DISCARD;
- }
- local_irq_restore(flags);
-
- return RELAY_WRITE_DISCARD_NONE;
-}
-
-/**
- * switch_buffers - switch over to a new sub-buffer
- * @rchan: the channel
- * @slot_len: the length of the slot needed for the current write
- * @offset: the offset calculated for the new index
- * @ts: timestamp
- * @tsc: the timestamp counter associated with time
- * @old_idx: the value of the buffer control index when we were called
- * @old_idx: the new calculated value of the buffer control index
- * @resetting: are we resetting the channel?
- */
-static inline void
-switch_buffers(struct rchan *rchan,
- u32 slot_len,
- u32 offset,
- struct timeval *ts,
- u32 *tsc,
- u32 new_idx,
- u32 old_idx,
- int resetting)
-{
- u32 size_lost = rchan->end_reserve;
- unsigned long int flags;
- u32 idx_mask = idx_mask(rchan);
- u8 offset_bits = offset_bits(rchan);
- char *cur_write_pos;
- u32 new_buf_no;
- u32 start_reserve = rchan->start_reserve;
-
- if (resetting)
- size_lost = RELAY_BUF_SIZE(offset_bits(rchan)) - old_idx % rchan->buf_size;
-
- if (offset > 0)
- size_lost += slot_len - offset;
- else
- old_idx += slot_len;
-
- local_irq_save(flags);
- if (!rchan->half_switch)
- finalize_buffer(rchan, old_idx & idx_mask, size_lost,
- ts, tsc, resetting);
- rchan->half_switch = 0;
- rchan->buf_start_time = *ts;
- rchan->buf_start_tsc = *tsc;
- local_irq_restore(flags);
-
- cur_write_pos = rchan->buf + RELAY_BUF_OFFSET_CLEAR((new_idx
- & idx_mask), offset_mask(rchan));
- if (resetting)
- rchan->buf_idx = 0;
- else
- rchan->buf_idx++;
- rchan->buf_id++;
-
- rchan->unused_bytes[rchan->buf_idx % rchan->n_bufs] = 0;
-
- rchan->callbacks->buffer_start(rchan->id, cur_write_pos,
- rchan->buf_id, *ts, *tsc, using_tsc(rchan));
- new_buf_no = RELAY_BUFNO_GET(new_idx & idx_mask, offset_bits);
- atomic_sub_volatile(&fill_count(rchan, new_buf_no),
- RELAY_BUF_SIZE(offset_bits) - start_reserve);
- if (atomic_read(&fill_count(rchan, new_buf_no)) < start_reserve)
- atomic_set_volatile(&fill_count(rchan, new_buf_no),
- start_reserve);
-}
-
-/**
- * lockless_reserve_slow - the slow reserve path in the lockless scheme
- * @rchan: the channel
- * @slot_len: the length of the slot to reserve
- * @ts: variable that will receive the time the slot was reserved
- * @tsc: the timestamp counter associated with time
- * @old_idx: the value of the buffer control index when we were called
- * @err: receives the result flags
- *
- * Returns pointer to the beginning of the reserved slot, NULL if error.
-
- * err values same as for lockless_reserve.
- */
-static inline char *
-lockless_reserve_slow(struct rchan *rchan,
- u32 slot_len,
- struct timeval *ts,
- u32 *tsc,
- u32 old_idx,
- int *err)
-{
- u32 new_idx, offset;
- unsigned long int flags;
- u32 offset_mask = offset_mask(rchan);
- u32 idx_mask = idx_mask(rchan);
- u32 start_reserve = rchan->start_reserve;
- u32 end_reserve = rchan->end_reserve;
- int discard_event;
- u32 reserved_idx;
- char *cur_write_pos;
- int initializing = 0;
-
- *err = RELAY_BUFFER_SWITCH_NONE;
-
- discard_event = discard_check(rchan, old_idx, slot_len, ts, tsc);
- if (discard_event != RELAY_WRITE_DISCARD_NONE) {
- *err = discard_event;
- return NULL;
- }
-
- local_irq_save(flags);
- if (rchan->initialized == 0) {
- rchan->initialized = initializing = 1;
- idx(rchan) = rchan->start_reserve + rchan->rchan_start_reserve;
- }
- local_irq_restore(flags);
-
- do {
- old_idx = idx(rchan);
- new_idx = old_idx + slot_len;
-
- offset = RELAY_BUF_OFFSET_GET(new_idx + end_reserve,
- offset_mask);
- if ((offset < slot_len) && (offset > 0)) {
- reserved_idx = RELAY_BUF_OFFSET_CLEAR(new_idx
- + end_reserve, offset_mask) + start_reserve;
- new_idx = reserved_idx + slot_len;
- } else if (offset < slot_len) {
- reserved_idx = old_idx;
- new_idx = RELAY_BUF_OFFSET_CLEAR(new_idx
- + end_reserve, offset_mask) + start_reserve;
- } else
- reserved_idx = old_idx;
- get_timestamp(ts, tsc, rchan);
- } while (!compare_and_store_volatile(&idx(rchan), old_idx, new_idx));
-
- reserved_idx &= idx_mask;
-
- if (initializing == 1) {
- cur_write_pos = rchan->buf
- + RELAY_BUF_OFFSET_CLEAR((old_idx & idx_mask),
- offset_mask(rchan));
- rchan->buf_start_time = *ts;
- rchan->buf_start_tsc = *tsc;
- rchan->unused_bytes[0] = 0;
-
- rchan->callbacks->buffer_start(rchan->id, cur_write_pos,
- rchan->buf_id, *ts, *tsc, using_tsc(rchan));
- }
-
- if (offset < slot_len) {
- switch_buffers(rchan, slot_len, offset, ts, tsc, new_idx,
- old_idx, 0);
- *err = RELAY_BUFFER_SWITCH;
- }
-
- /* If not using TSC, need to calc time delta */
- recalc_time_delta(ts, tsc, rchan);
-
- return rchan->buf + reserved_idx;
-}
-
-/**
- * lockless_reserve - reserve a slot in the buffer for an event.
- * @rchan: the channel
- * @slot_len: the length of the slot to reserve
- * @ts: variable that will receive the time the slot was reserved
- * @tsc: the timestamp counter associated with time
- * @err: receives the result flags
- * @interrupting: not used
- *
- * Returns pointer to the beginning of the reserved slot, NULL if error.
- *
- * The err value contains the result flags and is an ORed combination
- * of the following:
- *
- * RELAY_BUFFER_SWITCH_NONE - no buffer switch occurred
- * RELAY_EVENT_DISCARD_NONE - event should not be discarded
- * RELAY_BUFFER_SWITCH - buffer switch occurred
- * RELAY_EVENT_DISCARD - event should be discarded (all buffers are full)
- * RELAY_EVENT_TOO_LONG - event won't fit into even an empty buffer
- */
-inline char *
-lockless_reserve(struct rchan *rchan,
- u32 slot_len,
- struct timeval *ts,
- u32 *tsc,
- int *err,
- int *interrupting)
-{
- u32 old_idx, new_idx, offset;
- u32 offset_mask = offset_mask(rchan);
-
- do {
- old_idx = idx(rchan);
- new_idx = old_idx + slot_len;
-
- offset = RELAY_BUF_OFFSET_GET(new_idx + rchan->end_reserve,
- offset_mask);
- if (offset < slot_len)
- return lockless_reserve_slow(rchan, slot_len,
- ts, tsc, old_idx, err);
- get_time_or_tsc(ts, tsc, rchan);
- } while (!compare_and_store_volatile(&idx(rchan), old_idx, new_idx));
-
- /* If not using TSC, need to calc time delta */
- recalc_time_delta(ts, tsc, rchan);
-
- *err = RELAY_BUFFER_SWITCH_NONE;
-
- return rchan->buf + (old_idx & idx_mask(rchan));
-}
-
-/**
- * lockless_get_offset - get current and max channel offsets
- * @rchan: the channel
- * @max_offset: maximum channel offset
- *
- * Returns the current and maximum channel offsets.
- */
-u32
-lockless_get_offset(struct rchan *rchan,
- u32 *max_offset)
-{
- if (max_offset)
- *max_offset = rchan->buf_size * rchan->n_bufs - 1;
-
- return rchan->initialized ? idx(rchan) & idx_mask(rchan) : 0;
-}
-
-/**
- * lockless_reset - reset the channel
- * @rchan: the channel
- * @init: 1 if this is a first-time channel initialization
- */
-void lockless_reset(struct rchan *rchan, int init)
-{
- int i;
-
- /* Start first buffer at 0 - (end_reserve + 1) so that it
- gets initialized via buffer_start callback as well. */
- idx(rchan) = 0UL - (rchan->end_reserve + 1);
- idx_mask(rchan) =
- (1UL << (bufno_bits(rchan) + offset_bits(rchan))) - 1;
- atomic_set(&fill_count(rchan, 0),
- (int)rchan->start_reserve +
- (int)rchan->rchan_start_reserve);
- for (i = 1; i < rchan->n_bufs; i++)
- atomic_set(&fill_count(rchan, i),
- (int)RELAY_BUF_SIZE(offset_bits(rchan)));
-}
-
-/**
- * lockless_reset_index - atomically set channel index to the beginning
- * @rchan: the channel
- * @old_idx: the current index
- *
- * If this fails, it means that something else just logged something
- * and therefore we probably no longer want to do this. It's up to the
- * caller anyway...
- *
- * Returns 0 if the index was successfully set, negative otherwise
- */
-int
-lockless_reset_index(struct rchan *rchan, u32 old_idx)
-{
- struct timeval ts;
- u32 tsc;
- u32 new_idx;
-
- if (compare_and_store_volatile(&idx(rchan), old_idx, 0)) {
- new_idx = rchan->start_reserve;
- switch_buffers(rchan, 0, 0, &ts, &tsc, new_idx, old_idx, 1);
- return 0;
- } else
- return -1;
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
+++ /dev/null
-#ifndef _RELAY_LOCKLESS_H
-#define _RELAY_LOCKLESS_H
-
-extern char *
-lockless_reserve(struct rchan *rchan,
- u32 slot_len,
- struct timeval *time_stamp,
- u32 *tsc,
- int * interrupting,
- int * errcode);
-
-extern void
-lockless_commit(struct rchan *rchan,
- char * from,
- u32 len,
- int deliver,
- int interrupting);
-
-extern void
-lockless_resume(struct rchan *rchan);
-
-extern void
-lockless_finalize(struct rchan *rchan);
-
-extern u32
-lockless_get_offset(struct rchan *rchan, u32 *max_offset);
-
-extern void
-lockless_reset(struct rchan *rchan, int init);
-
-extern int
-lockless_reset_index(struct rchan *rchan, u32 old_idx);
-
-#endif/* _RELAY_LOCKLESS_H */
+++ /dev/null
-/*
- * RelayFS buffer management and resizing code.
- *
- * Copyright (C) 2002, 2003 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
- * Copyright (C) 1999, 2000, 2001, 2002 - Karim Yaghmour (karim@opersys.com)
- *
- * This file is released under the GPL.
- */
-
-#include <linux/module.h>
-#include <linux/vmalloc.h>
-#include <linux/mm.h>
-#include <asm/relay.h>
-#include "resize.h"
-
-/**
- * alloc_page_array - alloc array to hold pages, but not pages
- * @size: the total size of the memory represented by the page array
- * @page_count: the number of pages the array can hold
- * @err: 0 on success, negative otherwise
- *
- * Returns a pointer to the page array if successful, NULL otherwise.
- */
-static struct page **
-alloc_page_array(int size, int *page_count, int *err)
-{
- int n_pages;
- struct page **page_array;
- int page_array_size;
-
- *err = 0;
-
- size = PAGE_ALIGN(size);
- n_pages = size >> PAGE_SHIFT;
- page_array_size = n_pages * sizeof(struct page *);
- page_array = kmalloc(page_array_size, GFP_KERNEL);
- if (page_array == NULL) {
- *err = -ENOMEM;
- return NULL;
- }
- *page_count = n_pages;
- memset(page_array, 0, page_array_size);
-
- return page_array;
-}
-
-/**
- * free_page_array - free array to hold pages, but not pages
- * @page_array: pointer to the page array
- */
-static inline void
-free_page_array(struct page **page_array)
-{
- kfree(page_array);
-}
-
-/**
- * depopulate_page_array - free and unreserve all pages in the array
- * @page_array: pointer to the page array
- * @page_count: number of pages to free
- */
-static void
-depopulate_page_array(struct page **page_array, int page_count)
-{
- int i;
-
- for (i = 0; i < page_count; i++) {
- ClearPageReserved(page_array[i]);
- __free_page(page_array[i]);
- }
-}
-
-/**
- * populate_page_array - allocate and reserve pages
- * @page_array: pointer to the page array
- * @page_count: number of pages to allocate
- *
- * Returns 0 if successful, negative otherwise.
- */
-static int
-populate_page_array(struct page **page_array, int page_count)
-{
- int i;
-
- for (i = 0; i < page_count; i++) {
- page_array[i] = alloc_page(GFP_KERNEL);
- if (unlikely(!page_array[i])) {
- depopulate_page_array(page_array, i);
- return -ENOMEM;
- }
- SetPageReserved(page_array[i]);
- }
- return 0;
-}
-
-/**
- * alloc_rchan_buf - allocate the initial channel buffer
- * @size: total size of the buffer
- * @page_array: receives a pointer to the buffer's page array
- * @page_count: receives the number of pages allocated
- *
- * Returns a pointer to the resulting buffer, NULL if unsuccessful
- */
-void *
-alloc_rchan_buf(unsigned long size, struct page ***page_array, int *page_count)
-{
- void *mem;
- int err;
-
- *page_array = alloc_page_array(size, page_count, &err);
- if (!*page_array)
- return NULL;
-
- err = populate_page_array(*page_array, *page_count);
- if (err) {
- free_page_array(*page_array);
- *page_array = NULL;
- return NULL;
- }
-
- mem = vmap(*page_array, *page_count, GFP_KERNEL, PAGE_KERNEL);
- if (!mem) {
- depopulate_page_array(*page_array, *page_count);
- free_page_array(*page_array);
- *page_array = NULL;
- return NULL;
- }
- memset(mem, 0, size);
-
- return mem;
-}
-
-/**
- * free_rchan_buf - free a channel buffer
- * @buf: pointer to the buffer to free
- * @page_array: pointer to the buffer's page array
- * @page_count: number of pages in page array
- */
-void
-free_rchan_buf(void *buf, struct page **page_array, int page_count)
-{
- vunmap(buf);
- depopulate_page_array(page_array, page_count);
- free_page_array(page_array);
-}
-
-/**
- * expand_check - check whether the channel needs expanding
- * @rchan: the channel
- *
- * If the channel needs expanding, the needs_resize callback is
- * called with RELAY_RESIZE_EXPAND.
- *
- * Returns the suggested number of sub-buffers for the new
- * buffer.
- */
-void
-expand_check(struct rchan *rchan)
-{
- u32 active_bufs;
- u32 new_n_bufs = 0;
- u32 threshold = rchan->n_bufs * RESIZE_THRESHOLD;
-
- if (rchan->init_buf)
- return;
-
- if (rchan->resize_min == 0)
- return;
-
- if (rchan->resizing || rchan->replace_buffer)
- return;
-
- active_bufs = rchan->bufs_produced - rchan->bufs_consumed + 1;
-
- if (rchan->resize_max && active_bufs == threshold) {
- new_n_bufs = rchan->n_bufs * 2;
- }
-
- if (new_n_bufs && (new_n_bufs * rchan->buf_size <= rchan->resize_max))
- rchan->callbacks->needs_resize(rchan->id,
- RELAY_RESIZE_EXPAND,
- rchan->buf_size,
- new_n_bufs);
-}
-
-/**
- * can_shrink - check whether the channel can shrink
- * @rchan: the channel
- * @cur_idx: the current channel index
- *
- * Returns the suggested number of sub-buffers for the new
- * buffer, 0 if the buffer is not shrinkable.
- */
-static inline u32
-can_shrink(struct rchan *rchan, u32 cur_idx)
-{
- u32 active_bufs = rchan->bufs_produced - rchan->bufs_consumed + 1;
- u32 new_n_bufs = 0;
- u32 cur_bufno_bytes = cur_idx % rchan->buf_size;
-
- if (rchan->resize_min == 0 ||
- rchan->resize_min >= rchan->n_bufs * rchan->buf_size)
- goto out;
-
- if (active_bufs > 1)
- goto out;
-
- if (cur_bufno_bytes != rchan->bytes_consumed)
- goto out;
-
- new_n_bufs = rchan->resize_min / rchan->buf_size;
-out:
- return new_n_bufs;
-}
-
-/**
- * shrink_check: - timer function checking whether the channel can shrink
- * @data: unused
- *
- * Every SHRINK_TIMER_SECS, check whether the channel is shrinkable.
- * If so, we attempt to atomically reset the channel to the beginning.
- * The needs_resize callback is then called with RELAY_RESIZE_SHRINK.
- * If the reset fails, it means we really shouldn't be shrinking now
- * and need to wait until the next time around.
- */
-static void
-shrink_check(unsigned long data)
-{
- struct rchan *rchan = (struct rchan *)data;
- u32 shrink_to_nbufs, cur_idx;
-
- del_timer(&rchan->shrink_timer);
- rchan->shrink_timer.expires = jiffies + SHRINK_TIMER_SECS * HZ;
- add_timer(&rchan->shrink_timer);
-
- if (rchan->init_buf)
- return;
-
- if (rchan->resizing || rchan->replace_buffer)
- return;
-
- if (using_lockless(rchan))
- cur_idx = idx(rchan);
- else
- cur_idx = relay_get_offset(rchan, NULL);
-
- shrink_to_nbufs = can_shrink(rchan, cur_idx);
- if (shrink_to_nbufs != 0 && reset_index(rchan, cur_idx) == 0) {
- update_readers_consumed(rchan, rchan->bufs_consumed, 0);
- rchan->callbacks->needs_resize(rchan->id,
- RELAY_RESIZE_SHRINK,
- rchan->buf_size,
- shrink_to_nbufs);
- }
-}
-
-/**
- * init_shrink_timer: - Start timer used to check shrinkability.
- * @rchan: the channel
- */
-void
-init_shrink_timer(struct rchan *rchan)
-{
- if (rchan->resize_min) {
- init_timer(&rchan->shrink_timer);
- rchan->shrink_timer.function = shrink_check;
- rchan->shrink_timer.data = (unsigned long)rchan;
- rchan->shrink_timer.expires = jiffies + SHRINK_TIMER_SECS * HZ;
- add_timer(&rchan->shrink_timer);
- }
-}
-
-
-/**
- * alloc_new_pages - allocate new pages for expanding buffer
- * @rchan: the channel
- *
- * Returns 0 on success, negative otherwise.
- */
-static int
-alloc_new_pages(struct rchan *rchan)
-{
- int new_pages_size, err;
-
- if (unlikely(rchan->expand_page_array)) BUG();
-
- new_pages_size = rchan->resize_alloc_size - rchan->alloc_size;
- rchan->expand_page_array = alloc_page_array(new_pages_size,
- &rchan->expand_page_count, &err);
- if (rchan->expand_page_array == NULL) {
- rchan->resize_err = -ENOMEM;
- return -ENOMEM;
- }
-
- err = populate_page_array(rchan->expand_page_array,
- rchan->expand_page_count);
- if (err) {
- rchan->resize_err = -ENOMEM;
- free_page_array(rchan->expand_page_array);
- rchan->expand_page_array = NULL;
- }
-
- return err;
-}
-
-/**
- * clear_resize_offset - helper function for buffer resizing
- * @rchan: the channel
- *
- * Clear the saved offset change.
- */
-static inline void
-clear_resize_offset(struct rchan *rchan)
-{
- rchan->resize_offset.ge = 0UL;
- rchan->resize_offset.le = 0UL;
- rchan->resize_offset.delta = 0;
-}
-
-/**
- * save_resize_offset - helper function for buffer resizing
- * @rchan: the channel
- * @ge: affected region ge this
- * @le: affected region le this
- * @delta: apply this delta
- *
- * Save a resize offset.
- */
-static inline void
-save_resize_offset(struct rchan *rchan, u32 ge, u32 le, int delta)
-{
- rchan->resize_offset.ge = ge;
- rchan->resize_offset.le = le;
- rchan->resize_offset.delta = delta;
-}
-
-/**
- * update_file_offset - apply offset change to reader
- * @reader: the channel reader
- * @change_idx: the offset index into the offsets array
- *
- * Returns non-zero if the offset was applied.
- *
- * Apply the offset delta saved in change_idx to the reader's
- * current read position.
- */
-static inline int
-update_file_offset(struct rchan_reader *reader)
-{
- int applied = 0;
- struct rchan *rchan = reader->rchan;
- u32 f_pos;
- int delta = reader->rchan->resize_offset.delta;
-
- if (reader->vfs_reader)
- f_pos = (u32)reader->pos.file->f_pos;
- else
- f_pos = reader->pos.f_pos;
-
- if (f_pos == relay_get_offset(rchan, NULL))
- return 0;
-
- if ((f_pos >= rchan->resize_offset.ge - 1) &&
- (f_pos <= rchan->resize_offset.le)) {
- if (reader->vfs_reader)
- reader->pos.file->f_pos += delta;
- else
- reader->pos.f_pos += delta;
- applied = 1;
- }
-
- return applied;
-}
-
-/**
- * update_file_offsets - apply offset change to readers
- * @rchan: the channel
- *
- * Apply the saved offset deltas to all files open on the channel.
- */
-static inline void
-update_file_offsets(struct rchan *rchan)
-{
- struct list_head *p;
- struct rchan_reader *reader;
-
- read_lock(&rchan->open_readers_lock);
- list_for_each(p, &rchan->open_readers) {
- reader = list_entry(p, struct rchan_reader, list);
- if (update_file_offset(reader))
- reader->offset_changed = 1;
- }
- read_unlock(&rchan->open_readers_lock);
-}
-
-/**
- * setup_expand_buf - setup expand buffer for replacement
- * @rchan: the channel
- * @newsize: the size of the new buffer
- * @oldsize: the size of the old buffer
- * @old_n_bufs: the number of sub-buffers in the old buffer
- *
- * Inserts new pages into the old buffer to create a larger
- * new channel buffer, splitting them at old_cur_idx, the bottom
- * half of the old buffer going to the bottom of the new, likewise
- * for the top half.
- */
-static void
-setup_expand_buf(struct rchan *rchan, int newsize, int oldsize, u32 old_n_bufs)
-{
- u32 cur_idx;
- int cur_bufno, delta, i, j;
- u32 ge, le;
- int cur_pageno;
- u32 free_bufs, free_pages;
- u32 free_pages_in_cur_buf;
- u32 free_bufs_to_end;
- u32 cur_pages = rchan->alloc_size >> PAGE_SHIFT;
- u32 pages_per_buf = cur_pages / rchan->n_bufs;
- u32 bufs_ready = rchan->bufs_produced - rchan->bufs_consumed;
-
- if (!rchan->resize_page_array || !rchan->expand_page_array ||
- !rchan->buf_page_array)
- return;
-
- if (bufs_ready >= rchan->n_bufs) {
- bufs_ready = rchan->n_bufs;
- free_bufs = 0;
- } else
- free_bufs = rchan->n_bufs - bufs_ready - 1;
-
- cur_idx = relay_get_offset(rchan, NULL);
- cur_pageno = cur_idx / PAGE_SIZE;
- cur_bufno = cur_idx / rchan->buf_size;
-
- free_pages_in_cur_buf = (pages_per_buf - 1) - (cur_pageno % pages_per_buf);
- free_pages = free_bufs * pages_per_buf + free_pages_in_cur_buf;
- free_bufs_to_end = (rchan->n_bufs - 1) - cur_bufno;
- if (free_bufs >= free_bufs_to_end) {
- free_pages = free_bufs_to_end * pages_per_buf + free_pages_in_cur_buf;
- free_bufs = free_bufs_to_end;
- }
-
- for (i = 0, j = 0; i <= cur_pageno + free_pages; i++, j++)
- rchan->resize_page_array[j] = rchan->buf_page_array[i];
- for (i = 0; i < rchan->expand_page_count; i++, j++)
- rchan->resize_page_array[j] = rchan->expand_page_array[i];
- for (i = cur_pageno + free_pages + 1; i < rchan->buf_page_count; i++, j++)
- rchan->resize_page_array[j] = rchan->buf_page_array[i];
-
- delta = newsize - oldsize;
- ge = (cur_pageno + 1 + free_pages) * PAGE_SIZE;
- le = oldsize;
- save_resize_offset(rchan, ge, le, delta);
-
- rchan->expand_buf_id = rchan->buf_id + 1 + free_bufs;
-}
-
-/**
- * setup_shrink_buf - setup shrink buffer for replacement
- * @rchan: the channel
- *
- * Removes pages from the old buffer to create a smaller
- * new channel buffer.
- */
-static void
-setup_shrink_buf(struct rchan *rchan)
-{
- int i;
- int copy_end_page;
-
- if (!rchan->resize_page_array || !rchan->shrink_page_array ||
- !rchan->buf_page_array)
- return;
-
- copy_end_page = rchan->resize_alloc_size / PAGE_SIZE;
-
- for (i = 0; i < copy_end_page; i++)
- rchan->resize_page_array[i] = rchan->buf_page_array[i];
-}
-
-/**
- * cleanup_failed_alloc - relaybuf_alloc helper
- */
-static void
-cleanup_failed_alloc(struct rchan *rchan)
-{
- if (rchan->expand_page_array) {
- depopulate_page_array(rchan->expand_page_array,
- rchan->expand_page_count);
- free_page_array(rchan->expand_page_array);
- rchan->expand_page_array = NULL;
- rchan->expand_page_count = 0;
- } else if (rchan->shrink_page_array) {
- free_page_array(rchan->shrink_page_array);
- rchan->shrink_page_array = NULL;
- rchan->shrink_page_count = 0;
- }
-
- if (rchan->resize_page_array) {
- free_page_array(rchan->resize_page_array);
- rchan->resize_page_array = NULL;
- rchan->resize_page_count = 0;
- }
-}
-
-/**
- * relaybuf_alloc - allocate a new resized channel buffer
- * @private: pointer to the channel struct
- *
- * Internal - manages the allocation and remapping of new channel
- * buffers.
- */
-static void
-relaybuf_alloc(void *private)
-{
- struct rchan *rchan = (struct rchan *)private;
- int i, j, err;
- u32 old_cur_idx;
- int free_size;
- int free_start_page, free_end_page;
- u32 newsize, oldsize;
-
- if (rchan->resize_alloc_size > rchan->alloc_size) {
- err = alloc_new_pages(rchan);
- if (err) goto cleanup;
- } else {
- free_size = rchan->alloc_size - rchan->resize_alloc_size;
- BUG_ON(free_size <= 0);
- rchan->shrink_page_array = alloc_page_array(free_size,
- &rchan->shrink_page_count, &err);
- if (rchan->shrink_page_array == NULL)
- goto cleanup;
- free_start_page = rchan->resize_alloc_size / PAGE_SIZE;
- free_end_page = rchan->alloc_size / PAGE_SIZE;
- for (i = 0, j = free_start_page; j < free_end_page; i++, j++)
- rchan->shrink_page_array[i] = rchan->buf_page_array[j];
- }
-
- rchan->resize_page_array = alloc_page_array(rchan->resize_alloc_size,
- &rchan->resize_page_count, &err);
- if (rchan->resize_page_array == NULL)
- goto cleanup;
-
- old_cur_idx = relay_get_offset(rchan, NULL);
- clear_resize_offset(rchan);
- newsize = rchan->resize_alloc_size;
- oldsize = rchan->alloc_size;
- if (newsize > oldsize)
- setup_expand_buf(rchan, newsize, oldsize, rchan->n_bufs);
- else
- setup_shrink_buf(rchan);
-
- rchan->resize_buf = vmap(rchan->resize_page_array, rchan->resize_page_count, GFP_KERNEL, PAGE_KERNEL);
-
- if (rchan->resize_buf == NULL)
- goto cleanup;
-
- rchan->replace_buffer = 1;
- rchan->resizing = 0;
-
- rchan->callbacks->needs_resize(rchan->id, RELAY_RESIZE_REPLACE, 0, 0);
- return;
-
-cleanup:
- cleanup_failed_alloc(rchan);
- rchan->resize_err = -ENOMEM;
- return;
-}
-
-/**
- * relaybuf_free - free a resized channel buffer
- * @private: pointer to the channel struct
- *
- * Internal - manages the de-allocation and unmapping of old channel
- * buffers.
- */
-static void
-relaybuf_free(void *private)
-{
- struct free_rchan_buf *free_buf = (struct free_rchan_buf *)private;
- int i;
-
- if (free_buf->unmap_buf)
- vunmap(free_buf->unmap_buf);
-
- for (i = 0; i < 3; i++) {
- if (!free_buf->page_array[i].array)
- continue;
- if (free_buf->page_array[i].count)
- depopulate_page_array(free_buf->page_array[i].array,
- free_buf->page_array[i].count);
- free_page_array(free_buf->page_array[i].array);
- }
-
- kfree(free_buf);
-}
-
-/**
- * calc_order - determine the power-of-2 order of a resize
- * @high: the larger size
- * @low: the smaller size
- *
- * Returns order
- */
-static inline int
-calc_order(u32 high, u32 low)
-{
- int order = 0;
-
- if (!high || !low || high <= low)
- return 0;
-
- while (high > low) {
- order++;
- high /= 2;
- }
-
- return order;
-}
-
-/**
- * check_size - check the sanity of the requested channel size
- * @rchan: the channel
- * @nbufs: the new number of sub-buffers
- * @err: return code
- *
- * Returns the non-zero total buffer size if ok, otherwise 0 and
- * sets errcode if not.
- */
-static inline u32
-check_size(struct rchan *rchan, u32 nbufs, int *err)
-{
- u32 new_channel_size = 0;
-
- *err = 0;
-
- if (nbufs > rchan->n_bufs) {
- rchan->resize_order = calc_order(nbufs, rchan->n_bufs);
- if (!rchan->resize_order) {
- *err = -EINVAL;
- goto out;
- }
-
- new_channel_size = rchan->buf_size * nbufs;
- if (new_channel_size > rchan->resize_max) {
- *err = -EINVAL;
- goto out;
- }
- } else if (nbufs < rchan->n_bufs) {
- if (rchan->n_bufs < 2) {
- *err = -EINVAL;
- goto out;
- }
- rchan->resize_order = -calc_order(rchan->n_bufs, nbufs);
- if (!rchan->resize_order) {
- *err = -EINVAL;
- goto out;
- }
-
- new_channel_size = rchan->buf_size * nbufs;
- if (new_channel_size < rchan->resize_min) {
- *err = -EINVAL;
- goto out;
- }
- } else
- *err = -EINVAL;
-out:
- return new_channel_size;
-}
-
-/**
- * __relay_realloc_buffer - allocate a new channel buffer
- * @rchan: the channel
- * @new_nbufs: the new number of sub-buffers
- * @async: do the allocation using a work queue
- *
- * Internal - see relay_realloc_buffer() for details.
- */
-static int
-__relay_realloc_buffer(struct rchan *rchan, u32 new_nbufs, int async)
-{
- u32 new_channel_size;
- int err = 0;
-
- if (new_nbufs == rchan->n_bufs)
- return -EINVAL;
-
- if (down_trylock(&rchan->resize_sem))
- return -EBUSY;
-
- if (rchan->init_buf) {
- err = -EPERM;
- goto out;
- }
-
- if (rchan->replace_buffer) {
- err = -EBUSY;
- goto out;
- }
-
- if (rchan->resizing) {
- err = -EBUSY;
- goto out;
- } else
- rchan->resizing = 1;
-
- if (rchan->resize_failures > MAX_RESIZE_FAILURES) {
- err = -ENOMEM;
- goto out;
- }
-
- new_channel_size = check_size(rchan, new_nbufs, &err);
- if (err)
- goto out;
-
- rchan->resize_n_bufs = new_nbufs;
- rchan->resize_buf_size = rchan->buf_size;
- rchan->resize_alloc_size = FIX_SIZE(new_channel_size);
-
- if (async) {
- INIT_WORK(&rchan->work, relaybuf_alloc, rchan);
- schedule_delayed_work(&rchan->work, 1);
- } else
- relaybuf_alloc((void *)rchan);
-out:
- up(&rchan->resize_sem);
-
- return err;
-}
-
-/**
- * relay_realloc_buffer - allocate a new channel buffer
- * @rchan_id: the channel id
- * @bufsize: the new sub-buffer size
- * @nbufs: the new number of sub-buffers
- *
- * Allocates a new channel buffer using the specified sub-buffer size
- * and count. If async is non-zero, the allocation is done in the
- * background using a work queue. When the allocation has completed,
- * the needs_resize() callback is called with a resize_type of
- * RELAY_RESIZE_REPLACE. This function doesn't replace the old buffer
- * with the new - see relay_replace_buffer(). See
- * Documentation/filesystems/relayfs.txt for more details.
- *
- * Returns 0 on success, or errcode if the channel is busy or if
- * the allocation couldn't happen for some reason.
- */
-int
-relay_realloc_buffer(int rchan_id, u32 new_nbufs, int async)
-{
- int err;
-
- struct rchan *rchan;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- err = __relay_realloc_buffer(rchan, new_nbufs, async);
-
- rchan_put(rchan);
-
- return err;
-}
-
-/**
- * expand_cancel_check - check whether the current expand needs canceling
- * @rchan: the channel
- *
- * Returns 1 if the expand should be canceled, 0 otherwise.
- */
-static int
-expand_cancel_check(struct rchan *rchan)
-{
- if (rchan->buf_id >= rchan->expand_buf_id)
- return 1;
- else
- return 0;
-}
-
-/**
- * shrink_cancel_check - check whether the current shrink needs canceling
- * @rchan: the channel
- *
- * Returns 1 if the shrink should be canceled, 0 otherwise.
- */
-static int
-shrink_cancel_check(struct rchan *rchan, u32 newsize)
-{
- u32 active_bufs = rchan->bufs_produced - rchan->bufs_consumed + 1;
- u32 cur_idx = relay_get_offset(rchan, NULL);
-
- if (cur_idx >= newsize)
- return 1;
-
- if (active_bufs > 1)
- return 1;
-
- return 0;
-}
-
-/**
- * switch_rchan_buf - do_replace_buffer helper
- */
-static void
-switch_rchan_buf(struct rchan *rchan,
- int newsize,
- int oldsize,
- u32 old_nbufs,
- u32 cur_idx)
-{
- u32 newbufs, cur_bufno;
- int i;
-
- cur_bufno = cur_idx / rchan->buf_size;
-
- rchan->buf = rchan->resize_buf;
- rchan->alloc_size = rchan->resize_alloc_size;
- rchan->n_bufs = rchan->resize_n_bufs;
-
- if (newsize > oldsize) {
- u32 ge = rchan->resize_offset.ge;
- u32 moved_buf = ge / rchan->buf_size;
-
- newbufs = (newsize - oldsize) / rchan->buf_size;
- for (i = moved_buf; i < old_nbufs; i++) {
- if (using_lockless(rchan))
- atomic_set(&fill_count(rchan, i + newbufs),
- atomic_read(&fill_count(rchan, i)));
- rchan->unused_bytes[i + newbufs] = rchan->unused_bytes[i];
- }
- for (i = moved_buf; i < moved_buf + newbufs; i++) {
- if (using_lockless(rchan))
- atomic_set(&fill_count(rchan, i),
- (int)RELAY_BUF_SIZE(offset_bits(rchan)));
- rchan->unused_bytes[i] = 0;
- }
- }
-
- rchan->buf_idx = cur_bufno;
-
- if (!using_lockless(rchan)) {
- cur_write_pos(rchan) = rchan->buf + cur_idx;
- write_buf(rchan) = rchan->buf + cur_bufno * rchan->buf_size;
- write_buf_end(rchan) = write_buf(rchan) + rchan->buf_size;
- write_limit(rchan) = write_buf_end(rchan) - rchan->end_reserve;
- } else {
- idx(rchan) &= idx_mask(rchan);
- bufno_bits(rchan) += rchan->resize_order;
- idx_mask(rchan) =
- (1UL << (bufno_bits(rchan) + offset_bits(rchan))) - 1;
- }
-}
-
-/**
- * do_replace_buffer - does the work of channel buffer replacement
- * @rchan: the channel
- * @newsize: new channel buffer size
- * @oldsize: old channel buffer size
- * @old_n_bufs: old channel sub-buffer count
- *
- * Returns 0 if replacement happened, 1 if canceled
- *
- * Does the work of switching buffers and fixing everything up
- * so the channel can continue with a new size.
- */
-static int
-do_replace_buffer(struct rchan *rchan,
- int newsize,
- int oldsize,
- u32 old_nbufs)
-{
- u32 cur_idx;
- int err = 0;
- int canceled;
-
- cur_idx = relay_get_offset(rchan, NULL);
-
- if (newsize > oldsize)
- canceled = expand_cancel_check(rchan);
- else
- canceled = shrink_cancel_check(rchan, newsize);
-
- if (canceled) {
- err = -EAGAIN;
- goto out;
- }
-
- switch_rchan_buf(rchan, newsize, oldsize, old_nbufs, cur_idx);
-
- if (rchan->resize_offset.delta)
- update_file_offsets(rchan);
-
- atomic_set(&rchan->suspended, 0);
-
- rchan->old_buf_page_array = rchan->buf_page_array;
- rchan->buf_page_array = rchan->resize_page_array;
- rchan->buf_page_count = rchan->resize_page_count;
- rchan->resize_page_array = NULL;
- rchan->resize_page_count = 0;
- rchan->resize_buf = NULL;
- rchan->resize_buf_size = 0;
- rchan->resize_alloc_size = 0;
- rchan->resize_n_bufs = 0;
- rchan->resize_err = 0;
- rchan->resize_order = 0;
-out:
- rchan->callbacks->needs_resize(rchan->id,
- RELAY_RESIZE_REPLACED,
- rchan->buf_size,
- rchan->n_bufs);
- return err;
-}
-
-/**
- * add_free_page_array - add a page_array to be freed
- * @free_rchan_buf: the free_rchan_buf struct
- * @page_array: the page array to free
- * @page_count: the number of pages to free, 0 to free the array only
- *
- * Internal - Used add page_arrays to be freed asynchronously.
- */
-static inline void
-add_free_page_array(struct free_rchan_buf *free_rchan_buf,
- struct page **page_array, int page_count)
-{
- int cur = free_rchan_buf->cur++;
-
- free_rchan_buf->page_array[cur].array = page_array;
- free_rchan_buf->page_array[cur].count = page_count;
-}
-
-/**
- * free_replaced_buffer - free a channel's old buffer
- * @rchan: the channel
- * @oldbuf: the old buffer
- * @oldsize: old buffer size
- *
- * Frees a channel buffer via work queue.
- */
-static int
-free_replaced_buffer(struct rchan *rchan, char *oldbuf, int oldsize)
-{
- struct free_rchan_buf *free_buf;
-
- free_buf = kmalloc(sizeof(struct free_rchan_buf), GFP_ATOMIC);
- if (!free_buf)
- return -ENOMEM;
- memset(free_buf, 0, sizeof(struct free_rchan_buf));
-
- free_buf->unmap_buf = oldbuf;
- add_free_page_array(free_buf, rchan->old_buf_page_array, 0);
- rchan->old_buf_page_array = NULL;
- add_free_page_array(free_buf, rchan->expand_page_array, 0);
- add_free_page_array(free_buf, rchan->shrink_page_array, rchan->shrink_page_count);
-
- rchan->expand_page_array = NULL;
- rchan->expand_page_count = 0;
- rchan->shrink_page_array = NULL;
- rchan->shrink_page_count = 0;
-
- INIT_WORK(&free_buf->work, relaybuf_free, free_buf);
- schedule_delayed_work(&free_buf->work, 1);
-
- return 0;
-}
-
-/**
- * free_canceled_resize - free buffers allocated for a canceled resize
- * @rchan: the channel
- *
- * Frees canceled buffers via work queue.
- */
-static int
-free_canceled_resize(struct rchan *rchan)
-{
- struct free_rchan_buf *free_buf;
-
- free_buf = kmalloc(sizeof(struct free_rchan_buf), GFP_ATOMIC);
- if (!free_buf)
- return -ENOMEM;
- memset(free_buf, 0, sizeof(struct free_rchan_buf));
-
- if (rchan->resize_alloc_size > rchan->alloc_size)
- add_free_page_array(free_buf, rchan->expand_page_array, rchan->expand_page_count);
- else
- add_free_page_array(free_buf, rchan->shrink_page_array, 0);
-
- add_free_page_array(free_buf, rchan->resize_page_array, 0);
- free_buf->unmap_buf = rchan->resize_buf;
-
- rchan->expand_page_array = NULL;
- rchan->expand_page_count = 0;
- rchan->shrink_page_array = NULL;
- rchan->shrink_page_count = 0;
- rchan->resize_page_array = NULL;
- rchan->resize_page_count = 0;
- rchan->resize_buf = NULL;
-
- INIT_WORK(&free_buf->work, relaybuf_free, free_buf);
- schedule_delayed_work(&free_buf->work, 1);
-
- return 0;
-}
-
-/**
- * __relay_replace_buffer - replace channel buffer with new buffer
- * @rchan: the channel
- *
- * Internal - see relay_replace_buffer() for details.
- *
- * Returns 0 if successful, negative otherwise.
- */
-static int
-__relay_replace_buffer(struct rchan *rchan)
-{
- int oldsize;
- int err = 0;
- char *oldbuf;
-
- if (down_trylock(&rchan->resize_sem))
- return -EBUSY;
-
- if (rchan->init_buf) {
- err = -EPERM;
- goto out;
- }
-
- if (!rchan->replace_buffer)
- goto out;
-
- if (rchan->resizing) {
- err = -EBUSY;
- goto out;
- }
-
- if (rchan->resize_buf == NULL) {
- err = -EINVAL;
- goto out;
- }
-
- oldbuf = rchan->buf;
- oldsize = rchan->alloc_size;
-
- err = do_replace_buffer(rchan, rchan->resize_alloc_size,
- oldsize, rchan->n_bufs);
- if (err == 0)
- err = free_replaced_buffer(rchan, oldbuf, oldsize);
- else
- err = free_canceled_resize(rchan);
-out:
- rchan->replace_buffer = 0;
- up(&rchan->resize_sem);
-
- return err;
-}
-
-/**
- * relay_replace_buffer - replace channel buffer with new buffer
- * @rchan_id: the channel id
- *
- * Replaces the current channel buffer with the new buffer allocated
- * by relay_alloc_buffer and contained in the channel struct. When the
- * replacement is complete, the needs_resize() callback is called with
- * RELAY_RESIZE_REPLACED.
- *
- * Returns 0 on success, or errcode if the channel is busy or if
- * the replacement or previous allocation didn't happen for some reason.
- */
-int
-relay_replace_buffer(int rchan_id)
-{
- int err;
-
- struct rchan *rchan;
-
- rchan = rchan_get(rchan_id);
- if (rchan == NULL)
- return -EBADF;
-
- err = __relay_replace_buffer(rchan);
-
- rchan_put(rchan);
-
- return err;
-}
-
-EXPORT_SYMBOL(relay_realloc_buffer);
-EXPORT_SYMBOL(relay_replace_buffer);
-
+++ /dev/null
-#ifndef _RELAY_RESIZE_H
-#define _RELAY_RESIZE_H
-
-/*
- * If the channel usage has been below the low water mark for more than
- * this amount of time, we can shrink the buffer if necessary.
- */
-#define SHRINK_TIMER_SECS 60
-
-/* This inspired by rtai/shmem */
-#define FIX_SIZE(x) (((x) - 1) & PAGE_MASK) + PAGE_SIZE
-
-/* Don't attempt resizing again after this many failures */
-#define MAX_RESIZE_FAILURES 1
-
-/* Trigger resizing if a resizable channel is this full */
-#define RESIZE_THRESHOLD 3 / 4
-
-/*
- * Used for deferring resized channel free
- */
-struct free_rchan_buf
-{
- char *unmap_buf;
- struct
- {
- struct page **array;
- int count;
- } page_array[3];
-
- int cur;
- struct work_struct work; /* resize de-allocation work struct */
-};
-
-extern void *
-alloc_rchan_buf(unsigned long size,
- struct page ***page_array,
- int *page_count);
-
-extern void
-free_rchan_buf(void *buf,
- struct page **page_array,
- int page_count);
-
-extern void
-expand_check(struct rchan *rchan);
-
-extern void
-init_shrink_timer(struct rchan *rchan);
-
-#endif/* _RELAY_RESIZE_H */
git://git.onelab.eu / linux-2.6.git / commitdiff