| /* |
| * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. |
| * Copyright (C) 2007 The Regents of the University of California. |
| * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). |
| * Written by Brian Behlendorf <behlendorf1@llnl.gov>. |
| * UCRL-CODE-235197 |
| * |
| * This file is part of the SPL, Solaris Porting Layer. |
| * For details, see <http://zfsonlinux.org/>. |
| * |
| * The SPL is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License as published by the |
| * Free Software Foundation; either version 2 of the License, or (at your |
| * option) any later version. |
| * |
| * The SPL is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * for more details. |
| * |
| * You should have received a copy of the GNU General Public License along |
| * with the SPL. If not, see <http://www.gnu.org/licenses/>. |
| * |
| * Solaris Porting Layer (SPL) Credential Implementation. |
| */ |
| |
| #include <sys/condvar.h> |
| #include <sys/time.h> |
| #include <sys/sysmacros.h> |
| #include <linux/hrtimer.h> |
| #include <linux/compiler_compat.h> |
| #include <linux/mod_compat.h> |
| |
| #include <linux/sched.h> |
| |
| #ifdef HAVE_SCHED_SIGNAL_HEADER |
| #include <linux/sched/signal.h> |
| #endif |
| |
| #define MAX_HRTIMEOUT_SLACK_US 1000 |
| unsigned int spl_schedule_hrtimeout_slack_us = 0; |
| |
| static int |
| param_set_hrtimeout_slack(const char *buf, zfs_kernel_param_t *kp) |
| { |
| unsigned long val; |
| int error; |
| |
| error = kstrtoul(buf, 0, &val); |
| if (error) |
| return (error); |
| |
| if (val > MAX_HRTIMEOUT_SLACK_US) |
| return (-EINVAL); |
| |
| error = param_set_uint(buf, kp); |
| if (error < 0) |
| return (error); |
| |
| return (0); |
| } |
| |
| module_param_call(spl_schedule_hrtimeout_slack_us, param_set_hrtimeout_slack, |
| param_get_uint, &spl_schedule_hrtimeout_slack_us, 0644); |
| MODULE_PARM_DESC(spl_schedule_hrtimeout_slack_us, |
| "schedule_hrtimeout_range() delta/slack value in us, default(0)"); |
| |
| void |
| __cv_init(kcondvar_t *cvp, char *name, kcv_type_t type, void *arg) |
| { |
| ASSERT(cvp); |
| ASSERT(name == NULL); |
| ASSERT(type == CV_DEFAULT); |
| ASSERT(arg == NULL); |
| |
| cvp->cv_magic = CV_MAGIC; |
| init_waitqueue_head(&cvp->cv_event); |
| init_waitqueue_head(&cvp->cv_destroy); |
| atomic_set(&cvp->cv_waiters, 0); |
| atomic_set(&cvp->cv_refs, 1); |
| cvp->cv_mutex = NULL; |
| } |
| EXPORT_SYMBOL(__cv_init); |
| |
| static int |
| cv_destroy_wakeup(kcondvar_t *cvp) |
| { |
| if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) { |
| ASSERT(cvp->cv_mutex == NULL); |
| ASSERT(!waitqueue_active(&cvp->cv_event)); |
| return (1); |
| } |
| |
| return (0); |
| } |
| |
| void |
| __cv_destroy(kcondvar_t *cvp) |
| { |
| ASSERT(cvp); |
| ASSERT(cvp->cv_magic == CV_MAGIC); |
| |
| cvp->cv_magic = CV_DESTROY; |
| atomic_dec(&cvp->cv_refs); |
| |
| /* Block until all waiters are woken and references dropped. */ |
| while (cv_destroy_wakeup(cvp) == 0) |
| wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1); |
| |
| ASSERT3P(cvp->cv_mutex, ==, NULL); |
| ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0); |
| ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0); |
| ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0); |
| } |
| EXPORT_SYMBOL(__cv_destroy); |
| |
| static void |
| cv_wait_common(kcondvar_t *cvp, kmutex_t *mp, int state, int io) |
| { |
| DEFINE_WAIT(wait); |
| kmutex_t *m; |
| |
| ASSERT(cvp); |
| ASSERT(mp); |
| ASSERT(cvp->cv_magic == CV_MAGIC); |
| ASSERT(mutex_owned(mp)); |
| atomic_inc(&cvp->cv_refs); |
| |
| m = READ_ONCE(cvp->cv_mutex); |
| if (!m) |
| m = xchg(&cvp->cv_mutex, mp); |
| /* Ensure the same mutex is used by all callers */ |
| ASSERT(m == NULL || m == mp); |
| |
| prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); |
| atomic_inc(&cvp->cv_waiters); |
| |
| /* |
| * Mutex should be dropped after prepare_to_wait() this |
| * ensures we're linked in to the waiters list and avoids the |
| * race where 'cvp->cv_waiters > 0' but the list is empty. |
| */ |
| mutex_exit(mp); |
| if (io) |
| io_schedule(); |
| else |
| schedule(); |
| |
| /* No more waiters a different mutex could be used */ |
| if (atomic_dec_and_test(&cvp->cv_waiters)) { |
| /* |
| * This is set without any lock, so it's racy. But this is |
| * just for debug anyway, so make it best-effort |
| */ |
| cvp->cv_mutex = NULL; |
| wake_up(&cvp->cv_destroy); |
| } |
| |
| finish_wait(&cvp->cv_event, &wait); |
| atomic_dec(&cvp->cv_refs); |
| |
| /* |
| * Hold mutex after we release the cvp, otherwise we could dead lock |
| * with a thread holding the mutex and call cv_destroy. |
| */ |
| mutex_enter(mp); |
| } |
| |
| void |
| __cv_wait(kcondvar_t *cvp, kmutex_t *mp) |
| { |
| cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 0); |
| } |
| EXPORT_SYMBOL(__cv_wait); |
| |
| void |
| __cv_wait_io(kcondvar_t *cvp, kmutex_t *mp) |
| { |
| cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 1); |
| } |
| EXPORT_SYMBOL(__cv_wait_io); |
| |
| int |
| __cv_wait_io_sig(kcondvar_t *cvp, kmutex_t *mp) |
| { |
| cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 1); |
| |
| return (signal_pending(current) ? 0 : 1); |
| } |
| EXPORT_SYMBOL(__cv_wait_io_sig); |
| |
| int |
| __cv_wait_sig(kcondvar_t *cvp, kmutex_t *mp) |
| { |
| cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0); |
| |
| return (signal_pending(current) ? 0 : 1); |
| } |
| EXPORT_SYMBOL(__cv_wait_sig); |
| |
| #if defined(HAVE_IO_SCHEDULE_TIMEOUT) |
| #define spl_io_schedule_timeout(t) io_schedule_timeout(t) |
| #else |
| |
| struct spl_task_timer { |
| struct timer_list timer; |
| struct task_struct *task; |
| }; |
| |
| static void |
| __cv_wakeup(spl_timer_list_t t) |
| { |
| struct timer_list *tmr = (struct timer_list *)t; |
| struct spl_task_timer *task_timer = from_timer(task_timer, tmr, timer); |
| |
| wake_up_process(task_timer->task); |
| } |
| |
| static long |
| spl_io_schedule_timeout(long time_left) |
| { |
| long expire_time = jiffies + time_left; |
| struct spl_task_timer task_timer; |
| struct timer_list *timer = &task_timer.timer; |
| |
| task_timer.task = current; |
| |
| timer_setup(timer, __cv_wakeup, 0); |
| |
| timer->expires = expire_time; |
| add_timer(timer); |
| |
| io_schedule(); |
| |
| del_timer_sync(timer); |
| |
| time_left = expire_time - jiffies; |
| |
| return (time_left < 0 ? 0 : time_left); |
| } |
| #endif |
| |
| /* |
| * 'expire_time' argument is an absolute wall clock time in jiffies. |
| * Return value is time left (expire_time - now) or -1 if timeout occurred. |
| */ |
| static clock_t |
| __cv_timedwait_common(kcondvar_t *cvp, kmutex_t *mp, clock_t expire_time, |
| int state, int io) |
| { |
| DEFINE_WAIT(wait); |
| kmutex_t *m; |
| clock_t time_left; |
| |
| ASSERT(cvp); |
| ASSERT(mp); |
| ASSERT(cvp->cv_magic == CV_MAGIC); |
| ASSERT(mutex_owned(mp)); |
| |
| /* XXX - Does not handle jiffie wrap properly */ |
| time_left = expire_time - jiffies; |
| if (time_left <= 0) |
| return (-1); |
| |
| atomic_inc(&cvp->cv_refs); |
| m = READ_ONCE(cvp->cv_mutex); |
| if (!m) |
| m = xchg(&cvp->cv_mutex, mp); |
| /* Ensure the same mutex is used by all callers */ |
| ASSERT(m == NULL || m == mp); |
| |
| prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); |
| atomic_inc(&cvp->cv_waiters); |
| |
| /* |
| * Mutex should be dropped after prepare_to_wait() this |
| * ensures we're linked in to the waiters list and avoids the |
| * race where 'cvp->cv_waiters > 0' but the list is empty. |
| */ |
| mutex_exit(mp); |
| if (io) |
| time_left = spl_io_schedule_timeout(time_left); |
| else |
| time_left = schedule_timeout(time_left); |
| |
| /* No more waiters a different mutex could be used */ |
| if (atomic_dec_and_test(&cvp->cv_waiters)) { |
| /* |
| * This is set without any lock, so it's racy. But this is |
| * just for debug anyway, so make it best-effort |
| */ |
| cvp->cv_mutex = NULL; |
| wake_up(&cvp->cv_destroy); |
| } |
| |
| finish_wait(&cvp->cv_event, &wait); |
| atomic_dec(&cvp->cv_refs); |
| |
| /* |
| * Hold mutex after we release the cvp, otherwise we could dead lock |
| * with a thread holding the mutex and call cv_destroy. |
| */ |
| mutex_enter(mp); |
| return (time_left > 0 ? time_left : -1); |
| } |
| |
| clock_t |
| __cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) |
| { |
| return (__cv_timedwait_common(cvp, mp, exp_time, |
| TASK_UNINTERRUPTIBLE, 0)); |
| } |
| EXPORT_SYMBOL(__cv_timedwait); |
| |
| clock_t |
| __cv_timedwait_io(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) |
| { |
| return (__cv_timedwait_common(cvp, mp, exp_time, |
| TASK_UNINTERRUPTIBLE, 1)); |
| } |
| EXPORT_SYMBOL(__cv_timedwait_io); |
| |
| clock_t |
| __cv_timedwait_sig(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) |
| { |
| return (__cv_timedwait_common(cvp, mp, exp_time, |
| TASK_INTERRUPTIBLE, 0)); |
| } |
| EXPORT_SYMBOL(__cv_timedwait_sig); |
| |
| /* |
| * 'expire_time' argument is an absolute clock time in nanoseconds. |
| * Return value is time left (expire_time - now) or -1 if timeout occurred. |
| */ |
| static clock_t |
| __cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t expire_time, |
| hrtime_t res, int state) |
| { |
| DEFINE_WAIT(wait); |
| kmutex_t *m; |
| hrtime_t time_left; |
| ktime_t ktime_left; |
| u64 slack = 0; |
| |
| ASSERT(cvp); |
| ASSERT(mp); |
| ASSERT(cvp->cv_magic == CV_MAGIC); |
| ASSERT(mutex_owned(mp)); |
| |
| time_left = expire_time - gethrtime(); |
| if (time_left <= 0) |
| return (-1); |
| |
| atomic_inc(&cvp->cv_refs); |
| m = READ_ONCE(cvp->cv_mutex); |
| if (!m) |
| m = xchg(&cvp->cv_mutex, mp); |
| /* Ensure the same mutex is used by all callers */ |
| ASSERT(m == NULL || m == mp); |
| |
| prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); |
| atomic_inc(&cvp->cv_waiters); |
| |
| /* |
| * Mutex should be dropped after prepare_to_wait() this |
| * ensures we're linked in to the waiters list and avoids the |
| * race where 'cvp->cv_waiters > 0' but the list is empty. |
| */ |
| mutex_exit(mp); |
| |
| ktime_left = ktime_set(0, time_left); |
| slack = MIN(MAX(res, spl_schedule_hrtimeout_slack_us * NSEC_PER_USEC), |
| MAX_HRTIMEOUT_SLACK_US * NSEC_PER_USEC); |
| schedule_hrtimeout_range(&ktime_left, slack, HRTIMER_MODE_REL); |
| |
| /* No more waiters a different mutex could be used */ |
| if (atomic_dec_and_test(&cvp->cv_waiters)) { |
| /* |
| * This is set without any lock, so it's racy. But this is |
| * just for debug anyway, so make it best-effort |
| */ |
| cvp->cv_mutex = NULL; |
| wake_up(&cvp->cv_destroy); |
| } |
| |
| finish_wait(&cvp->cv_event, &wait); |
| atomic_dec(&cvp->cv_refs); |
| |
| mutex_enter(mp); |
| time_left = expire_time - gethrtime(); |
| return (time_left > 0 ? NSEC_TO_TICK(time_left) : -1); |
| } |
| |
| /* |
| * Compatibility wrapper for the cv_timedwait_hires() Illumos interface. |
| */ |
| static clock_t |
| cv_timedwait_hires_common(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, |
| hrtime_t res, int flag, int state) |
| { |
| if (!(flag & CALLOUT_FLAG_ABSOLUTE)) |
| tim += gethrtime(); |
| |
| return (__cv_timedwait_hires(cvp, mp, tim, res, state)); |
| } |
| |
| clock_t |
| cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, hrtime_t res, |
| int flag) |
| { |
| return (cv_timedwait_hires_common(cvp, mp, tim, res, flag, |
| TASK_UNINTERRUPTIBLE)); |
| } |
| EXPORT_SYMBOL(cv_timedwait_hires); |
| |
| clock_t |
| cv_timedwait_sig_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, |
| hrtime_t res, int flag) |
| { |
| return (cv_timedwait_hires_common(cvp, mp, tim, res, flag, |
| TASK_INTERRUPTIBLE)); |
| } |
| EXPORT_SYMBOL(cv_timedwait_sig_hires); |
| |
| void |
| __cv_signal(kcondvar_t *cvp) |
| { |
| ASSERT(cvp); |
| ASSERT(cvp->cv_magic == CV_MAGIC); |
| atomic_inc(&cvp->cv_refs); |
| |
| /* |
| * All waiters are added with WQ_FLAG_EXCLUSIVE so only one |
| * waiter will be set runnable with each call to wake_up(). |
| * Additionally wake_up() holds a spin_lock associated with |
| * the wait queue to ensure we don't race waking up processes. |
| */ |
| if (atomic_read(&cvp->cv_waiters) > 0) |
| wake_up(&cvp->cv_event); |
| |
| atomic_dec(&cvp->cv_refs); |
| } |
| EXPORT_SYMBOL(__cv_signal); |
| |
| void |
| __cv_broadcast(kcondvar_t *cvp) |
| { |
| ASSERT(cvp); |
| ASSERT(cvp->cv_magic == CV_MAGIC); |
| atomic_inc(&cvp->cv_refs); |
| |
| /* |
| * Wake_up_all() will wake up all waiters even those which |
| * have the WQ_FLAG_EXCLUSIVE flag set. |
| */ |
| if (atomic_read(&cvp->cv_waiters) > 0) |
| wake_up_all(&cvp->cv_event); |
| |
| atomic_dec(&cvp->cv_refs); |
| } |
| EXPORT_SYMBOL(__cv_broadcast); |