diff options
Diffstat (limited to 'linux')
| -rw-r--r-- | linux/blkdev.c | 4 | ||||
| -rw-r--r-- | linux/mean_and_variance.c | 2 | ||||
| -rw-r--r-- | linux/pretty-printers.c | 60 | ||||
| -rw-r--r-- | linux/printbuf.c | 368 | ||||
| -rw-r--r-- | linux/printbuf_userspace.c | 34 | ||||
| -rw-r--r-- | linux/seq_buf.c | 152 | ||||
| -rw-r--r-- | linux/six.c | 230 | ||||
| -rw-r--r-- | linux/string_helpers.c | 1 |
8 files changed, 316 insertions, 535 deletions
diff --git a/linux/blkdev.c b/linux/blkdev.c index 54cd6e9c..0a5cedfe 100644 --- a/linux/blkdev.c +++ b/linux/blkdev.c @@ -184,7 +184,7 @@ struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, if (buffered_fd < 0) return ERR_PTR(-errno); - fd = open(path, flags|O_DIRECT); + fd = open(path, flags); if (fd < 0) fd = dup(buffered_fd); if (fd < 0) { @@ -192,7 +192,7 @@ struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, return ERR_PTR(-errno); } - sync_fd = open(path, flags|O_DIRECT|O_SYNC); + sync_fd = open(path, flags|O_SYNC); if (sync_fd < 0) sync_fd = open(path, flags|O_SYNC); if (sync_fd < 0) { diff --git a/linux/mean_and_variance.c b/linux/mean_and_variance.c index 55d46c9d..bd08da5f 100644 --- a/linux/mean_and_variance.c +++ b/linux/mean_and_variance.c @@ -42,8 +42,6 @@ #include <linux/math64.h> #include <linux/mean_and_variance.h> #include <linux/module.h> -#include <linux/printbuf.h> - /** * fast_divpow2() - fast approximation for n / (1 << d) diff --git a/linux/pretty-printers.c b/linux/pretty-printers.c deleted file mode 100644 index addbac95..00000000 --- a/linux/pretty-printers.c +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: LGPL-2.1+ -/* Copyright (C) 2022 Kent Overstreet */ - -#include <linux/bitops.h> -#include <linux/kernel.h> -#include <linux/printbuf.h> -#include <linux/pretty-printers.h> - -/** - * prt_string_option - Given a list of strings, print out the list and indicate - * which option is selected, with square brackets (sysfs style) - * - * @out: The printbuf to output to - * @list: List of strings to choose from - * @selected: The option to highlight, with square brackets - */ -void prt_string_option(struct printbuf *out, - const char * const list[], - size_t selected) -{ - size_t i; - - for (i = 0; list[i]; i++) { - if (i) - prt_char(out, ' '); - if (i == selected) - prt_char(out, '['); - prt_str(out, list[i]); - if (i == selected) - prt_char(out, ']'); - } -} -EXPORT_SYMBOL(prt_string_option); - -/** - * prt_bitflags: Given a bitmap and a list of names for each bit, print out which - * bits are on, comma separated - * - * @out: The printbuf to output to - * @list: List of names for each bit - * @flags: Bits to print - */ -void prt_bitflags(struct printbuf *out, - const char * const list[], u64 flags) -{ - unsigned bit, nr = 0; - bool first = true; - - while (list[nr]) - nr++; - - while (flags && (bit = __ffs(flags)) < nr) { - if (!first) - prt_char(out, ','); - first = false; - prt_str(out, list[bit]); - flags ^= 1 << bit; - } -} -EXPORT_SYMBOL(prt_bitflags); diff --git a/linux/printbuf.c b/linux/printbuf.c deleted file mode 100644 index 5cf79d43..00000000 --- a/linux/printbuf.c +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-License-Identifier: LGPL-2.1+ -/* Copyright (C) 2022 Kent Overstreet */ - -#include <linux/err.h> -#include <linux/export.h> -#include <linux/kernel.h> -#include <linux/printbuf.h> -#include <linux/slab.h> -#include <linux/string_helpers.h> - -static inline unsigned printbuf_linelen(struct printbuf *buf) -{ - return buf->pos - buf->last_newline; -} - -int printbuf_make_room(struct printbuf *out, unsigned extra) -{ - unsigned new_size; - char *buf; - - if (!out->heap_allocated) - return 0; - - /* Reserved space for terminating nul: */ - extra += 1; - - if (out->pos + extra < out->size) - return 0; - - new_size = roundup_pow_of_two(out->size + extra); - - /* - * Note: output buffer must be freeable with kfree(), it's not required - * that the user use printbuf_exit(). - */ - buf = krealloc(out->buf, new_size, !out->atomic ? GFP_KERNEL : GFP_NOWAIT); - - if (!buf) { - out->allocation_failure = true; - return -ENOMEM; - } - - out->buf = buf; - out->size = new_size; - return 0; -} -EXPORT_SYMBOL(printbuf_make_room); - -/** - * printbuf_str - returns printbuf's buf as a C string, guaranteed to be null - * terminated - */ -const char *printbuf_str(const struct printbuf *buf) -{ - /* - * If we've written to a printbuf then it's guaranteed to be a null - * terminated string - but if we haven't, then we might not have - * allocated a buffer at all: - */ - return buf->pos - ? buf->buf - : ""; -} -EXPORT_SYMBOL(printbuf_str); - -/** - * printbuf_exit - exit a printbuf, freeing memory it owns and poisoning it - * against accidental use. - */ -void printbuf_exit(struct printbuf *buf) -{ - if (buf->heap_allocated) { - kfree(buf->buf); - buf->buf = ERR_PTR(-EINTR); /* poison value */ - } -} -EXPORT_SYMBOL(printbuf_exit); - -void printbuf_tabstops_reset(struct printbuf *buf) -{ - buf->nr_tabstops = 0; -} -EXPORT_SYMBOL(printbuf_tabstops_reset); - -void printbuf_tabstop_pop(struct printbuf *buf) -{ - if (buf->nr_tabstops) - --buf->nr_tabstops; -} -EXPORT_SYMBOL(printbuf_tabstop_pop); - -/* - * printbuf_tabstop_set - add a tabstop, n spaces from the previous tabstop - * - * @buf: printbuf to control - * @spaces: number of spaces from previous tabpstop - * - * In the future this function may allocate memory if setting more than - * PRINTBUF_INLINE_TABSTOPS or setting tabstops more than 255 spaces from start - * of line. - */ -int printbuf_tabstop_push(struct printbuf *buf, unsigned spaces) -{ - unsigned prev_tabstop = buf->nr_tabstops - ? buf->_tabstops[buf->nr_tabstops - 1] - : 0; - - if (WARN_ON(buf->nr_tabstops >= ARRAY_SIZE(buf->_tabstops))) - return -EINVAL; - - buf->_tabstops[buf->nr_tabstops++] = prev_tabstop + spaces; - buf->has_indent_or_tabstops = true; - return 0; -} -EXPORT_SYMBOL(printbuf_tabstop_push); - -/** - * printbuf_indent_add - add to the current indent level - * - * @buf: printbuf to control - * @spaces: number of spaces to add to the current indent level - * - * Subsequent lines, and the current line if the output position is at the start - * of the current line, will be indented by @spaces more spaces. - */ -void printbuf_indent_add(struct printbuf *buf, unsigned spaces) -{ - if (WARN_ON_ONCE(buf->indent + spaces < buf->indent)) - spaces = 0; - - buf->indent += spaces; - prt_chars(buf, ' ', spaces); - - buf->has_indent_or_tabstops = true; -} -EXPORT_SYMBOL(printbuf_indent_add); - -/** - * printbuf_indent_sub - subtract from the current indent level - * - * @buf: printbuf to control - * @spaces: number of spaces to subtract from the current indent level - * - * Subsequent lines, and the current line if the output position is at the start - * of the current line, will be indented by @spaces less spaces. - */ -void printbuf_indent_sub(struct printbuf *buf, unsigned spaces) -{ - if (WARN_ON_ONCE(spaces > buf->indent)) - spaces = buf->indent; - - if (buf->last_newline + buf->indent == buf->pos) { - buf->pos -= spaces; - printbuf_nul_terminate(buf); - } - buf->indent -= spaces; - - if (!buf->indent && !buf->nr_tabstops) - buf->has_indent_or_tabstops = false; -} -EXPORT_SYMBOL(printbuf_indent_sub); - -void prt_newline(struct printbuf *buf) -{ - unsigned i; - - printbuf_make_room(buf, 1 + buf->indent); - - __prt_char(buf, '\n'); - - buf->last_newline = buf->pos; - - for (i = 0; i < buf->indent; i++) - __prt_char(buf, ' '); - - printbuf_nul_terminate(buf); - - buf->last_field = buf->pos; - buf->cur_tabstop = 0; -} -EXPORT_SYMBOL(prt_newline); - -/* - * Returns spaces from start of line, if set, or 0 if unset: - */ -static inline unsigned cur_tabstop(struct printbuf *buf) -{ - return buf->cur_tabstop < buf->nr_tabstops - ? buf->_tabstops[buf->cur_tabstop] - : 0; -} - -static void __prt_tab(struct printbuf *out) -{ - int spaces = max_t(int, 0, cur_tabstop(out) - printbuf_linelen(out)); - - prt_chars(out, ' ', spaces); - - out->last_field = out->pos; - out->cur_tabstop++; -} - -/** - * prt_tab - Advance printbuf to the next tabstop - * - * @buf: printbuf to control - * - * Advance output to the next tabstop by printing spaces. - */ -void prt_tab(struct printbuf *out) -{ - if (WARN_ON(!cur_tabstop(out))) - return; - - __prt_tab(out); -} -EXPORT_SYMBOL(prt_tab); - -static void __prt_tab_rjust(struct printbuf *buf) -{ - unsigned move = buf->pos - buf->last_field; - int pad = (int) cur_tabstop(buf) - (int) printbuf_linelen(buf); - - if (pad > 0) { - printbuf_make_room(buf, pad); - - if (buf->last_field + pad < buf->size) - memmove(buf->buf + buf->last_field + pad, - buf->buf + buf->last_field, - min(move, buf->size - 1 - buf->last_field - pad)); - - if (buf->last_field < buf->size) - memset(buf->buf + buf->last_field, ' ', - min((unsigned) pad, buf->size - buf->last_field)); - - buf->pos += pad; - printbuf_nul_terminate(buf); - } - - buf->last_field = buf->pos; - buf->cur_tabstop++; -} - -/** - * prt_tab_rjust - Advance printbuf to the next tabstop, right justifying - * previous output - * - * @buf: printbuf to control - * - * Advance output to the next tabstop by inserting spaces immediately after the - * previous tabstop, right justifying previously outputted text. - */ -void prt_tab_rjust(struct printbuf *buf) -{ - if (WARN_ON(!cur_tabstop(buf))) - return; - - __prt_tab_rjust(buf); -} -EXPORT_SYMBOL(prt_tab_rjust); - -/** - * prt_bytes_indented - Print an array of chars, handling embedded control characters - * - * @out: printbuf to output to - * @str: string to print - * @count: number of bytes to print - * - * The following contol characters are handled as so: - * \n: prt_newline newline that obeys current indent level - * \t: prt_tab advance to next tabstop - * \r: prt_tab_rjust advance to next tabstop, with right justification - */ -void prt_bytes_indented(struct printbuf *out, const char *str, unsigned count) -{ - const char *unprinted_start = str; - const char *end = str + count; - - if (!out->has_indent_or_tabstops || out->suppress_indent_tabstop_handling) { - prt_bytes(out, str, count); - return; - } - - while (str != end) { - switch (*str) { - case '\n': - prt_bytes(out, unprinted_start, str - unprinted_start); - unprinted_start = str + 1; - prt_newline(out); - break; - case '\t': - if (likely(cur_tabstop(out))) { - prt_bytes(out, unprinted_start, str - unprinted_start); - unprinted_start = str + 1; - __prt_tab(out); - } - break; - case '\r': - if (likely(cur_tabstop(out))) { - prt_bytes(out, unprinted_start, str - unprinted_start); - unprinted_start = str + 1; - __prt_tab_rjust(out); - } - break; - } - - str++; - } - - prt_bytes(out, unprinted_start, str - unprinted_start); -} -EXPORT_SYMBOL(prt_bytes_indented); - -/** - * prt_human_readable_u64 - Print out a u64 in human readable units - * - * Units of 2^10 (default) or 10^3 are controlled via @buf->si_units - */ -void prt_human_readable_u64(struct printbuf *buf, u64 v) -{ - printbuf_make_room(buf, 10); - buf->pos += string_get_size(v, 1, !buf->si_units, - buf->buf + buf->pos, - printbuf_remaining_size(buf)); -} -EXPORT_SYMBOL(prt_human_readable_u64); - -/** - * prt_human_readable_s64 - Print out a s64 in human readable units - * - * Units of 2^10 (default) or 10^3 are controlled via @buf->si_units - */ -void prt_human_readable_s64(struct printbuf *buf, s64 v) -{ - if (v < 0) - prt_char(buf, '-'); - prt_human_readable_u64(buf, abs(v)); -} -EXPORT_SYMBOL(prt_human_readable_s64); - -/** - * prt_units_u64 - Print out a u64 according to printbuf unit options - * - * Units are either raw (default), or human reabable units (controlled via - * @buf->human_readable_units) - */ -void prt_units_u64(struct printbuf *out, u64 v) -{ - if (out->human_readable_units) - prt_human_readable_u64(out, v); - else - prt_printf(out, "%llu", v); -} -EXPORT_SYMBOL(prt_units_u64); - -/** - * prt_units_s64 - Print out a s64 according to printbuf unit options - * - * Units are either raw (default), or human reabable units (controlled via - * @buf->human_readable_units) - */ -void prt_units_s64(struct printbuf *out, s64 v) -{ - if (v < 0) - prt_char(out, '-'); - prt_units_u64(out, abs(v)); -} -EXPORT_SYMBOL(prt_units_s64); diff --git a/linux/printbuf_userspace.c b/linux/printbuf_userspace.c deleted file mode 100644 index 0ae56ee1..00000000 --- a/linux/printbuf_userspace.c +++ /dev/null @@ -1,34 +0,0 @@ - -#include <stdio.h> -#include <linux/printbuf.h> - -void prt_vprintf(struct printbuf *out, const char *fmt, va_list args) -{ - int len; - - do { - va_list args2; - - va_copy(args2, args); - len = vsnprintf(out->buf + out->pos, printbuf_remaining(out), fmt, args2); - } while (len + 1 >= printbuf_remaining(out) && - !printbuf_make_room(out, len + 1)); - - len = min_t(size_t, len, - printbuf_remaining(out) ? printbuf_remaining(out) - 1 : 0); - out->pos += len; -} - -void prt_printf(struct printbuf *out, const char *fmt, ...) -{ - va_list args; - - va_start(args, fmt); - prt_vprintf(out, fmt, args); - va_end(args); -} - -void prt_u64(struct printbuf *out, u64 v) -{ - prt_printf(out, "%llu", v); -} diff --git a/linux/seq_buf.c b/linux/seq_buf.c new file mode 100644 index 00000000..cf8709ad --- /dev/null +++ b/linux/seq_buf.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * seq_buf.c + * + * Copyright (C) 2014 Red Hat Inc, Steven Rostedt <srostedt@redhat.com> + * + * The seq_buf is a handy tool that allows you to pass a descriptor around + * to a buffer that other functions can write to. It is similar to the + * seq_file functionality but has some differences. + * + * To use it, the seq_buf must be initialized with seq_buf_init(). + * This will set up the counters within the descriptor. You can call + * seq_buf_init() more than once to reset the seq_buf to start + * from scratch. + */ +#include <linux/seq_buf.h> +#include <stdio.h> + +/** + * seq_buf_can_fit - can the new data fit in the current buffer? + * @s: the seq_buf descriptor + * @len: The length to see if it can fit in the current buffer + * + * Returns true if there's enough unused space in the seq_buf buffer + * to fit the amount of new data according to @len. + */ +static bool seq_buf_can_fit(struct seq_buf *s, size_t len) +{ + return s->len + len <= s->size; +} + +/** + * seq_buf_vprintf - sequence printing of information. + * @s: seq_buf descriptor + * @fmt: printf format string + * @args: va_list of arguments from a printf() type function + * + * Writes a vnprintf() format into the sequencce buffer. + * + * Returns zero on success, -1 on overflow. + */ +int seq_buf_vprintf(struct seq_buf *s, const char *fmt, va_list args) +{ + int len; + + WARN_ON(s->size == 0); + + if (s->len < s->size) { + len = vsnprintf(s->buffer + s->len, s->size - s->len, fmt, args); + if (s->len + len < s->size) { + s->len += len; + return 0; + } + } + seq_buf_set_overflow(s); + return -1; +} + +/** + * seq_buf_printf - sequence printing of information + * @s: seq_buf descriptor + * @fmt: printf format string + * + * Writes a printf() format into the sequence buffer. + * + * Returns zero on success, -1 on overflow. + */ +int seq_buf_printf(struct seq_buf *s, const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = seq_buf_vprintf(s, fmt, ap); + va_end(ap); + + return ret; +} + +/** + * seq_buf_puts - sequence printing of simple string + * @s: seq_buf descriptor + * @str: simple string to record + * + * Copy a simple string into the sequence buffer. + * + * Returns zero on success, -1 on overflow + */ +int seq_buf_puts(struct seq_buf *s, const char *str) +{ + size_t len = strlen(str); + + WARN_ON(s->size == 0); + + /* Add 1 to len for the trailing null byte which must be there */ + len += 1; + + if (seq_buf_can_fit(s, len)) { + memcpy(s->buffer + s->len, str, len); + /* Don't count the trailing null byte against the capacity */ + s->len += len - 1; + return 0; + } + seq_buf_set_overflow(s); + return -1; +} + +/** + * seq_buf_putc - sequence printing of simple character + * @s: seq_buf descriptor + * @c: simple character to record + * + * Copy a single character into the sequence buffer. + * + * Returns zero on success, -1 on overflow + */ +int seq_buf_putc(struct seq_buf *s, unsigned char c) +{ + WARN_ON(s->size == 0); + + if (seq_buf_can_fit(s, 1)) { + s->buffer[s->len++] = c; + return 0; + } + seq_buf_set_overflow(s); + return -1; +} + +/** + * seq_buf_putmem - write raw data into the sequenc buffer + * @s: seq_buf descriptor + * @mem: The raw memory to copy into the buffer + * @len: The length of the raw memory to copy (in bytes) + * + * There may be cases where raw memory needs to be written into the + * buffer and a strcpy() would not work. Using this function allows + * for such cases. + * + * Returns zero on success, -1 on overflow + */ +int seq_buf_putmem(struct seq_buf *s, const void *mem, unsigned int len) +{ + WARN_ON(s->size == 0); + + if (seq_buf_can_fit(s, len)) { + memcpy(s->buffer + s->len, mem, len); + s->len += len; + return 0; + } + seq_buf_set_overflow(s); + return -1; +} diff --git a/linux/six.c b/linux/six.c index 39a9bd6e..41337a7f 100644 --- a/linux/six.c +++ b/linux/six.c @@ -11,14 +11,16 @@ #include <linux/six.h> #include <linux/slab.h> +#include <trace/events/lock.h> + #ifdef DEBUG #define EBUG_ON(cond) BUG_ON(cond) #else #define EBUG_ON(cond) do {} while (0) #endif -#define six_acquire(l, t, r) lock_acquire(l, 0, t, r, 1, NULL, _RET_IP_) -#define six_release(l) lock_release(l, _RET_IP_) +#define six_acquire(l, t, r, ip) lock_acquire(l, 0, t, r, 1, NULL, ip) +#define six_release(l, ip) lock_release(l, ip) static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type); @@ -278,19 +280,20 @@ static bool do_six_trylock_type(struct six_lock *lock, } __always_inline __flatten -static bool __six_trylock_type(struct six_lock *lock, enum six_lock_type type) +static bool __six_trylock_type(struct six_lock *lock, enum six_lock_type type, + unsigned long ip) { if (!do_six_trylock_type(lock, type, true)) return false; if (type != SIX_LOCK_write) - six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read); + six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip); return true; } __always_inline __flatten static bool __six_relock_type(struct six_lock *lock, enum six_lock_type type, - unsigned seq) + unsigned seq, unsigned long ip) { const struct six_lock_vals l[] = LOCK_VALS; union six_lock_state old; @@ -321,7 +324,7 @@ static bool __six_relock_type(struct six_lock *lock, enum six_lock_type type, six_lock_wakeup(lock, old, SIX_LOCK_write); if (ret) - six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read); + six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip); return ret; } @@ -338,36 +341,48 @@ static bool __six_relock_type(struct six_lock *lock, enum six_lock_type type, six_set_owner(lock, type, old, current); if (type != SIX_LOCK_write) - six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read); + six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip); return true; } -/* - * We don't see stable performance with SIX_LOCK_SPIN_ON_OWNER enabled, so it's - * off for now: - */ -#ifdef SIX_LOCK_SPIN_ON_OWNER +#ifdef CONFIG_LOCK_SPIN_ON_OWNER -static inline bool six_optimistic_spin(struct six_lock *lock, - struct six_lock_waiter *wait) +static inline bool six_can_spin_on_owner(struct six_lock *lock) { - struct task_struct *owner, *task = current; + struct task_struct *owner; + bool ret; - switch (wait->lock_want) { - case SIX_LOCK_read: - break; - case SIX_LOCK_intent: - if (lock->wait_list.next != &wait->list) - return false; - break; - case SIX_LOCK_write: + if (need_resched()) return false; - } rcu_read_lock(); owner = READ_ONCE(lock->owner); + ret = !owner || owner_on_cpu(owner); + rcu_read_unlock(); + + return ret; +} + +static inline void six_set_nospin(struct six_lock *lock) +{ + union six_lock_state old, new; + u64 v = READ_ONCE(lock->state.v); + + do { + new.v = old.v = v; + new.nospin = true; + } while ((v = atomic64_cmpxchg(&lock->state.counter, old.v, new.v)) != old.v); +} - while (owner && lock->owner == owner) { +static inline bool six_spin_on_owner(struct six_lock *lock, + struct task_struct *owner, + u64 end_time) +{ + bool ret = true; + unsigned loop = 0; + + rcu_read_lock(); + while (lock->owner == owner) { /* * Ensure we emit the owner->on_cpu, dereference _after_ * checking lock->owner still matches owner. If that fails, @@ -376,27 +391,94 @@ static inline bool six_optimistic_spin(struct six_lock *lock, */ barrier(); + if (!owner_on_cpu(owner) || need_resched()) { + ret = false; + break; + } + + if (!(++loop & 0xf) && (time_after64(sched_clock(), end_time))) { + six_set_nospin(lock); + ret = false; + break; + } + + cpu_relax(); + } + rcu_read_unlock(); + + return ret; +} + +static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type type) +{ + struct task_struct *task = current; + u64 end_time; + + if (type == SIX_LOCK_write) + return false; + + preempt_disable(); + if (!six_can_spin_on_owner(lock)) + goto fail; + + if (!osq_lock(&lock->osq)) + goto fail; + + end_time = sched_clock() + 10 * NSEC_PER_USEC; + + while (1) { + struct task_struct *owner; + + /* + * If there's an owner, wait for it to either + * release the lock or go to sleep. + */ + owner = READ_ONCE(lock->owner); + if (owner && !six_spin_on_owner(lock, owner, end_time)) + break; + + if (do_six_trylock_type(lock, type, false)) { + osq_unlock(&lock->osq); + preempt_enable(); + return true; + } + /* - * If we're an RT task that will live-lock because we won't let + * When there's no owner, we might have preempted between the + * owner acquiring the lock and setting the owner field. If + * we're an RT task that will live-lock because we won't let * the owner complete. */ - if (wait->lock_acquired || - !owner->on_cpu || - rt_task(task) || - need_resched()) + if (!owner && (need_resched() || rt_task(task))) break; + /* + * The cpu_relax() call is a compiler barrier which forces + * everything in this loop to be re-loaded. We don't need + * memory barriers as we'll eventually observe the right + * values at the cost of a few extra spins. + */ cpu_relax(); } - rcu_read_unlock(); - return wait->lock_acquired; + osq_unlock(&lock->osq); +fail: + preempt_enable(); + + /* + * If we fell out of the spin path because of need_resched(), + * reschedule now, before we try-lock again. This avoids getting + * scheduled out right after we obtained the lock. + */ + if (need_resched()) + schedule(); + + return false; } #else /* CONFIG_LOCK_SPIN_ON_OWNER */ -static inline bool six_optimistic_spin(struct six_lock *lock, - struct six_lock_waiter *wait) +static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type type) { return false; } @@ -406,7 +488,8 @@ static inline bool six_optimistic_spin(struct six_lock *lock, noinline static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type type, struct six_lock_waiter *wait, - six_lock_should_sleep_fn should_sleep_fn, void *p) + six_lock_should_sleep_fn should_sleep_fn, void *p, + unsigned long ip) { union six_lock_state old; int ret = 0; @@ -417,7 +500,11 @@ static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type ty smp_mb__after_atomic(); } - lock_contended(&lock->dep_map, _RET_IP_); + trace_contention_begin(lock, 0); + lock_contended(&lock->dep_map, ip); + + if (six_optimistic_spin(lock, type)) + goto out; wait->task = current; wait->lock_want = type; @@ -457,9 +544,6 @@ static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type ty ret = 0; } - if (six_optimistic_spin(lock, wait)) - goto out; - while (1) { set_current_state(TASK_UNINTERRUPTIBLE); @@ -488,6 +572,7 @@ out: &lock->state.counter); six_lock_wakeup(lock, old, SIX_LOCK_read); } + trace_contention_end(lock, 0); return ret; } @@ -495,33 +580,35 @@ out: __always_inline __flatten static int __six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type, struct six_lock_waiter *wait, - six_lock_should_sleep_fn should_sleep_fn, void *p) + six_lock_should_sleep_fn should_sleep_fn, void *p, + unsigned long ip) { int ret; wait->start_time = 0; if (type != SIX_LOCK_write) - six_acquire(&lock->dep_map, 0, type == SIX_LOCK_read); + six_acquire(&lock->dep_map, 0, type == SIX_LOCK_read, ip); ret = do_six_trylock_type(lock, type, true) ? 0 - : __six_lock_type_slowpath(lock, type, wait, should_sleep_fn, p); + : __six_lock_type_slowpath(lock, type, wait, should_sleep_fn, p, ip); if (ret && type != SIX_LOCK_write) - six_release(&lock->dep_map); + six_release(&lock->dep_map, ip); if (!ret) - lock_acquired(&lock->dep_map, _RET_IP_); + lock_acquired(&lock->dep_map, ip); return ret; } __always_inline static int __six_lock_type(struct six_lock *lock, enum six_lock_type type, - six_lock_should_sleep_fn should_sleep_fn, void *p) + six_lock_should_sleep_fn should_sleep_fn, void *p, + unsigned long ip) { struct six_lock_waiter wait; - return __six_lock_type_waiter(lock, type, &wait, should_sleep_fn, p); + return __six_lock_type_waiter(lock, type, &wait, should_sleep_fn, p, ip); } __always_inline __flatten @@ -540,16 +627,21 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type) smp_mb(); /* between unlocking and checking for waiters */ state.v = READ_ONCE(lock->state.v); } else { + u64 v = l[type].unlock_val; + + if (type != SIX_LOCK_read) + v -= lock->state.v & __SIX_VAL(nospin, 1); + EBUG_ON(!(lock->state.v & l[type].held_mask)); - state.v = atomic64_add_return_release(l[type].unlock_val, - &lock->state.counter); + state.v = atomic64_add_return_release(v, &lock->state.counter); } six_lock_wakeup(lock, state, l[type].unlock_wakeup); } __always_inline __flatten -static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type) +static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type, + unsigned long ip) { EBUG_ON(type == SIX_LOCK_write && !(lock->state.v & __SIX_LOCK_HELD_intent)); @@ -558,7 +650,7 @@ static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type) lock->owner != current); if (type != SIX_LOCK_write) - six_release(&lock->dep_map); + six_release(&lock->dep_map, ip); if (type == SIX_LOCK_intent && lock->intent_lock_recurse) { @@ -570,38 +662,40 @@ static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type) } #define __SIX_LOCK(type) \ -bool six_trylock_##type(struct six_lock *lock) \ +bool six_trylock_ip_##type(struct six_lock *lock, unsigned long ip) \ { \ - return __six_trylock_type(lock, SIX_LOCK_##type); \ + return __six_trylock_type(lock, SIX_LOCK_##type, ip); \ } \ -EXPORT_SYMBOL_GPL(six_trylock_##type); \ +EXPORT_SYMBOL_GPL(six_trylock_ip_##type); \ \ -bool six_relock_##type(struct six_lock *lock, u32 seq) \ +bool six_relock_ip_##type(struct six_lock *lock, u32 seq, unsigned long ip)\ { \ - return __six_relock_type(lock, SIX_LOCK_##type, seq); \ + return __six_relock_type(lock, SIX_LOCK_##type, seq, ip); \ } \ -EXPORT_SYMBOL_GPL(six_relock_##type); \ +EXPORT_SYMBOL_GPL(six_relock_ip_##type); \ \ -int six_lock_##type(struct six_lock *lock, \ - six_lock_should_sleep_fn should_sleep_fn, void *p) \ +int six_lock_ip_##type(struct six_lock *lock, \ + six_lock_should_sleep_fn should_sleep_fn, void *p, \ + unsigned long ip) \ { \ - return __six_lock_type(lock, SIX_LOCK_##type, should_sleep_fn, p);\ + return __six_lock_type(lock, SIX_LOCK_##type, should_sleep_fn, p, ip);\ } \ -EXPORT_SYMBOL_GPL(six_lock_##type); \ +EXPORT_SYMBOL_GPL(six_lock_ip_##type); \ \ -int six_lock_waiter_##type(struct six_lock *lock, \ +int six_lock_ip_waiter_##type(struct six_lock *lock, \ struct six_lock_waiter *wait, \ - six_lock_should_sleep_fn should_sleep_fn, void *p)\ + six_lock_should_sleep_fn should_sleep_fn, void *p,\ + unsigned long ip) \ { \ - return __six_lock_type_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p);\ + return __six_lock_type_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p, ip);\ } \ -EXPORT_SYMBOL_GPL(six_lock_waiter_##type); \ +EXPORT_SYMBOL_GPL(six_lock_ip_waiter_##type); \ \ -void six_unlock_##type(struct six_lock *lock) \ +void six_unlock_ip_##type(struct six_lock *lock, unsigned long ip) \ { \ - __six_unlock_type(lock, SIX_LOCK_##type); \ + __six_unlock_type(lock, SIX_LOCK_##type, ip); \ } \ -EXPORT_SYMBOL_GPL(six_unlock_##type); +EXPORT_SYMBOL_GPL(six_unlock_ip_##type); __SIX_LOCK(read) __SIX_LOCK(intent) @@ -672,7 +766,7 @@ void six_lock_increment(struct six_lock *lock, enum six_lock_type type) { const struct six_lock_vals l[] = LOCK_VALS; - six_acquire(&lock->dep_map, 0, type == SIX_LOCK_read); + six_acquire(&lock->dep_map, 0, type == SIX_LOCK_read, _RET_IP_); /* XXX: assert already locked, and that we don't overflow: */ diff --git a/linux/string_helpers.c b/linux/string_helpers.c index 29c498ad..0810ca13 100644 --- a/linux/string_helpers.c +++ b/linux/string_helpers.c @@ -14,7 +14,6 @@ #include <linux/errno.h> #include <linux/fs.h> #include <linux/limits.h> -#include <linux/printbuf.h> #include <linux/slab.h> #include <linux/string.h> #include <linux/string_helpers.h> |