summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Documentation/admin-guide/sysctl/kernel.rst8
-rw-r--r--arch/alpha/include/asm/timex.h1
-rw-r--r--arch/arm/include/asm/timex.h1
-rw-r--r--arch/ia64/include/asm/timex.h1
-rw-r--r--arch/m68k/include/asm/timex.h2
-rw-r--r--arch/mips/include/asm/timex.h17
-rw-r--r--arch/nios2/include/asm/timex.h3
-rw-r--r--arch/openrisc/include/asm/timex.h1
-rw-r--r--arch/openrisc/kernel/head.S9
-rw-r--r--arch/parisc/include/asm/timex.h3
-rw-r--r--arch/powerpc/include/asm/timex.h1
-rw-r--r--arch/riscv/include/asm/timex.h2
-rw-r--r--arch/s390/include/asm/timex.h1
-rw-r--r--arch/sparc/include/asm/timex_32.h4
-rw-r--r--arch/um/include/asm/timex.h9
-rw-r--r--arch/x86/include/asm/timex.h9
-rw-r--r--arch/x86/include/asm/tsc.h7
-rw-r--r--arch/xtensa/include/asm/timex.h6
-rw-r--r--drivers/char/random.c1346
-rw-r--r--include/linux/mm.h1
-rw-r--r--include/linux/prandom.h61
-rw-r--r--include/linux/random.h90
-rw-r--r--include/linux/siphash.h28
-rw-r--r--include/linux/timex.h8
-rw-r--r--init/main.c13
-rw-r--r--kernel/time/timekeeping.c15
-rw-r--r--kernel/time/timer.c2
-rw-r--r--lib/Kconfig.debug3
-rw-r--r--lib/random32.c347
-rw-r--r--lib/siphash.c32
-rw-r--r--lib/vsprintf.c67
-rw-r--r--mm/util.c32
-rw-r--r--net/core/dev.c3
-rw-r--r--net/ipv4/devinet.c4
-rw-r--r--net/ipv6/addrconf.c2
35 files changed, 784 insertions, 1355 deletions
diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index 1144ea3229a3..e9c18dabc552 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -994,6 +994,9 @@ This is a directory, with the following entries:
* ``boot_id``: a UUID generated the first time this is retrieved, and
unvarying after that;
+* ``uuid``: a UUID generated every time this is retrieved (this can
+ thus be used to generate UUIDs at will);
+
* ``entropy_avail``: the pool's entropy count, in bits;
* ``poolsize``: the entropy pool size, in bits;
@@ -1001,10 +1004,7 @@ This is a directory, with the following entries:
* ``urandom_min_reseed_secs``: obsolete (used to determine the minimum
number of seconds between urandom pool reseeding). This file is
writable for compatibility purposes, but writing to it has no effect
- on any RNG behavior.
-
-* ``uuid``: a UUID generated every time this is retrieved (this can
- thus be used to generate UUIDs at will);
+ on any RNG behavior;
* ``write_wakeup_threshold``: when the entropy count drops below this
(as a number of bits), processes waiting to write to ``/dev/random``
diff --git a/arch/alpha/include/asm/timex.h b/arch/alpha/include/asm/timex.h
index b565cc6f408e..f89798da8a14 100644
--- a/arch/alpha/include/asm/timex.h
+++ b/arch/alpha/include/asm/timex.h
@@ -28,5 +28,6 @@ static inline cycles_t get_cycles (void)
__asm__ __volatile__ ("rpcc %0" : "=r"(ret));
return ret;
}
+#define get_cycles get_cycles
#endif
diff --git a/arch/arm/include/asm/timex.h b/arch/arm/include/asm/timex.h
index 7c3b3671d6c2..6d1337c169cd 100644
--- a/arch/arm/include/asm/timex.h
+++ b/arch/arm/include/asm/timex.h
@@ -11,5 +11,6 @@
typedef unsigned long cycles_t;
#define get_cycles() ({ cycles_t c; read_current_timer(&c) ? 0 : c; })
+#define random_get_entropy() (((unsigned long)get_cycles()) ?: random_get_entropy_fallback())
#endif
diff --git a/arch/ia64/include/asm/timex.h b/arch/ia64/include/asm/timex.h
index 869a3ac6bf23..7ccc077a60be 100644
--- a/arch/ia64/include/asm/timex.h
+++ b/arch/ia64/include/asm/timex.h
@@ -39,6 +39,7 @@ get_cycles (void)
ret = ia64_getreg(_IA64_REG_AR_ITC);
return ret;
}
+#define get_cycles get_cycles
extern void ia64_cpu_local_tick (void);
extern unsigned long long ia64_native_sched_clock (void);
diff --git a/arch/m68k/include/asm/timex.h b/arch/m68k/include/asm/timex.h
index 6a21d9358280..f4a7a340f4ca 100644
--- a/arch/m68k/include/asm/timex.h
+++ b/arch/m68k/include/asm/timex.h
@@ -35,7 +35,7 @@ static inline unsigned long random_get_entropy(void)
{
if (mach_random_get_entropy)
return mach_random_get_entropy();
- return 0;
+ return random_get_entropy_fallback();
}
#define random_get_entropy random_get_entropy
diff --git a/arch/mips/include/asm/timex.h b/arch/mips/include/asm/timex.h
index 8026baf46e72..2e107886f97a 100644
--- a/arch/mips/include/asm/timex.h
+++ b/arch/mips/include/asm/timex.h
@@ -76,25 +76,24 @@ static inline cycles_t get_cycles(void)
else
return 0; /* no usable counter */
}
+#define get_cycles get_cycles
/*
* Like get_cycles - but where c0_count is not available we desperately
* use c0_random in an attempt to get at least a little bit of entropy.
- *
- * R6000 and R6000A neither have a count register nor a random register.
- * That leaves no entropy source in the CPU itself.
*/
static inline unsigned long random_get_entropy(void)
{
- unsigned int prid = read_c0_prid();
- unsigned int imp = prid & PRID_IMP_MASK;
+ unsigned int c0_random;
- if (can_use_mips_counter(prid))
+ if (can_use_mips_counter(read_c0_prid()))
return read_c0_count();
- else if (likely(imp != PRID_IMP_R6000 && imp != PRID_IMP_R6000A))
- return read_c0_random();
+
+ if (cpu_has_3kex)
+ c0_random = (read_c0_random() >> 8) & 0x3f;
else
- return 0; /* no usable register */
+ c0_random = read_c0_random() & 0x3f;
+ return (random_get_entropy_fallback() << 6) | (0x3f - c0_random);
}
#define random_get_entropy random_get_entropy
diff --git a/arch/nios2/include/asm/timex.h b/arch/nios2/include/asm/timex.h
index a769f871b28d..40a1adc9bd03 100644
--- a/arch/nios2/include/asm/timex.h
+++ b/arch/nios2/include/asm/timex.h
@@ -8,5 +8,8 @@
typedef unsigned long cycles_t;
extern cycles_t get_cycles(void);
+#define get_cycles get_cycles
+
+#define random_get_entropy() (((unsigned long)get_cycles()) ?: random_get_entropy_fallback())
#endif
diff --git a/arch/openrisc/include/asm/timex.h b/arch/openrisc/include/asm/timex.h
index d52b4e536e3f..5487fa93dd9b 100644
--- a/arch/openrisc/include/asm/timex.h
+++ b/arch/openrisc/include/asm/timex.h
@@ -23,6 +23,7 @@ static inline cycles_t get_cycles(void)
{
return mfspr(SPR_TTCR);
}
+#define get_cycles get_cycles
/* This isn't really used any more */
#define CLOCK_TICK_RATE 1000
diff --git a/arch/openrisc/kernel/head.S b/arch/openrisc/kernel/head.S
index 15f1b38dfe03..2fa6cefa62ca 100644
--- a/arch/openrisc/kernel/head.S
+++ b/arch/openrisc/kernel/head.S
@@ -521,6 +521,15 @@ _start:
l.ori r3,r0,0x1
l.mtspr r0,r3,SPR_SR
+ /*
+ * Start the TTCR as early as possible, so that the RNG can make use of
+ * measurements of boot time from the earliest opportunity. Especially
+ * important is that the TTCR does not return zero by the time we reach
+ * random_init().
+ */
+ l.movhi r3,hi(SPR_TTMR_CR)
+ l.mtspr r0,r3,SPR_TTMR
+
CLEAR_GPR(r1)
CLEAR_GPR(r2)
CLEAR_GPR(r3)
diff --git a/arch/parisc/include/asm/timex.h b/arch/parisc/include/asm/timex.h
index 06b510f8172e..b4622cb06a75 100644
--- a/arch/parisc/include/asm/timex.h
+++ b/arch/parisc/include/asm/timex.h
@@ -13,9 +13,10 @@
typedef unsigned long cycles_t;
-static inline cycles_t get_cycles (void)
+static inline cycles_t get_cycles(void)
{
return mfctl(16);
}
+#define get_cycles get_cycles
#endif
diff --git a/arch/powerpc/include/asm/timex.h b/arch/powerpc/include/asm/timex.h
index fa2e76e4093a..14b4489de52c 100644
--- a/arch/powerpc/include/asm/timex.h
+++ b/arch/powerpc/include/asm/timex.h
@@ -19,6 +19,7 @@ static inline cycles_t get_cycles(void)
{
return mftb();
}
+#define get_cycles get_cycles
#endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_TIMEX_H */
diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h
index 507cae273bc6..d6a7428f6248 100644
--- a/arch/riscv/include/asm/timex.h
+++ b/arch/riscv/include/asm/timex.h
@@ -41,7 +41,7 @@ static inline u32 get_cycles_hi(void)
static inline unsigned long random_get_entropy(void)
{
if (unlikely(clint_time_val == NULL))
- return 0;
+ return random_get_entropy_fallback();
return get_cycles();
}
#define random_get_entropy() random_get_entropy()
diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h
index 2cfce42aa7fc..ce878e85b6e4 100644
--- a/arch/s390/include/asm/timex.h
+++ b/arch/s390/include/asm/timex.h
@@ -197,6 +197,7 @@ static inline cycles_t get_cycles(void)
{
return (cycles_t) get_tod_clock() >> 2;
}
+#define get_cycles get_cycles
int get_phys_clock(unsigned long *clock);
void init_cpu_timer(void);
diff --git a/arch/sparc/include/asm/timex_32.h b/arch/sparc/include/asm/timex_32.h
index 542915b46209..f86326a6f89e 100644
--- a/arch/sparc/include/asm/timex_32.h
+++ b/arch/sparc/include/asm/timex_32.h
@@ -9,8 +9,6 @@
#define CLOCK_TICK_RATE 1193180 /* Underlying HZ */
-/* XXX Maybe do something better at some point... -DaveM */
-typedef unsigned long cycles_t;
-#define get_cycles() (0)
+#include <asm-generic/timex.h>
#endif
diff --git a/arch/um/include/asm/timex.h b/arch/um/include/asm/timex.h
index e392a9a5bc9b..9f27176adb26 100644
--- a/arch/um/include/asm/timex.h
+++ b/arch/um/include/asm/timex.h
@@ -2,13 +2,8 @@
#ifndef __UM_TIMEX_H
#define __UM_TIMEX_H
-typedef unsigned long cycles_t;
-
-static inline cycles_t get_cycles (void)
-{
- return 0;
-}
-
#define CLOCK_TICK_RATE (HZ)
+#include <asm-generic/timex.h>
+
#endif
diff --git a/arch/x86/include/asm/timex.h b/arch/x86/include/asm/timex.h
index a4a8b1b16c0c..956e4145311b 100644
--- a/arch/x86/include/asm/timex.h
+++ b/arch/x86/include/asm/timex.h
@@ -5,6 +5,15 @@
#include <asm/processor.h>
#include <asm/tsc.h>
+static inline unsigned long random_get_entropy(void)
+{
+ if (!IS_ENABLED(CONFIG_X86_TSC) &&
+ !cpu_feature_enabled(X86_FEATURE_TSC))
+ return random_get_entropy_fallback();
+ return rdtsc();
+}
+#define random_get_entropy random_get_entropy
+
/* Assume we use the PIT time source for the clock tick */
#define CLOCK_TICK_RATE PIT_TICK_RATE
diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index 01a300a9700b..fbdc3d951494 100644
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -20,13 +20,12 @@ extern void disable_TSC(void);
static inline cycles_t get_cycles(void)
{
-#ifndef CONFIG_X86_TSC
- if (!boot_cpu_has(X86_FEATURE_TSC))
+ if (!IS_ENABLED(CONFIG_X86_TSC) &&
+ !cpu_feature_enabled(X86_FEATURE_TSC))
return 0;
-#endif
-
return rdtsc();
}
+#define get_cycles get_cycles
extern struct system_counterval_t convert_art_to_tsc(u64 art);
extern struct system_counterval_t convert_art_ns_to_tsc(u64 art_ns);
diff --git a/arch/xtensa/include/asm/timex.h b/arch/xtensa/include/asm/timex.h
index 233ec75e60c6..3f2462f2d027 100644
--- a/arch/xtensa/include/asm/timex.h
+++ b/arch/xtensa/include/asm/timex.h
@@ -29,10 +29,6 @@
extern unsigned long ccount_freq;
-typedef unsigned long long cycles_t;
-
-#define get_cycles() (0)
-
void local_timer_setup(unsigned cpu);
/*
@@ -59,4 +55,6 @@ static inline void set_linux_timer (unsigned long ccompare)
xtensa_set_sr(ccompare, SREG_CCOMPARE + LINUX_TIMER);
}
+#include <asm-generic/timex.h>
+
#endif /* _XTENSA_TIMEX_H */
diff --git a/drivers/char/random.c b/drivers/char/random.c
index 4c9adb4f3d5d..b691b9d59503 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -15,14 +15,12 @@
* - Sysctl interface.
*
* The high level overview is that there is one input pool, into which
- * various pieces of data are hashed. Some of that data is then "credited" as
- * having a certain number of bits of entropy. When enough bits of entropy are
- * available, the hash is finalized and handed as a key to a stream cipher that
- * expands it indefinitely for various consumers. This key is periodically
- * refreshed as the various entropy collectors, described below, add data to the
- * input pool and credit it. There is currently no Fortuna-like scheduler
- * involved, which can lead to malicious entropy sources causing a premature
- * reseed, and the entropy estimates are, at best, conservative guesses.
+ * various pieces of data are hashed. Prior to initialization, some of that
+ * data is then "credited" as having a certain number of bits of entropy.
+ * When enough bits of entropy are available, the hash is finalized and
+ * handed as a key to a stream cipher that expands it indefinitely for
+ * various consumers. This key is periodically refreshed as the various
+ * entropy collectors, described below, add data to the input pool.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -53,6 +51,8 @@
#include <linux/completion.h>
#include <linux/uuid.h>
#include <linux/uaccess.h>
+#include <linux/suspend.h>
+#include <linux/siphash.h>
#include <crypto/chacha.h>
#include <crypto/blake2s.h>
#include <asm/processor.h>
@@ -71,27 +71,25 @@
*********************************************************************/
/*
- * crng_init = 0 --> Uninitialized
- * 1 --> Initialized
- * 2 --> Initialized from input_pool
- *
* crng_init is protected by base_crng->lock, and only increases
- * its value (from 0->1->2).
+ * its value (from empty->early->ready).
*/
-static int crng_init = 0;
-#define crng_ready() (likely(crng_init > 1))
-/* Various types of waiters for crng_init->2 transition. */
+static enum {
+ CRNG_EMPTY = 0, /* Little to no entropy collected */
+ CRNG_EARLY = 1, /* At least POOL_EARLY_BITS collected */
+ CRNG_READY = 2 /* Fully initialized with POOL_READY_BITS collected */
+} crng_init __read_mostly = CRNG_EMPTY;
+static DEFINE_STATIC_KEY_FALSE(crng_is_ready);
+#define crng_ready() (static_branch_likely(&crng_is_ready) || crng_init >= CRNG_READY)
+/* Various types of waiters for crng_init->CRNG_READY transition. */
static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
static struct fasync_struct *fasync;
-static DEFINE_SPINLOCK(random_ready_chain_lock);
-static RAW_NOTIFIER_HEAD(random_ready_chain);
/* Control how we warn userspace. */
-static struct ratelimit_state unseeded_warning =
- RATELIMIT_STATE_INIT("warn_unseeded_randomness", HZ, 3);
static struct ratelimit_state urandom_warning =
RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3);
-static int ratelimit_disable __read_mostly;
+static int ratelimit_disable __read_mostly =
+ IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM);
module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
@@ -110,6 +108,11 @@ bool rng_is_initialized(void)
}
EXPORT_SYMBOL(rng_is_initialized);
+static void __cold crng_set_ready(struct work_struct *work)
+{
+ static_branch_enable(&crng_is_ready);
+}
+
/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */
static void try_to_generate_entropy(void);
@@ -137,73 +140,10 @@ int wait_for_random_bytes(void)
}
EXPORT_SYMBOL(wait_for_random_bytes);
-/*
- * Add a callback function that will be invoked when the input
- * pool is initialised.
- *
- * returns: 0 if callback is successfully added
- * -EALREADY if pool is already initialised (callback not called)
- */
-int register_random_ready_notifier(struct notifier_block *nb)
-{
- unsigned long flags;
- int ret = -EALREADY;
-
- if (crng_ready())
- return ret;
-
- spin_lock_irqsave(&random_ready_chain_lock, flags);
- if (!crng_ready())
- ret = raw_notifier_chain_register(&random_ready_chain, nb);
- spin_unlock_irqrestore(&random_ready_chain_lock, flags);
- return ret;
-}
-
-/*
- * Delete a previously registered readiness callback function.
- */
-int unregister_random_ready_notifier(struct notifier_block *nb)
-{
- unsigned long flags;
- int ret;
-
- spin_lock_irqsave(&random_ready_chain_lock, flags);
- ret = raw_notifier_chain_unregister(&random_ready_chain, nb);
- spin_unlock_irqrestore(&random_ready_chain_lock, flags);
- return ret;
-}
-
-static void process_random_ready_list(void)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&random_ready_chain_lock, flags);
- raw_notifier_call_chain(&random_ready_chain, 0, NULL);
- spin_unlock_irqrestore(&random_ready_chain_lock, flags);
-}
-
-#define warn_unseeded_randomness(previous) \
- _warn_unseeded_randomness(__func__, (void *)_RET_IP_, (previous))
-
-static void _warn_unseeded_randomness(const char *func_name, void *caller, void **previous)
-{
-#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM
- const bool print_once = false;
-#else
- static bool print_once __read_mostly;
-#endif
-
- if (print_once || crng_ready() ||
- (previous && (caller == READ_ONCE(*previous))))
- return;
- WRITE_ONCE(*previous, caller);
-#ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM
- print_once = true;
-#endif
- if (__ratelimit(&unseeded_warning))
- printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n",
- func_name, caller, crng_init);
-}
+#define warn_unseeded_randomness() \
+ if (IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM) && !crng_ready()) \
+ printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", \
+ __func__, (void *)_RET_IP_, crng_init)
/*********************************************************************
@@ -216,7 +156,7 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller, void
*
* There are a few exported interfaces for use by other drivers:
*
- * void get_random_bytes(void *buf, size_t nbytes)
+ * void get_random_bytes(void *buf, size_t len)
* u32 get_random_u32()
* u64 get_random_u64()
* unsigned int get_random_int()
@@ -232,8 +172,8 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller, void
*********************************************************************/
enum {
- CRNG_RESEED_INTERVAL = 300 * HZ,
- CRNG_INIT_CNT_THRESH = 2 * CHACHA_KEY_SIZE
+ CRNG_RESEED_START_INTERVAL = HZ,
+ CRNG_RESEED_INTERVAL = 60 * HZ
};
static struct {
@@ -256,24 +196,17 @@ static DEFINE_PER_CPU(struct crng, crngs) = {
.lock = INIT_LOCAL_LOCK(crngs.lock),
};
-/* Used by crng_reseed() to extract a new seed from the input pool. */
-static bool drain_entropy(void *buf, size_t nbytes, bool force);
+/* Used by crng_reseed() and crng_make_state() to extract a new seed from the input pool. */
+static void extract_entropy(void *buf, size_t len);
-/*
- * This extracts a new crng key from the input pool, but only if there is a
- * sufficient amount of entropy available or force is true, in order to
- * mitigate bruteforcing of newly added bits.
- */
-static void crng_reseed(bool force)
+/* This extracts a new crng key from the input pool. */
+static void crng_reseed(void)
{
unsigned long flags;
unsigned long next_gen;
u8 key[CHACHA_KEY_SIZE];
- bool finalize_init = false;
- /* Only reseed if we can, to prevent brute forcing a small amount of new bits. */
- if (!drain_entropy(key, sizeof(key), force))
- return;
+ extract_entropy(key, sizeof(key));
/*
* We copy the new key into the base_crng, overwriting the old one,
@@ -288,28 +221,10 @@ static void crng_reseed(bool force)
++next_gen;
WRITE_ONCE(base_crng.generation, next_gen);
WRITE_ONCE(base_crng.birth, jiffies);
- if (!crng_ready()) {
- crng_init = 2;
- finalize_init = true;
- }
+ if (!static_branch_likely(&crng_is_ready))
+ crng_init = CRNG_READY;
spin_unlock_irqrestore(&base_crng.lock, flags);
memzero_explicit(key, sizeof(key));
- if (finalize_init) {
- process_random_ready_list();
- wake_up_interruptible(&crng_init_wait);
- kill_fasync(&fasync, SIGIO, POLL_IN);
- pr_notice("crng init done\n");
- if (unseeded_warning.missed) {
- pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n",
- unseeded_warning.missed);
- unseeded_warning.missed = 0;
- }
- if (urandom_warning.missed) {
- pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
- urandom_warning.missed);
- urandom_warning.missed = 0;
- }
- }
}
/*
@@ -345,10 +260,10 @@ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE],
}
/*
- * Return whether the crng seed is considered to be sufficiently
- * old that a reseeding might be attempted. This happens if the last
- * reseeding was CRNG_RESEED_INTERVAL ago, or during early boot, at
- * an interval proportional to the uptime.
+ * Return whether the crng seed is considered to be sufficiently old
+ * that a reseeding is needed. This happens if the last reseeding
+ * was CRNG_RESEED_INTERVAL ago, or during early boot, at an interval
+ * proportional to the uptime.
*/
static bool crng_has_old_seed(void)
{
@@ -360,10 +275,10 @@ static bool crng_has_old_seed(void)
if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2)
WRITE_ONCE(early_boot, false);
else
- interval = max_t(unsigned int, 5 * HZ,
+ interval = max_t(unsigned int, CRNG_RESEED_START_INTERVAL,
(unsigned int)uptime / 2 * HZ);
}
- return time_after(jiffies, READ_ONCE(base_crng.birth) + interval);
+ return time_is_before_jiffies(READ_ONCE(base_crng.birth) + interval);
}
/*
@@ -382,28 +297,31 @@ static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS],
/*
* For the fast path, we check whether we're ready, unlocked first, and
* then re-check once locked later. In the case where we're really not
- * ready, we do fast key erasure with the base_crng directly, because
- * this is what crng_pre_init_inject() mutates during early init.
+ * ready, we do fast key erasure with the base_crng directly, extracting
+ * when crng_init is CRNG_EMPTY.
*/
if (!crng_ready()) {
bool ready;
spin_lock_irqsave(&base_crng.lock, flags);
ready = crng_ready();
- if (!ready)
+ if (!ready) {
+ if (crng_init == CRNG_EMPTY)
+ extract_entropy(base_crng.key, sizeof(base_crng.key));
crng_fast_key_erasure(base_crng.key, chacha_state,
random_data, random_data_len);
+ }
spin_unlock_irqrestore(&base_crng.lock, flags);
if (!ready)
return;
}
/*
- * If the base_crng is old enough, we try to reseed, which in turn
- * bumps the generation counter that we check below.
+ * If the base_crng is old enough, we reseed, which in turn bumps the
+ * generation counter that we check below.
*/
if (unlikely(crng_has_old_seed()))
- crng_reseed(false);
+ crng_reseed();
local_lock_irqsave(&crngs.lock, flags);
crng = raw_cpu_ptr(&crngs);
@@ -433,68 +351,24 @@ static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS],
local_unlock_irqrestore(&crngs.lock, flags);
}
-/*
- * This function is for crng_init == 0 only. It loads entropy directly
- * into the crng's key, without going through the input pool. It is,
- * generally speaking, not very safe, but we use this only at early
- * boot time when it's better to have something there rather than
- * nothing.
- *
- * If account is set, then the crng_init_cnt counter is incremented.
- * This shouldn't be set by functions like add_device_randomness(),
- * where we can't trust the buffer passed to it is guaranteed to be
- * unpredictable (so it might not have any entropy at all).
- */
-static void crng_pre_init_inject(const void *input, size_t len, bool account)
-{
- static int crng_init_cnt = 0;
- struct blake2s_state hash;
- unsigned long flags;
-
- blake2s_init(&hash, sizeof(base_crng.key));
-
- spin_lock_irqsave(&base_crng.lock, flags);
- if (crng_init != 0) {
- spin_unlock_irqrestore(&base_crng.lock, flags);
- return;
- }
-
- blake2s_update(&hash, base_crng.key, sizeof(base_crng.key));
- blake2s_update(&hash, input, len);
- blake2s_final(&hash, base_crng.key);
-
- if (account) {
- crng_init_cnt += min_t(size_t, len, CRNG_INIT_CNT_THRESH - crng_init_cnt);
- if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) {
- ++base_crng.generation;
- crng_init = 1;
- }
- }
-
- spin_unlock_irqrestore(&base_crng.lock, flags);
-
- if (crng_init == 1)
- pr_notice("fast init done\n");
-}
-
-static void _get_random_bytes(void *buf, size_t nbytes)
+static void _get_random_bytes(void *buf, size_t len)
{
u32 chacha_state[CHACHA_STATE_WORDS];
u8 tmp[CHACHA_BLOCK_SIZE];
- size_t len;
+ size_t first_block_len;
- if (!nbytes)
+ if (!len)
return;
- len = min_t(size_t, 32, nbytes);
- crng_make_state(chacha_state, buf, len);
- nbytes -= len;
- buf += len;
+ first_block_len = min_t(size_t, 32, len);
+ crng_make_state(chacha_state, buf, first_block_len);
+ len -= first_block_len;
+ buf += first_block_len;
- while (nbytes) {
- if (nbytes < CHACHA_BLOCK_SIZE) {
+ while (len) {
+ if (len < CHACHA_BLOCK_SIZE) {
chacha20_block(chacha_state, tmp);
- memcpy(buf, tmp, nbytes);
+ memcpy(buf, tmp, len);
memzero_explicit(tmp, sizeof(tmp));
break;
}
@@ -502,7 +376,7 @@ static void _get_random_bytes(void *buf, size_t nbytes)
chacha20_block(chacha_state, buf);
if (unlikely(chacha_state[12] == 0))
++chacha_state[13];
- nbytes -= CHACHA_BLOCK_SIZE;
+ len -= CHACHA_BLOCK_SIZE;
buf += CHACHA_BLOCK_SIZE;
}
@@ -512,29 +386,24 @@ static void _get_random_bytes(void *buf, size_t nbytes)
/*
* This function is the exported kernel interface. It returns some
* number of good random numbers, suitable for key generation, seeding
- * TCP sequence numbers, etc. It does not rely on the hardware random
- * number generator. For random bytes direct from the hardware RNG
- * (when available), use get_random_bytes_arch(). In order to ensure
- * that the randomness provided by this function is okay, the function
- * wait_for_random_bytes() should be called and return 0 at least once
- * at any point prior.
+ * TCP sequence numbers, etc. In order to ensure that the randomness
+ * by this function is okay, the function wait_for_random_bytes()
+ * should be called and return 0 at least once at any point prior.
*/
-void get_random_bytes(void *buf, size_t nbytes)
+void get_random_bytes(void *buf, size_t len)
{
- static void *previous;
-
- warn_unseeded_randomness(&previous);
- _get_random_bytes(buf, nbytes);
+ warn_unseeded_randomness();
+ _get_random_bytes(buf, len);
}
EXPORT_SYMBOL(get_random_bytes);
-static ssize_t get_random_bytes_user(void __user *buf, size_t nbytes)
+static ssize_t get_random_bytes_user(struct iov_iter *iter)
{
- size_t len, left, ret = 0;
u32 chacha_state[CHACHA_STATE_WORDS];
- u8 output[CHACHA_BLOCK_SIZE];
+ u8 block[CHACHA_BLOCK_SIZE];
+ size_t ret = 0, copied;
- if (!nbytes)
+ if (unlikely(!iov_iter_count(iter)))
return 0;
/*
@@ -548,30 +417,22 @@ static ssize_t get_random_bytes_user(void __user *buf, size_t nbytes)
* use chacha_state after, so we can simply return those bytes to
* the user directly.
*/
- if (nbytes <= CHACHA_KEY_SIZE) {
- ret = nbytes - copy_to_user(buf, &chacha_state[4], nbytes);
+ if (iov_iter_count(iter) <= CHACHA_KEY_SIZE) {
+ ret = copy_to_iter(&chacha_state[4], CHACHA_KEY_SIZE, iter);
goto out_zero_chacha;
}
for (;;) {
- chacha20_block(chacha_state, output);
+ chacha20_block(chacha_state, block);
if (unlikely(chacha_state[12] == 0))
++chacha_state[13];
- len = min_t(size_t, nbytes, CHACHA_BLOCK_SIZE);
- left = copy_to_user(buf, output, len);
- if (left) {
- ret += len - left;
- break;
- }
-
- buf += len;
- ret += len;
- nbytes -= len;
- if (!nbytes)
+ copied = copy_to_iter(block, sizeof(block), iter);
+ ret += copied;
+ if (!iov_iter_count(iter) || copied != sizeof(block))
break;
- BUILD_BUG_ON(PAGE_SIZE % CHACHA_BLOCK_SIZE != 0);
+ BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
if (ret % PAGE_SIZE == 0) {
if (signal_pending(current))
break;
@@ -579,7 +440,7 @@ static ssize_t get_random_bytes_user(void __user *buf, size_t nbytes)
}
}
- memzero_explicit(output, sizeof(output));
+ memzero_explicit(block, sizeof(block));
out_zero_chacha:
memzero_explicit(chacha_state, sizeof(chacha_state));
return ret ? ret : -EFAULT;
@@ -591,98 +452,69 @@ out_zero_chacha:
* provided by this function is okay, the function wait_for_random_bytes()
* should be called and return 0 at least once at any point prior.
*/
-struct batched_entropy {
- union {
- /*
- * We make this 1.5x a ChaCha block, so that we get the
- * remaining 32 bytes from fast key erasure, plus one full
- * block from the detached ChaCha state. We can increase
- * the size of this later if needed so long as we keep the
- * formula of (integer_blocks + 0.5) * CHACHA_BLOCK_SIZE.
- */
- u64 entropy_u64[CHACHA_BLOCK_SIZE * 3 / (2 * sizeof(u64))];
- u32 entropy_u32[CHACHA_BLOCK_SIZE * 3 / (2 * sizeof(u32))];
- };
- local_lock_t lock;
- unsigned long generation;
- unsigned int position;
-};
-
-static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = {
- .lock = INIT_LOCAL_LOCK(batched_entropy_u64.lock),
- .position = UINT_MAX
-};
-
-u64 get_random_u64(void)
-{
- u64 ret;
- unsigned long flags;
- struct batched_entropy *batch;
- static void *previous;
- unsigned long next_gen;
-
- warn_unseeded_randomness(&previous);
-
- local_lock_irqsave(&batched_entropy_u64.lock, flags);
- batch = raw_cpu_ptr(&batched_entropy_u64);
-
- next_gen = READ_ONCE(base_crng.generation);
- if (batch->position >= ARRAY_SIZE(batch->entropy_u64) ||
- next_gen != batch->generation) {
- _get_random_bytes(batch->entropy_u64, sizeof(batch->entropy_u64));
- batch->position = 0;
- batch->generation = next_gen;
- }
-
- ret = batch->entropy_u64[batch->position];
- batch->entropy_u64[batch->position] = 0;
- ++batch->position;
- local_unlock_irqrestore(&batched_entropy_u64.lock, flags);
- return ret;
-}
-EXPORT_SYMBOL(get_random_u64);
-
-static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = {
- .lock = INIT_LOCAL_LOCK(batched_entropy_u32.lock),
- .position = UINT_MAX
-};
-
-u32 get_random_u32(void)
-{
- u32 ret;
- unsigned long flags;
- struct batched_entropy *batch;
- static void *previous;
- unsigned long next_gen;
-
- warn_unseeded_randomness(&previous);
-
- local_lock_irqsave(&batched_entropy_u32.lock, flags);
- batch = raw_cpu_ptr(&batched_entropy_u32);
-
- next_gen = READ_ONCE(base_crng.generation);
- if (batch->position >= ARRAY_SIZE(batch->entropy_u32) ||
- next_gen != batch->generation) {
- _get_random_bytes(batch->entropy_u32, sizeof(batch->entropy_u32));
- batch->position = 0;
- batch->generation = next_gen;
- }
-
- ret = batch->entropy_u32[batch->position];
- batch->entropy_u32[batch->position] = 0;
- ++batch->position;
- local_unlock_irqrestore(&batched_entropy_u32.lock, flags);
- return ret;
-}
-EXPORT_SYMBOL(get_random_u32);
+#define DEFINE_BATCHED_ENTROPY(type) \
+struct batch_ ##type { \
+ /* \
+ * We make this 1.5x a ChaCha block, so that we get the \
+ * remaining 32 bytes from fast key erasure, plus one full \
+ * block from the detached ChaCha state. We can increase \
+ * the size of this later if needed so long as we keep the \
+ * formula of (integer_blocks + 0.5) * CHACHA_BLOCK_SIZE. \
+ */ \
+ type entropy[CHACHA_BLOCK_SIZE * 3 / (2 * sizeof(type))]; \
+ local_lock_t lock; \
+ unsigned long generation; \
+ unsigned int position; \
+}; \
+ \
+static DEFINE_PER_CPU(struct batch_ ##type, batched_entropy_ ##type) = { \
+ .lock = INIT_LOCAL_LOCK(batched_entropy_ ##type.lock), \
+ .position = UINT_MAX \
+}; \
+ \
+type get_random_ ##type(void) \
+{ \
+ type ret; \
+ unsigned long flags; \
+ struct batch_ ##type *batch; \
+ unsigned long next_gen; \
+ \
+ warn_unseeded_randomness(); \
+ \
+ if (!crng_ready()) { \
+ _get_random_bytes(&ret, sizeof(ret)); \
+ return ret; \
+ } \
+ \
+ local_lock_irqsave(&batched_entropy_ ##type.lock, flags); \
+ batch = raw_cpu_ptr(&batched_entropy_##type); \
+ \
+ next_gen = READ_ONCE(base_crng.generation); \
+ if (batch->position >= ARRAY_SIZE(batch->entropy) || \
+ next_gen != batch->generation) { \
+ _get_random_bytes(batch->entropy, sizeof(batch->entropy)); \
+ batch->position = 0; \
+ batch->generation = next_gen; \
+ } \
+ \
+ ret = batch->entropy[batch->position]; \
+ batch->entropy[batch->position] = 0; \
+ ++batch->position; \
+ local_unlock_irqrestore(&batched_entropy_ ##type.lock, flags); \
+ return ret; \
+} \
+EXPORT_SYMBOL(get_random_ ##type);
+
+DEFINE_BATCHED_ENTROPY(u64)
+DEFINE_BATCHED_ENTROPY(u32)
#ifdef CONFIG_SMP
/*
* This function is called when the CPU is coming up, with entry
* CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP.
*/
-int random_prepare_cpu(unsigned int cpu)
+int __cold random_prepare_cpu(unsigned int cpu)
{
/*
* When the cpu comes back online, immediately invalidate both
@@ -696,65 +528,6 @@ int random_prepare_cpu(unsigned int cpu)
}
#endif
-/**
- * randomize_page - Generate a random, page aligned address
- * @start: The smallest acceptable address the caller will take.
- * @range: The size of the area, starting at @start, within which the
- * random address must fall.
- *
- * If @start + @range would overflow, @range is capped.
- *
- * NOTE: Historical use of randomize_range, which this replaces, presumed that
- * @start was already page aligned. We now align it regardless.
- *
- * Return: A page aligned address within [start, start + range). On error,
- * @start is returned.
- */
-unsigned long randomize_page(unsigned long start, unsigned long range)
-{
- if (!PAGE_ALIGNED(start)) {
- range -= PAGE_ALIGN(start) - start;
- start = PAGE_ALIGN(start);
- }
-
- if (start > ULONG_MAX - range)
- range = ULONG_MAX - start;
-
- range >>= PAGE_SHIFT;
-
- if (range == 0)
- return start;
-
- return start + (get_random_long() % range << PAGE_SHIFT);
-}
-
-/*
- * This function will use the architecture-specific hardware random
- * number generator if it is available. It is not recommended for
- * use. Use get_random_bytes() instead. It returns the number of
- * bytes filled in.
- */
-size_t __must_check get_random_bytes_arch(void *buf, size_t nbytes)
-{
- size_t left = nbytes;
- u8 *p = buf;
-
- while (left) {
- unsigned long v;
- size_t chunk = min_t(size_t, left, sizeof(unsigned long));
-
- if (!arch_get_random_long(&v))
- break;
-
- memcpy(p, &v, chunk);
- p += chunk;
- left -= chunk;
- }
-
- return nbytes - left;
-}
-EXPORT_SYMBOL(get_random_bytes_arch);
-
/**********************************************************************
*
@@ -762,33 +535,28 @@ EXPORT_SYMBOL(get_random_bytes_arch);
*
* Callers may add entropy via:
*
- * static void mix_pool_bytes(const void *in, size_t nbytes)
+ * static void mix_pool_bytes(const void *buf, size_t len)
*
* After which, if added entropy should be credited:
*
- * static void credit_entropy_bits(size_t nbits)
+ * static void credit_init_bits(size_t bits)
*
- * Finally, extract entropy via these two, with the latter one
- * setting the entropy count to zero and extracting only if there
- * is POOL_MIN_BITS entropy credited prior or force is true:
+ * Finally, extract entropy via:
*
- * static void extract_entropy(void *buf, size_t nbytes)
- * static bool drain_entropy(void *buf, size_t nbytes, bool force)
+ * static void extract_entropy(void *buf, size_t len)
*
**********************************************************************/
enum {
POOL_BITS = BLAKE2S_HASH_SIZE * 8,
- POOL_MIN_BITS = POOL_BITS /* No point in settling for less. */
+ POOL_READY_BITS = POOL_BITS, /* When crng_init->CRNG_READY */
+ POOL_EARLY_BITS = POOL_READY_BITS / 2 /* When crng_init->CRNG_EARLY */
};
-/* For notifying userspace should write into /dev/random. */
-static DECLARE_WAIT_QUEUE_HEAD(random_write_wait);
-
static struct {
struct blake2s_state hash;
spinlock_t lock;
- unsigned int entropy_count;
+ unsigned int init_bits;
} input_pool = {
.hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE),
BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4,
@@ -797,48 +565,30 @@ static struct {
.lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
};
-static void _mix_pool_bytes(const void *in, size_t nbytes)
+static void _mix_pool_bytes(const void *buf, size_t len)
{
- blake2s_update(&input_pool.hash, in, nbytes);
+ blake2s_update(&input_pool.hash, buf, len);
}
/*
- * This function adds bytes into the entropy "pool". It does not
- * update the entropy estimate. The caller should call
- * credit_entropy_bits if this is appropriate.
+ * This function adds bytes into the input pool. It does not
+ * update the initialization bit counter; the caller should call
+ * credit_init_bits if this is appropriate.
*/
-static void mix_pool_bytes(const void *in, size_t nbytes)
+static void mix_pool_bytes(const void *buf, size_t len)
{
unsigned long flags;
spin_lock_irqsave(&input_pool.lock, flags);
- _mix_pool_bytes(in, nbytes);
+ _mix_pool_bytes(buf, len);
spin_unlock_irqrestore(&input_pool.lock, flags);
}
-static void credit_entropy_bits(size_t nbits)
-{
- unsigned int entropy_count, orig, add;
-
- if (!nbits)
- return;
-
- add = min_t(size_t, nbits, POOL_BITS);
-
- do {
- orig = READ_ONCE(input_pool.entropy_count);
- entropy_count = min_t(unsigned int, POOL_BITS, orig + add);
- } while (cmpxchg(&input_pool.entropy_count, orig, entropy_count) != orig);
-
- if (!crng_ready() && entropy_count >= POOL_MIN_BITS)
- crng_reseed(false);
-}
-
/*
* This is an HKDF-like construction for using the hashed collected entropy
* as a PRF key, that's then expanded block-by-block.
*/
-static void extract_entropy(void *buf, size_t nbytes)
+static void extract_entropy(void *buf, size_t len)
{
unsigned long flags;
u8 seed[BLAKE2S_HASH_SIZE], next_key[BLAKE2S_HASH_SIZE];
@@ -867,12 +617,12 @@ static void extract_entropy(void *buf, size_t nbytes)
spin_unlock_irqrestore(&input_pool.lock, flags);
memzero_explicit(next_key, sizeof(next_key));
- while (nbytes) {
- i = min_t(size_t, nbytes, BLAKE2S_HASH_SIZE);
+ while (len) {
+ i = min_t(size_t, len, BLAKE2S_HASH_SIZE);
/* output = HASHPRF(seed, RDSEED || ++counter) */
++block.counter;
blake2s(buf, (u8 *)&block, seed, i, sizeof(block), sizeof(seed));
- nbytes -= i;
+ len -= i;
buf += i;
}
@@ -880,23 +630,42 @@ static void extract_entropy(void *buf, size_t nbytes)
memzero_explicit(&block, sizeof(block));
}
-/*
- * First we make sure we have POOL_MIN_BITS of entropy in the pool unless force
- * is true, and then we set the entropy count to zero (but don't actually touch
- * any data). Only then can we extract a new key with extract_entropy().
- */
-static bool drain_entropy(void *buf, size_t nbytes, bool force)
+#define credit_init_bits(bits) if (!crng_ready()) _credit_init_bits(bits)
+
+static void __cold _credit_init_bits(size_t bits)
{
- unsigned int entropy_count;
+ static struct execute_work set_ready;
+ unsigned int new, orig, add;
+ unsigned long flags;
+
+ if (!bits)
+ return;
+
+ add = min_t(size_t, bits, POOL_BITS);
+
do {
- entropy_count = READ_ONCE(input_pool.entropy_count);
- if (!force && entropy_count < POOL_MIN_BITS)
- return false;
- } while (cmpxchg(&input_pool.entropy_count, entropy_count, 0) != entropy_count);
- extract_entropy(buf, nbytes);
- wake_up_interruptible(&random_write_wait);
- kill_fasync(&fasync, SIGIO, POLL_OUT);
- return true;
+ orig = READ_ONCE(input_pool.init_bits);
+ new = min_t(unsigned int, POOL_BITS, orig + add);
+ } while (cmpxchg(&input_pool.init_bits, orig, new) != orig);
+
+ if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
+ crng_reseed(); /* Sets crng_init to CRNG_READY under base_crng.lock. */
+ execute_in_process_context(crng_set_ready, &set_ready);
+ wake_up_interruptible(&crng_init_wait);
+ kill_fasync(&fasync, SIGIO, POLL_IN);
+ pr_notice("crng init done\n");
+ if (urandom_warning.missed)
+ pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
+ urandom_warning.missed);
+ } else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
+ spin_lock_irqsave(&base_crng.lock, flags);
+ /* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
+ if (crng_init == CRNG_EMPTY) {
+ extract_entropy(base_crng.key, sizeof(base_crng.key));
+ crng_init = CRNG_EARLY;
+ }
+ spin_unlock_irqrestore(&base_crng.lock, flags);
+ }
}
@@ -907,15 +676,13 @@ static bool drain_entropy(void *buf, size_t nbytes, bool force)
* The following exported functions are used for pushing entropy into
* the above entropy accumulation routines:
*
- * void add_device_randomness(const void *buf, size_t size);
- * void add_input_randomness(unsigned int type, unsigned int code,
- * unsigned int value);
- * void add_disk_randomness(struct gendisk *disk);
- * void add_hwgenerator_randomness(const void *buffer, size_t count,
- * size_t entropy);
- * void add_bootloader_randomness(const void *buf, size_t size);
- * void add_vmfork_randomness(const void *unique_vm_id, size_t size);
+ * void add_device_randomness(const void *buf, size_t len);
+ * void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy);
+ * void add_bootloader_randomness(const void *buf, size_t len);
+ * void add_vmfork_randomness(const void *unique_vm_id, size_t len);
* void add_interrupt_randomness(int irq);
+ * void add_input_randomness(unsigned int type, unsigned int code, unsigned int value);
+ * void add_disk_randomness(struct gendisk *disk);
*
* add_device_randomness() adds data to the input pool that
* is likely to differ between two devices (or possibly even per boot).
@@ -925,26 +692,13 @@ static bool drain_entropy(void *buf, size_t nbytes, bool force)
* that might otherwise be identical and have very little entropy
* available to them (particularly common in the embedded world).
*
- * add_input_randomness() uses the input layer interrupt timing, as well
- * as the event type information from the hardware.
- *
- * add_disk_randomness() uses what amounts to the seek time of block
- * layer request events, on a per-disk_devt basis, as input to the
- * entropy pool. Note that high-speed solid state drives with very low
- * seek times do not make for good sources of entropy, as their seek
- * times are usually fairly consistent.
- *
- * The above two routines try to estimate how many bits of entropy
- * to credit. They do this by keeping track of the first and second
- * order deltas of the event timings.
- *
* add_hwgenerator_randomness() is for true hardware RNGs, and will credit
* entropy as specified by the caller. If the entropy pool is full it will
* block until more entropy is needed.
*
- * add_bootloader_randomness() is the same as add_hwgenerator_randomness() or
- * add_device_randomness(), depending on whether or not the configuration
- * option CONFIG_RANDOM_TRUST_BOOTLOADER is set.
+ * add_bootloader_randomness() is called by bootloader drivers, such as EFI
+ * and device tree, and credits its input depending on whether or not the
+ * configuration option CONFIG_RANDOM_TRUST_BOOTLOADER is set.
*
* add_vmfork_randomness() adds a unique (but not necessarily secret) ID
* representing the current instance of a VM to the pool, without crediting,
@@ -955,8 +709,22 @@ static bool drain_entropy(void *buf, size_t nbytes, bool force)
* as inputs, it feeds the input pool roughly once a second or after 64
* interrupts, crediting 1 bit of entropy for whichever comes first.
*
+ * add_input_randomness() uses the input layer interrupt timing, as well
+ * as the event type information from the hardware.
+ *
+ * add_disk_randomness() uses what amounts to the seek time of block
+ * layer request events, on a per-disk_devt basis, as input to the
+ * entropy pool. Note that high-speed solid state drives with very low
+ * seek times do not make for good sources of entropy, as their seek
+ * times are usually fairly consistent.
+ *
+ * The last two routines try to estimate how many bits of entropy
+ * to credit. They do this by keeping track of the first and second
+ * order deltas of the event timings.
+ *
**********************************************************************/
+static bool used_arch_random;
static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU);
static bool trust_bootloader __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER);
static int __init parse_trust_cpu(char *arg)
@@ -970,52 +738,91 @@ static int __init parse_trust_bootloader(char *arg)
early_param("random.trust_cpu", parse_trust_cpu);
early_param("random.trust_bootloader", parse_trust_bootloader);
+static int random_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
+{
+ unsigned long flags, entropy = random_get_entropy();
+
+ /*
+ * Encode a representation of how long the system has been suspended,
+ * in a way that is distinct from prior system suspends.
+ */
+ ktime_t stamps[] = { ktime_get(), ktime_get_boottime(), ktime_get_real() };
+
+ spin_lock_irqsave(&input_pool.lock, flags);
+ _mix_pool_bytes(&action, sizeof(action));
+ _mix_pool_bytes(stamps, sizeof(stamps));
+ _mix_pool_bytes(&entropy, sizeof(entropy));
+ spin_unlock_irqrestore(&input_pool.lock, flags);
+
+ if (crng_ready() && (action == PM_RESTORE_PREPARE ||
+ (action == PM_POST_SUSPEND &&
+ !IS_ENABLED(CONFIG_PM_AUTOSLEEP) && !IS_ENABLED(CONFIG_ANDROID)))) {
+ crng_reseed();
+ pr_notice("crng reseeded on system resumption\n");
+ }
+ return 0;
+}
+
+static struct notifier_block pm_notifier = { .notifier_call = random_pm_notification };
+
/*
* The first collection of entropy occurs at system boot while interrupts
- * are still turned off. Here we push in RDSEED, a timestamp, and utsname().
- * Depending on the above configuration knob, RDSEED may be considered
- * sufficient for initialization. Note that much earlier setup may already
- * have pushed entropy into the input pool by the time we get here.
+ * are still turned off. Here we push in latent entropy, RDSEED, a timestamp,
+ * utsname(), and the command line. Depending on the above configuration knob,
+ * RDSEED may be considered sufficient for initialization. Note that much
+ * earlier setup may already have pushed entropy into the input pool by the
+ * time we get here.
*/
-int __init rand_initialize(void)
+int __init random_init(const char *command_line)
{
- size_t i;
ktime_t now = ktime_get_real();
- bool arch_init = true;
- unsigned long rv;
+ unsigned int i, arch_bytes;
+ unsigned long entropy;
#if defined(LATENT_ENTROPY_PLUGIN)
static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy;
_mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed));
#endif
- for (i = 0; i < BLAKE2S_BLOCK_SIZE; i += sizeof(rv)) {
- if (!arch_get_random_seed_long_early(&rv) &&
- !arch_get_random_long_early(&rv)) {
- rv = random_get_entropy();
- arch_init = false;
+ for (i = 0, arch_bytes = BLAKE2S_BLOCK_SIZE;
+ i < BLAKE2S_BLOCK_SIZE; i += sizeof(entropy)) {
+ if (!arch_get_random_seed_long_early(&entropy) &&
+ !arch_get_random_long_early(&entropy)) {
+ entropy = random_get_entropy();
+ arch_bytes -= sizeof(entropy);
}
- _mix_pool_bytes(&rv, sizeof(rv));
+ _mix_pool_bytes(&entropy, sizeof(entropy));
}
_mix_pool_bytes(&now, sizeof(now));
_mix_pool_bytes(utsname(), sizeof(*(utsname())));
+ _mix_pool_bytes(command_line, strlen(command_line));
+ add_latent_entropy();
- extract_entropy(base_crng.key, sizeof(base_crng.key));
- ++base_crng.generation;
+ if (crng_ready())
+ crng_reseed();
+ else if (trust_cpu)
+ credit_init_bits(arch_bytes * 8);
+ used_arch_random = arch_bytes * 8 >= POOL_READY_BITS;
- if (arch_init && trust_cpu && !crng_ready()) {
- crng_init = 2;
- pr_notice("crng init done (trusting CPU's manufacturer)\n");
- }
+ WARN_ON(register_pm_notifier(&pm_notifier));
- if (ratelimit_disable) {
- urandom_warning.interval = 0;
- unseeded_warning.interval = 0;
- }
+ WARN(!random_get_entropy(), "Missing cycle counter and fallback timer; RNG "
+ "entropy collection will consequently suffer.");
return 0;
}
/*
+ * Returns whether arch randomness has been mixed into the initial
+ * state of the RNG, regardless of whether or not that randomness
+ * was credited. Knowing this is only good for a very limited set
+ * of uses, such as early init printk pointer obfuscation.
+ */
+bool rng_has_arch_random(void)
+{
+ return used_arch_random;
+}
+
+/*
* Add device- or boot-specific data to the input pool to help
* initialize it.
*
@@ -1023,164 +830,46 @@ int __init rand_initialize(void)
* the entropy pool having similar initial state across largely
* identical devices.
*/
-void add_device_randomness(const void *buf, size_t size)
+void add_device_randomness(const void *buf, size_t len)
{
- unsigned long cycles = random_get_entropy();
- unsigned long flags, now = jiffies;
-
- if (crng_init == 0 && size)
- crng_pre_init_inject(buf, size, false);
+ unsigned long entropy = random_get_entropy();
+ unsigned long flags;
spin_lock_irqsave(&input_pool.lock, flags);
- _mix_pool_bytes(&cycles, sizeof(cycles));
- _mix_pool_bytes(&now, sizeof(now));
- _mix_pool_bytes(buf, size);
+ _mix_pool_bytes(&entropy, sizeof(entropy));
+ _mix_pool_bytes(buf, len);
spin_unlock_irqrestore(&input_pool.lock, flags);
}
EXPORT_SYMBOL(add_device_randomness);
-/* There is one of these per entropy source */
-struct timer_rand_state {
- unsigned long last_time;
- long last_delta, last_delta2;
-};
-
-/*
- * This function adds entropy to the entropy "pool" by using timing
- * delays. It uses the timer_rand_state structure to make an estimate
- * of how many bits of entropy this call has added to the pool.
- *
- * The number "num" is also added to the pool - it should somehow describe
- * the type of event which just happened. This is currently 0-255 for
- * keyboard scan codes, and 256 upwards for interrupts.
- */
-static void add_timer_randomness(struct timer_rand_state *state, unsigned int num)
-{
- unsigned long cycles = random_get_entropy(), now = jiffies, flags;
- long delta, delta2, delta3;
-
- spin_lock_irqsave(&input_pool.lock, flags);
- _mix_pool_bytes(&cycles, sizeof(cycles));
- _mix_pool_bytes(&now, sizeof(now));
- _mix_pool_bytes(&num, sizeof(num));
- spin_unlock_irqrestore(&input_pool.lock, flags);
-
- /*
- * Calculate number of bits of randomness we probably added.
- * We take into account the first, second and third-order deltas
- * in order to make our estimate.
- */
- delta = now - READ_ONCE(state->last_time);
- WRITE_ONCE(state->last_time, now);
-
- delta2 = delta - READ_ONCE(state->last_delta);
- WRITE_ONCE(state->last_delta, delta);
-
- delta3 = delta2 - READ_ONCE(state->last_delta2);
- WRITE_ONCE(state->last_delta2, delta2);
-
- if (delta < 0)
- delta = -delta;
- if (delta2 < 0)
- delta2 = -delta2;
- if (delta3 < 0)
- delta3 = -delta3;
- if (delta > delta2)
- delta = delta2;
- if (delta > delta3)
- delta = delta3;
-
- /*
- * delta is now minimum absolute delta.
- * Round down by 1 bit on general principles,
- * and limit entropy estimate to 12 bits.
- */
- credit_entropy_bits(min_t(unsigned int, fls(delta >> 1), 11));
-}
-
-void add_input_randomness(unsigned int type, unsigned int code,
- unsigned int value)
-{
- static unsigned char last_value;
- static struct timer_rand_state input_timer_state = { INITIAL_JIFFIES };
-
- /* Ignore autorepeat and the like. */
- if (value == last_value)
- return;
-
- last_value = value;
- add_timer_randomness(&input_timer_state,
- (type << 4) ^ code ^ (code >> 4) ^ value);
-}
-EXPORT_SYMBOL_GPL(add_input_randomness);
-
-#ifdef CONFIG_BLOCK
-void add_disk_randomness(struct gendisk *disk)
-{
- if (!disk || !disk->random)
- return;
- /* First major is 1, so we get >= 0x200 here. */
- add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
-}
-EXPORT_SYMBOL_GPL(add_disk_randomness);
-
-void rand_initialize_disk(struct gendisk *disk)
-{
- struct timer_rand_state *state;
-
- /*
- * If kzalloc returns null, we just won't use that entropy
- * source.
- */
- state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL);
- if (state) {
- state->last_time = INITIAL_JIFFIES;
- disk->random = state;
- }
-}
-#endif
-
/*
* Interface for in-kernel drivers of true hardware RNGs.
* Those devices may produce endless random bits and will be throttled
* when our pool is full.
*/
-void add_hwgenerator_randomness(const void *buffer, size_t count,
- size_t entropy)
+void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy)
{
- if (unlikely(crng_init == 0 && entropy < POOL_MIN_BITS)) {
- crng_pre_init_inject(buffer, count, true);
- mix_pool_bytes(buffer, count);
- return;
- }
+ mix_pool_bytes(buf, len);
+ credit_init_bits(entropy);
/*
- * Throttle writing if we're above the trickle threshold.
- * We'll be woken up again once below POOL_MIN_BITS, when
- * the calling thread is about to terminate, or once
- * CRNG_RESEED_INTERVAL has elapsed.
+ * Throttle writing to once every CRNG_RESEED_INTERVAL, unless
+ * we're not yet initialized.
*/
- wait_event_interruptible_timeout(random_write_wait,
- !system_wq || kthread_should_stop() ||
- input_pool.entropy_count < POOL_MIN_BITS,
- CRNG_RESEED_INTERVAL);
- mix_pool_bytes(buffer, count);
- credit_entropy_bits(entropy);
+ if (!kthread_should_stop() && crng_ready())
+ schedule_timeout_interruptible(CRNG_RESEED_INTERVAL);
}
EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
/*
- * Handle random seed passed by bootloader.
- * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise
- * it would be regarded as device data.
- * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER.
+ * Handle random seed passed by bootloader, and credit it if
+ * CONFIG_RANDOM_TRUST_BOOTLOADER is set.
*/
-void add_bootloader_randomness(const void *buf, size_t size)
+void __cold add_bootloader_randomness(const void *buf, size_t len)
{
+ mix_pool_bytes(buf, len);
if (trust_bootloader)
- add_hwgenerator_randomness(buf, size, size * 8);
- else
- add_device_randomness(buf, size);
+ credit_init_bits(len * 8);
}
EXPORT_SYMBOL_GPL(add_bootloader_randomness);
@@ -1192,11 +881,11 @@ static BLOCKING_NOTIFIER_HEAD(vmfork_chain);
* don't credit it, but we do immediately force a reseed after so
* that it's used by the crng posthaste.
*/
-void add_vmfork_randomness(const void *unique_vm_id, size_t size)
+void __cold add_vmfork_randomness(const void *unique_vm_id, size_t len)
{
- add_device_randomness(unique_vm_id, size);
+ add_device_randomness(unique_vm_id, len);
if (crng_ready()) {
- crng_reseed(true);
+ crng_reseed();
pr_notice("crng reseeded due to virtual machine fork\n");
}
blocking_notifier_call_chain(&vmfork_chain, 0, NULL);
@@ -1205,13 +894,13 @@ void add_vmfork_randomness(const void *unique_vm_id, size_t size)
EXPORT_SYMBOL_GPL(add_vmfork_randomness);
#endif
-int register_random_vmfork_notifier(struct notifier_block *nb)
+int __cold register_random_vmfork_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_register(&vmfork_chain, nb);
}
EXPORT_SYMBOL_GPL(register_random_vmfork_notifier);
-int unregister_random_vmfork_notifier(struct notifier_block *nb)
+int __cold unregister_random_vmfork_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_unregister(&vmfork_chain, nb);
}
@@ -1223,17 +912,15 @@ struct fast_pool {
unsigned long pool[4];
unsigned long last;
unsigned int count;
- u16 reg_idx;
};
static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = {
#ifdef CONFIG_64BIT
- /* SipHash constants */
- .pool = { 0x736f6d6570736575UL, 0x646f72616e646f6dUL,
- 0x6c7967656e657261UL, 0x7465646279746573UL }
+#define FASTMIX_PERM SIPHASH_PERMUTATION
+ .pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 }
#else
- /* HalfSipHash constants */
- .pool = { 0, 0, 0x6c796765U, 0x74656462U }
+#define FASTMIX_PERM HSIPHASH_PERMUTATION
+ .pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 }
#endif
};
@@ -1241,27 +928,16 @@ static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = {
* This is [Half]SipHash-1-x, starting from an empty key. Because
* the key is fixed, it assumes that its inputs are non-malicious,
* and therefore this has no security on its own. s represents the
- * 128 or 256-bit SipHash state, while v represents a 128-bit input.
+ * four-word SipHash state, while v represents a two-word input.
*/
-static void fast_mix(unsigned long s[4], const unsigned long *v)
+static void fast_mix(unsigned long s[4], unsigned long v1, unsigned long v2)
{
- size_t i;
-
- for (i = 0; i < 16 / sizeof(long); ++i) {
- s[3] ^= v[i];
-#ifdef CONFIG_64BIT
- s[0] += s[1]; s[1] = rol64(s[1], 13); s[1] ^= s[0]; s[0] = rol64(s[0], 32);
- s[2] += s[3]; s[3] = rol64(s[3], 16); s[3] ^= s[2];
- s[0] += s[3]; s[3] = rol64(s[3], 21); s[3] ^= s[0];
- s[2] += s[1]; s[1] = rol64(s[1], 17); s[1] ^= s[2]; s[2] = rol64(s[2], 32);
-#else
- s[0] += s[1]; s[1] = rol32(s[1], 5); s[1] ^= s[0]; s[0] = rol32(s[0], 16);
- s[2] += s[3]; s[3] = rol32(s[3], 8); s[3] ^= s[2];
- s[0] += s[3]; s[3] = rol32(s[3], 7); s[3] ^= s[0];
- s[2] += s[1]; s[1] = rol32(s[1], 13); s[1] ^= s[2]; s[2] = rol32(s[2], 16);
-#endif
- s[0] ^= v[i];
- }
+ s[3] ^= v1;
+ FASTMIX_PERM(s[0], s[1], s[2], s[3]);
+ s[0] ^= v1;
+ s[3] ^= v2;
+ FASTMIX_PERM(s[0], s[1], s[2], s[3]);
+ s[0] ^= v2;
}
#ifdef CONFIG_SMP
@@ -1269,7 +945,7 @@ static void fast_mix(unsigned long s[4], const unsigned long *v)
* This function is called when the CPU has just come online, with
* entry CPUHP_AP_RANDOM_ONLINE, just after CPUHP_AP_WORKQUEUE_ONLINE.
*/
-int random_online_cpu(unsigned int cpu)
+int __cold random_online_cpu(unsigned int cpu)
{
/*
* During CPU shutdown and before CPU onlining, add_interrupt_
@@ -1287,33 +963,18 @@ int random_online_cpu(unsigned int cpu)
}
#endif
-static unsigned long get_reg(struct fast_pool *f, struct pt_regs *regs)
-{
- unsigned long *ptr = (unsigned long *)regs;
- unsigned int idx;
-
- if (regs == NULL)
- return 0;
- idx = READ_ONCE(f->reg_idx);
- if (idx >= sizeof(struct pt_regs) / sizeof(unsigned long))
- idx = 0;
- ptr += idx++;
- WRITE_ONCE(f->reg_idx, idx);
- return *ptr;
-}
-
static void mix_interrupt_randomness(struct work_struct *work)
{
struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix);
/*
- * The size of the copied stack pool is explicitly 16 bytes so that we
- * tax mix_pool_byte()'s compression function the same amount on all
- * platforms. This means on 64-bit we copy half the pool into this,
- * while on 32-bit we copy all of it. The entropy is supposed to be
- * sufficiently dispersed between bits that in the sponge-like
- * half case, on average we don't wind up "losing" some.
+ * The size of the copied stack pool is explicitly 2 longs so that we
+ * only ever ingest half of the siphash output each time, retaining
+ * the other half as the next "key" that carries over. The entropy is
+ * supposed to be sufficiently dispersed between bits so on average
+ * we don't wind up "losing" some.
*/
- u8 pool[16];
+ unsigned long pool[2];
+ unsigned int count;
/* Check to see if we're running on the wrong CPU due to hotplug. */
local_irq_disable();
@@ -1327,17 +988,13 @@ static void mix_interrupt_randomness(struct work_struct *work)
* consistent view, before we reenable irqs again.
*/
memcpy(pool, fast_pool->pool, sizeof(pool));
+ count = fast_pool->count;
fast_pool->count = 0;
fast_pool->last = jiffies;
local_irq_enable();
- if (unlikely(crng_init == 0)) {
- crng_pre_init_inject(pool, sizeof(pool), true);
- mix_pool_bytes(pool, sizeof(pool));
- } else {
- mix_pool_bytes(pool, sizeof(pool));
- credit_entropy_bits(1);
- }
+ mix_pool_bytes(pool, sizeof(pool));
+ credit_init_bits(max(1u, (count & U16_MAX) / 64));
memzero_explicit(pool, sizeof(pool));
}
@@ -1345,37 +1002,19 @@ static void mix_interrupt_randomness(struct work_struct *work)
void add_interrupt_randomness(int irq)
{
enum { MIX_INFLIGHT = 1U << 31 };
- unsigned long cycles = random_get_entropy(), now = jiffies;
+ unsigned long entropy = random_get_entropy();
struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
struct pt_regs *regs = get_irq_regs();
unsigned int new_count;
- union {
- u32 u32[4];
- u64 u64[2];
- unsigned long longs[16 / sizeof(long)];
- } irq_data;
-
- if (cycles == 0)
- cycles = get_reg(fast_pool, regs);
-
- if (sizeof(unsigned long) == 8) {
- irq_data.u64[0] = cycles ^ rol64(now, 32) ^ irq;
- irq_data.u64[1] = regs ? instruction_pointer(regs) : _RET_IP_;
- } else {
- irq_data.u32[0] = cycles ^ irq;
- irq_data.u32[1] = now;
- irq_data.u32[2] = regs ? instruction_pointer(regs) : _RET_IP_;
- irq_data.u32[3] = get_reg(fast_pool, regs);
- }
- fast_mix(fast_pool->pool, irq_data.longs);
+ fast_mix(fast_pool->pool, entropy,
+ (regs ? instruction_pointer(regs) : _RET_IP_) ^ swab(irq));
new_count = ++fast_pool->count;
if (new_count & MIX_INFLIGHT)
return;
- if (new_count < 64 && (!time_after(now, fast_pool->last + HZ) ||
- unlikely(crng_init == 0)))
+ if (new_count < 64 && !time_is_before_jiffies(fast_pool->last + HZ))
return;
if (unlikely(!fast_pool->mix.func))
@@ -1385,6 +1024,132 @@ void add_interrupt_randomness(int irq)
}
EXPORT_SYMBOL_GPL(add_interrupt_randomness);
+/* There is one of these per entropy source */
+struct timer_rand_state {
+ unsigned long last_time;
+ long last_delta, last_delta2;
+};
+
+/*
+ * This function adds entropy to the entropy "pool" by using timing
+ * delays. It uses the timer_rand_state structure to make an estimate
+ * of how many bits of entropy this call has added to the pool. The
+ * value "num" is also added to the pool; it should somehow describe
+ * the type of event that just happened.
+ */
+static void add_timer_randomness(struct timer_rand_state *state, unsigned int num)
+{
+ unsigned long entropy = random_get_entropy(), now = jiffies, flags;
+ long delta, delta2, delta3;
+ unsigned int bits;
+
+ /*
+ * If we're in a hard IRQ, add_interrupt_randomness() will be called
+ * sometime after, so mix into the fast pool.
+ */
+ if (in_hardirq()) {
+ fast_mix(this_cpu_ptr(&irq_randomness)->pool, entropy, num);
+ } else {
+ spin_lock_irqsave(&input_pool.lock, flags);
+ _mix_pool_bytes(&entropy, sizeof(entropy));
+ _mix_pool_bytes(&num, sizeof(num));
+ spin_unlock_irqrestore(&input_pool.lock, flags);
+ }
+
+ if (crng_ready())
+ return;
+
+ /*
+ * Calculate number of bits of randomness we probably added.
+ * We take into account the first, second and third-order deltas
+ * in order to make our estimate.
+ */
+ delta = now - READ_ONCE(state->last_time);
+ WRITE_ONCE(state->last_time, now);
+
+ delta2 = delta - READ_ONCE(state->last_delta);
+ WRITE_ONCE(state->last_delta, delta);
+
+ delta3 = delta2 - READ_ONCE(state->last_delta2);
+ WRITE_ONCE(state->last_delta2, delta2);
+
+ if (delta < 0)
+ delta = -delta;
+ if (delta2 < 0)
+ delta2 = -delta2;
+ if (delta3 < 0)
+ delta3 = -delta3;
+ if (delta > delta2)
+ delta = delta2;
+ if (delta > delta3)
+ delta = delta3;
+
+ /*
+ * delta is now minimum absolute delta. Round down by 1 bit
+ * on general principles, and limit entropy estimate to 11 bits.
+ */
+ bits = min(fls(delta >> 1), 11);
+
+ /*
+ * As mentioned above, if we're in a hard IRQ, add_interrupt_randomness()
+ * will run after this, which uses a different crediting scheme of 1 bit
+ * per every 64 interrupts. In order to let that function do accounting
+ * close to the one in this function, we credit a full 64/64 bit per bit,
+ * and then subtract one to account for the extra one added.
+ */
+ if (in_hardirq())
+ this_cpu_ptr(&irq_randomness)->count += max(1u, bits * 64) - 1;
+ else
+ _credit_init_bits(bits);
+}
+
+void add_input_randomness(unsigned int type, unsigned int code, unsigned int value)
+{
+ static unsigned char last_value;
+ static struct timer_rand_state input_timer_state = { INITIAL_JIFFIES };
+
+ /* Ignore autorepeat and the like. */
+ if (value == last_value)
+ return;
+
+ last_value = value;
+ add_timer_randomness(&input_timer_state,
+ (type << 4) ^ code ^ (code >> 4) ^ value);
+}
+EXPORT_SYMBOL_GPL(add_input_randomness);
+
+#ifdef CONFIG_BLOCK
+void add_disk_randomness(struct gendisk *disk)
+{
+ if (!disk || !disk->random)
+ return;
+ /* First major is 1, so we get >= 0x200 here. */
+ add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
+}
+EXPORT_SYMBOL_GPL(add_disk_randomness);
+
+void __cold rand_initialize_disk(struct gendisk *disk)
+{
+ struct timer_rand_state *state;
+
+ /*
+ * If kzalloc returns null, we just won't use that entropy
+ * source.
+ */
+ state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL);
+ if (state) {
+ state->last_time = INITIAL_JIFFIES;
+ disk->random = state;
+ }
+}
+#endif
+
+struct entropy_timer_state {
+ unsigned long entropy;
+ struct timer_list timer;
+ unsigned int samples, samples_per_bit;
+};
+
/*
* Each time the timer fires, we expect that we got an unpredictable
* jump in the cycle counter. Even if the timer is running on another
@@ -1398,40 +1163,50 @@ EXPORT_SYMBOL_GPL(add_interrupt_randomness);
*
* So the re-arming always happens in the entropy loop itself.
*/
-static void entropy_timer(struct timer_list *t)
+static void __cold entropy_timer(struct timer_list *timer)
{
- credit_entropy_bits(1);
+ struct entropy_timer_state *state = container_of(timer, struct entropy_timer_state, timer);
+
+ if (++state->samples == state->samples_per_bit) {
+ credit_init_bits(1);
+ state->samples = 0;
+ }
}
/*
* If we have an actual cycle counter, see if we can
* generate enough entropy with timing noise
*/
-static void try_to_generate_entropy(void)
+static void __cold try_to_generate_entropy(void)
{
- struct {
- unsigned long cycles;
- struct timer_list timer;
- } stack;
-
- stack.cycles = random_get_entropy();
-
- /* Slow counter - or none. Don't even bother */
- if (stack.cycles == random_get_entropy())
+ enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = 32 };
+ struct entropy_timer_state stack;
+ unsigned int i, num_different = 0;
+ unsigned long last = random_get_entropy();
+
+ for (i = 0; i < NUM_TRIAL_SAMPLES - 1; ++i) {
+ stack.entropy = random_get_entropy();
+ if (stack.entropy != last)
+ ++num_different;
+ last = stack.entropy;
+ }
+ stack.samples_per_bit = DIV_ROUND_UP(NUM_TRIAL_SAMPLES, num_different + 1);
+ if (stack.samples_per_bit > MAX_SAMPLES_PER_BIT)
return;
+ stack.samples = 0;
timer_setup_on_stack(&stack.timer, entropy_timer, 0);
while (!crng_ready() && !signal_pending(current)) {
if (!timer_pending(&stack.timer))
mod_timer(&stack.timer, jiffies + 1);
- mix_pool_bytes(&stack.cycles, sizeof(stack.cycles));
+ mix_pool_bytes(&stack.entropy, sizeof(stack.entropy));
schedule();
- stack.cycles = random_get_entropy();
+ stack.entropy = random_get_entropy();
}
del_timer_sync(&stack.timer);
destroy_timer_on_stack(&stack.timer);
- mix_pool_bytes(&stack.cycles, sizeof(stack.cycles));
+ mix_pool_bytes(&stack.entropy, sizeof(stack.entropy));
}
@@ -1463,9 +1238,12 @@ static void try_to_generate_entropy(void)
*
**********************************************************************/
-SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, unsigned int,
- flags)
+SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags)
{
+ struct iov_iter iter;
+ struct iovec iov;
+ int ret;
+
if (flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE))
return -EINVAL;
@@ -1476,72 +1254,60 @@ SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, unsigned int,
if ((flags & (GRND_INSECURE | GRND_RANDOM)) == (GRND_INSECURE | GRND_RANDOM))
return -EINVAL;
- if (count > INT_MAX)
- count = INT_MAX;
-
- if (!(flags & GRND_INSECURE) && !crng_ready()) {
- int ret;
-
+ if (!crng_ready() && !(flags & GRND_INSECURE)) {
if (flags & GRND_NONBLOCK)
return -EAGAIN;
ret = wait_for_random_bytes();
if (unlikely(ret))
return ret;
}
- return get_random_bytes_user(buf, count);
+
+ ret = import_single_range(READ, ubuf, len, &iov, &iter);
+ if (unlikely(ret))
+ return ret;
+ return get_random_bytes_user(&iter);
}
static __poll_t random_poll(struct file *file, poll_table *wait)
{
- __poll_t mask;
-
poll_wait(file, &crng_init_wait, wait);
- poll_wait(file, &random_write_wait, wait);
- mask = 0;
- if (crng_ready())
- mask |= EPOLLIN | EPOLLRDNORM;
- if (input_pool.entropy_count < POOL_MIN_BITS)
- mask |= EPOLLOUT | EPOLLWRNORM;
- return mask;
+ return crng_ready() ? EPOLLIN | EPOLLRDNORM : EPOLLOUT | EPOLLWRNORM;
}
-static int write_pool(const char __user *ubuf, size_t count)
+static ssize_t write_pool_user(struct iov_iter *iter)
{
- size_t len;
- int ret = 0;
u8 block[BLAKE2S_BLOCK_SIZE];
+ ssize_t ret = 0;
+ size_t copied;
+
+ if (unlikely(!iov_iter_count(iter)))
+ return 0;
+
+ for (;;) {
+ copied = copy_from_iter(block, sizeof(block), iter);
+ ret += copied;
+ mix_pool_bytes(block, copied);
+ if (!iov_iter_count(iter) || copied != sizeof(block))
+ break;
- while (count) {
- len = min(count, sizeof(block));
- if (copy_from_user(block, ubuf, len)) {
- ret = -EFAULT;
- goto out;
+ BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
+ if (ret % PAGE_SIZE == 0) {
+ if (signal_pending(current))
+ break;
+ cond_resched();
}
- count -= len;
- ubuf += len;
- mix_pool_bytes(block, len);
- cond_resched();
}
-out:
memzero_explicit(block, sizeof(block));
- return ret;
+ return ret ? ret : -EFAULT;
}
-static ssize_t random_write(struct file *file, const char __user *buffer,
- size_t count, loff_t *ppos)
+static ssize_t random_write_iter(struct kiocb *kiocb, struct iov_iter *iter)
{
- int ret;
-
- ret = write_pool(buffer, count);
- if (ret)
- return ret;
-
- return (ssize_t)count;
+ return write_pool_user(iter);
}
-static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes,
- loff_t *ppos)
+static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
{
static int maxwarn = 10;
@@ -1552,37 +1318,38 @@ static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes,
if (!crng_ready())
try_to_generate_entropy();
- if (!crng_ready() && maxwarn > 0) {
- maxwarn--;
- if (__ratelimit(&urandom_warning))
- pr_notice("%s: uninitialized urandom read (%zd bytes read)\n",
- current->comm, nbytes);
+ if (!crng_ready()) {
+ if (!ratelimit_disable && maxwarn <= 0)
+ ++urandom_warning.missed;
+ else if (ratelimit_disable || __ratelimit(&urandom_warning)) {
+ --maxwarn;
+ pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
+ current->comm, iov_iter_count(iter));
+ }
}
- return get_random_bytes_user(buf, nbytes);
+ return get_random_bytes_user(iter);
}
-static ssize_t random_read(struct file *file, char __user *buf, size_t nbytes,
- loff_t *ppos)
+static ssize_t random_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
{
int ret;
ret = wait_for_random_bytes();
if (ret != 0)
return ret;
- return get_random_bytes_user(buf, nbytes);
+ return get_random_bytes_user(iter);
}
static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
{
- int size, ent_count;
int __user *p = (int __user *)arg;
- int retval;
+ int ent_count;
switch (cmd) {
case RNDGETENTCNT:
/* Inherently racy, no point locking. */
- if (put_user(input_pool.entropy_count, p))
+ if (put_user(input_pool.init_bits, p))
return -EFAULT;
return 0;
case RNDADDTOENTCNT:
@@ -1592,41 +1359,46 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
return -EFAULT;
if (ent_count < 0)
return -EINVAL;
- credit_entropy_bits(ent_count);
+ credit_init_bits(ent_count);
return 0;
- case RNDADDENTROPY:
+ case RNDADDENTROPY: {
+ struct iov_iter iter;
+ struct iovec iov;
+ ssize_t ret;
+ int len;
+
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (get_user(ent_count, p++))
return -EFAULT;
if (ent_count < 0)
return -EINVAL;
- if (get_user(size, p++))
+ if (get_user(len, p++))
return -EFAULT;
- retval = write_pool((const char __user *)p, size);
- if (retval < 0)
- return retval;
- credit_entropy_bits(ent_count);
+ ret = import_single_range(WRITE, p, len, &iov, &iter);
+ if (unlikely(ret))
+ return ret;
+ ret = write_pool_user(&iter);
+ if (unlikely(ret < 0))
+ return ret;
+ /* Since we're crediting, enforce that it was all written into the pool. */
+ if (unlikely(ret != len))
+ return -EFAULT;
+ credit_init_bits(ent_count);
return 0;
+ }
case RNDZAPENTCNT:
case RNDCLEARPOOL:
- /*
- * Clear the entropy pool counters. We no longer clear
- * the entropy pool, as that's silly.
- */
+ /* No longer has any effect. */
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
- if (xchg(&input_pool.entropy_count, 0) >= POOL_MIN_BITS) {
- wake_up_interruptible(&random_write_wait);
- kill_fasync(&fasync, SIGIO, POLL_OUT);
- }
return 0;
case RNDRESEEDCRNG:
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (!crng_ready())
return -ENODATA;
- crng_reseed(false);
+ crng_reseed();
return 0;
default:
return -EINVAL;
@@ -1639,22 +1411,26 @@ static int random_fasync(int fd, struct file *filp, int on)
}
const struct file_operations random_fops = {
- .read = random_read,
- .write = random_write,
+ .read_iter = random_read_iter,
+ .write_iter = random_write_iter,
.poll = random_poll,
.unlocked_ioctl = random_ioctl,
.compat_ioctl = compat_ptr_ioctl,
.fasync = random_fasync,
.llseek = noop_llseek,
+ .splice_read = generic_file_splice_read,
+ .splice_write = iter_file_splice_write,
};
const struct file_operations urandom_fops = {
- .read = urandom_read,
- .write = random_write,
+ .read_iter = urandom_read_iter,
+ .write_iter = random_write_iter,
.unlocked_ioctl = random_ioctl,
.compat_ioctl = compat_ptr_ioctl,
.fasync = random_fasync,
.llseek = noop_llseek,
+ .splice_read = generic_file_splice_read,
+ .splice_write = iter_file_splice_write,
};
@@ -1678,7 +1454,7 @@ const struct file_operations urandom_fops = {
*
* - write_wakeup_threshold - the amount of entropy in the input pool
* below which write polls to /dev/random will unblock, requesting
- * more entropy, tied to the POOL_MIN_BITS constant. It is writable
+ * more entropy, tied to the POOL_READY_BITS constant. It is writable
* to avoid breaking old userspaces, but writing to it does not
* change any behavior of the RNG.
*
@@ -1693,7 +1469,7 @@ const struct file_operations urandom_fops = {
#include <linux/sysctl.h>
static int sysctl_random_min_urandom_seed = CRNG_RESEED_INTERVAL / HZ;
-static int sysctl_random_write_wakeup_bits = POOL_MIN_BITS;
+static int sysctl_random_write_wakeup_bits = POOL_READY_BITS;
static int sysctl_poolsize = POOL_BITS;
static u8 sysctl_bootid[UUID_SIZE];
@@ -1702,7 +1478,7 @@ static u8 sysctl_bootid[UUID_SIZE];
* UUID. The difference is in whether table->data is NULL; if it is,
* then a new UUID is generated and returned to the user.
*/
-static int proc_do_uuid(struct ctl_table *table, int write, void *buffer,
+static int proc_do_uuid(struct ctl_table *table, int write, void *buf,
size_t *lenp, loff_t *ppos)
{
u8 tmp_uuid[UUID_SIZE], *uuid;
@@ -1729,14 +1505,14 @@ static int proc_do_uuid(struct ctl_table *table, int write, void *buffer,
}
snprintf(uuid_string, sizeof(uuid_string), "%pU", uuid);
- return proc_dostring(&fake_table, 0, buffer, lenp, ppos);
+ return proc_dostring(&fake_table, 0, buf, lenp, ppos);
}
/* The same as proc_dointvec, but writes don't change anything. */
-static int proc_do_rointvec(struct ctl_table *table, int write, void *buffer,
+static int proc_do_rointvec(struct ctl_table *table, int write, void *buf,
size_t *lenp, loff_t *ppos)
{
- return write ? 0 : proc_dointvec(table, 0, buffer, lenp, ppos);
+ return write ? 0 : proc_dointvec(table, 0, buf, lenp, ppos);
}
static struct ctl_table random_table[] = {
@@ -1749,7 +1525,7 @@ static struct ctl_table random_table[] = {
},
{
.procname = "entropy_avail",
- .data = &input_pool.entropy_count,
+ .data = &input_pool.init_bits,
.maxlen = sizeof(int),
.mode = 0444,
.proc_handler = proc_dointvec,
@@ -1783,8 +1559,8 @@ static struct ctl_table random_table[] = {
};
/*
- * rand_initialize() is called before sysctl_init(),
- * so we cannot call register_sysctl_init() in rand_initialize()
+ * random_init() is called before sysctl_init(),
+ * so we cannot call register_sysctl_init() in random_init()
*/
static int __init random_sysctls_init(void)
{
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 9f44254af8ce..b0183450e484 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2677,6 +2677,7 @@ extern int install_special_mapping(struct mm_struct *mm,
unsigned long flags, struct page **pages);
unsigned long randomize_stack_top(unsigned long stack_top);
+unsigned long randomize_page(unsigned long start, unsigned long range);
extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
diff --git a/include/linux/prandom.h b/include/linux/prandom.h
index 056d31317e49..deace5fb4e62 100644
--- a/include/linux/prandom.h
+++ b/include/linux/prandom.h
@@ -10,62 +10,16 @@
#include <linux/types.h>
#include <linux/percpu.h>
+#include <linux/random.h>
-u32 prandom_u32(void);
-void prandom_bytes(void *buf, size_t nbytes);
-void prandom_seed(u32 seed);
-void prandom_reseed_late(void);
-
-DECLARE_PER_CPU(unsigned long, net_rand_noise);
-
-#define PRANDOM_ADD_NOISE(a, b, c, d) \
- prandom_u32_add_noise((unsigned long)(a), (unsigned long)(b), \
- (unsigned long)(c), (unsigned long)(d))
-
-#if BITS_PER_LONG == 64
-/*
- * The core SipHash round function. Each line can be executed in
- * parallel given enough CPU resources.
- */
-#define PRND_SIPROUND(v0, v1, v2, v3) ( \
- v0 += v1, v1 = rol64(v1, 13), v2 += v3, v3 = rol64(v3, 16), \
- v1 ^= v0, v0 = rol64(v0, 32), v3 ^= v2, \
- v0 += v3, v3 = rol64(v3, 21), v2 += v1, v1 = rol64(v1, 17), \
- v3 ^= v0, v1 ^= v2, v2 = rol64(v2, 32) \
-)
-
-#define PRND_K0 (0x736f6d6570736575 ^ 0x6c7967656e657261)
-#define PRND_K1 (0x646f72616e646f6d ^ 0x7465646279746573)
-
-#elif BITS_PER_LONG == 32
-/*
- * On 32-bit machines, we use HSipHash, a reduced-width version of SipHash.
- * This is weaker, but 32-bit machines are not used for high-traffic
- * applications, so there is less output for an attacker to analyze.
- */
-#define PRND_SIPROUND(v0, v1, v2, v3) ( \
- v0 += v1, v1 = rol32(v1, 5), v2 += v3, v3 = rol32(v3, 8), \
- v1 ^= v0, v0 = rol32(v0, 16), v3 ^= v2, \
- v0 += v3, v3 = rol32(v3, 7), v2 += v1, v1 = rol32(v1, 13), \
- v3 ^= v0, v1 ^= v2, v2 = rol32(v2, 16) \
-)
-#define PRND_K0 0x6c796765
-#define PRND_K1 0x74656462
-
-#else
-#error Unsupported BITS_PER_LONG
-#endif
+static inline u32 prandom_u32(void)
+{
+ return get_random_u32();
+}
-static inline void prandom_u32_add_noise(unsigned long a, unsigned long b,
- unsigned long c, unsigned long d)
+static inline void prandom_bytes(void *buf, size_t nbytes)
{
- /*
- * This is not used cryptographically; it's just
- * a convenient 4-word hash function. (3 xor, 2 add, 2 rol)
- */
- a ^= raw_cpu_read(net_rand_noise);
- PRND_SIPROUND(a, b, c, d);
- raw_cpu_write(net_rand_noise, d);
+ return get_random_bytes(buf, nbytes);
}
struct rnd_state {
@@ -117,7 +71,6 @@ static inline void prandom_seed_state(struct rnd_state *state, u64 seed)
state->s2 = __seed(i, 8U);
state->s3 = __seed(i, 16U);
state->s4 = __seed(i, 128U);
- PRANDOM_ADD_NOISE(state, i, 0, 0);
}
/* Pseudo random number generator from numerical recipes. */
diff --git a/include/linux/random.h b/include/linux/random.h
index f673fbb838b3..fae0c84027fd 100644
--- a/include/linux/random.h
+++ b/include/linux/random.h
@@ -12,45 +12,32 @@
struct notifier_block;
-extern void add_device_randomness(const void *, size_t);
-extern void add_bootloader_randomness(const void *, size_t);
+void add_device_randomness(const void *buf, size_t len);
+void add_bootloader_randomness(const void *buf, size_t len);
+void add_input_randomness(unsigned int type, unsigned int code,
+ unsigned int value) __latent_entropy;
+void add_interrupt_randomness(int irq) __latent_entropy;
+void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy);
#if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__)
static inline void add_latent_entropy(void)
{
- add_device_randomness((const void *)&latent_entropy,
- sizeof(latent_entropy));
+ add_device_randomness((const void *)&latent_entropy, sizeof(latent_entropy));
}
#else
-static inline void add_latent_entropy(void) {}
+static inline void add_latent_entropy(void) { }
#endif
-extern void add_input_randomness(unsigned int type, unsigned int code,
- unsigned int value) __latent_entropy;
-extern void add_interrupt_randomness(int irq) __latent_entropy;
-extern void add_hwgenerator_randomness(const void *buffer, size_t count,
- size_t entropy);
#if IS_ENABLED(CONFIG_VMGENID)
-extern void add_vmfork_randomness(const void *unique_vm_id, size_t size);
-extern int register_random_vmfork_notifier(struct notifier_block *nb);
-extern int unregister_random_vmfork_notifier(struct notifier_block *nb);
+void add_vmfork_randomness(const void *unique_vm_id, size_t len);
+int register_random_vmfork_notifier(struct notifier_block *nb);
+int unregister_random_vmfork_notifier(struct notifier_block *nb);
#else
static inline int register_random_vmfork_notifier(struct notifier_block *nb) { return 0; }
static inline int unregister_random_vmfork_notifier(struct notifier_block *nb) { return 0; }
#endif
-extern void get_random_bytes(void *buf, size_t nbytes);
-extern int wait_for_random_bytes(void);
-extern int __init rand_initialize(void);
-extern bool rng_is_initialized(void);
-extern int register_random_ready_notifier(struct notifier_block *nb);
-extern int unregister_random_ready_notifier(struct notifier_block *nb);
-extern size_t __must_check get_random_bytes_arch(void *buf, size_t nbytes);
-
-#ifndef MODULE
-extern const struct file_operations random_fops, urandom_fops;
-#endif
-
+void get_random_bytes(void *buf, size_t len);
u32 get_random_u32(void);
u64 get_random_u64(void);
static inline unsigned int get_random_int(void)
@@ -82,11 +69,14 @@ static inline unsigned long get_random_long(void)
static inline unsigned long get_random_canary(void)
{
- unsigned long val = get_random_long();
-
- return val & CANARY_MASK;
+ return get_random_long() & CANARY_MASK;
}
+int __init random_init(const char *command_line);
+bool rng_is_initialized(void);
+bool rng_has_arch_random(void);
+int wait_for_random_bytes(void);
+
/* Calls wait_for_random_bytes() and then calls get_random_bytes(buf, nbytes).
* Returns the result of the call to wait_for_random_bytes. */
static inline int get_random_bytes_wait(void *buf, size_t nbytes)
@@ -96,22 +86,20 @@ static inline int get_random_bytes_wait(void *buf, size_t nbytes)
return ret;
}
-#define declare_get_random_var_wait(var) \
- static inline int get_random_ ## var ## _wait(var *out) { \
+#define declare_get_random_var_wait(name, ret_type) \
+ static inline int get_random_ ## name ## _wait(ret_type *out) { \
int ret = wait_for_random_bytes(); \
if (unlikely(ret)) \
return ret; \
- *out = get_random_ ## var(); \
+ *out = get_random_ ## name(); \
return 0; \
}
-declare_get_random_var_wait(u32)
-declare_get_random_var_wait(u64)
-declare_get_random_var_wait(int)
-declare_get_random_var_wait(long)
+declare_get_random_var_wait(u32, u32)
+declare_get_random_var_wait(u64, u32)
+declare_get_random_var_wait(int, unsigned int)
+declare_get_random_var_wait(long, unsigned long)
#undef declare_get_random_var
-unsigned long randomize_page(unsigned long start, unsigned long range);
-
/*
* This is designed to be standalone for just prandom
* users, but for now we include it from <linux/random.h>
@@ -122,22 +110,10 @@ unsigned long randomize_page(unsigned long start, unsigned long range);
#ifdef CONFIG_ARCH_RANDOM
# include <asm/archrandom.h>
#else
-static inline bool __must_check arch_get_random_long(unsigned long *v)
-{
- return false;
-}
-static inline bool __must_check arch_get_random_int(unsigned int *v)
-{
- return false;
-}
-static inline bool __must_check arch_get_random_seed_long(unsigned long *v)
-{
- return false;
-}
-static inline bool __must_check arch_get_random_seed_int(unsigned int *v)
-{
- return false;
-}
+static inline bool __must_check arch_get_random_long(unsigned long *v) { return false; }
+static inline bool __must_check arch_get_random_int(unsigned int *v) { return false; }
+static inline bool __must_check arch_get_random_seed_long(unsigned long *v) { return false; }
+static inline bool __must_check arch_get_random_seed_int(unsigned int *v) { return false; }
#endif
/*
@@ -161,8 +137,12 @@ static inline bool __init arch_get_random_long_early(unsigned long *v)
#endif
#ifdef CONFIG_SMP
-extern int random_prepare_cpu(unsigned int cpu);
-extern int random_online_cpu(unsigned int cpu);
+int random_prepare_cpu(unsigned int cpu);
+int random_online_cpu(unsigned int cpu);
+#endif
+
+#ifndef MODULE
+extern const struct file_operations random_fops, urandom_fops;
#endif
#endif /* _LINUX_RANDOM_H */
diff --git a/include/linux/siphash.h b/include/linux/siphash.h
index cce8a9acc76c..3af1428da559 100644
--- a/include/linux/siphash.h
+++ b/include/linux/siphash.h
@@ -138,4 +138,32 @@ static inline u32 hsiphash(const void *data, size_t len,
return ___hsiphash_aligned(data, len, key);
}
+/*
+ * These macros expose the raw SipHash and HalfSipHash permutations.
+ * Do not use them directly! If you think you have a use for them,
+ * be sure to CC the maintainer of this file explaining why.
+ */
+
+#define SIPHASH_PERMUTATION(a, b, c, d) ( \
+ (a) += (b), (b) = rol64((b), 13), (b) ^= (a), (a) = rol64((a), 32), \
+ (c) += (d), (d) = rol64((d), 16), (d) ^= (c), \
+ (a) += (d), (d) = rol64((d), 21), (d) ^= (a), \
+ (c) += (b), (b) = rol64((b), 17), (b) ^= (c), (c) = rol64((c), 32))
+
+#define SIPHASH_CONST_0 0x736f6d6570736575ULL
+#define SIPHASH_CONST_1 0x646f72616e646f6dULL
+#define SIPHASH_CONST_2 0x6c7967656e657261ULL
+#define SIPHASH_CONST_3 0x7465646279746573ULL
+
+#define HSIPHASH_PERMUTATION(a, b, c, d) ( \
+ (a) += (b), (b) = rol32((b), 5), (b) ^= (a), (a) = rol32((a), 16), \
+ (c) += (d), (d) = rol32((d), 8), (d) ^= (c), \
+ (a) += (d), (d) = rol32((d), 7), (d) ^= (a), \
+ (c) += (b), (b) = rol32((b), 13), (b) ^= (c), (c) = rol32((c), 16))
+
+#define HSIPHASH_CONST_0 0U
+#define HSIPHASH_CONST_1 0U
+#define HSIPHASH_CONST_2 0x6c796765U
+#define HSIPHASH_CONST_3 0x74656462U
+
#endif /* _LINUX_SIPHASH_H */
diff --git a/include/linux/timex.h b/include/linux/timex.h
index 5745c90c8800..3871b06bd302 100644
--- a/include/linux/timex.h
+++ b/include/linux/timex.h
@@ -62,6 +62,8 @@
#include <linux/types.h>
#include <linux/param.h>
+unsigned long random_get_entropy_fallback(void);
+
#include <asm/timex.h>
#ifndef random_get_entropy
@@ -74,8 +76,14 @@
*
* By default we use get_cycles() for this purpose, but individual
* architectures may override this in their asm/timex.h header file.
+ * If a given arch does not have get_cycles(), then we fallback to
+ * using random_get_entropy_fallback().
*/
+#ifdef get_cycles
#define random_get_entropy() ((unsigned long)get_cycles())
+#else
+#define random_get_entropy() random_get_entropy_fallback()
+#endif
#endif
/*
diff --git a/init/main.c b/init/main.c
index 98182c3c2c4b..f057c49f1d9d 100644
--- a/init/main.c
+++ b/init/main.c
@@ -1035,21 +1035,18 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
softirq_init();
timekeeping_init();
kfence_init();
+ time_init();
/*
* For best initial stack canary entropy, prepare it after:
* - setup_arch() for any UEFI RNG entropy and boot cmdline access
- * - timekeeping_init() for ktime entropy used in rand_initialize()
- * - rand_initialize() to get any arch-specific entropy like RDRAND
- * - add_latent_entropy() to get any latent entropy
- * - adding command line entropy
+ * - timekeeping_init() for ktime entropy used in random_init()
+ * - time_init() for making random_get_entropy() work on some platforms
+ * - random_init() to initialize the RNG from from early entropy sources
*/
- rand_initialize();
- add_latent_entropy();
- add_device_randomness(command_line, strlen(command_line));
+ random_init(command_line);
boot_init_stack_canary();
- time_init();
perf_event_init();
profile_init();
call_function_init();
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 4ab9949772d5..8e4b3c32fcf9 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -17,6 +17,7 @@
#include <linux/clocksource.h>
#include <linux/jiffies.h>
#include <linux/time.h>
+#include <linux/timex.h>
#include <linux/tick.h>
#include <linux/stop_machine.h>
#include <linux/pvclock_gtod.h>
@@ -2397,6 +2398,20 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc)
return 0;
}
+/**
+ * random_get_entropy_fallback - Returns the raw clock source value,
+ * used by random.c for platforms with no valid random_get_entropy().
+ */
+unsigned long random_get_entropy_fallback(void)
+{
+ struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
+ struct clocksource *clock = READ_ONCE(tkr->clock);
+
+ if (unlikely(timekeeping_suspended || !clock))
+ return 0;
+ return clock->read(clock);
+}
+EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
/**
* do_adjtimex() - Accessor function to NTP __do_adjtimex function
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index a0666d948147..717fcb9fb14a 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -1833,8 +1833,6 @@ void update_process_times(int user_tick)
{
struct task_struct *p = current;
- PRANDOM_ADD_NOISE(jiffies, user_tick, p, 0);
-
/* Note: this timer irq context must be accounted for as well. */
account_process_tick(p, user_tick);
run_local_timers();
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 55b9acb2f524..a30d5279efda 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1617,8 +1617,7 @@ config WARN_ALL_UNSEEDED_RANDOM
so architecture maintainers really need to do what they can
to get the CRNG seeded sooner after the system is booted.
However, since users cannot do anything actionable to
- address this, by default the kernel will issue only a single
- warning for the first use of unseeded randomness.
+ address this, by default this option is disabled.
Say Y here if you want to receive warnings for all uses of
unseeded randomness. This will be of use primarily for
diff --git a/lib/random32.c b/lib/random32.c
index 976632003ec6..d5d9029362cb 100644
--- a/lib/random32.c
+++ b/lib/random32.c
@@ -245,25 +245,13 @@ static struct prandom_test2 {
{ 407983964U, 921U, 728767059U },
};
-static u32 __extract_hwseed(void)
-{
- unsigned int val = 0;
-
- (void)(arch_get_random_seed_int(&val) ||
- arch_get_random_int(&val));
-
- return val;
-}
-
-static void prandom_seed_early(struct rnd_state *state, u32 seed,
- bool mix_with_hwseed)
+static void prandom_state_selftest_seed(struct rnd_state *state, u32 seed)
{
#define LCG(x) ((x) * 69069U) /* super-duper LCG */
-#define HWSEED() (mix_with_hwseed ? __extract_hwseed() : 0)
- state->s1 = __seed(HWSEED() ^ LCG(seed), 2U);
- state->s2 = __seed(HWSEED() ^ LCG(state->s1), 8U);
- state->s3 = __seed(HWSEED() ^ LCG(state->s2), 16U);
- state->s4 = __seed(HWSEED() ^ LCG(state->s3), 128U);
+ state->s1 = __seed(LCG(seed), 2U);
+ state->s2 = __seed(LCG(state->s1), 8U);
+ state->s3 = __seed(LCG(state->s2), 16U);
+ state->s4 = __seed(LCG(state->s3), 128U);
}
static int __init prandom_state_selftest(void)
@@ -274,7 +262,7 @@ static int __init prandom_state_selftest(void)
for (i = 0; i < ARRAY_SIZE(test1); i++) {
struct rnd_state state;
- prandom_seed_early(&state, test1[i].seed, false);
+ prandom_state_selftest_seed(&state, test1[i].seed);
prandom_warmup(&state);
if (test1[i].result != prandom_u32_state(&state))
@@ -289,7 +277,7 @@ static int __init prandom_state_selftest(void)
for (i = 0; i < ARRAY_SIZE(test2); i++) {
struct rnd_state state;
- prandom_seed_early(&state, test2[i].seed, false);
+ prandom_state_selftest_seed(&state, test2[i].seed);
prandom_warmup(&state);
for (j = 0; j < test2[i].iteration - 1; j++)
@@ -310,324 +298,3 @@ static int __init prandom_state_selftest(void)
}
core_initcall(prandom_state_selftest);
#endif
-
-/*
- * The prandom_u32() implementation is now completely separate from the
- * prandom_state() functions, which are retained (for now) for compatibility.
- *
- * Because of (ab)use in the networking code for choosing random TCP/UDP port
- * numbers, which open DoS possibilities if guessable, we want something
- * stronger than a standard PRNG. But the performance requirements of
- * the network code do not allow robust crypto for this application.
- *
- * So this is a homebrew Junior Spaceman implementation, based on the
- * lowest-latency trustworthy crypto primitive available, SipHash.
- * (The authors of SipHash have not been consulted about this abuse of
- * their work.)
- *
- * Standard SipHash-2-4 uses 2n+4 rounds to hash n words of input to
- * one word of output. This abbreviated version uses 2 rounds per word
- * of output.
- */
-
-struct siprand_state {
- unsigned long v0;
- unsigned long v1;
- unsigned long v2;
- unsigned long v3;
-};
-
-static DEFINE_PER_CPU(struct siprand_state, net_rand_state) __latent_entropy;
-DEFINE_PER_CPU(unsigned long, net_rand_noise);
-EXPORT_PER_CPU_SYMBOL(net_rand_noise);
-
-/*
- * This is the core CPRNG function. As "pseudorandom", this is not used
- * for truly valuable things, just intended to be a PITA to guess.
- * For maximum speed, we do just two SipHash rounds per word. This is
- * the same rate as 4 rounds per 64 bits that SipHash normally uses,
- * so hopefully it's reasonably secure.
- *
- * There are two changes from the official SipHash finalization:
- * - We omit some constants XORed with v2 in the SipHash spec as irrelevant;
- * they are there only to make the output rounds distinct from the input
- * rounds, and this application has no input rounds.
- * - Rather than returning v0^v1^v2^v3, return v1+v3.
- * If you look at the SipHash round, the last operation on v3 is
- * "v3 ^= v0", so "v0 ^ v3" just undoes that, a waste of time.
- * Likewise "v1 ^= v2". (The rotate of v2 makes a difference, but
- * it still cancels out half of the bits in v2 for no benefit.)
- * Second, since the last combining operation was xor, continue the
- * pattern of alternating xor/add for a tiny bit of extra non-linearity.
- */
-static inline u32 siprand_u32(struct siprand_state *s)
-{
- unsigned long v0 = s->v0, v1 = s->v1, v2 = s->v2, v3 = s->v3;
- unsigned long n = raw_cpu_read(net_rand_noise);
-
- v3 ^= n;
- PRND_SIPROUND(v0, v1, v2, v3);
- PRND_SIPROUND(v0, v1, v2, v3);
- v0 ^= n;
- s->v0 = v0; s->v1 = v1; s->v2 = v2; s->v3 = v3;
- return v1 + v3;
-}
-
-
-/**
- * prandom_u32 - pseudo random number generator
- *
- * A 32 bit pseudo-random number is generated using a fast
- * algorithm suitable for simulation. This algorithm is NOT
- * considered safe for cryptographic use.
- */
-u32 prandom_u32(void)
-{
- struct siprand_state *state = get_cpu_ptr(&net_rand_state);
- u32 res = siprand_u32(state);
-
- put_cpu_ptr(&net_rand_state);
- return res;
-}
-EXPORT_SYMBOL(prandom_u32);
-
-/**
- * prandom_bytes - get the requested number of pseudo-random bytes
- * @buf: where to copy the pseudo-random bytes to
- * @bytes: the requested number of bytes
- */
-void prandom_bytes(void *buf, size_t bytes)
-{
- struct siprand_state *state = get_cpu_ptr(&net_rand_state);
- u8 *ptr = buf;
-
- while (bytes >= sizeof(u32)) {
- put_unaligned(siprand_u32(state), (u32 *)ptr);
- ptr += sizeof(u32);
- bytes -= sizeof(u32);
- }
-
- if (bytes > 0) {
- u32 rem = siprand_u32(state);
-
- do {
- *ptr++ = (u8)rem;
- rem >>= BITS_PER_BYTE;
- } while (--bytes > 0);
- }
- put_cpu_ptr(&net_rand_state);
-}
-EXPORT_SYMBOL(prandom_bytes);
-
-/**
- * prandom_seed - add entropy to pseudo random number generator
- * @entropy: entropy value
- *
- * Add some additional seed material to the prandom pool.
- * The "entropy" is actually our IP address (the only caller is
- * the network code), not for unpredictability, but to ensure that
- * different machines are initialized differently.
- */
-void prandom_seed(u32 entropy)
-{
- int i;
-
- add_device_randomness(&entropy, sizeof(entropy));
-
- for_each_possible_cpu(i) {
- struct siprand_state *state = per_cpu_ptr(&net_rand_state, i);
- unsigned long v0 = state->v0, v1 = state->v1;
- unsigned long v2 = state->v2, v3 = state->v3;
-
- do {
- v3 ^= entropy;
- PRND_SIPROUND(v0, v1, v2, v3);
- PRND_SIPROUND(v0, v1, v2, v3);
- v0 ^= entropy;
- } while (unlikely(!v0 || !v1 || !v2 || !v3));
-
- WRITE_ONCE(state->v0, v0);
- WRITE_ONCE(state->v1, v1);
- WRITE_ONCE(state->v2, v2);
- WRITE_ONCE(state->v3, v3);
- }
-}
-EXPORT_SYMBOL(prandom_seed);
-
-/*
- * Generate some initially weak seeding values to allow
- * the prandom_u32() engine to be started.
- */
-static int __init prandom_init_early(void)
-{
- int i;
- unsigned long v0, v1, v2, v3;
-
- if (!arch_get_random_long(&v0))
- v0 = jiffies;
- if (!arch_get_random_long(&v1))
- v1 = random_get_entropy();
- v2 = v0 ^ PRND_K0;
- v3 = v1 ^ PRND_K1;
-
- for_each_possible_cpu(i) {
- struct siprand_state *state;
-
- v3 ^= i;
- PRND_SIPROUND(v0, v1, v2, v3);
- PRND_SIPROUND(v0, v1, v2, v3);
- v0 ^= i;
-
- state = per_cpu_ptr(&net_rand_state, i);
- state->v0 = v0; state->v1 = v1;
- state->v2 = v2; state->v3 = v3;
- }
-
- return 0;
-}
-core_initcall(prandom_init_early);
-
-
-/* Stronger reseeding when available, and periodically thereafter. */
-static void prandom_reseed(struct timer_list *unused);
-
-static DEFINE_TIMER(seed_timer, prandom_reseed);
-
-static void prandom_reseed(struct timer_list *unused)
-{
- unsigned long expires;
- int i;
-
- /*
- * Reinitialize each CPU's PRNG with 128 bits of key.
- * No locking on the CPUs, but then somewhat random results are,
- * well, expected.
- */
- for_each_possible_cpu(i) {
- struct siprand_state *state;
- unsigned long v0 = get_random_long(), v2 = v0 ^ PRND_K0;
- unsigned long v1 = get_random_long(), v3 = v1 ^ PRND_K1;
-#if BITS_PER_LONG == 32
- int j;
-
- /*
- * On 32-bit machines, hash in two extra words to
- * approximate 128-bit key length. Not that the hash
- * has that much security, but this prevents a trivial
- * 64-bit brute force.
- */
- for (j = 0; j < 2; j++) {
- unsigned long m = get_random_long();
-
- v3 ^= m;
- PRND_SIPROUND(v0, v1, v2, v3);
- PRND_SIPROUND(v0, v1, v2, v3);
- v0 ^= m;
- }
-#endif
- /*
- * Probably impossible in practice, but there is a
- * theoretical risk that a race between this reseeding
- * and the target CPU writing its state back could
- * create the all-zero SipHash fixed point.
- *
- * To ensure that never happens, ensure the state
- * we write contains no zero words.
- */
- state = per_cpu_ptr(&net_rand_state, i);
- WRITE_ONCE(state->v0, v0 ? v0 : -1ul);
- WRITE_ONCE(state->v1, v1 ? v1 : -1ul);
- WRITE_ONCE(state->v2, v2 ? v2 : -1ul);
- WRITE_ONCE(state->v3, v3 ? v3 : -1ul);
- }
-
- /* reseed every ~60 seconds, in [40 .. 80) interval with slack */
- expires = round_jiffies(jiffies + 40 * HZ + prandom_u32_max(40 * HZ));
- mod_timer(&seed_timer, expires);
-}
-
-/*
- * The random ready callback can be called from almost any interrupt.
- * To avoid worrying about whether it's safe to delay that interrupt
- * long enough to seed all CPUs, just schedule an immediate timer event.
- */
-static int prandom_timer_start(struct notifier_block *nb,
- unsigned long action, void *data)
-{
- mod_timer(&seed_timer, jiffies);
- return 0;
-}
-
-#ifdef CONFIG_RANDOM32_SELFTEST
-/* Principle: True 32-bit random numbers will all have 16 differing bits on
- * average. For each 32-bit number, there are 601M numbers differing by 16
- * bits, and 89% of the numbers differ by at least 12 bits. Note that more
- * than 16 differing bits also implies a correlation with inverted bits. Thus
- * we take 1024 random numbers and compare each of them to the other ones,
- * counting the deviation of correlated bits to 16. Constants report 32,
- * counters 32-log2(TEST_SIZE), and pure randoms, around 6 or lower. With the
- * u32 total, TEST_SIZE may be as large as 4096 samples.
- */
-#define TEST_SIZE 1024
-static int __init prandom32_state_selftest(void)
-{
- unsigned int x, y, bits, samples;
- u32 xor, flip;
- u32 total;
- u32 *data;
-
- data = kmalloc(sizeof(*data) * TEST_SIZE, GFP_KERNEL);
- if (!data)
- return 0;
-
- for (samples = 0; samples < TEST_SIZE; samples++)
- data[samples] = prandom_u32();
-
- flip = total = 0;
- for (x = 0; x < samples; x++) {
- for (y = 0; y < samples; y++) {
- if (x == y)
- continue;
- xor = data[x] ^ data[y];
- flip |= xor;
- bits = hweight32(xor);
- total += (bits - 16) * (bits - 16);
- }
- }
-
- /* We'll return the average deviation as 2*sqrt(corr/samples), which
- * is also sqrt(4*corr/samples) which provides a better resolution.
- */
- bits = int_sqrt(total / (samples * (samples - 1)) * 4);
- if (bits > 6)
- pr_warn("prandom32: self test failed (at least %u bits"
- " correlated, fixed_mask=%#x fixed_value=%#x\n",
- bits, ~flip, data[0] & ~flip);
- else
- pr_info("prandom32: self test passed (less than %u bits"
- " correlated)\n",
- bits+1);
- kfree(data);
- return 0;
-}
-core_initcall(prandom32_state_selftest);
-#endif /* CONFIG_RANDOM32_SELFTEST */
-
-/*
- * Start periodic full reseeding as soon as strong
- * random numbers are available.
- */
-static int __init prandom_init_late(void)
-{
- static struct notifier_block random_ready = {
- .notifier_call = prandom_timer_start
- };
- int ret = register_random_ready_notifier(&random_ready);
-
- if (ret == -EALREADY) {
- prandom_timer_start(&random_ready, 0, NULL);
- ret = 0;
- }
- return ret;
-}
-late_initcall(prandom_init_late);
diff --git a/lib/siphash.c b/lib/siphash.c
index 72b9068ab57b..71d315a6ad62 100644
--- a/lib/siphash.c
+++ b/lib/siphash.c
@@ -18,19 +18,13 @@
#include <asm/word-at-a-time.h>
#endif
-#define SIPROUND \
- do { \
- v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \
- v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \
- v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \
- v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \
- } while (0)
+#define SIPROUND SIPHASH_PERMUTATION(v0, v1, v2, v3)
#define PREAMBLE(len) \
- u64 v0 = 0x736f6d6570736575ULL; \
- u64 v1 = 0x646f72616e646f6dULL; \
- u64 v2 = 0x6c7967656e657261ULL; \
- u64 v3 = 0x7465646279746573ULL; \
+ u64 v0 = SIPHASH_CONST_0; \
+ u64 v1 = SIPHASH_CONST_1; \
+ u64 v2 = SIPHASH_CONST_2; \
+ u64 v3 = SIPHASH_CONST_3; \
u64 b = ((u64)(len)) << 56; \
v3 ^= key->key[1]; \
v2 ^= key->key[0]; \
@@ -389,19 +383,13 @@ u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
}
EXPORT_SYMBOL(hsiphash_4u32);
#else
-#define HSIPROUND \
- do { \
- v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \
- v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \
- v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \
- v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \
- } while (0)
+#define HSIPROUND HSIPHASH_PERMUTATION(v0, v1, v2, v3)
#define HPREAMBLE(len) \
- u32 v0 = 0; \
- u32 v1 = 0; \
- u32 v2 = 0x6c796765U; \
- u32 v3 = 0x74656462U; \
+ u32 v0 = HSIPHASH_CONST_0; \
+ u32 v1 = HSIPHASH_CONST_1; \
+ u32 v2 = HSIPHASH_CONST_2; \
+ u32 v3 = HSIPHASH_CONST_3; \
u32 b = ((u32)(len)) << 24; \
v3 ^= key->key[1]; \
v2 ^= key->key[0]; \
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 40d26a07a133..fb77f7bfd126 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -750,61 +750,38 @@ static int __init debug_boot_weak_hash_enable(char *str)
}
early_param("debug_boot_weak_hash", debug_boot_weak_hash_enable);
-static DEFINE_STATIC_KEY_TRUE(not_filled_random_ptr_key);
-static siphash_key_t ptr_key __read_mostly;
+static DEFINE_STATIC_KEY_FALSE(filled_random_ptr_key);
static void enable_ptr_key_workfn(struct work_struct *work)
{
- get_random_bytes(&ptr_key, sizeof(ptr_key));
- /* Needs to run from preemptible context */
- static_branch_disable(&not_filled_random_ptr_key);
+ static_branch_enable(&filled_random_ptr_key);
}
-static DECLARE_WORK(enable_ptr_key_work, enable_ptr_key_workfn);
-
-static int fill_random_ptr_key(struct notifier_block *nb,
- unsigned long action, void *data)
-{
- /* This may be in an interrupt handler. */
- queue_work(system_unbound_wq, &enable_ptr_key_work);
- return 0;
-}
-
-static struct notifier_block random_ready = {
- .notifier_call = fill_random_ptr_key
-};
-
-static int __init initialize_ptr_random(void)
-{
- int key_size = sizeof(ptr_key);
- int ret;
-
- /* Use hw RNG if available. */
- if (get_random_bytes_arch(&ptr_key, key_size) == key_size) {
- static_branch_disable(&not_filled_random_ptr_key);
- return 0;
- }
-
- ret = register_random_ready_notifier(&random_ready);
- if (!ret) {
- return 0;
- } else if (ret == -EALREADY) {
- /* This is in preemptible context */
- enable_ptr_key_workfn(&enable_ptr_key_work);
- return 0;
- }
-
- return ret;
-}
-early_initcall(initialize_ptr_random);
-
/* Maps a pointer to a 32 bit unique identifier. */
static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
{
+ static siphash_key_t ptr_key __read_mostly;
unsigned long hashval;
- if (static_branch_unlikely(&not_filled_random_ptr_key))
- return -EAGAIN;
+ if (!static_branch_likely(&filled_random_ptr_key)) {
+ static bool filled = false;
+ static DEFINE_SPINLOCK(filling);
+ static DECLARE_WORK(enable_ptr_key_work, enable_ptr_key_workfn);
+ unsigned long flags;
+
+ if (!system_unbound_wq ||
+ (!rng_is_initialized() && !rng_has_arch_random()) ||
+ !spin_trylock_irqsave(&filling, flags))
+ return -EAGAIN;
+
+ if (!filled) {
+ get_random_bytes(&ptr_key, sizeof(ptr_key));
+ queue_work(system_unbound_wq, &enable_ptr_key_work);
+ filled = true;
+ }
+ spin_unlock_irqrestore(&filling, flags);
+ }
+
#ifdef CONFIG_64BIT
hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key);
diff --git a/mm/util.c b/mm/util.c
index 3492a9e81aa3..ac63e5ca8b21 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -343,6 +343,38 @@ unsigned long randomize_stack_top(unsigned long stack_top)
#endif
}
+/**
+ * randomize_page - Generate a random, page aligned address
+ * @start: The smallest acceptable address the caller will take.
+ * @range: The size of the area, starting at @start, within which the
+ * random address must fall.
+ *
+ * If @start + @range would overflow, @range is capped.
+ *
+ * NOTE: Historical use of randomize_range, which this replaces, presumed that
+ * @start was already page aligned. We now align it regardless.
+ *
+ * Return: A page aligned address within [start, start + range). On error,
+ * @start is returned.
+ */
+unsigned long randomize_page(unsigned long start, unsigned long range)
+{
+ if (!PAGE_ALIGNED(start)) {
+ range -= PAGE_ALIGN(start) - start;
+ start = PAGE_ALIGN(start);
+ }
+
+ if (start > ULONG_MAX - range)
+ range = ULONG_MAX - start;
+
+ range >>= PAGE_SHIFT;
+
+ if (range == 0)
+ return start;
+
+ return start + (get_random_long() % range << PAGE_SHIFT);
+}
+
#ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
unsigned long arch_randomize_brk(struct mm_struct *mm)
{
diff --git a/net/core/dev.c b/net/core/dev.c
index 2771fd22dc6a..191ec76d4c3b 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3527,7 +3527,6 @@ static int xmit_one(struct sk_buff *skb, struct net_device *dev,
dev_queue_xmit_nit(skb, dev);
len = skb->len;
- PRANDOM_ADD_NOISE(skb, dev, txq, len + jiffies);
trace_net_dev_start_xmit(skb, dev);
rc = netdev_start_xmit(skb, dev, txq, more);
trace_net_dev_xmit(skb, rc, dev, len);
@@ -4168,7 +4167,6 @@ static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
if (!skb)
goto out;
- PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
HARD_TX_LOCK(dev, txq, cpu);
if (!netif_xmit_stopped(txq)) {
@@ -4234,7 +4232,6 @@ int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
skb_set_queue_mapping(skb, queue_id);
txq = skb_get_tx_queue(dev, skb);
- PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
local_bh_disable();
diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c
index 53a6b14dc50a..3d6d33ac20cc 100644
--- a/net/ipv4/devinet.c
+++ b/net/ipv4/devinet.c
@@ -536,10 +536,8 @@ static int __inet_insert_ifa(struct in_ifaddr *ifa, struct nlmsghdr *nlh,
return ret;
}
- if (!(ifa->ifa_flags & IFA_F_SECONDARY)) {
- prandom_seed((__force u32) ifa->ifa_local);
+ if (!(ifa->ifa_flags & IFA_F_SECONDARY))
ifap = last_primary;
- }
rcu_assign_pointer(ifa->ifa_next, *ifap);
rcu_assign_pointer(*ifap, ifa);
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index b22504176588..e7c68fa12fae 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -3972,8 +3972,6 @@ static void addrconf_dad_begin(struct inet6_ifaddr *ifp)
addrconf_join_solict(dev, &ifp->addr);
- prandom_seed((__force u32) ifp->addr.s6_addr32[3]);
-
read_lock_bh(&idev->lock);
spin_lock(&ifp->lock);
if (ifp->state == INET6_IFADDR_STATE_DEAD)