1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/*
27 * Software based random number provider for the Kernel Cryptographic
28 * Framework (KCF). This provider periodically collects unpredictable input
29 * from external sources and processes it into a pool of entropy (randomness)
30 * in order to satisfy requests for random bits from kCF. It implements
31 * software-based mixing, extraction, and generation algorithms.
32 *
33 * A history note: The software-based algorithms in this file used to be
34 * part of the /dev/random driver.
35 */
36
37#include <sys/types.h>
38#include <sys/errno.h>
39#include <sys/debug.h>
40#include <vm/seg_kmem.h>
41#include <vm/hat.h>
42#include <sys/systm.h>
43#include <sys/memlist.h>
44#include <sys/cmn_err.h>
45#include <sys/ksynch.h>
46#include <sys/random.h>
47#include <sys/ddi.h>
48#include <sys/mman.h>
49#include <sys/sysmacros.h>
50#include <sys/mem_config.h>
51#include <sys/time.h>
52#include <sys/crypto/spi.h>
53#include <sys/sha1.h>
54#include <sys/sunddi.h>
55#include <sys/modctl.h>
56#include <sys/hold_page.h>
57#include <rng/fips_random.h>
58
59#define	RNDPOOLSIZE		1024	/* Pool size in bytes */
60#define	HASHBUFSIZE		64	/* Buffer size used for pool mixing */
61#define	MAXMEMBLOCKS		16384	/* Number of memory blocks to scan */
62#define	MEMBLOCKSIZE		4096	/* Size of memory block to read */
63#define	MINEXTRACTBITS		160	/* Min entropy level for extraction */
64#define	TIMEOUT_INTERVAL	5	/* Periodic mixing interval in secs */
65
66/* Hash-algo generic definitions. For now, they are SHA1's. */
67#define	HASHSIZE		20
68#define	HASH_CTX		SHA1_CTX
69#define	HashInit(ctx)		SHA1Init((ctx))
70#define	HashUpdate(ctx, p, s)	SHA1Update((ctx), (p), (s))
71#define	HashFinal(d, ctx)	SHA1Final((d), (ctx))
72
73/* Physical memory entropy source */
74typedef struct physmem_entsrc_s {
75	uint8_t *parity;		/* parity bit vector */
76	caddr_t pmbuf;			/* buffer for memory block */
77	uint32_t nblocks;		/* number of  memory blocks */
78	int entperblock;		/* entropy bits per block read */
79	hrtime_t last_diff;		/* previous time to process a block */
80	hrtime_t last_delta;		/* previous time delta */
81	hrtime_t last_delta2;		/* previous 2nd order time delta */
82} physmem_entsrc_t;
83
84static uint32_t srndpool[RNDPOOLSIZE/4];	/* Pool of random bits */
85static uint32_t buffer[RNDPOOLSIZE/4];	/* entropy mixed in later */
86static int buffer_bytes;		/* bytes written to buffer */
87static uint32_t entropy_bits;		/* pool's current amount of entropy */
88static kmutex_t srndpool_lock;		/* protects r/w accesses to the pool, */
89					/* and the global variables */
90static kmutex_t buffer_lock;		/* protects r/w accesses to buffer */
91static kcondvar_t srndpool_read_cv;	/* serializes poll/read syscalls */
92static int pindex;			/* Global index for adding/extracting */
93					/* from the pool */
94static int bstart, bindex;		/* Global vars for adding/extracting */
95					/* from the buffer */
96static uint8_t leftover[HASHSIZE];	/* leftover output */
97static uint32_t	swrand_XKEY[6];		/* one extra word for getentropy */
98static int leftover_bytes;		/* leftover length */
99static uint32_t previous_bytes[HASHSIZE/BYTES_IN_WORD];	/* prev random bytes */
100
101static physmem_entsrc_t entsrc;		/* Physical mem as an entropy source */
102static timeout_id_t rnd_timeout_id;
103static int snum_waiters;
104static crypto_kcf_provider_handle_t swrand_prov_handle = NULL;
105swrand_stats_t swrand_stats;
106
107static int physmem_ent_init(physmem_entsrc_t *);
108static void physmem_ent_fini(physmem_entsrc_t *);
109static void physmem_ent_gen(physmem_entsrc_t *);
110static int physmem_parity_update(uint8_t *, uint32_t, int);
111static void physmem_count_blocks();
112static void rnd_dr_callback_post_add(void *, pgcnt_t);
113static int rnd_dr_callback_pre_del(void *, pgcnt_t);
114static void rnd_dr_callback_post_del(void *, pgcnt_t, int);
115static void rnd_handler(void *arg);
116static void swrand_init();
117static void swrand_schedule_timeout(void);
118static int swrand_get_entropy(uint8_t *ptr, size_t len, boolean_t);
119static void swrand_add_entropy(uint8_t *ptr, size_t len, uint16_t entropy_est);
120static void swrand_add_entropy_later(uint8_t *ptr, size_t len);
121
122/* Dynamic Reconfiguration related declarations */
123kphysm_setup_vector_t rnd_dr_callback_vec = {
124	KPHYSM_SETUP_VECTOR_VERSION,
125	rnd_dr_callback_post_add,
126	rnd_dr_callback_pre_del,
127	rnd_dr_callback_post_del
128};
129
130extern struct mod_ops mod_cryptoops;
131
132/*
133 * Module linkage information for the kernel.
134 */
135static struct modlcrypto modlcrypto = {
136	&mod_cryptoops,
137	"Kernel Random number Provider"
138};
139
140static struct modlinkage modlinkage = {
141	MODREV_1,
142	(void *)&modlcrypto,
143	NULL
144};
145
146/*
147 * CSPI information (entry points, provider info, etc.)
148 */
149static void swrand_provider_status(crypto_provider_handle_t, uint_t *);
150
151static crypto_control_ops_t swrand_control_ops = {
152	swrand_provider_status
153};
154
155static int swrand_seed_random(crypto_provider_handle_t, crypto_session_id_t,
156    uchar_t *, size_t, uint_t, uint32_t, crypto_req_handle_t);
157static int swrand_generate_random(crypto_provider_handle_t,
158    crypto_session_id_t, uchar_t *, size_t, crypto_req_handle_t);
159
160static crypto_random_number_ops_t swrand_random_number_ops = {
161	swrand_seed_random,
162	swrand_generate_random
163};
164
165static void swrand_POST(int *);
166
167static crypto_fips140_ops_t swrand_fips140_ops = {
168	swrand_POST
169};
170
171static crypto_ops_t swrand_crypto_ops = {
172	&swrand_control_ops,
173	NULL,
174	NULL,
175	NULL,
176	NULL,
177	NULL,
178	NULL,
179	NULL,
180	&swrand_random_number_ops,
181	NULL,
182	NULL,
183	NULL,
184	NULL,
185	NULL,
186	NULL,
187	NULL,
188	&swrand_fips140_ops
189};
190
191static crypto_provider_info_t swrand_prov_info = {
192	CRYPTO_SPI_VERSION_4,
193	"Kernel Random Number Provider",
194	CRYPTO_SW_PROVIDER,
195	{&modlinkage},
196	NULL,
197	&swrand_crypto_ops,
198	0,
199	NULL
200};
201
202int
203_init(void)
204{
205	int ret;
206	hrtime_t ts;
207	time_t now;
208
209	mutex_init(&srndpool_lock, NULL, MUTEX_DEFAULT, NULL);
210	mutex_init(&buffer_lock, NULL, MUTEX_DEFAULT, NULL);
211	cv_init(&srndpool_read_cv, NULL, CV_DEFAULT, NULL);
212	entropy_bits = 0;
213	pindex = 0;
214	bindex = 0;
215	bstart = 0;
216	snum_waiters = 0;
217	leftover_bytes = 0;
218	buffer_bytes = 0;
219
220	/*
221	 * Initialize the pool using
222	 * . 2 unpredictable times: high resolution time since the boot-time,
223	 *   and the current time-of-the day.
224	 * . The initial physical memory state.
225	 */
226	ts = gethrtime();
227	swrand_add_entropy((uint8_t *)&ts, sizeof (ts), 0);
228
229	(void) drv_getparm(TIME, &now);
230	swrand_add_entropy((uint8_t *)&now, sizeof (now), 0);
231
232	ret = kphysm_setup_func_register(&rnd_dr_callback_vec, NULL);
233	ASSERT(ret == 0);
234
235	if (physmem_ent_init(&entsrc) != 0) {
236		ret = ENOMEM;
237		goto exit1;
238	}
239
240	if ((ret = mod_install(&modlinkage)) != 0)
241		goto exit2;
242
243	/* Schedule periodic mixing of the pool. */
244	mutex_enter(&srndpool_lock);
245	swrand_schedule_timeout();
246	mutex_exit(&srndpool_lock);
247	(void) swrand_get_entropy((uint8_t *)swrand_XKEY, HASHSIZE, B_TRUE);
248	bcopy(swrand_XKEY, previous_bytes, HASHSIZE);
249
250	/* Register with KCF. If the registration fails, return error. */
251	if (crypto_register_provider(&swrand_prov_info, &swrand_prov_handle)) {
252		(void) mod_remove(&modlinkage);
253		ret = EACCES;
254		goto exit2;
255	}
256
257	return (0);
258
259exit2:
260	physmem_ent_fini(&entsrc);
261exit1:
262	mutex_destroy(&srndpool_lock);
263	mutex_destroy(&buffer_lock);
264	cv_destroy(&srndpool_read_cv);
265	return (ret);
266}
267
268int
269_info(struct modinfo *modinfop)
270{
271	return (mod_info(&modlinkage, modinfop));
272}
273
274/*
275 * Control entry points.
276 */
277/* ARGSUSED */
278static void
279swrand_provider_status(crypto_provider_handle_t provider, uint_t *status)
280{
281	*status = CRYPTO_PROVIDER_READY;
282}
283
284/*
285 * Random number entry points.
286 */
287/* ARGSUSED */
288static int
289swrand_seed_random(crypto_provider_handle_t provider, crypto_session_id_t sid,
290    uchar_t *buf, size_t len, uint_t entropy_est, uint32_t flags,
291    crypto_req_handle_t req)
292{
293	/* The entropy estimate is always 0 in this path */
294	if (flags & CRYPTO_SEED_NOW)
295		swrand_add_entropy(buf, len, 0);
296	else
297		swrand_add_entropy_later(buf, len);
298	return (CRYPTO_SUCCESS);
299}
300
301/* ARGSUSED */
302static int
303swrand_generate_random(crypto_provider_handle_t provider,
304    crypto_session_id_t sid, uchar_t *buf, size_t len, crypto_req_handle_t req)
305{
306	if (crypto_kmflag(req) == KM_NOSLEEP)
307		(void) swrand_get_entropy(buf, len, B_TRUE);
308	else
309		(void) swrand_get_entropy(buf, len, B_FALSE);
310
311	return (CRYPTO_SUCCESS);
312}
313
314/*
315 * Extraction of entropy from the pool.
316 *
317 * Returns "len" random bytes in *ptr.
318 * Try to gather some more entropy by calling physmem_ent_gen() when less than
319 * MINEXTRACTBITS are present in the pool.
320 * Will block if not enough entropy was available and the call is blocking.
321 */
322static int
323swrand_get_entropy(uint8_t *ptr, size_t len, boolean_t nonblock)
324{
325	int i, bytes;
326	HASH_CTX hashctx;
327	uint8_t digest[HASHSIZE], *pool;
328	uint32_t tempout[HASHSIZE/BYTES_IN_WORD];
329	int size;
330
331	mutex_enter(&srndpool_lock);
332	if (leftover_bytes > 0) {
333		bytes = min(len, leftover_bytes);
334		bcopy(leftover, ptr, bytes);
335		len -= bytes;
336		ptr += bytes;
337		leftover_bytes -= bytes;
338		if (leftover_bytes > 0)
339			ovbcopy(leftover+bytes, leftover, leftover_bytes);
340	}
341
342	while (len > 0) {
343		/* Check if there is enough entropy */
344		while (entropy_bits < MINEXTRACTBITS) {
345
346			physmem_ent_gen(&entsrc);
347
348			if (entropy_bits < MINEXTRACTBITS &&
349			    nonblock == B_TRUE) {
350				mutex_exit(&srndpool_lock);
351				return (EAGAIN);
352			}
353
354			if (entropy_bits < MINEXTRACTBITS) {
355				ASSERT(nonblock == B_FALSE);
356				snum_waiters++;
357				if (cv_wait_sig(&srndpool_read_cv,
358				    &srndpool_lock) == 0) {
359					snum_waiters--;
360					mutex_exit(&srndpool_lock);
361					return (EINTR);
362				}
363				snum_waiters--;
364			}
365		}
366
367		/* Figure out how many bytes to extract */
368		bytes = min(HASHSIZE, len);
369		bytes = min(bytes, CRYPTO_BITS2BYTES(entropy_bits));
370		entropy_bits -= CRYPTO_BYTES2BITS(bytes);
371		BUMP_SWRAND_STATS(ss_entOut, CRYPTO_BYTES2BITS(bytes));
372		swrand_stats.ss_entEst = entropy_bits;
373
374		/* Extract entropy by hashing pool content */
375		HashInit(&hashctx);
376		HashUpdate(&hashctx, (uint8_t *)srndpool, RNDPOOLSIZE);
377		HashFinal(digest, &hashctx);
378
379		/*
380		 * Feed the digest back into the pool so next
381		 * extraction produces different result
382		 */
383		pool = (uint8_t *)srndpool;
384		for (i = 0; i < HASHSIZE; i++) {
385			pool[pindex++] ^= digest[i];
386			/* pindex modulo RNDPOOLSIZE */
387			pindex &= (RNDPOOLSIZE - 1);
388		}
389
390		/* LINTED E_BAD_PTR_CAST_ALIGN */
391		fips_random_inner(swrand_XKEY, tempout, (uint32_t *)digest);
392
393		if (len >= HASHSIZE) {
394			size = HASHSIZE;
395		} else {
396			size = min(bytes, HASHSIZE);
397		}
398
399		/*
400		 * FIPS 140-2: Continuous RNG test - each generation
401		 * of an n-bit block shall be compared with the previously
402		 * generated block. Test shall fail if any two compared
403		 * n-bit blocks are equal.
404		 */
405		for (i = 0; i < HASHSIZE/BYTES_IN_WORD; i++) {
406			if (tempout[i] != previous_bytes[i])
407				break;
408		}
409
410		if (i == HASHSIZE/BYTES_IN_WORD) {
411			cmn_err(CE_WARN, "swrand: The value of 160-bit block "
412			    "random bytes are same as the previous one.\n");
413			/* discard random bytes and return error */
414			return (EIO);
415		}
416
417		bcopy(tempout, previous_bytes, HASHSIZE);
418
419		bcopy(tempout, ptr, size);
420		if (len < HASHSIZE) {
421			leftover_bytes = HASHSIZE - bytes;
422			bcopy((uint8_t *)tempout + bytes, leftover,
423			    leftover_bytes);
424		}
425
426		ptr += size;
427		len -= size;
428		BUMP_SWRAND_STATS(ss_bytesOut, size);
429	}
430
431	/* Zero out sensitive information */
432	bzero(digest, HASHSIZE);
433	bzero(tempout, HASHSIZE);
434	mutex_exit(&srndpool_lock);
435	return (0);
436}
437
438#define	SWRAND_ADD_BYTES(ptr, len, i, pool)		\
439	ASSERT((ptr) != NULL && (len) > 0);		\
440	BUMP_SWRAND_STATS(ss_bytesIn, (len));		\
441	while ((len)--) {				\
442		(pool)[(i)++] ^= *(ptr);		\
443		(ptr)++;				\
444		(i) &= (RNDPOOLSIZE - 1);		\
445	}
446
447/* Write some more user-provided entropy to the pool */
448static void
449swrand_add_bytes(uint8_t *ptr, size_t len)
450{
451	uint8_t *pool = (uint8_t *)srndpool;
452
453	ASSERT(MUTEX_HELD(&srndpool_lock));
454	SWRAND_ADD_BYTES(ptr, len, pindex, pool);
455}
456
457/*
458 * Add bytes to buffer. Adding the buffer to the random pool
459 * is deferred until the random pool is mixed.
460 */
461static void
462swrand_add_bytes_later(uint8_t *ptr, size_t len)
463{
464	uint8_t *pool = (uint8_t *)buffer;
465
466	ASSERT(MUTEX_HELD(&buffer_lock));
467	SWRAND_ADD_BYTES(ptr, len, bindex, pool);
468	buffer_bytes += len;
469}
470
471#undef SWRAND_ADD_BYTES
472
473/* Mix the pool */
474static void
475swrand_mix_pool(uint16_t entropy_est)
476{
477	int i, j, k, start;
478	HASH_CTX hashctx;
479	uint8_t digest[HASHSIZE];
480	uint8_t *pool = (uint8_t *)srndpool;
481	uint8_t *bp = (uint8_t *)buffer;
482
483	ASSERT(MUTEX_HELD(&srndpool_lock));
484
485	/* add deferred bytes */
486	mutex_enter(&buffer_lock);
487	if (buffer_bytes > 0) {
488		if (buffer_bytes >= RNDPOOLSIZE) {
489			for (i = 0; i < RNDPOOLSIZE/4; i++) {
490				srndpool[i] ^= buffer[i];
491				buffer[i] = 0;
492			}
493			bstart = bindex = 0;
494		} else {
495			for (i = 0; i < buffer_bytes; i++) {
496				pool[pindex++] ^= bp[bstart];
497				bp[bstart++] = 0;
498				pindex &= (RNDPOOLSIZE - 1);
499				bstart &= (RNDPOOLSIZE - 1);
500			}
501			ASSERT(bstart == bindex);
502		}
503		buffer_bytes = 0;
504	}
505	mutex_exit(&buffer_lock);
506
507	start = 0;
508	for (i = 0; i < RNDPOOLSIZE/HASHSIZE + 1; i++) {
509		HashInit(&hashctx);
510
511		/* Hash a buffer centered on a block in the pool */
512		if (start + HASHBUFSIZE <= RNDPOOLSIZE)
513			HashUpdate(&hashctx, &pool[start], HASHBUFSIZE);
514		else {
515			HashUpdate(&hashctx, &pool[start],
516			    RNDPOOLSIZE - start);
517			HashUpdate(&hashctx, pool,
518			    HASHBUFSIZE - RNDPOOLSIZE + start);
519		}
520		HashFinal(digest, &hashctx);
521
522		/* XOR the hash result back into the block */
523		k = (start + HASHSIZE) & (RNDPOOLSIZE - 1);
524		for (j = 0; j < HASHSIZE; j++) {
525			pool[k++] ^= digest[j];
526			k &= (RNDPOOLSIZE - 1);
527		}
528
529		/* Slide the hash buffer and repeat with next block */
530		start = (start + HASHSIZE) & (RNDPOOLSIZE - 1);
531	}
532
533	entropy_bits += entropy_est;
534	if (entropy_bits > CRYPTO_BYTES2BITS(RNDPOOLSIZE))
535		entropy_bits = CRYPTO_BYTES2BITS(RNDPOOLSIZE);
536
537	swrand_stats.ss_entEst = entropy_bits;
538	BUMP_SWRAND_STATS(ss_entIn, entropy_est);
539}
540
541static void
542swrand_add_entropy_later(uint8_t *ptr, size_t len)
543{
544	mutex_enter(&buffer_lock);
545	swrand_add_bytes_later(ptr, len);
546	mutex_exit(&buffer_lock);
547}
548
549static void
550swrand_add_entropy(uint8_t *ptr, size_t len, uint16_t entropy_est)
551{
552	mutex_enter(&srndpool_lock);
553	swrand_add_bytes(ptr, len);
554	swrand_mix_pool(entropy_est);
555	mutex_exit(&srndpool_lock);
556}
557
558/*
559 * The physmem_* routines below generate entropy by reading blocks of
560 * physical memory.  Entropy is gathered in a couple of ways:
561 *
562 *  - By reading blocks of physical memory and detecting if changes
563 *    occurred in the blocks read.
564 *
565 *  - By measuring the time it takes to load and hash a block of memory
566 *    and computing the differences in the measured time.
567 *
568 * The first method was used in the CryptoRand implementation.  Physical
569 * memory is divided into blocks of fixed size.  A block of memory is
570 * chosen from the possible blocks and hashed to produce a digest.  This
571 * digest is then mixed into the pool.  A single bit from the digest is
572 * used as a parity bit or "checksum" and compared against the previous
573 * "checksum" computed for the block.  If the single-bit checksum has not
574 * changed, no entropy is credited to the pool.  If there is a change,
575 * then the assumption is that at least one bit in the block has changed.
576 * The possible locations within the memory block of where the bit change
577 * occurred is used as a measure of entropy.  For example, if a block
578 * size of 4096 bytes is used, about log_2(4096*8)=15 bits worth of
579 * entropy is available.  Because the single-bit checksum will miss half
580 * of the changes, the amount of entropy credited to the pool is doubled
581 * when a change is detected.  With a 4096 byte block size, a block
582 * change will add a total of 30 bits of entropy to the pool.
583 *
584 * The second method measures the amount of time it takes to read and
585 * hash a physical memory block (as described above).  The time measured
586 * can vary depending on system load, scheduling and other factors.
587 * Differences between consecutive measurements are computed to come up
588 * with an entropy estimate.  The first, second, and third order delta is
589 * calculated to determine the minimum delta value.  The number of bits
590 * present in this minimum delta value is the entropy estimate.  This
591 * entropy estimation technique using time deltas is similar to that used
592 * in /dev/random implementations from Linux/BSD.
593 */
594
595static int
596physmem_ent_init(physmem_entsrc_t *entsrc)
597{
598	uint8_t *ptr;
599	int i;
600
601	bzero(entsrc, sizeof (*entsrc));
602
603	/*
604	 * The maximum entropy amount in bits per block of memory read is
605	 * log_2(MEMBLOCKSIZE * 8);
606	 */
607	i = CRYPTO_BYTES2BITS(MEMBLOCKSIZE);
608	while (i >>= 1)
609		entsrc->entperblock++;
610
611	/* Initialize entsrc->nblocks */
612	physmem_count_blocks();
613
614	if (entsrc->nblocks == 0) {
615		cmn_err(CE_WARN, "no memory blocks to scan!");
616		return (-1);
617	}
618
619	/* Allocate space for the parity vector and memory page */
620	entsrc->parity = kmem_alloc(howmany(entsrc->nblocks, 8),
621	    KM_SLEEP);
622	entsrc->pmbuf = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
623
624
625	/* Initialize parity vector with bits from the pool */
626	i = howmany(entsrc->nblocks, 8);
627	ptr = entsrc->parity;
628	while (i > 0) {
629		if (i > RNDPOOLSIZE) {
630			bcopy(srndpool, ptr, RNDPOOLSIZE);
631			mutex_enter(&srndpool_lock);
632			swrand_mix_pool(0);
633			mutex_exit(&srndpool_lock);
634			ptr += RNDPOOLSIZE;
635			i -= RNDPOOLSIZE;
636		} else {
637			bcopy(srndpool, ptr, i);
638			break;
639		}
640	}
641
642	/* Generate some entropy to further initialize the pool */
643	mutex_enter(&srndpool_lock);
644	physmem_ent_gen(entsrc);
645	entropy_bits = 0;
646	mutex_exit(&srndpool_lock);
647
648	return (0);
649}
650
651static void
652physmem_ent_fini(physmem_entsrc_t *entsrc)
653{
654	if (entsrc->pmbuf != NULL)
655		vmem_free(heap_arena, entsrc->pmbuf, PAGESIZE);
656	if (entsrc->parity != NULL)
657		kmem_free(entsrc->parity, howmany(entsrc->nblocks, 8));
658	bzero(entsrc, sizeof (*entsrc));
659}
660
661static void
662physmem_ent_gen(physmem_entsrc_t *entsrc)
663{
664	struct memlist *pmem;
665	offset_t offset, poffset;
666	pfn_t pfn;
667	int i, nbytes, len, ent = 0;
668	uint32_t block, oblock;
669	hrtime_t ts1, ts2, diff, delta, delta2, delta3;
670	uint8_t digest[HASHSIZE];
671	HASH_CTX ctx;
672	page_t *pp;
673
674	/*
675	 * Use each 32-bit quantity in the pool to pick a memory
676	 * block to read.
677	 */
678	for (i = 0; i < RNDPOOLSIZE/4; i++) {
679
680		/* If the pool is "full", stop after one block */
681		if (entropy_bits + ent >= CRYPTO_BYTES2BITS(RNDPOOLSIZE)) {
682			if (i > 0)
683				break;
684		}
685
686		/*
687		 * This lock protects reading of phys_install.
688		 * Any changes to this list, by DR, are done while
689		 * holding this lock. So, holding this lock is sufficient
690		 * to handle DR also.
691		 */
692		memlist_read_lock();
693
694		/* We're left with less than 4K of memory after DR */
695		ASSERT(entsrc->nblocks > 0);
696
697		/* Pick a memory block to read */
698		block = oblock = srndpool[i] % entsrc->nblocks;
699
700		for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) {
701			if (block < pmem->ml_size / MEMBLOCKSIZE)
702				break;
703			block -= pmem->ml_size / MEMBLOCKSIZE;
704		}
705
706		ASSERT(pmem != NULL);
707
708		offset = pmem->ml_address + block * MEMBLOCKSIZE;
709
710		if (!address_in_memlist(phys_install, offset, MEMBLOCKSIZE)) {
711			memlist_read_unlock();
712			continue;
713		}
714
715		/*
716		 * Do an initial check to see if the address is safe
717		 */
718		if (plat_hold_page(offset >> PAGESHIFT, PLAT_HOLD_NO_LOCK, NULL)
719		    == PLAT_HOLD_FAIL) {
720			memlist_read_unlock();
721			continue;
722		}
723
724		/*
725		 * Figure out which page to load to read the
726		 * memory block.  Load the page and compute the
727		 * hash of the memory block.
728		 */
729		len = MEMBLOCKSIZE;
730		ts1 = gethrtime();
731		HashInit(&ctx);
732		while (len) {
733			pfn = offset >> PAGESHIFT;
734			poffset = offset & PAGEOFFSET;
735			nbytes = PAGESIZE - poffset < len ?
736			    PAGESIZE - poffset : len;
737
738			/*
739			 * Re-check the offset, and lock the frame.  If the
740			 * page was given away after the above check, we'll
741			 * just bail out.
742			 */
743			if (plat_hold_page(pfn, PLAT_HOLD_LOCK, &pp) ==
744			    PLAT_HOLD_FAIL)
745				break;
746
747			hat_devload(kas.a_hat, entsrc->pmbuf,
748			    PAGESIZE, pfn, PROT_READ,
749			    HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
750
751			HashUpdate(&ctx, (uint8_t *)entsrc->pmbuf + poffset,
752			    nbytes);
753
754			hat_unload(kas.a_hat, entsrc->pmbuf, PAGESIZE,
755			    HAT_UNLOAD_UNLOCK);
756
757			plat_release_page(pp);
758
759			len -= nbytes;
760			offset += nbytes;
761		}
762		/* We got our pages. Let the DR roll */
763		memlist_read_unlock();
764
765		/* See if we had to bail out due to a page being given away */
766		if (len)
767			continue;
768
769		HashFinal(digest, &ctx);
770		ts2 = gethrtime();
771
772		/*
773		 * Compute the time it took to load and hash the
774		 * block and compare it against the previous
775		 * measurement. The delta of the time values
776		 * provides a small amount of entropy.  The
777		 * minimum of the first, second, and third order
778		 * delta is used to estimate how much entropy
779		 * is present.
780		 */
781		diff = ts2 - ts1;
782		delta = diff - entsrc->last_diff;
783		if (delta < 0)
784			delta = -delta;
785		delta2 = delta - entsrc->last_delta;
786		if (delta2 < 0)
787			delta2 = -delta2;
788		delta3 = delta2 - entsrc->last_delta2;
789		if (delta3 < 0)
790			delta3 = -delta3;
791		entsrc->last_diff = diff;
792		entsrc->last_delta = delta;
793		entsrc->last_delta2 = delta2;
794
795		if (delta > delta2)
796			delta = delta2;
797		if (delta > delta3)
798			delta = delta3;
799		delta2 = 0;
800		while (delta >>= 1)
801			delta2++;
802		ent += delta2;
803
804		/*
805		 * If the memory block has changed, credit the pool with
806		 * the entropy estimate.  The entropy estimate is doubled
807		 * because the single-bit checksum misses half the change
808		 * on average.
809		 */
810		if (physmem_parity_update(entsrc->parity, oblock,
811		    digest[0] & 1))
812			ent += 2 * entsrc->entperblock;
813
814		/* Add the entropy bytes to the pool */
815		swrand_add_bytes(digest, HASHSIZE);
816		swrand_add_bytes((uint8_t *)&ts1, sizeof (ts1));
817		swrand_add_bytes((uint8_t *)&ts2, sizeof (ts2));
818	}
819
820	swrand_mix_pool(ent);
821}
822
823static int
824physmem_parity_update(uint8_t *parity_vec, uint32_t block, int parity)
825{
826	/* Test and set the parity bit, return 1 if changed */
827	if (parity == ((parity_vec[block >> 3] >> (block & 7)) & 1))
828		return (0);
829	parity_vec[block >> 3] ^= 1 << (block & 7);
830	return (1);
831}
832
833/* Compute number of memory blocks available to scan */
834static void
835physmem_count_blocks()
836{
837	struct memlist *pmem;
838
839	memlist_read_lock();
840	entsrc.nblocks = 0;
841	for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) {
842		entsrc.nblocks += pmem->ml_size / MEMBLOCKSIZE;
843		if (entsrc.nblocks > MAXMEMBLOCKS) {
844			entsrc.nblocks = MAXMEMBLOCKS;
845			break;
846		}
847	}
848	memlist_read_unlock();
849}
850
851/*
852 * Dynamic Reconfiguration call-back functions
853 */
854
855/* ARGSUSED */
856static void
857rnd_dr_callback_post_add(void *arg, pgcnt_t delta)
858{
859	/* More memory is available now, so update entsrc->nblocks. */
860	physmem_count_blocks();
861}
862
863/* Call-back routine invoked before the DR starts a memory removal. */
864/* ARGSUSED */
865static int
866rnd_dr_callback_pre_del(void *arg, pgcnt_t delta)
867{
868	return (0);
869}
870
871/* Call-back routine invoked after the DR starts a memory removal. */
872/* ARGSUSED */
873static void
874rnd_dr_callback_post_del(void *arg, pgcnt_t delta, int cancelled)
875{
876	/* Memory has shrunk, so update entsrc->nblocks. */
877	physmem_count_blocks();
878}
879
880/* Timeout handling to gather entropy from physmem events */
881static void
882swrand_schedule_timeout(void)
883{
884	clock_t ut;	/* time in microseconds */
885
886	ASSERT(MUTEX_HELD(&srndpool_lock));
887	/*
888	 * The new timeout value is taken from the pool of random bits.
889	 * We're merely reading the first 32 bits from the pool here, not
890	 * consuming any entropy.
891	 * This routine is usually called right after stirring the pool, so
892	 * srndpool[0] will have a *fresh* random value each time.
893	 * The timeout multiplier value is a random value between 0.7 sec and
894	 * 1.748575 sec (0.7 sec + 0xFFFFF microseconds).
895	 * The new timeout is TIMEOUT_INTERVAL times that multiplier.
896	 */
897	ut = 700000 + (clock_t)(srndpool[0] & 0xFFFFF);
898	rnd_timeout_id = timeout(rnd_handler, NULL,
899	    TIMEOUT_INTERVAL * drv_usectohz(ut));
900}
901
902/*ARGSUSED*/
903static void
904rnd_handler(void *arg)
905{
906	mutex_enter(&srndpool_lock);
907
908	physmem_ent_gen(&entsrc);
909	if (snum_waiters > 0)
910		cv_broadcast(&srndpool_read_cv);
911	swrand_schedule_timeout();
912
913	mutex_exit(&srndpool_lock);
914}
915
916/*
917 * Swrand Power-Up Self-Test
918 */
919void
920swrand_POST(int *rc)
921{
922
923	*rc = fips_rng_post();
924
925}
926