memnode.c revision 414:c62c3f13a640 
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#pragma ident	"%Z%%M%	%I%	%E% SMI"
28
29#include <sys/systm.h>
30#include <sys/platform_module.h>
31#include <sys/sysmacros.h>
32#include <sys/atomic.h>
33#include <sys/memlist.h>
34#include <sys/memnode.h>
35#include <vm/vm_dep.h>
36
37int max_mem_nodes = 1;		/* max memory nodes on this system */
38
39struct mem_node_conf mem_node_config[MAX_MEM_NODES];
40int mem_node_pfn_shift;
41/*
42 * num_memnodes should be updated atomically and always >=
43 * the number of bits in memnodes_mask or the algorithm may fail.
44 */
45uint16_t num_memnodes;
46mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */
47
48/*
49 * If set, mem_node_physalign should be a power of two, and
50 * should reflect the minimum address alignment of each node.
51 */
52uint64_t mem_node_physalign;
53
54/*
55 * Platform hooks we will need.
56 */
57
58#pragma weak plat_build_mem_nodes
59#pragma weak plat_slice_add
60#pragma weak plat_slice_del
61
62/*
63 * Adjust the memnode config after a DR operation.
64 *
65 * It is rather tricky to do these updates since we can't
66 * protect the memnode structures with locks, so we must
67 * be mindful of the order in which updates and reads to
68 * these values can occur.
69 */
70void
71mem_node_add_slice(pfn_t start, pfn_t end)
72{
73	int mnode;
74	mnodeset_t newmask, oldmask;
75
76	/*
77	 * DR will pass us the first pfn that is allocatable.
78	 * We need to round down to get the real start of
79	 * the slice.
80	 */
81	if (mem_node_physalign) {
82		start &= ~(btop(mem_node_physalign) - 1);
83		end = roundup(end, btop(mem_node_physalign)) - 1;
84	}
85
86	if (&plat_slice_add)
87		plat_slice_add(start, end);
88
89	mnode = PFN_2_MEM_NODE(start);
90	ASSERT(mnode < max_mem_nodes);
91
92	if (cas32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) {
93		/*
94		 * Add slice to existing node.
95		 */
96		if (start < mem_node_config[mnode].physbase)
97			mem_node_config[mnode].physbase = start;
98		if (end > mem_node_config[mnode].physmax)
99			mem_node_config[mnode].physmax = end;
100	} else {
101		mem_node_config[mnode].physbase = start;
102		mem_node_config[mnode].physmax = end;
103		atomic_add_16(&num_memnodes, 1);
104		do {
105			oldmask = memnodes_mask;
106			newmask = memnodes_mask | (1ull << mnode);
107		} while (cas64(&memnodes_mask, oldmask, newmask) != oldmask);
108	}
109	/*
110	 * Let the common lgrp framework know about the new memory
111	 */
112	lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode));
113}
114
115/* ARGSUSED */
116void
117mem_node_pre_del_slice(pfn_t start, pfn_t end)
118{
119	int mnode = PFN_2_MEM_NODE(start);
120
121	ASSERT(mnode < max_mem_nodes);
122	ASSERT(mem_node_config[mnode].exists == 1);
123}
124
125/*
126 * Remove a PFN range from a memnode.  On some platforms,
127 * the memnode will be created with physbase at the first
128 * allocatable PFN, but later deleted with the MC slice
129 * base address converted to a PFN, in which case we need
130 * to assume physbase and up.
131 */
132void
133mem_node_post_del_slice(pfn_t start, pfn_t end, int cancelled)
134{
135	int mnode;
136	pgcnt_t delta_pgcnt, node_size;
137	mnodeset_t omask, nmask;
138
139	if (mem_node_physalign) {
140		start &= ~(btop(mem_node_physalign) - 1);
141		end = roundup(end, btop(mem_node_physalign)) - 1;
142	}
143	mnode = PFN_2_MEM_NODE(start);
144
145	ASSERT(mnode < max_mem_nodes);
146	ASSERT(mem_node_config[mnode].exists == 1);
147
148	if (!cancelled) {
149		delta_pgcnt = end - start;
150		node_size = mem_node_config[mnode].physmax -
151				mem_node_config[mnode].physbase;
152
153		if (node_size > delta_pgcnt) {
154			/*
155			 * Subtract the slice from the memnode.
156			 */
157			if (start <= mem_node_config[mnode].physbase)
158				mem_node_config[mnode].physbase = end + 1;
159			ASSERT(end <= mem_node_config[mnode].physmax);
160			if (end == mem_node_config[mnode].physmax)
161				mem_node_config[mnode].physmax = start - 1;
162		} else {
163
164			/*
165			 * Let the common lgrp framework know the mnode is
166			 * leaving
167			 */
168			lgrp_config(LGRP_CONFIG_MEM_DEL, mnode,
169			    MEM_NODE_2_LGRPHAND(mnode));
170
171			/*
172			 * Delete the whole node.
173			 */
174			ASSERT(MNODE_PGCNT(mnode) == 0);
175			do {
176				omask = memnodes_mask;
177				nmask = omask & ~(1ull << mnode);
178			} while (cas64(&memnodes_mask, omask, nmask) != omask);
179			atomic_add_16(&num_memnodes, -1);
180			mem_node_config[mnode].exists = 0;
181		}
182
183		if (&plat_slice_del)
184			plat_slice_del(start, end);
185	}
186}
187
188void
189startup_build_mem_nodes(u_longlong_t *list, size_t nelems)
190{
191	size_t	elem;
192	pfn_t	basepfn;
193	pgcnt_t	npgs;
194
195	/* LINTED: ASSERT will always true or false */
196	ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes);
197
198	if (&plat_build_mem_nodes) {
199		plat_build_mem_nodes(list, nelems);
200	} else {
201		/*
202		 * Boot install lists are arranged <addr, len>, ...
203		 */
204		for (elem = 0; elem < nelems; elem += 2) {
205			basepfn = btop(list[elem]);
206			npgs = btop(list[elem+1]);
207			mem_node_add_slice(basepfn, basepfn + npgs - 1);
208		}
209		mem_node_physalign = 0;
210		mem_node_pfn_shift = 0;
211	}
212}
213
214/*
215 * Allocate an unassigned memnode.
216 */
217int
218mem_node_alloc()
219{
220	int mnode;
221	mnodeset_t newmask, oldmask;
222
223	/*
224	 * Find an unused memnode.  Update it atomically to prevent
225	 * a first time memnode creation race.
226	 */
227	for (mnode = 0; mnode < max_mem_nodes; mnode++)
228		if (cas32((uint32_t *)&mem_node_config[mnode].exists,
229			0, 1) == 0)
230			break;
231
232	if (mnode >= max_mem_nodes)
233			panic("Out of free memnodes\n");
234
235	mem_node_config[mnode].physbase = (uint64_t)-1;
236	mem_node_config[mnode].physmax = 0;
237	atomic_add_16(&num_memnodes, 1);
238	do {
239		oldmask = memnodes_mask;
240		newmask = memnodes_mask | (1ull << mnode);
241	} while (cas64(&memnodes_mask, oldmask, newmask) != oldmask);
242
243	return (mnode);
244}
245
246/*
247 * Find the intersection between a memnode and a memlist
248 * and returns the number of pages that overlap.
249 *
250 * Assumes the list is protected from DR operations by
251 * the memlist lock.
252 */
253pgcnt_t
254mem_node_memlist_pages(int mnode, struct memlist *mlist)
255{
256	pfn_t		base, end;
257	pfn_t		cur_base, cur_end;
258	pgcnt_t		npgs;
259	struct memlist	*pmem;
260
261	base = mem_node_config[mnode].physbase;
262	end = mem_node_config[mnode].physmax;
263	npgs = 0;
264
265	memlist_read_lock();
266
267	for (pmem = mlist; pmem; pmem = pmem->next) {
268		cur_base = btop(pmem->address);
269		cur_end = cur_base + btop(pmem->size) - 1;
270		if (end <= cur_base || base >= cur_end)
271			continue;
272		npgs = npgs + (MIN(cur_end, end) -
273		    MAX(cur_base, base)) + 1;
274	}
275
276	memlist_read_unlock();
277
278	return (npgs);
279}
280