1/*
2 * Copyright (C) 2001-2006 Silicon Graphics, Inc.  All rights
3 * reserved.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of version 2 of the GNU General Public License
7 * as published by the Free Software Foundation.
8 */
9
10/*
11 * SN Platform Special Memory (mspec) Support
12 *
13 * This driver exports the SN special memory (mspec) facility to user
14 * processes.
15 * There are three types of memory made available thru this driver:
16 * fetchops, uncached and cached.
17 *
18 * Fetchops are atomic memory operations that are implemented in the
19 * memory controller on SGI SN hardware.
20 *
21 * Uncached are used for memory write combining feature of the ia64
22 * cpu.
23 *
24 * Cached are used for areas of memory that are used as cached addresses
25 * on our partition and used as uncached addresses from other partitions.
26 * Due to a design constraint of the SN2 Shub, you can not have processors
27 * on the same FSB perform both a cached and uncached reference to the
28 * same cache line.  These special memory cached regions prevent the
29 * kernel from ever dropping in a TLB entry and therefore prevent the
30 * processor from ever speculating a cache line from this page.
31 */
32
33#include <linux/types.h>
34#include <linux/kernel.h>
35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/errno.h>
38#include <linux/miscdevice.h>
39#include <linux/spinlock.h>
40#include <linux/mm.h>
41#include <linux/vmalloc.h>
42#include <linux/string.h>
43#include <linux/slab.h>
44#include <linux/numa.h>
45#include <asm/page.h>
46#include <asm/system.h>
47#include <asm/pgtable.h>
48#include <asm/atomic.h>
49#include <asm/tlbflush.h>
50#include <asm/uncached.h>
51#include <asm/sn/addrs.h>
52#include <asm/sn/arch.h>
53#include <asm/sn/mspec.h>
54#include <asm/sn/sn_cpuid.h>
55#include <asm/sn/io.h>
56#include <asm/sn/bte.h>
57#include <asm/sn/shubio.h>
58
59
60#define FETCHOP_ID	"SGI Fetchop,"
61#define CACHED_ID	"Cached,"
62#define UNCACHED_ID	"Uncached"
63#define REVISION	"4.0"
64#define MSPEC_BASENAME	"mspec"
65
66/*
67 * Page types allocated by the device.
68 */
69enum {
70	MSPEC_FETCHOP = 1,
71	MSPEC_CACHED,
72	MSPEC_UNCACHED
73};
74
75#ifdef CONFIG_SGI_SN
76static int is_sn2;
77#else
78#define is_sn2		0
79#endif
80
81/*
82 * One of these structures is allocated when an mspec region is mmaped. The
83 * structure is pointed to by the vma->vm_private_data field in the vma struct.
84 * This structure is used to record the addresses of the mspec pages.
85 */
86struct vma_data {
87	atomic_t refcnt;	/* Number of vmas sharing the data. */
88	spinlock_t lock;	/* Serialize access to the vma. */
89	int count;		/* Number of pages allocated. */
90	int type;		/* Type of pages allocated. */
91	unsigned long maddr[0];	/* Array of MSPEC addresses. */
92};
93
94/* used on shub2 to clear FOP cache in the HUB */
95static unsigned long scratch_page[MAX_NUMNODES];
96#define SH2_AMO_CACHE_ENTRIES	4
97
98static inline int
99mspec_zero_block(unsigned long addr, int len)
100{
101	int status;
102
103	if (is_sn2) {
104		if (is_shub2()) {
105			int nid;
106			void *p;
107			int i;
108
109			nid = nasid_to_cnodeid(get_node_number(__pa(addr)));
110			p = (void *)TO_AMO(scratch_page[nid]);
111
112			for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {
113				FETCHOP_LOAD_OP(p, FETCHOP_LOAD);
114				p += FETCHOP_VAR_SIZE;
115			}
116		}
117
118		status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,
119				  BTE_WACQUIRE | BTE_ZERO_FILL, NULL);
120	} else {
121		memset((char *) addr, 0, len);
122		status = 0;
123	}
124	return status;
125}
126
127/*
128 * mspec_open
129 *
130 * Called when a device mapping is created by a means other than mmap
131 * (via fork, etc.).  Increments the reference count on the underlying
132 * mspec data so it is not freed prematurely.
133 */
134static void
135mspec_open(struct vm_area_struct *vma)
136{
137	struct vma_data *vdata;
138
139	vdata = vma->vm_private_data;
140	atomic_inc(&vdata->refcnt);
141}
142
143/*
144 * mspec_close
145 *
146 * Called when unmapping a device mapping. Frees all mspec pages
147 * belonging to the vma.
148 */
149static void
150mspec_close(struct vm_area_struct *vma)
151{
152	struct vma_data *vdata;
153	int i, pages, result, vdata_size;
154
155	vdata = vma->vm_private_data;
156	if (!atomic_dec_and_test(&vdata->refcnt))
157		return;
158
159	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
160	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
161	for (i = 0; i < pages; i++) {
162		if (vdata->maddr[i] == 0)
163			continue;
164		/*
165		 * Clear the page before sticking it back
166		 * into the pool.
167		 */
168		result = mspec_zero_block(vdata->maddr[i], PAGE_SIZE);
169		if (!result)
170			uncached_free_page(vdata->maddr[i]);
171		else
172			printk(KERN_WARNING "mspec_close(): "
173			       "failed to zero page %i\n",
174			       result);
175	}
176
177	if (vdata_size <= PAGE_SIZE)
178		kfree(vdata);
179	else
180		vfree(vdata);
181}
182
183
184/*
185 * mspec_nopfn
186 *
187 * Creates a mspec page and maps it to user space.
188 */
189static unsigned long
190mspec_nopfn(struct vm_area_struct *vma, unsigned long address)
191{
192	unsigned long paddr, maddr;
193	unsigned long pfn;
194	int index;
195	struct vma_data *vdata = vma->vm_private_data;
196
197	index = (address - vma->vm_start) >> PAGE_SHIFT;
198	maddr = (volatile unsigned long) vdata->maddr[index];
199	if (maddr == 0) {
200		maddr = uncached_alloc_page(numa_node_id());
201		if (maddr == 0)
202			return NOPFN_OOM;
203
204		spin_lock(&vdata->lock);
205		if (vdata->maddr[index] == 0) {
206			vdata->count++;
207			vdata->maddr[index] = maddr;
208		} else {
209			uncached_free_page(maddr);
210			maddr = vdata->maddr[index];
211		}
212		spin_unlock(&vdata->lock);
213	}
214
215	if (vdata->type == MSPEC_FETCHOP)
216		paddr = TO_AMO(maddr);
217	else
218		paddr = maddr & ~__IA64_UNCACHED_OFFSET;
219
220	pfn = paddr >> PAGE_SHIFT;
221
222	return pfn;
223}
224
225static struct vm_operations_struct mspec_vm_ops = {
226	.open = mspec_open,
227	.close = mspec_close,
228	.nopfn = mspec_nopfn
229};
230
231/*
232 * mspec_mmap
233 *
234 * Called when mmaping the device.  Initializes the vma with a fault handler
235 * and private data structure necessary to allocate, track, and free the
236 * underlying pages.
237 */
238static int
239mspec_mmap(struct file *file, struct vm_area_struct *vma, int type)
240{
241	struct vma_data *vdata;
242	int pages, vdata_size;
243
244	if (vma->vm_pgoff != 0)
245		return -EINVAL;
246
247	if ((vma->vm_flags & VM_SHARED) == 0)
248		return -EINVAL;
249
250	if ((vma->vm_flags & VM_WRITE) == 0)
251		return -EPERM;
252
253	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
254	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
255	if (vdata_size <= PAGE_SIZE)
256		vdata = kmalloc(vdata_size, GFP_KERNEL);
257	else
258		vdata = vmalloc(vdata_size);
259	if (!vdata)
260		return -ENOMEM;
261	memset(vdata, 0, vdata_size);
262
263	vdata->type = type;
264	spin_lock_init(&vdata->lock);
265	vdata->refcnt = ATOMIC_INIT(1);
266	vma->vm_private_data = vdata;
267
268	vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED | VM_PFNMAP);
269	if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)
270		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
271	vma->vm_ops = &mspec_vm_ops;
272
273	return 0;
274}
275
276static int
277fetchop_mmap(struct file *file, struct vm_area_struct *vma)
278{
279	return mspec_mmap(file, vma, MSPEC_FETCHOP);
280}
281
282static int
283cached_mmap(struct file *file, struct vm_area_struct *vma)
284{
285	return mspec_mmap(file, vma, MSPEC_CACHED);
286}
287
288static int
289uncached_mmap(struct file *file, struct vm_area_struct *vma)
290{
291	return mspec_mmap(file, vma, MSPEC_UNCACHED);
292}
293
294static const struct file_operations fetchop_fops = {
295	.owner = THIS_MODULE,
296	.mmap = fetchop_mmap
297};
298
299static struct miscdevice fetchop_miscdev = {
300	.minor = MISC_DYNAMIC_MINOR,
301	.name = "sgi_fetchop",
302	.fops = &fetchop_fops
303};
304
305static const struct file_operations cached_fops = {
306	.owner = THIS_MODULE,
307	.mmap = cached_mmap
308};
309
310static struct miscdevice cached_miscdev = {
311	.minor = MISC_DYNAMIC_MINOR,
312	.name = "mspec_cached",
313	.fops = &cached_fops
314};
315
316static const struct file_operations uncached_fops = {
317	.owner = THIS_MODULE,
318	.mmap = uncached_mmap
319};
320
321static struct miscdevice uncached_miscdev = {
322	.minor = MISC_DYNAMIC_MINOR,
323	.name = "mspec_uncached",
324	.fops = &uncached_fops
325};
326
327/*
328 * mspec_init
329 *
330 * Called at boot time to initialize the mspec facility.
331 */
332static int __init
333mspec_init(void)
334{
335	int ret;
336	int nid;
337
338	/*
339	 * The fetchop device only works on SN2 hardware, uncached and cached
340	 * memory drivers should both be valid on all ia64 hardware
341	 */
342#ifdef CONFIG_SGI_SN
343	if (ia64_platform_is("sn2")) {
344		is_sn2 = 1;
345		if (is_shub2()) {
346			ret = -ENOMEM;
347			for_each_online_node(nid) {
348				int actual_nid;
349				int nasid;
350				unsigned long phys;
351
352				scratch_page[nid] = uncached_alloc_page(nid);
353				if (scratch_page[nid] == 0)
354					goto free_scratch_pages;
355				phys = __pa(scratch_page[nid]);
356				nasid = get_node_number(phys);
357				actual_nid = nasid_to_cnodeid(nasid);
358				if (actual_nid != nid)
359					goto free_scratch_pages;
360			}
361		}
362
363		ret = misc_register(&fetchop_miscdev);
364		if (ret) {
365			printk(KERN_ERR
366			       "%s: failed to register device %i\n",
367			       FETCHOP_ID, ret);
368			goto free_scratch_pages;
369		}
370	}
371#endif
372	ret = misc_register(&cached_miscdev);
373	if (ret) {
374		printk(KERN_ERR "%s: failed to register device %i\n",
375		       CACHED_ID, ret);
376		if (is_sn2)
377			misc_deregister(&fetchop_miscdev);
378		goto free_scratch_pages;
379	}
380	ret = misc_register(&uncached_miscdev);
381	if (ret) {
382		printk(KERN_ERR "%s: failed to register device %i\n",
383		       UNCACHED_ID, ret);
384		misc_deregister(&cached_miscdev);
385		if (is_sn2)
386			misc_deregister(&fetchop_miscdev);
387		goto free_scratch_pages;
388	}
389
390	printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",
391	       MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",
392	       CACHED_ID, UNCACHED_ID);
393
394	return 0;
395
396 free_scratch_pages:
397	for_each_node(nid) {
398		if (scratch_page[nid] != 0)
399			uncached_free_page(scratch_page[nid]);
400	}
401	return ret;
402}
403
404static void __exit
405mspec_exit(void)
406{
407	int nid;
408
409	misc_deregister(&uncached_miscdev);
410	misc_deregister(&cached_miscdev);
411	if (is_sn2) {
412		misc_deregister(&fetchop_miscdev);
413
414		for_each_node(nid) {
415			if (scratch_page[nid] != 0)
416				uncached_free_page(scratch_page[nid]);
417		}
418	}
419}
420
421module_init(mspec_init);
422module_exit(mspec_exit);
423
424MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
425MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
426MODULE_LICENSE("GPL");
427