1/* 2 * "Cell Reference Set" HTAB support. 3 * 4 * (C) Copyright 2006-2007 TOSHIBA CORPORATION 5 * 6 * This code is based on arch/powerpc/platforms/pseries/lpar.c: 7 * Copyright (C) 2001 Todd Inglett, IBM Corporation 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published by 11 * the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License along 20 * with this program; if not, write to the Free Software Foundation, Inc., 21 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 22 */ 23 24#undef DEBUG_LOW 25 26#include <linux/kernel.h> 27#include <linux/spinlock.h> 28 29#include <asm/mmu.h> 30#include <asm/page.h> 31#include <asm/pgtable.h> 32#include <asm/machdep.h> 33#include <asm/udbg.h> 34 35#include "beat_wrapper.h" 36 37#ifdef DEBUG_LOW 38#define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0) 39#else 40#define DBG_LOW(fmt...) do { } while (0) 41#endif 42 43static DEFINE_RAW_SPINLOCK(beat_htab_lock); 44 45static inline unsigned int beat_read_mask(unsigned hpte_group) 46{ 47 unsigned long rmask = 0; 48 u64 hpte_v[5]; 49 50 beat_read_htab_entries(0, hpte_group + 0, hpte_v); 51 if (!(hpte_v[0] & HPTE_V_BOLTED)) 52 rmask |= 0x8000; 53 if (!(hpte_v[1] & HPTE_V_BOLTED)) 54 rmask |= 0x4000; 55 if (!(hpte_v[2] & HPTE_V_BOLTED)) 56 rmask |= 0x2000; 57 if (!(hpte_v[3] & HPTE_V_BOLTED)) 58 rmask |= 0x1000; 59 beat_read_htab_entries(0, hpte_group + 4, hpte_v); 60 if (!(hpte_v[0] & HPTE_V_BOLTED)) 61 rmask |= 0x0800; 62 if (!(hpte_v[1] & HPTE_V_BOLTED)) 63 rmask |= 0x0400; 64 if (!(hpte_v[2] & HPTE_V_BOLTED)) 65 rmask |= 0x0200; 66 if (!(hpte_v[3] & HPTE_V_BOLTED)) 67 rmask |= 0x0100; 68 hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP); 69 beat_read_htab_entries(0, hpte_group + 0, hpte_v); 70 if (!(hpte_v[0] & HPTE_V_BOLTED)) 71 rmask |= 0x80; 72 if (!(hpte_v[1] & HPTE_V_BOLTED)) 73 rmask |= 0x40; 74 if (!(hpte_v[2] & HPTE_V_BOLTED)) 75 rmask |= 0x20; 76 if (!(hpte_v[3] & HPTE_V_BOLTED)) 77 rmask |= 0x10; 78 beat_read_htab_entries(0, hpte_group + 4, hpte_v); 79 if (!(hpte_v[0] & HPTE_V_BOLTED)) 80 rmask |= 0x08; 81 if (!(hpte_v[1] & HPTE_V_BOLTED)) 82 rmask |= 0x04; 83 if (!(hpte_v[2] & HPTE_V_BOLTED)) 84 rmask |= 0x02; 85 if (!(hpte_v[3] & HPTE_V_BOLTED)) 86 rmask |= 0x01; 87 return rmask; 88} 89 90static long beat_lpar_hpte_insert(unsigned long hpte_group, 91 unsigned long va, unsigned long pa, 92 unsigned long rflags, unsigned long vflags, 93 int psize, int ssize) 94{ 95 unsigned long lpar_rc; 96 u64 hpte_v, hpte_r, slot; 97 98 /* same as iseries */ 99 if (vflags & HPTE_V_SECONDARY) 100 return -1; 101 102 if (!(vflags & HPTE_V_BOLTED)) 103 DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, " 104 "rflags=%lx, vflags=%lx, psize=%d)\n", 105 hpte_group, va, pa, rflags, vflags, psize); 106 107 hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) | 108 vflags | HPTE_V_VALID; 109 hpte_r = hpte_encode_r(pa, psize) | rflags; 110 111 if (!(vflags & HPTE_V_BOLTED)) 112 DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); 113 114 if (rflags & _PAGE_NO_CACHE) 115 hpte_r &= ~_PAGE_COHERENT; 116 117 raw_spin_lock(&beat_htab_lock); 118 lpar_rc = beat_read_mask(hpte_group); 119 if (lpar_rc == 0) { 120 if (!(vflags & HPTE_V_BOLTED)) 121 DBG_LOW(" full\n"); 122 raw_spin_unlock(&beat_htab_lock); 123 return -1; 124 } 125 126 lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48, 127 hpte_v, hpte_r, &slot); 128 raw_spin_unlock(&beat_htab_lock); 129 130 /* 131 * Since we try and ioremap PHBs we don't own, the pte insert 132 * will fail. However we must catch the failure in hash_page 133 * or we will loop forever, so return -2 in this case. 134 */ 135 if (unlikely(lpar_rc != 0)) { 136 if (!(vflags & HPTE_V_BOLTED)) 137 DBG_LOW(" lpar err %lx\n", lpar_rc); 138 return -2; 139 } 140 if (!(vflags & HPTE_V_BOLTED)) 141 DBG_LOW(" -> slot: %lx\n", slot); 142 143 /* We have to pass down the secondary bucket bit here as well */ 144 return (slot ^ hpte_group) & 15; 145} 146 147static long beat_lpar_hpte_remove(unsigned long hpte_group) 148{ 149 DBG_LOW("hpte_remove(group=%lx)\n", hpte_group); 150 return -1; 151} 152 153static unsigned long beat_lpar_hpte_getword0(unsigned long slot) 154{ 155 unsigned long dword0; 156 unsigned long lpar_rc; 157 u64 dword[5]; 158 159 lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword); 160 161 dword0 = dword[slot&3]; 162 163 BUG_ON(lpar_rc != 0); 164 165 return dword0; 166} 167 168static void beat_lpar_hptab_clear(void) 169{ 170 unsigned long size_bytes = 1UL << ppc64_pft_size; 171 unsigned long hpte_count = size_bytes >> 4; 172 int i; 173 u64 dummy0, dummy1; 174 175 /* TODO: Use bulk call */ 176 for (i = 0; i < hpte_count; i++) 177 beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1); 178} 179 180/* 181 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and 182 * the low 3 bits of flags happen to line up. So no transform is needed. 183 * We can probably optimize here and assume the high bits of newpp are 184 * already zero. For now I am paranoid. 185 */ 186static long beat_lpar_hpte_updatepp(unsigned long slot, 187 unsigned long newpp, 188 unsigned long va, 189 int psize, int ssize, int local) 190{ 191 unsigned long lpar_rc; 192 u64 dummy0, dummy1; 193 unsigned long want_v; 194 195 want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); 196 197 DBG_LOW(" update: " 198 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ", 199 want_v & HPTE_V_AVPN, slot, psize, newpp); 200 201 raw_spin_lock(&beat_htab_lock); 202 dummy0 = beat_lpar_hpte_getword0(slot); 203 if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) { 204 DBG_LOW("not found !\n"); 205 raw_spin_unlock(&beat_htab_lock); 206 return -1; 207 } 208 209 lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0, 210 &dummy1); 211 raw_spin_unlock(&beat_htab_lock); 212 if (lpar_rc != 0 || dummy0 == 0) { 213 DBG_LOW("not found !\n"); 214 return -1; 215 } 216 217 DBG_LOW("ok %lx %lx\n", dummy0, dummy1); 218 219 BUG_ON(lpar_rc != 0); 220 221 return 0; 222} 223 224static long beat_lpar_hpte_find(unsigned long va, int psize) 225{ 226 unsigned long hash; 227 unsigned long i, j; 228 long slot; 229 unsigned long want_v, hpte_v; 230 231 hash = hpt_hash(va, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M); 232 want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); 233 234 for (j = 0; j < 2; j++) { 235 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 236 for (i = 0; i < HPTES_PER_GROUP; i++) { 237 hpte_v = beat_lpar_hpte_getword0(slot); 238 239 if (HPTE_V_COMPARE(hpte_v, want_v) 240 && (hpte_v & HPTE_V_VALID) 241 && (!!(hpte_v & HPTE_V_SECONDARY) == j)) { 242 /* HPTE matches */ 243 if (j) 244 slot = -slot; 245 return slot; 246 } 247 ++slot; 248 } 249 hash = ~hash; 250 } 251 252 return -1; 253} 254 255static void beat_lpar_hpte_updateboltedpp(unsigned long newpp, 256 unsigned long ea, 257 int psize, int ssize) 258{ 259 unsigned long lpar_rc, slot, vsid, va; 260 u64 dummy0, dummy1; 261 262 vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M); 263 va = (vsid << 28) | (ea & 0x0fffffff); 264 265 raw_spin_lock(&beat_htab_lock); 266 slot = beat_lpar_hpte_find(va, psize); 267 BUG_ON(slot == -1); 268 269 lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, 270 &dummy0, &dummy1); 271 raw_spin_unlock(&beat_htab_lock); 272 273 BUG_ON(lpar_rc != 0); 274} 275 276static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long va, 277 int psize, int ssize, int local) 278{ 279 unsigned long want_v; 280 unsigned long lpar_rc; 281 u64 dummy1, dummy2; 282 unsigned long flags; 283 284 DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", 285 slot, va, psize, local); 286 want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); 287 288 raw_spin_lock_irqsave(&beat_htab_lock, flags); 289 dummy1 = beat_lpar_hpte_getword0(slot); 290 291 if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) { 292 DBG_LOW("not found !\n"); 293 raw_spin_unlock_irqrestore(&beat_htab_lock, flags); 294 return; 295 } 296 297 lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0, 298 &dummy1, &dummy2); 299 raw_spin_unlock_irqrestore(&beat_htab_lock, flags); 300 301 BUG_ON(lpar_rc != 0); 302} 303 304void __init hpte_init_beat(void) 305{ 306 ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate; 307 ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp; 308 ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; 309 ppc_md.hpte_insert = beat_lpar_hpte_insert; 310 ppc_md.hpte_remove = beat_lpar_hpte_remove; 311 ppc_md.hpte_clear_all = beat_lpar_hptab_clear; 312} 313 314static long beat_lpar_hpte_insert_v3(unsigned long hpte_group, 315 unsigned long va, unsigned long pa, 316 unsigned long rflags, unsigned long vflags, 317 int psize, int ssize) 318{ 319 unsigned long lpar_rc; 320 u64 hpte_v, hpte_r, slot; 321 322 /* same as iseries */ 323 if (vflags & HPTE_V_SECONDARY) 324 return -1; 325 326 if (!(vflags & HPTE_V_BOLTED)) 327 DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, " 328 "rflags=%lx, vflags=%lx, psize=%d)\n", 329 hpte_group, va, pa, rflags, vflags, psize); 330 331 hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) | 332 vflags | HPTE_V_VALID; 333 hpte_r = hpte_encode_r(pa, psize) | rflags; 334 335 if (!(vflags & HPTE_V_BOLTED)) 336 DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); 337 338 if (rflags & _PAGE_NO_CACHE) 339 hpte_r &= ~_PAGE_COHERENT; 340 341 /* insert into not-volted entry */ 342 lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r, 343 HPTE_V_BOLTED, 0, &slot); 344 /* 345 * Since we try and ioremap PHBs we don't own, the pte insert 346 * will fail. However we must catch the failure in hash_page 347 * or we will loop forever, so return -2 in this case. 348 */ 349 if (unlikely(lpar_rc != 0)) { 350 if (!(vflags & HPTE_V_BOLTED)) 351 DBG_LOW(" lpar err %lx\n", lpar_rc); 352 return -2; 353 } 354 if (!(vflags & HPTE_V_BOLTED)) 355 DBG_LOW(" -> slot: %lx\n", slot); 356 357 /* We have to pass down the secondary bucket bit here as well */ 358 return (slot ^ hpte_group) & 15; 359} 360 361/* 362 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and 363 * the low 3 bits of flags happen to line up. So no transform is needed. 364 * We can probably optimize here and assume the high bits of newpp are 365 * already zero. For now I am paranoid. 366 */ 367static long beat_lpar_hpte_updatepp_v3(unsigned long slot, 368 unsigned long newpp, 369 unsigned long va, 370 int psize, int ssize, int local) 371{ 372 unsigned long lpar_rc; 373 unsigned long want_v; 374 unsigned long pss; 375 376 want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); 377 pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc; 378 379 DBG_LOW(" update: " 380 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ", 381 want_v & HPTE_V_AVPN, slot, psize, newpp); 382 383 lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp); 384 385 if (lpar_rc == 0xfffffff7) { 386 DBG_LOW("not found !\n"); 387 return -1; 388 } 389 390 DBG_LOW("ok\n"); 391 392 BUG_ON(lpar_rc != 0); 393 394 return 0; 395} 396 397static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long va, 398 int psize, int ssize, int local) 399{ 400 unsigned long want_v; 401 unsigned long lpar_rc; 402 unsigned long pss; 403 404 DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", 405 slot, va, psize, local); 406 want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); 407 pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc; 408 409 lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss); 410 411 /* E_busy can be valid output: page may be already replaced */ 412 BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7); 413} 414 415static int64_t _beat_lpar_hptab_clear_v3(void) 416{ 417 return beat_clear_htab3(0); 418} 419 420static void beat_lpar_hptab_clear_v3(void) 421{ 422 _beat_lpar_hptab_clear_v3(); 423} 424 425void __init hpte_init_beat_v3(void) 426{ 427 if (_beat_lpar_hptab_clear_v3() == 0) { 428 ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate_v3; 429 ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp_v3; 430 ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; 431 ppc_md.hpte_insert = beat_lpar_hpte_insert_v3; 432 ppc_md.hpte_remove = beat_lpar_hpte_remove; 433 ppc_md.hpte_clear_all = beat_lpar_hptab_clear_v3; 434 } else { 435 ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate; 436 ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp; 437 ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; 438 ppc_md.hpte_insert = beat_lpar_hpte_insert; 439 ppc_md.hpte_remove = beat_lpar_hpte_remove; 440 ppc_md.hpte_clear_all = beat_lpar_hptab_clear; 441 } 442} 443