1/*
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Cavium, Inc. nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33/*$FreeBSD: stable/11/sys/dev/liquidio/base/lio_mem_ops.c 325618 2017-11-09 19:52:56Z sbruno $*/
34
35#include "lio_bsd.h"
36#include "lio_common.h"
37#include "lio_droq.h"
38#include "lio_iq.h"
39#include "lio_response_manager.h"
40#include "lio_device.h"
41#include "lio_mem_ops.h"
42
43#define MEMOPS_IDX   LIO_MAX_BAR1_MAP_INDEX
44
45#if BYTE_ORDER == BIG_ENDIAN
46static inline void
47lio_toggle_bar1_swapmode(struct octeon_device *oct, uint32_t idx)
48{
49	uint32_t mask;
50
51	mask = oct->fn_list.bar1_idx_read(oct, idx);
52	mask = (mask & 0x2) ? (mask & ~2) : (mask | 2);
53	oct->fn_list.bar1_idx_write(oct, idx, mask);
54}
55
56#else	/* BYTE_ORDER != BIG_ENDIAN */
57#define lio_toggle_bar1_swapmode(oct, idx)
58#endif	/* BYTE_ORDER == BIG_ENDIAN */
59
60static inline void
61lio_write_bar1_mem8(struct octeon_device *oct, uint32_t reg, uint64_t val)
62{
63
64	bus_space_write_1(oct->mem_bus_space[1].tag,
65			  oct->mem_bus_space[1].handle, reg, val);
66}
67
68#ifdef __i386__
69static inline uint32_t
70lio_read_bar1_mem32(struct octeon_device *oct, uint32_t reg)
71{
72
73	return (bus_space_read_4(oct->mem_bus_space[1].tag,
74				 oct->mem_bus_space[1].handle, reg));
75}
76
77static inline void
78lio_write_bar1_mem32(struct octeon_device *oct, uint32_t reg, uint32_t val)
79{
80
81	bus_space_write_4(oct->mem_bus_space[1].tag,
82			  oct->mem_bus_space[1].handle, reg, val);
83}
84#endif
85
86static inline uint64_t
87lio_read_bar1_mem64(struct octeon_device *oct, uint32_t reg)
88{
89
90#ifdef __i386__
91	return (lio_read_bar1_mem32(oct, reg) |
92			((uint64_t)lio_read_bar1_mem32(oct, reg + 4) << 32));
93#else
94	return (bus_space_read_8(oct->mem_bus_space[1].tag,
95				 oct->mem_bus_space[1].handle, reg));
96#endif
97}
98
99static inline void
100lio_write_bar1_mem64(struct octeon_device *oct, uint32_t reg, uint64_t val)
101{
102
103#ifdef __i386__
104	lio_write_bar1_mem32(oct, reg, (uint32_t)val);
105	lio_write_bar1_mem32(oct, reg + 4, val >> 32);
106#else
107	bus_space_write_8(oct->mem_bus_space[1].tag,
108			  oct->mem_bus_space[1].handle, reg, val);
109#endif
110}
111
112static void
113lio_pci_fastwrite(struct octeon_device *oct, uint32_t offset,
114		  uint8_t *hostbuf, uint32_t len)
115{
116
117	while ((len) && ((unsigned long)offset) & 7) {
118		lio_write_bar1_mem8(oct, offset++, *(hostbuf++));
119		len--;
120	}
121
122	lio_toggle_bar1_swapmode(oct, MEMOPS_IDX);
123
124	while (len >= 8) {
125		lio_write_bar1_mem64(oct, offset, *((uint64_t *)hostbuf));
126		offset += 8;
127		hostbuf += 8;
128		len -= 8;
129	}
130
131	lio_toggle_bar1_swapmode(oct, MEMOPS_IDX);
132
133	while (len--)
134		lio_write_bar1_mem8(oct, offset++, *(hostbuf++));
135}
136
137static inline uint64_t
138lio_read_bar1_mem8(struct octeon_device *oct, uint32_t reg)
139{
140
141	return (bus_space_read_1(oct->mem_bus_space[1].tag,
142				 oct->mem_bus_space[1].handle, reg));
143}
144
145static void
146lio_pci_fastread(struct octeon_device *oct, uint32_t offset,
147		 uint8_t *hostbuf, uint32_t len)
148{
149
150	while ((len) && ((unsigned long)offset) & 7) {
151		*(hostbuf++) = lio_read_bar1_mem8(oct, offset++);
152		len--;
153	}
154
155	lio_toggle_bar1_swapmode(oct, MEMOPS_IDX);
156
157	while (len >= 8) {
158		*((uint64_t *)hostbuf) = lio_read_bar1_mem64(oct, offset);
159		offset += 8;
160		hostbuf += 8;
161		len -= 8;
162	}
163
164	lio_toggle_bar1_swapmode(oct, MEMOPS_IDX);
165
166	while (len--)
167		*(hostbuf++) = lio_read_bar1_mem8(oct, offset++);
168}
169
170/* Core mem read/write with temporary bar1 settings. */
171/* op = 1 to read, op = 0 to write. */
172static void
173lio_pci_rw_core_mem(struct octeon_device *oct, uint64_t addr,
174		    uint8_t *hostbuf, uint32_t len, uint32_t op)
175{
176	uint64_t	static_mapping_base;
177	uint32_t	copy_len = 0, index_reg_val = 0;
178	uint32_t	offset;
179
180	static_mapping_base = oct->console_nb_info.dram_region_base;
181
182	if (static_mapping_base && static_mapping_base ==
183	    (addr & 0xFFFFFFFFFFC00000ULL)) {
184		int	bar1_index = oct->console_nb_info.bar1_index;
185
186		offset = (bar1_index << 22) + (addr & 0x3fffff);
187
188		if (op)
189			lio_pci_fastread(oct, offset, hostbuf, len);
190		else
191			lio_pci_fastwrite(oct, offset, hostbuf, len);
192
193		return;
194	}
195	mtx_lock(&oct->mem_access_lock);
196
197	/* Save the original index reg value. */
198	index_reg_val = oct->fn_list.bar1_idx_read(oct, MEMOPS_IDX);
199	do {
200		oct->fn_list.bar1_idx_setup(oct, addr, MEMOPS_IDX, 1);
201		offset = (MEMOPS_IDX << 22) + (addr & 0x3fffff);
202
203		/*
204		 * If operation crosses a 4MB boundary, split the transfer
205		 * at the 4MB boundary.
206		 */
207		if (((addr + len - 1) & ~(0x3fffff)) != (addr & ~(0x3fffff))) {
208			copy_len = (uint32_t)(((addr & ~(0x3fffff)) +
209					       (MEMOPS_IDX << 22)) - addr);
210		} else {
211			copy_len = len;
212		}
213
214		if (op) {	/* read from core */
215			lio_pci_fastread(oct, offset, hostbuf,
216					 copy_len);
217		} else {
218			lio_pci_fastwrite(oct, offset, hostbuf,
219					  copy_len);
220		}
221
222		len -= copy_len;
223		addr += copy_len;
224		hostbuf += copy_len;
225
226	} while (len);
227
228	oct->fn_list.bar1_idx_write(oct, MEMOPS_IDX, index_reg_val);
229
230	mtx_unlock(&oct->mem_access_lock);
231}
232
233void
234lio_pci_read_core_mem(struct octeon_device *oct, uint64_t coreaddr,
235		      uint8_t *buf, uint32_t len)
236{
237
238	lio_pci_rw_core_mem(oct, coreaddr, buf, len, 1);
239}
240
241void
242lio_pci_write_core_mem(struct octeon_device *oct, uint64_t coreaddr,
243		       uint8_t *buf, uint32_t len)
244{
245
246	lio_pci_rw_core_mem(oct, coreaddr, buf, len, 0);
247}
248
249uint64_t
250lio_read_device_mem64(struct octeon_device *oct, uint64_t coreaddr)
251{
252	__be64	ret;
253
254	lio_pci_rw_core_mem(oct, coreaddr, (uint8_t *)&ret, 8, 1);
255
256	return (be64toh(ret));
257}
258
259uint32_t
260lio_read_device_mem32(struct octeon_device *oct, uint64_t coreaddr)
261{
262	__be32	ret;
263
264	lio_pci_rw_core_mem(oct, coreaddr, (uint8_t *)&ret, 4, 1);
265
266	return (be32toh(ret));
267}
268
269void
270lio_write_device_mem32(struct octeon_device *oct, uint64_t coreaddr,
271		       uint32_t val)
272{
273	__be32	t = htobe32(val);
274
275	lio_pci_rw_core_mem(oct, coreaddr, (uint8_t *)&t, 4, 0);
276}
277