1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2/* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
3
4#ifndef VCHIQ_CORE_H
5#define VCHIQ_CORE_H
6
7#include <linux/mutex.h>
8#include <linux/completion.h>
9#include <linux/debugfs.h>
10#include <linux/dev_printk.h>
11#include <linux/kthread.h>
12#include <linux/kref.h>
13#include <linux/rcupdate.h>
14#include <linux/wait.h>
15
16#include "../../include/linux/raspberrypi/vchiq.h"
17#include "vchiq_cfg.h"
18
19/* Do this so that we can test-build the code on non-rpi systems */
20#if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE)
21
22#else
23
24#ifndef dsb
25#define dsb(a)
26#endif
27
28#endif	/* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */
29
30#define VCHIQ_SERVICE_HANDLE_INVALID 0
31
32#define VCHIQ_SLOT_SIZE     4096
33#define VCHIQ_MAX_MSG_SIZE  (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header))
34
35#define VCHIQ_SLOT_MASK        (VCHIQ_SLOT_SIZE - 1)
36#define VCHIQ_SLOT_QUEUE_MASK  (VCHIQ_MAX_SLOTS_PER_SIDE - 1)
37#define VCHIQ_SLOT_ZERO_SLOTS  DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \
38					    VCHIQ_SLOT_SIZE)
39
40#define BITSET_SIZE(b)        ((b + 31) >> 5)
41#define BITSET_WORD(b)        (b >> 5)
42#define BITSET_BIT(b)         (1 << (b & 31))
43#define BITSET_IS_SET(bs, b)  (bs[BITSET_WORD(b)] & BITSET_BIT(b))
44#define BITSET_SET(bs, b)     (bs[BITSET_WORD(b)] |= BITSET_BIT(b))
45
46enum {
47	DEBUG_ENTRIES,
48#if VCHIQ_ENABLE_DEBUG
49	DEBUG_SLOT_HANDLER_COUNT,
50	DEBUG_SLOT_HANDLER_LINE,
51	DEBUG_PARSE_LINE,
52	DEBUG_PARSE_HEADER,
53	DEBUG_PARSE_MSGID,
54	DEBUG_AWAIT_COMPLETION_LINE,
55	DEBUG_DEQUEUE_MESSAGE_LINE,
56	DEBUG_SERVICE_CALLBACK_LINE,
57	DEBUG_MSG_QUEUE_FULL_COUNT,
58	DEBUG_COMPLETION_QUEUE_FULL_COUNT,
59#endif
60	DEBUG_MAX
61};
62
63#if VCHIQ_ENABLE_DEBUG
64
65#define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug
66#define DEBUG_TRACE(d) \
67	do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0)
68#define DEBUG_VALUE(d, v) \
69	do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0)
70#define DEBUG_COUNT(d) \
71	do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0)
72
73#else /* VCHIQ_ENABLE_DEBUG */
74
75#define DEBUG_INITIALISE(local)
76#define DEBUG_TRACE(d)
77#define DEBUG_VALUE(d, v)
78#define DEBUG_COUNT(d)
79
80#endif /* VCHIQ_ENABLE_DEBUG */
81
82enum vchiq_connstate {
83	VCHIQ_CONNSTATE_DISCONNECTED,
84	VCHIQ_CONNSTATE_CONNECTING,
85	VCHIQ_CONNSTATE_CONNECTED,
86	VCHIQ_CONNSTATE_PAUSING,
87	VCHIQ_CONNSTATE_PAUSE_SENT,
88	VCHIQ_CONNSTATE_PAUSED,
89	VCHIQ_CONNSTATE_RESUMING,
90	VCHIQ_CONNSTATE_PAUSE_TIMEOUT,
91	VCHIQ_CONNSTATE_RESUME_TIMEOUT
92};
93
94enum {
95	VCHIQ_SRVSTATE_FREE,
96	VCHIQ_SRVSTATE_HIDDEN,
97	VCHIQ_SRVSTATE_LISTENING,
98	VCHIQ_SRVSTATE_OPENING,
99	VCHIQ_SRVSTATE_OPEN,
100	VCHIQ_SRVSTATE_OPENSYNC,
101	VCHIQ_SRVSTATE_CLOSESENT,
102	VCHIQ_SRVSTATE_CLOSERECVD,
103	VCHIQ_SRVSTATE_CLOSEWAIT,
104	VCHIQ_SRVSTATE_CLOSED
105};
106
107enum vchiq_bulk_dir {
108	VCHIQ_BULK_TRANSMIT,
109	VCHIQ_BULK_RECEIVE
110};
111
112struct vchiq_bulk {
113	short mode;
114	short dir;
115	void *userdata;
116	dma_addr_t data;
117	int size;
118	void *remote_data;
119	int remote_size;
120	int actual;
121};
122
123struct vchiq_bulk_queue {
124	int local_insert;  /* Where to insert the next local bulk */
125	int remote_insert; /* Where to insert the next remote bulk (master) */
126	int process;       /* Bulk to transfer next */
127	int remote_notify; /* Bulk to notify the remote client of next (mstr) */
128	int remove;        /* Bulk to notify the local client of, and remove, next */
129	struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS];
130};
131
132/*
133 * Remote events provide a way of presenting several virtual doorbells to a
134 * peer (ARM host to VPU) using only one physical doorbell. They can be thought
135 * of as a way for the peer to signal a semaphore, in this case implemented as
136 * a workqueue.
137 *
138 * Remote events remain signalled until acknowledged by the receiver, and they
139 * are non-counting. They are designed in such a way as to minimise the number
140 * of interrupts and avoid unnecessary waiting.
141 *
142 * A remote_event is as small data structures that live in shared memory. It
143 * comprises two booleans - armed and fired:
144 *
145 * The sender sets fired when they signal the receiver.
146 * If fired is set, the receiver has been signalled and need not wait.
147 * The receiver sets the armed field before they begin to wait.
148 * If armed is set, the receiver is waiting and wishes to be woken by interrupt.
149 */
150struct remote_event {
151	int armed;
152	int fired;
153	u32 __unused;
154};
155
156struct opaque_platform_state;
157
158struct vchiq_slot {
159	char data[VCHIQ_SLOT_SIZE];
160};
161
162struct vchiq_slot_info {
163	/* Use two counters rather than one to avoid the need for a mutex. */
164	short use_count;
165	short release_count;
166};
167
168struct vchiq_service {
169	struct vchiq_service_base base;
170	unsigned int handle;
171	struct kref ref_count;
172	struct rcu_head rcu;
173	int srvstate;
174	void (*userdata_term)(void *userdata);
175	unsigned int localport;
176	unsigned int remoteport;
177	int public_fourcc;
178	int client_id;
179	char auto_close;
180	char sync;
181	char closing;
182	char trace;
183	atomic_t poll_flags;
184	short version;
185	short version_min;
186	short peer_version;
187
188	struct vchiq_state *state;
189	struct vchiq_instance *instance;
190
191	int service_use_count;
192
193	struct vchiq_bulk_queue bulk_tx;
194	struct vchiq_bulk_queue bulk_rx;
195
196	struct completion remove_event;
197	struct completion bulk_remove_event;
198	struct mutex bulk_mutex;
199
200	struct service_stats_struct {
201		int quota_stalls;
202		int slot_stalls;
203		int bulk_stalls;
204		int error_count;
205		int ctrl_tx_count;
206		int ctrl_rx_count;
207		int bulk_tx_count;
208		int bulk_rx_count;
209		int bulk_aborted_count;
210		u64 ctrl_tx_bytes;
211		u64 ctrl_rx_bytes;
212		u64 bulk_tx_bytes;
213		u64 bulk_rx_bytes;
214	} stats;
215
216	int msg_queue_read;
217	int msg_queue_write;
218	struct completion msg_queue_pop;
219	struct completion msg_queue_push;
220	struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS];
221};
222
223/*
224 * The quota information is outside struct vchiq_service so that it can
225 * be statically allocated, since for accounting reasons a service's slot
226 * usage is carried over between users of the same port number.
227 */
228struct vchiq_service_quota {
229	unsigned short slot_quota;
230	unsigned short slot_use_count;
231	unsigned short message_quota;
232	unsigned short message_use_count;
233	struct completion quota_event;
234	int previous_tx_index;
235};
236
237struct vchiq_shared_state {
238	/* A non-zero value here indicates that the content is valid. */
239	int initialised;
240
241	/* The first and last (inclusive) slots allocated to the owner. */
242	int slot_first;
243	int slot_last;
244
245	/* The slot allocated to synchronous messages from the owner. */
246	int slot_sync;
247
248	/*
249	 * Signalling this event indicates that owner's slot handler thread
250	 * should run.
251	 */
252	struct remote_event trigger;
253
254	/*
255	 * Indicates the byte position within the stream where the next message
256	 * will be written. The least significant bits are an index into the
257	 * slot. The next bits are the index of the slot in slot_queue.
258	 */
259	int tx_pos;
260
261	/* This event should be signalled when a slot is recycled. */
262	struct remote_event recycle;
263
264	/* The slot_queue index where the next recycled slot will be written. */
265	int slot_queue_recycle;
266
267	/* This event should be signalled when a synchronous message is sent. */
268	struct remote_event sync_trigger;
269
270	/*
271	 * This event should be signalled when a synchronous message has been
272	 * released.
273	 */
274	struct remote_event sync_release;
275
276	/* A circular buffer of slot indexes. */
277	int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE];
278
279	/* Debugging state */
280	int debug[DEBUG_MAX];
281};
282
283struct vchiq_slot_zero {
284	int magic;
285	short version;
286	short version_min;
287	int slot_zero_size;
288	int slot_size;
289	int max_slots;
290	int max_slots_per_side;
291	int platform_data[2];
292	struct vchiq_shared_state master;
293	struct vchiq_shared_state slave;
294	struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS];
295};
296
297struct vchiq_state {
298	struct device *dev;
299	int id;
300	int initialised;
301	enum vchiq_connstate conn_state;
302	short version_common;
303
304	struct vchiq_shared_state *local;
305	struct vchiq_shared_state *remote;
306	struct vchiq_slot *slot_data;
307
308	unsigned short default_slot_quota;
309	unsigned short default_message_quota;
310
311	/* Event indicating connect message received */
312	struct completion connect;
313
314	/* Mutex protecting services */
315	struct mutex mutex;
316	struct vchiq_instance **instance;
317
318	/* Processes incoming messages */
319	struct task_struct *slot_handler_thread;
320
321	/* Processes recycled slots */
322	struct task_struct *recycle_thread;
323
324	/* Processes synchronous messages */
325	struct task_struct *sync_thread;
326
327	/* Local implementation of the trigger remote event */
328	wait_queue_head_t trigger_event;
329
330	/* Local implementation of the recycle remote event */
331	wait_queue_head_t recycle_event;
332
333	/* Local implementation of the sync trigger remote event */
334	wait_queue_head_t sync_trigger_event;
335
336	/* Local implementation of the sync release remote event */
337	wait_queue_head_t sync_release_event;
338
339	char *tx_data;
340	char *rx_data;
341	struct vchiq_slot_info *rx_info;
342
343	struct mutex slot_mutex;
344
345	struct mutex recycle_mutex;
346
347	struct mutex sync_mutex;
348
349	struct mutex bulk_transfer_mutex;
350
351	/*
352	 * Indicates the byte position within the stream from where the next
353	 * message will be read. The least significant bits are an index into
354	 * the slot.The next bits are the index of the slot in
355	 * remote->slot_queue.
356	 */
357	int rx_pos;
358
359	/*
360	 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read
361	 * from remote->tx_pos.
362	 */
363	int local_tx_pos;
364
365	/* The slot_queue index of the slot to become available next. */
366	int slot_queue_available;
367
368	/* A flag to indicate if any poll has been requested */
369	int poll_needed;
370
371	/* Ths index of the previous slot used for data messages. */
372	int previous_data_index;
373
374	/* The number of slots occupied by data messages. */
375	unsigned short data_use_count;
376
377	/* The maximum number of slots to be occupied by data messages. */
378	unsigned short data_quota;
379
380	/* An array of bit sets indicating which services must be polled. */
381	atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
382
383	/* The number of the first unused service */
384	int unused_service;
385
386	/* Signalled when a free slot becomes available. */
387	struct completion slot_available_event;
388
389	struct completion slot_remove_event;
390
391	/* Signalled when a free data slot becomes available. */
392	struct completion data_quota_event;
393
394	struct state_stats_struct {
395		int slot_stalls;
396		int data_stalls;
397		int ctrl_tx_count;
398		int ctrl_rx_count;
399		int error_count;
400	} stats;
401
402	struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES];
403	struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES];
404	struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS];
405
406	struct opaque_platform_state *platform_state;
407};
408
409struct bulk_waiter {
410	struct vchiq_bulk *bulk;
411	struct completion event;
412	int actual;
413};
414
415struct vchiq_config {
416	unsigned int max_msg_size;
417	unsigned int bulk_threshold;	/* The message size above which it
418					 * is better to use a bulk transfer
419					 * (<= max_msg_size)
420					 */
421	unsigned int max_outstanding_bulks;
422	unsigned int max_services;
423	short version;      /* The version of VCHIQ */
424	short version_min;  /* The minimum compatible version of VCHIQ */
425};
426
427extern spinlock_t bulk_waiter_spinlock;
428
429extern const char *
430get_conn_state_name(enum vchiq_connstate conn_state);
431
432extern struct vchiq_slot_zero *
433vchiq_init_slots(struct device *dev, void *mem_base, int mem_size);
434
435extern int
436vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero, struct device *dev);
437
438extern int
439vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance);
440
441struct vchiq_service *
442vchiq_add_service_internal(struct vchiq_state *state,
443			   const struct vchiq_service_params_kernel *params,
444			   int srvstate, struct vchiq_instance *instance,
445			   void (*userdata_term)(void *userdata));
446
447extern int
448vchiq_open_service_internal(struct vchiq_service *service, int client_id);
449
450extern int
451vchiq_close_service_internal(struct vchiq_service *service, int close_recvd);
452
453extern void
454vchiq_terminate_service_internal(struct vchiq_service *service);
455
456extern void
457vchiq_free_service_internal(struct vchiq_service *service);
458
459extern void
460vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance);
461
462extern void
463remote_event_pollall(struct vchiq_state *state);
464
465extern int
466vchiq_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *offset,
467		    void __user *uoffset, int size, void *userdata, enum vchiq_bulk_mode mode,
468		    enum vchiq_bulk_dir dir);
469
470extern void
471vchiq_dump_state(struct seq_file *f, struct vchiq_state *state);
472
473extern void
474vchiq_loud_error_header(void);
475
476extern void
477vchiq_loud_error_footer(void);
478
479extern void
480request_poll(struct vchiq_state *state, struct vchiq_service *service,
481	     int poll_type);
482
483struct vchiq_service *handle_to_service(struct vchiq_instance *instance, unsigned int handle);
484
485extern struct vchiq_service *
486find_service_by_handle(struct vchiq_instance *instance, unsigned int handle);
487
488extern struct vchiq_service *
489find_service_by_port(struct vchiq_state *state, unsigned int localport);
490
491extern struct vchiq_service *
492find_service_for_instance(struct vchiq_instance *instance, unsigned int handle);
493
494extern struct vchiq_service *
495find_closed_service_for_instance(struct vchiq_instance *instance, unsigned int handle);
496
497extern struct vchiq_service *
498__next_service_by_instance(struct vchiq_state *state,
499			   struct vchiq_instance *instance,
500			   int *pidx);
501
502extern struct vchiq_service *
503next_service_by_instance(struct vchiq_state *state,
504			 struct vchiq_instance *instance,
505			 int *pidx);
506
507extern void
508vchiq_service_get(struct vchiq_service *service);
509
510extern void
511vchiq_service_put(struct vchiq_service *service);
512
513extern int
514vchiq_queue_message(struct vchiq_instance *instance, unsigned int handle,
515		    ssize_t (*copy_callback)(void *context, void *dest,
516					     size_t offset, size_t maxsize),
517		    void *context,
518		    size_t size);
519
520int vchiq_prepare_bulk_data(struct vchiq_instance *instance, struct vchiq_bulk *bulk, void *offset,
521			    void __user *uoffset, int size, int dir);
522
523void vchiq_complete_bulk(struct vchiq_instance *instance, struct vchiq_bulk *bulk);
524
525void remote_event_signal(struct remote_event *event);
526
527void vchiq_dump_platform_state(struct seq_file *f);
528
529void vchiq_dump_platform_instances(struct seq_file *f);
530
531void vchiq_dump_platform_service_state(struct seq_file *f, struct vchiq_service *service);
532
533int vchiq_use_service_internal(struct vchiq_service *service);
534
535int vchiq_release_service_internal(struct vchiq_service *service);
536
537void vchiq_on_remote_use(struct vchiq_state *state);
538
539void vchiq_on_remote_release(struct vchiq_state *state);
540
541int vchiq_platform_init_state(struct vchiq_state *state);
542
543int vchiq_check_service(struct vchiq_service *service);
544
545void vchiq_on_remote_use_active(struct vchiq_state *state);
546
547int vchiq_send_remote_use(struct vchiq_state *state);
548
549int vchiq_send_remote_use_active(struct vchiq_state *state);
550
551void vchiq_platform_conn_state_changed(struct vchiq_state *state,
552				       enum vchiq_connstate oldstate,
553				  enum vchiq_connstate newstate);
554
555void vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate);
556
557void vchiq_log_dump_mem(struct device *dev, const char *label, u32 addr,
558			const void *void_mem, size_t num_bytes);
559
560int vchiq_remove_service(struct vchiq_instance *instance, unsigned int service);
561
562int vchiq_get_client_id(struct vchiq_instance *instance, unsigned int service);
563
564void vchiq_get_config(struct vchiq_config *config);
565
566int vchiq_set_service_option(struct vchiq_instance *instance, unsigned int service,
567			     enum vchiq_service_option option, int value);
568
569#endif
570