1// SPDX-License-Identifier: GPL-2.0
2/*
3 * NVMe over Fabrics common host code.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7#include <linux/init.h>
8#include <linux/miscdevice.h>
9#include <linux/module.h>
10#include <linux/mutex.h>
11#include <linux/parser.h>
12#include <linux/seq_file.h>
13#include "nvme.h"
14#include "fabrics.h"
15
16static LIST_HEAD(nvmf_transports);
17static DECLARE_RWSEM(nvmf_transports_rwsem);
18
19static LIST_HEAD(nvmf_hosts);
20static DEFINE_MUTEX(nvmf_hosts_mutex);
21
22static struct nvmf_host *nvmf_default_host;
23
24static struct nvmf_host *__nvmf_host_find(const char *hostnqn)
25{
26	struct nvmf_host *host;
27
28	list_for_each_entry(host, &nvmf_hosts, list) {
29		if (!strcmp(host->nqn, hostnqn))
30			return host;
31	}
32
33	return NULL;
34}
35
36static struct nvmf_host *nvmf_host_add(const char *hostnqn)
37{
38	struct nvmf_host *host;
39
40	mutex_lock(&nvmf_hosts_mutex);
41	host = __nvmf_host_find(hostnqn);
42	if (host) {
43		kref_get(&host->ref);
44		goto out_unlock;
45	}
46
47	host = kmalloc(sizeof(*host), GFP_KERNEL);
48	if (!host)
49		goto out_unlock;
50
51	kref_init(&host->ref);
52	strlcpy(host->nqn, hostnqn, NVMF_NQN_SIZE);
53
54	list_add_tail(&host->list, &nvmf_hosts);
55out_unlock:
56	mutex_unlock(&nvmf_hosts_mutex);
57	return host;
58}
59
60static struct nvmf_host *nvmf_host_default(void)
61{
62	struct nvmf_host *host;
63
64	host = kmalloc(sizeof(*host), GFP_KERNEL);
65	if (!host)
66		return NULL;
67
68	kref_init(&host->ref);
69	uuid_gen(&host->id);
70	snprintf(host->nqn, NVMF_NQN_SIZE,
71		"nqn.2014-08.org.nvmexpress:uuid:%pUb", &host->id);
72
73	mutex_lock(&nvmf_hosts_mutex);
74	list_add_tail(&host->list, &nvmf_hosts);
75	mutex_unlock(&nvmf_hosts_mutex);
76
77	return host;
78}
79
80static void nvmf_host_destroy(struct kref *ref)
81{
82	struct nvmf_host *host = container_of(ref, struct nvmf_host, ref);
83
84	mutex_lock(&nvmf_hosts_mutex);
85	list_del(&host->list);
86	mutex_unlock(&nvmf_hosts_mutex);
87
88	kfree(host);
89}
90
91static void nvmf_host_put(struct nvmf_host *host)
92{
93	if (host)
94		kref_put(&host->ref, nvmf_host_destroy);
95}
96
97/**
98 * nvmf_get_address() -  Get address/port
99 * @ctrl:	Host NVMe controller instance which we got the address
100 * @buf:	OUTPUT parameter that will contain the address/port
101 * @size:	buffer size
102 */
103int nvmf_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
104{
105	int len = 0;
106
107	if (ctrl->opts->mask & NVMF_OPT_TRADDR)
108		len += scnprintf(buf, size, "traddr=%s", ctrl->opts->traddr);
109	if (ctrl->opts->mask & NVMF_OPT_TRSVCID)
110		len += scnprintf(buf + len, size - len, "%strsvcid=%s",
111				(len) ? "," : "", ctrl->opts->trsvcid);
112	if (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)
113		len += scnprintf(buf + len, size - len, "%shost_traddr=%s",
114				(len) ? "," : "", ctrl->opts->host_traddr);
115	if (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)
116		len += scnprintf(buf + len, size - len, "%shost_iface=%s",
117				(len) ? "," : "", ctrl->opts->host_iface);
118	len += scnprintf(buf + len, size - len, "\n");
119
120	return len;
121}
122EXPORT_SYMBOL_GPL(nvmf_get_address);
123
124/**
125 * nvmf_reg_read32() -  NVMe Fabrics "Property Get" API function.
126 * @ctrl:	Host NVMe controller instance maintaining the admin
127 *		queue used to submit the property read command to
128 *		the allocated NVMe controller resource on the target system.
129 * @off:	Starting offset value of the targeted property
130 *		register (see the fabrics section of the NVMe standard).
131 * @val:	OUTPUT parameter that will contain the value of
132 *		the property after a successful read.
133 *
134 * Used by the host system to retrieve a 32-bit capsule property value
135 * from an NVMe controller on the target system.
136 *
137 * ("Capsule property" is an "PCIe register concept" applied to the
138 * NVMe fabrics space.)
139 *
140 * Return:
141 *	0: successful read
142 *	> 0: NVMe error status code
143 *	< 0: Linux errno error code
144 */
145int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
146{
147	struct nvme_command cmd;
148	union nvme_result res;
149	int ret;
150
151	memset(&cmd, 0, sizeof(cmd));
152	cmd.prop_get.opcode = nvme_fabrics_command;
153	cmd.prop_get.fctype = nvme_fabrics_type_property_get;
154	cmd.prop_get.offset = cpu_to_le32(off);
155
156	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res, NULL, 0, 0,
157			NVME_QID_ANY, 0, 0);
158
159	if (ret >= 0)
160		*val = le64_to_cpu(res.u64);
161	if (unlikely(ret != 0))
162		dev_err(ctrl->device,
163			"Property Get error: %d, offset %#x\n",
164			ret > 0 ? ret & ~NVME_SC_DNR : ret, off);
165
166	return ret;
167}
168EXPORT_SYMBOL_GPL(nvmf_reg_read32);
169
170/**
171 * nvmf_reg_read64() -  NVMe Fabrics "Property Get" API function.
172 * @ctrl:	Host NVMe controller instance maintaining the admin
173 *		queue used to submit the property read command to
174 *		the allocated controller resource on the target system.
175 * @off:	Starting offset value of the targeted property
176 *		register (see the fabrics section of the NVMe standard).
177 * @val:	OUTPUT parameter that will contain the value of
178 *		the property after a successful read.
179 *
180 * Used by the host system to retrieve a 64-bit capsule property value
181 * from an NVMe controller on the target system.
182 *
183 * ("Capsule property" is an "PCIe register concept" applied to the
184 * NVMe fabrics space.)
185 *
186 * Return:
187 *	0: successful read
188 *	> 0: NVMe error status code
189 *	< 0: Linux errno error code
190 */
191int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
192{
193	struct nvme_command cmd = { };
194	union nvme_result res;
195	int ret;
196
197	cmd.prop_get.opcode = nvme_fabrics_command;
198	cmd.prop_get.fctype = nvme_fabrics_type_property_get;
199	cmd.prop_get.attrib = 1;
200	cmd.prop_get.offset = cpu_to_le32(off);
201
202	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res, NULL, 0, 0,
203			NVME_QID_ANY, 0, 0);
204
205	if (ret >= 0)
206		*val = le64_to_cpu(res.u64);
207	if (unlikely(ret != 0))
208		dev_err(ctrl->device,
209			"Property Get error: %d, offset %#x\n",
210			ret > 0 ? ret & ~NVME_SC_DNR : ret, off);
211	return ret;
212}
213EXPORT_SYMBOL_GPL(nvmf_reg_read64);
214
215/**
216 * nvmf_reg_write32() -  NVMe Fabrics "Property Write" API function.
217 * @ctrl:	Host NVMe controller instance maintaining the admin
218 *		queue used to submit the property read command to
219 *		the allocated NVMe controller resource on the target system.
220 * @off:	Starting offset value of the targeted property
221 *		register (see the fabrics section of the NVMe standard).
222 * @val:	Input parameter that contains the value to be
223 *		written to the property.
224 *
225 * Used by the NVMe host system to write a 32-bit capsule property value
226 * to an NVMe controller on the target system.
227 *
228 * ("Capsule property" is an "PCIe register concept" applied to the
229 * NVMe fabrics space.)
230 *
231 * Return:
232 *	0: successful write
233 *	> 0: NVMe error status code
234 *	< 0: Linux errno error code
235 */
236int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
237{
238	struct nvme_command cmd = { };
239	int ret;
240
241	cmd.prop_set.opcode = nvme_fabrics_command;
242	cmd.prop_set.fctype = nvme_fabrics_type_property_set;
243	cmd.prop_set.attrib = 0;
244	cmd.prop_set.offset = cpu_to_le32(off);
245	cmd.prop_set.value = cpu_to_le64(val);
246
247	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, NULL, NULL, 0, 0,
248			NVME_QID_ANY, 0, 0);
249	if (unlikely(ret))
250		dev_err(ctrl->device,
251			"Property Set error: %d, offset %#x\n",
252			ret > 0 ? ret & ~NVME_SC_DNR : ret, off);
253	return ret;
254}
255EXPORT_SYMBOL_GPL(nvmf_reg_write32);
256
257/**
258 * nvmf_log_connect_error() - Error-parsing-diagnostic print out function for
259 * 				connect() errors.
260 * @ctrl:	The specific /dev/nvmeX device that had the error.
261 * @errval:	Error code to be decoded in a more human-friendly
262 * 		printout.
263 * @offset:	For use with the NVMe error code
264 * 		NVME_SC_CONNECT_INVALID_PARAM.
265 * @cmd:	This is the SQE portion of a submission capsule.
266 * @data:	This is the "Data" portion of a submission capsule.
267 */
268static void nvmf_log_connect_error(struct nvme_ctrl *ctrl,
269		int errval, int offset, struct nvme_command *cmd,
270		struct nvmf_connect_data *data)
271{
272	int err_sctype = errval & ~NVME_SC_DNR;
273
274	switch (err_sctype) {
275	case (NVME_SC_CONNECT_INVALID_PARAM):
276		if (offset >> 16) {
277			char *inv_data = "Connect Invalid Data Parameter";
278
279			switch (offset & 0xffff) {
280			case (offsetof(struct nvmf_connect_data, cntlid)):
281				dev_err(ctrl->device,
282					"%s, cntlid: %d\n",
283					inv_data, data->cntlid);
284				break;
285			case (offsetof(struct nvmf_connect_data, hostnqn)):
286				dev_err(ctrl->device,
287					"%s, hostnqn \"%s\"\n",
288					inv_data, data->hostnqn);
289				break;
290			case (offsetof(struct nvmf_connect_data, subsysnqn)):
291				dev_err(ctrl->device,
292					"%s, subsysnqn \"%s\"\n",
293					inv_data, data->subsysnqn);
294				break;
295			default:
296				dev_err(ctrl->device,
297					"%s, starting byte offset: %d\n",
298				       inv_data, offset & 0xffff);
299				break;
300			}
301		} else {
302			char *inv_sqe = "Connect Invalid SQE Parameter";
303
304			switch (offset) {
305			case (offsetof(struct nvmf_connect_command, qid)):
306				dev_err(ctrl->device,
307				       "%s, qid %d\n",
308					inv_sqe, cmd->connect.qid);
309				break;
310			default:
311				dev_err(ctrl->device,
312					"%s, starting byte offset: %d\n",
313					inv_sqe, offset);
314			}
315		}
316		break;
317	case NVME_SC_CONNECT_INVALID_HOST:
318		dev_err(ctrl->device,
319			"Connect for subsystem %s is not allowed, hostnqn: %s\n",
320			data->subsysnqn, data->hostnqn);
321		break;
322	case NVME_SC_CONNECT_CTRL_BUSY:
323		dev_err(ctrl->device,
324			"Connect command failed: controller is busy or not available\n");
325		break;
326	case NVME_SC_CONNECT_FORMAT:
327		dev_err(ctrl->device,
328			"Connect incompatible format: %d",
329			cmd->connect.recfmt);
330		break;
331	case NVME_SC_HOST_PATH_ERROR:
332		dev_err(ctrl->device,
333			"Connect command failed: host path error\n");
334		break;
335	default:
336		dev_err(ctrl->device,
337			"Connect command failed, error wo/DNR bit: %d\n",
338			err_sctype);
339		break;
340	}
341}
342
343/**
344 * nvmf_connect_admin_queue() - NVMe Fabrics Admin Queue "Connect"
345 *				API function.
346 * @ctrl:	Host nvme controller instance used to request
347 *              a new NVMe controller allocation on the target
348 *              system and  establish an NVMe Admin connection to
349 *              that controller.
350 *
351 * This function enables an NVMe host device to request a new allocation of
352 * an NVMe controller resource on a target system as well establish a
353 * fabrics-protocol connection of the NVMe Admin queue between the
354 * host system device and the allocated NVMe controller on the
355 * target system via a NVMe Fabrics "Connect" command.
356 *
357 * Return:
358 *	0: success
359 *	> 0: NVMe error status code
360 *	< 0: Linux errno error code
361 *
362 */
363int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
364{
365	struct nvme_command cmd = { };
366	union nvme_result res;
367	struct nvmf_connect_data *data;
368	int ret;
369
370	cmd.connect.opcode = nvme_fabrics_command;
371	cmd.connect.fctype = nvme_fabrics_type_connect;
372	cmd.connect.qid = 0;
373	cmd.connect.sqsize = cpu_to_le16(NVME_AQ_DEPTH - 1);
374
375	/*
376	 * Set keep-alive timeout in seconds granularity (ms * 1000)
377	 */
378	cmd.connect.kato = cpu_to_le32(ctrl->kato * 1000);
379
380	if (ctrl->opts->disable_sqflow)
381		cmd.connect.cattr |= NVME_CONNECT_DISABLE_SQFLOW;
382
383	data = kzalloc(sizeof(*data), GFP_KERNEL);
384	if (!data)
385		return -ENOMEM;
386
387	uuid_copy(&data->hostid, &ctrl->opts->host->id);
388	data->cntlid = cpu_to_le16(0xffff);
389	strncpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
390	strncpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
391
392	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res,
393			data, sizeof(*data), 0, NVME_QID_ANY, 1,
394			BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
395	if (ret) {
396		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
397				       &cmd, data);
398		goto out_free_data;
399	}
400
401	ctrl->cntlid = le16_to_cpu(res.u16);
402
403out_free_data:
404	kfree(data);
405	return ret;
406}
407EXPORT_SYMBOL_GPL(nvmf_connect_admin_queue);
408
409/**
410 * nvmf_connect_io_queue() - NVMe Fabrics I/O Queue "Connect"
411 *			     API function.
412 * @ctrl:	Host nvme controller instance used to establish an
413 *		NVMe I/O queue connection to the already allocated NVMe
414 *		controller on the target system.
415 * @qid:	NVMe I/O queue number for the new I/O connection between
416 *		host and target (note qid == 0 is illegal as this is
417 *		the Admin queue, per NVMe standard).
418 *
419 * This function issues a fabrics-protocol connection
420 * of a NVMe I/O queue (via NVMe Fabrics "Connect" command)
421 * between the host system device and the allocated NVMe controller
422 * on the target system.
423 *
424 * Return:
425 *	0: success
426 *	> 0: NVMe error status code
427 *	< 0: Linux errno error code
428 */
429int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
430{
431	struct nvme_command cmd = { };
432	struct nvmf_connect_data *data;
433	union nvme_result res;
434	int ret;
435
436	cmd.connect.opcode = nvme_fabrics_command;
437	cmd.connect.fctype = nvme_fabrics_type_connect;
438	cmd.connect.qid = cpu_to_le16(qid);
439	cmd.connect.sqsize = cpu_to_le16(ctrl->sqsize);
440
441	if (ctrl->opts->disable_sqflow)
442		cmd.connect.cattr |= NVME_CONNECT_DISABLE_SQFLOW;
443
444	data = kzalloc(sizeof(*data), GFP_KERNEL);
445	if (!data)
446		return -ENOMEM;
447
448	uuid_copy(&data->hostid, &ctrl->opts->host->id);
449	data->cntlid = cpu_to_le16(ctrl->cntlid);
450	strncpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
451	strncpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
452
453	ret = __nvme_submit_sync_cmd(ctrl->connect_q, &cmd, &res,
454			data, sizeof(*data), 0, qid, 1,
455			BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
456	if (ret) {
457		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
458				       &cmd, data);
459	}
460	kfree(data);
461	return ret;
462}
463EXPORT_SYMBOL_GPL(nvmf_connect_io_queue);
464
465bool nvmf_should_reconnect(struct nvme_ctrl *ctrl)
466{
467	if (ctrl->opts->max_reconnects == -1 ||
468	    ctrl->nr_reconnects < ctrl->opts->max_reconnects)
469		return true;
470
471	return false;
472}
473EXPORT_SYMBOL_GPL(nvmf_should_reconnect);
474
475/**
476 * nvmf_register_transport() - NVMe Fabrics Library registration function.
477 * @ops:	Transport ops instance to be registered to the
478 *		common fabrics library.
479 *
480 * API function that registers the type of specific transport fabric
481 * being implemented to the common NVMe fabrics library. Part of
482 * the overall init sequence of starting up a fabrics driver.
483 */
484int nvmf_register_transport(struct nvmf_transport_ops *ops)
485{
486	if (!ops->create_ctrl)
487		return -EINVAL;
488
489	down_write(&nvmf_transports_rwsem);
490	list_add_tail(&ops->entry, &nvmf_transports);
491	up_write(&nvmf_transports_rwsem);
492
493	return 0;
494}
495EXPORT_SYMBOL_GPL(nvmf_register_transport);
496
497/**
498 * nvmf_unregister_transport() - NVMe Fabrics Library unregistration function.
499 * @ops:	Transport ops instance to be unregistered from the
500 *		common fabrics library.
501 *
502 * Fabrics API function that unregisters the type of specific transport
503 * fabric being implemented from the common NVMe fabrics library.
504 * Part of the overall exit sequence of unloading the implemented driver.
505 */
506void nvmf_unregister_transport(struct nvmf_transport_ops *ops)
507{
508	down_write(&nvmf_transports_rwsem);
509	list_del(&ops->entry);
510	up_write(&nvmf_transports_rwsem);
511}
512EXPORT_SYMBOL_GPL(nvmf_unregister_transport);
513
514static struct nvmf_transport_ops *nvmf_lookup_transport(
515		struct nvmf_ctrl_options *opts)
516{
517	struct nvmf_transport_ops *ops;
518
519	lockdep_assert_held(&nvmf_transports_rwsem);
520
521	list_for_each_entry(ops, &nvmf_transports, entry) {
522		if (strcmp(ops->name, opts->transport) == 0)
523			return ops;
524	}
525
526	return NULL;
527}
528
529static const match_table_t opt_tokens = {
530	{ NVMF_OPT_TRANSPORT,		"transport=%s"		},
531	{ NVMF_OPT_TRADDR,		"traddr=%s"		},
532	{ NVMF_OPT_TRSVCID,		"trsvcid=%s"		},
533	{ NVMF_OPT_NQN,			"nqn=%s"		},
534	{ NVMF_OPT_QUEUE_SIZE,		"queue_size=%d"		},
535	{ NVMF_OPT_NR_IO_QUEUES,	"nr_io_queues=%d"	},
536	{ NVMF_OPT_RECONNECT_DELAY,	"reconnect_delay=%d"	},
537	{ NVMF_OPT_CTRL_LOSS_TMO,	"ctrl_loss_tmo=%d"	},
538	{ NVMF_OPT_KATO,		"keep_alive_tmo=%d"	},
539	{ NVMF_OPT_HOSTNQN,		"hostnqn=%s"		},
540	{ NVMF_OPT_HOST_TRADDR,		"host_traddr=%s"	},
541	{ NVMF_OPT_HOST_IFACE,		"host_iface=%s"		},
542	{ NVMF_OPT_HOST_ID,		"hostid=%s"		},
543	{ NVMF_OPT_DUP_CONNECT,		"duplicate_connect"	},
544	{ NVMF_OPT_DISABLE_SQFLOW,	"disable_sqflow"	},
545	{ NVMF_OPT_HDR_DIGEST,		"hdr_digest"		},
546	{ NVMF_OPT_DATA_DIGEST,		"data_digest"		},
547	{ NVMF_OPT_NR_WRITE_QUEUES,	"nr_write_queues=%d"	},
548	{ NVMF_OPT_NR_POLL_QUEUES,	"nr_poll_queues=%d"	},
549	{ NVMF_OPT_TOS,			"tos=%d"		},
550	{ NVMF_OPT_FAIL_FAST_TMO,	"fast_io_fail_tmo=%d"	},
551	{ NVMF_OPT_DISCOVERY,		"discovery"		},
552	{ NVMF_OPT_ERR,			NULL			}
553};
554
555static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
556		const char *buf)
557{
558	substring_t args[MAX_OPT_ARGS];
559	char *options, *o, *p;
560	int token, ret = 0;
561	size_t nqnlen  = 0;
562	int ctrl_loss_tmo = NVMF_DEF_CTRL_LOSS_TMO;
563	uuid_t hostid;
564
565	/* Set defaults */
566	opts->queue_size = NVMF_DEF_QUEUE_SIZE;
567	opts->nr_io_queues = num_online_cpus();
568	opts->reconnect_delay = NVMF_DEF_RECONNECT_DELAY;
569	opts->kato = 0;
570	opts->duplicate_connect = false;
571	opts->fast_io_fail_tmo = NVMF_DEF_FAIL_FAST_TMO;
572	opts->hdr_digest = false;
573	opts->data_digest = false;
574	opts->tos = -1; /* < 0 == use transport default */
575
576	options = o = kstrdup(buf, GFP_KERNEL);
577	if (!options)
578		return -ENOMEM;
579
580	uuid_gen(&hostid);
581
582	while ((p = strsep(&o, ",\n")) != NULL) {
583		if (!*p)
584			continue;
585
586		token = match_token(p, opt_tokens, args);
587		opts->mask |= token;
588		switch (token) {
589		case NVMF_OPT_TRANSPORT:
590			p = match_strdup(args);
591			if (!p) {
592				ret = -ENOMEM;
593				goto out;
594			}
595			kfree(opts->transport);
596			opts->transport = p;
597			break;
598		case NVMF_OPT_NQN:
599			p = match_strdup(args);
600			if (!p) {
601				ret = -ENOMEM;
602				goto out;
603			}
604			kfree(opts->subsysnqn);
605			opts->subsysnqn = p;
606			nqnlen = strlen(opts->subsysnqn);
607			if (nqnlen >= NVMF_NQN_SIZE) {
608				pr_err("%s needs to be < %d bytes\n",
609					opts->subsysnqn, NVMF_NQN_SIZE);
610				ret = -EINVAL;
611				goto out;
612			}
613			opts->discovery_nqn =
614				!(strcmp(opts->subsysnqn,
615					 NVME_DISC_SUBSYS_NAME));
616			break;
617		case NVMF_OPT_TRADDR:
618			p = match_strdup(args);
619			if (!p) {
620				ret = -ENOMEM;
621				goto out;
622			}
623			kfree(opts->traddr);
624			opts->traddr = p;
625			break;
626		case NVMF_OPT_TRSVCID:
627			p = match_strdup(args);
628			if (!p) {
629				ret = -ENOMEM;
630				goto out;
631			}
632			kfree(opts->trsvcid);
633			opts->trsvcid = p;
634			break;
635		case NVMF_OPT_QUEUE_SIZE:
636			if (match_int(args, &token)) {
637				ret = -EINVAL;
638				goto out;
639			}
640			if (token < NVMF_MIN_QUEUE_SIZE ||
641			    token > NVMF_MAX_QUEUE_SIZE) {
642				pr_err("Invalid queue_size %d\n", token);
643				ret = -EINVAL;
644				goto out;
645			}
646			opts->queue_size = token;
647			break;
648		case NVMF_OPT_NR_IO_QUEUES:
649			if (match_int(args, &token)) {
650				ret = -EINVAL;
651				goto out;
652			}
653			if (token <= 0) {
654				pr_err("Invalid number of IOQs %d\n", token);
655				ret = -EINVAL;
656				goto out;
657			}
658			if (opts->discovery_nqn) {
659				pr_debug("Ignoring nr_io_queues value for discovery controller\n");
660				break;
661			}
662
663			opts->nr_io_queues = min_t(unsigned int,
664					num_online_cpus(), token);
665			break;
666		case NVMF_OPT_KATO:
667			if (match_int(args, &token)) {
668				ret = -EINVAL;
669				goto out;
670			}
671
672			if (token < 0) {
673				pr_err("Invalid keep_alive_tmo %d\n", token);
674				ret = -EINVAL;
675				goto out;
676			} else if (token == 0 && !opts->discovery_nqn) {
677				/* Allowed for debug */
678				pr_warn("keep_alive_tmo 0 won't execute keep alives!!!\n");
679			}
680			opts->kato = token;
681			break;
682		case NVMF_OPT_CTRL_LOSS_TMO:
683			if (match_int(args, &token)) {
684				ret = -EINVAL;
685				goto out;
686			}
687
688			if (token < 0)
689				pr_warn("ctrl_loss_tmo < 0 will reconnect forever\n");
690			ctrl_loss_tmo = token;
691			break;
692		case NVMF_OPT_FAIL_FAST_TMO:
693			if (match_int(args, &token)) {
694				ret = -EINVAL;
695				goto out;
696			}
697
698			if (token >= 0)
699				pr_warn("I/O fail on reconnect controller after %d sec\n",
700					token);
701			else
702				token = -1;
703
704			opts->fast_io_fail_tmo = token;
705			break;
706		case NVMF_OPT_HOSTNQN:
707			if (opts->host) {
708				pr_err("hostnqn already user-assigned: %s\n",
709				       opts->host->nqn);
710				ret = -EADDRINUSE;
711				goto out;
712			}
713			p = match_strdup(args);
714			if (!p) {
715				ret = -ENOMEM;
716				goto out;
717			}
718			nqnlen = strlen(p);
719			if (nqnlen >= NVMF_NQN_SIZE) {
720				pr_err("%s needs to be < %d bytes\n",
721					p, NVMF_NQN_SIZE);
722				kfree(p);
723				ret = -EINVAL;
724				goto out;
725			}
726			opts->host = nvmf_host_add(p);
727			kfree(p);
728			if (!opts->host) {
729				ret = -ENOMEM;
730				goto out;
731			}
732			break;
733		case NVMF_OPT_RECONNECT_DELAY:
734			if (match_int(args, &token)) {
735				ret = -EINVAL;
736				goto out;
737			}
738			if (token <= 0) {
739				pr_err("Invalid reconnect_delay %d\n", token);
740				ret = -EINVAL;
741				goto out;
742			}
743			opts->reconnect_delay = token;
744			break;
745		case NVMF_OPT_HOST_TRADDR:
746			p = match_strdup(args);
747			if (!p) {
748				ret = -ENOMEM;
749				goto out;
750			}
751			kfree(opts->host_traddr);
752			opts->host_traddr = p;
753			break;
754		case NVMF_OPT_HOST_IFACE:
755			p = match_strdup(args);
756			if (!p) {
757				ret = -ENOMEM;
758				goto out;
759			}
760			kfree(opts->host_iface);
761			opts->host_iface = p;
762			break;
763		case NVMF_OPT_HOST_ID:
764			p = match_strdup(args);
765			if (!p) {
766				ret = -ENOMEM;
767				goto out;
768			}
769			ret = uuid_parse(p, &hostid);
770			if (ret) {
771				pr_err("Invalid hostid %s\n", p);
772				ret = -EINVAL;
773				kfree(p);
774				goto out;
775			}
776			kfree(p);
777			break;
778		case NVMF_OPT_DUP_CONNECT:
779			opts->duplicate_connect = true;
780			break;
781		case NVMF_OPT_DISABLE_SQFLOW:
782			opts->disable_sqflow = true;
783			break;
784		case NVMF_OPT_HDR_DIGEST:
785			opts->hdr_digest = true;
786			break;
787		case NVMF_OPT_DATA_DIGEST:
788			opts->data_digest = true;
789			break;
790		case NVMF_OPT_NR_WRITE_QUEUES:
791			if (match_int(args, &token)) {
792				ret = -EINVAL;
793				goto out;
794			}
795			if (token <= 0) {
796				pr_err("Invalid nr_write_queues %d\n", token);
797				ret = -EINVAL;
798				goto out;
799			}
800			opts->nr_write_queues = token;
801			break;
802		case NVMF_OPT_NR_POLL_QUEUES:
803			if (match_int(args, &token)) {
804				ret = -EINVAL;
805				goto out;
806			}
807			if (token <= 0) {
808				pr_err("Invalid nr_poll_queues %d\n", token);
809				ret = -EINVAL;
810				goto out;
811			}
812			opts->nr_poll_queues = token;
813			break;
814		case NVMF_OPT_TOS:
815			if (match_int(args, &token)) {
816				ret = -EINVAL;
817				goto out;
818			}
819			if (token < 0) {
820				pr_err("Invalid type of service %d\n", token);
821				ret = -EINVAL;
822				goto out;
823			}
824			if (token > 255) {
825				pr_warn("Clamping type of service to 255\n");
826				token = 255;
827			}
828			opts->tos = token;
829			break;
830		case NVMF_OPT_DISCOVERY:
831			opts->discovery_nqn = true;
832			break;
833		default:
834			pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n",
835				p);
836			ret = -EINVAL;
837			goto out;
838		}
839	}
840
841	if (opts->discovery_nqn) {
842		opts->nr_io_queues = 0;
843		opts->nr_write_queues = 0;
844		opts->nr_poll_queues = 0;
845		opts->duplicate_connect = true;
846	} else {
847		if (!opts->kato)
848			opts->kato = NVME_DEFAULT_KATO;
849	}
850	if (ctrl_loss_tmo < 0) {
851		opts->max_reconnects = -1;
852	} else {
853		opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
854						opts->reconnect_delay);
855		if (ctrl_loss_tmo < opts->fast_io_fail_tmo)
856			pr_warn("failfast tmo (%d) larger than controller loss tmo (%d)\n",
857				opts->fast_io_fail_tmo, ctrl_loss_tmo);
858	}
859
860	if (!opts->host) {
861		kref_get(&nvmf_default_host->ref);
862		opts->host = nvmf_default_host;
863	}
864
865	uuid_copy(&opts->host->id, &hostid);
866
867out:
868	kfree(options);
869	return ret;
870}
871
872static int nvmf_check_required_opts(struct nvmf_ctrl_options *opts,
873		unsigned int required_opts)
874{
875	if ((opts->mask & required_opts) != required_opts) {
876		int i;
877
878		for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
879			if ((opt_tokens[i].token & required_opts) &&
880			    !(opt_tokens[i].token & opts->mask)) {
881				pr_warn("missing parameter '%s'\n",
882					opt_tokens[i].pattern);
883			}
884		}
885
886		return -EINVAL;
887	}
888
889	return 0;
890}
891
892bool nvmf_ip_options_match(struct nvme_ctrl *ctrl,
893		struct nvmf_ctrl_options *opts)
894{
895	if (!nvmf_ctlr_matches_baseopts(ctrl, opts) ||
896	    strcmp(opts->traddr, ctrl->opts->traddr) ||
897	    strcmp(opts->trsvcid, ctrl->opts->trsvcid))
898		return false;
899
900	/*
901	 * Checking the local address is rough. In most cases, none is specified
902	 * and the host port is selected by the stack.
903	 *
904	 * Assume no match if:
905	 * -  local address is specified and address is not the same
906	 * -  local address is not specified but remote is, or vice versa
907	 *    (admin using specific host_traddr when it matters).
908	 */
909	if ((opts->mask & NVMF_OPT_HOST_TRADDR) &&
910	    (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) {
911		if (strcmp(opts->host_traddr, ctrl->opts->host_traddr))
912			return false;
913	} else if ((opts->mask & NVMF_OPT_HOST_TRADDR) ||
914		   (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) {
915		return false;
916	}
917
918	return true;
919}
920EXPORT_SYMBOL_GPL(nvmf_ip_options_match);
921
922static int nvmf_check_allowed_opts(struct nvmf_ctrl_options *opts,
923		unsigned int allowed_opts)
924{
925	if (opts->mask & ~allowed_opts) {
926		int i;
927
928		for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
929			if ((opt_tokens[i].token & opts->mask) &&
930			    (opt_tokens[i].token & ~allowed_opts)) {
931				pr_warn("invalid parameter '%s'\n",
932					opt_tokens[i].pattern);
933			}
934		}
935
936		return -EINVAL;
937	}
938
939	return 0;
940}
941
942void nvmf_free_options(struct nvmf_ctrl_options *opts)
943{
944	nvmf_host_put(opts->host);
945	kfree(opts->transport);
946	kfree(opts->traddr);
947	kfree(opts->trsvcid);
948	kfree(opts->subsysnqn);
949	kfree(opts->host_traddr);
950	kfree(opts->host_iface);
951	kfree(opts);
952}
953EXPORT_SYMBOL_GPL(nvmf_free_options);
954
955#define NVMF_REQUIRED_OPTS	(NVMF_OPT_TRANSPORT | NVMF_OPT_NQN)
956#define NVMF_ALLOWED_OPTS	(NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \
957				 NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \
958				 NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\
959				 NVMF_OPT_DISABLE_SQFLOW | NVMF_OPT_DISCOVERY |\
960				 NVMF_OPT_FAIL_FAST_TMO)
961
962static struct nvme_ctrl *
963nvmf_create_ctrl(struct device *dev, const char *buf)
964{
965	struct nvmf_ctrl_options *opts;
966	struct nvmf_transport_ops *ops;
967	struct nvme_ctrl *ctrl;
968	int ret;
969
970	opts = kzalloc(sizeof(*opts), GFP_KERNEL);
971	if (!opts)
972		return ERR_PTR(-ENOMEM);
973
974	ret = nvmf_parse_options(opts, buf);
975	if (ret)
976		goto out_free_opts;
977
978
979	request_module("nvme-%s", opts->transport);
980
981	/*
982	 * Check the generic options first as we need a valid transport for
983	 * the lookup below.  Then clear the generic flags so that transport
984	 * drivers don't have to care about them.
985	 */
986	ret = nvmf_check_required_opts(opts, NVMF_REQUIRED_OPTS);
987	if (ret)
988		goto out_free_opts;
989	opts->mask &= ~NVMF_REQUIRED_OPTS;
990
991	down_read(&nvmf_transports_rwsem);
992	ops = nvmf_lookup_transport(opts);
993	if (!ops) {
994		pr_info("no handler found for transport %s.\n",
995			opts->transport);
996		ret = -EINVAL;
997		goto out_unlock;
998	}
999
1000	if (!try_module_get(ops->module)) {
1001		ret = -EBUSY;
1002		goto out_unlock;
1003	}
1004	up_read(&nvmf_transports_rwsem);
1005
1006	ret = nvmf_check_required_opts(opts, ops->required_opts);
1007	if (ret)
1008		goto out_module_put;
1009	ret = nvmf_check_allowed_opts(opts, NVMF_ALLOWED_OPTS |
1010				ops->allowed_opts | ops->required_opts);
1011	if (ret)
1012		goto out_module_put;
1013
1014	ctrl = ops->create_ctrl(dev, opts);
1015	if (IS_ERR(ctrl)) {
1016		ret = PTR_ERR(ctrl);
1017		goto out_module_put;
1018	}
1019
1020	module_put(ops->module);
1021	return ctrl;
1022
1023out_module_put:
1024	module_put(ops->module);
1025	goto out_free_opts;
1026out_unlock:
1027	up_read(&nvmf_transports_rwsem);
1028out_free_opts:
1029	nvmf_free_options(opts);
1030	return ERR_PTR(ret);
1031}
1032
1033static struct class *nvmf_class;
1034static struct device *nvmf_device;
1035static DEFINE_MUTEX(nvmf_dev_mutex);
1036
1037static ssize_t nvmf_dev_write(struct file *file, const char __user *ubuf,
1038		size_t count, loff_t *pos)
1039{
1040	struct seq_file *seq_file = file->private_data;
1041	struct nvme_ctrl *ctrl;
1042	const char *buf;
1043	int ret = 0;
1044
1045	if (count > PAGE_SIZE)
1046		return -ENOMEM;
1047
1048	buf = memdup_user_nul(ubuf, count);
1049	if (IS_ERR(buf))
1050		return PTR_ERR(buf);
1051
1052	mutex_lock(&nvmf_dev_mutex);
1053	if (seq_file->private) {
1054		ret = -EINVAL;
1055		goto out_unlock;
1056	}
1057
1058	ctrl = nvmf_create_ctrl(nvmf_device, buf);
1059	if (IS_ERR(ctrl)) {
1060		ret = PTR_ERR(ctrl);
1061		goto out_unlock;
1062	}
1063
1064	seq_file->private = ctrl;
1065
1066out_unlock:
1067	mutex_unlock(&nvmf_dev_mutex);
1068	kfree(buf);
1069	return ret ? ret : count;
1070}
1071
1072static void __nvmf_concat_opt_tokens(struct seq_file *seq_file)
1073{
1074	const struct match_token *tok;
1075	int idx;
1076
1077	/*
1078	 * Add dummy entries for instance and cntlid to
1079	 * signal an invalid/non-existing controller
1080	 */
1081	seq_puts(seq_file, "instance=-1,cntlid=-1");
1082	for (idx = 0; idx < ARRAY_SIZE(opt_tokens); idx++) {
1083		tok = &opt_tokens[idx];
1084		if (tok->token == NVMF_OPT_ERR)
1085			continue;
1086		seq_puts(seq_file, ",");
1087		seq_puts(seq_file, tok->pattern);
1088	}
1089	seq_puts(seq_file, "\n");
1090}
1091
1092static int nvmf_dev_show(struct seq_file *seq_file, void *private)
1093{
1094	struct nvme_ctrl *ctrl;
1095	int ret = 0;
1096
1097	mutex_lock(&nvmf_dev_mutex);
1098	ctrl = seq_file->private;
1099	if (!ctrl) {
1100		__nvmf_concat_opt_tokens(seq_file);
1101		goto out_unlock;
1102	}
1103
1104	seq_printf(seq_file, "instance=%d,cntlid=%d\n",
1105			ctrl->instance, ctrl->cntlid);
1106
1107out_unlock:
1108	mutex_unlock(&nvmf_dev_mutex);
1109	return ret;
1110}
1111
1112static int nvmf_dev_open(struct inode *inode, struct file *file)
1113{
1114	/*
1115	 * The miscdevice code initializes file->private_data, but doesn't
1116	 * make use of it later.
1117	 */
1118	file->private_data = NULL;
1119	return single_open(file, nvmf_dev_show, NULL);
1120}
1121
1122static int nvmf_dev_release(struct inode *inode, struct file *file)
1123{
1124	struct seq_file *seq_file = file->private_data;
1125	struct nvme_ctrl *ctrl = seq_file->private;
1126
1127	if (ctrl)
1128		nvme_put_ctrl(ctrl);
1129	return single_release(inode, file);
1130}
1131
1132static const struct file_operations nvmf_dev_fops = {
1133	.owner		= THIS_MODULE,
1134	.write		= nvmf_dev_write,
1135	.read		= seq_read,
1136	.open		= nvmf_dev_open,
1137	.release	= nvmf_dev_release,
1138};
1139
1140static struct miscdevice nvmf_misc = {
1141	.minor		= MISC_DYNAMIC_MINOR,
1142	.name           = "nvme-fabrics",
1143	.fops		= &nvmf_dev_fops,
1144};
1145
1146static int __init nvmf_init(void)
1147{
1148	int ret;
1149
1150	nvmf_default_host = nvmf_host_default();
1151	if (!nvmf_default_host)
1152		return -ENOMEM;
1153
1154	nvmf_class = class_create(THIS_MODULE, "nvme-fabrics");
1155	if (IS_ERR(nvmf_class)) {
1156		pr_err("couldn't register class nvme-fabrics\n");
1157		ret = PTR_ERR(nvmf_class);
1158		goto out_free_host;
1159	}
1160
1161	nvmf_device =
1162		device_create(nvmf_class, NULL, MKDEV(0, 0), NULL, "ctl");
1163	if (IS_ERR(nvmf_device)) {
1164		pr_err("couldn't create nvme-fabris device!\n");
1165		ret = PTR_ERR(nvmf_device);
1166		goto out_destroy_class;
1167	}
1168
1169	ret = misc_register(&nvmf_misc);
1170	if (ret) {
1171		pr_err("couldn't register misc device: %d\n", ret);
1172		goto out_destroy_device;
1173	}
1174
1175	return 0;
1176
1177out_destroy_device:
1178	device_destroy(nvmf_class, MKDEV(0, 0));
1179out_destroy_class:
1180	class_destroy(nvmf_class);
1181out_free_host:
1182	nvmf_host_put(nvmf_default_host);
1183	return ret;
1184}
1185
1186static void __exit nvmf_exit(void)
1187{
1188	misc_deregister(&nvmf_misc);
1189	device_destroy(nvmf_class, MKDEV(0, 0));
1190	class_destroy(nvmf_class);
1191	nvmf_host_put(nvmf_default_host);
1192
1193	BUILD_BUG_ON(sizeof(struct nvmf_common_command) != 64);
1194	BUILD_BUG_ON(sizeof(struct nvmf_connect_command) != 64);
1195	BUILD_BUG_ON(sizeof(struct nvmf_property_get_command) != 64);
1196	BUILD_BUG_ON(sizeof(struct nvmf_property_set_command) != 64);
1197	BUILD_BUG_ON(sizeof(struct nvmf_connect_data) != 1024);
1198}
1199
1200MODULE_LICENSE("GPL v2");
1201
1202module_init(nvmf_init);
1203module_exit(nvmf_exit);
1204