sfc/siena/efx_channels.c

157 	 * We need a channel per event queue, plus a VI per tx queue.
274 			/* Fall back to single channel MSI */
367 	struct efx_channel *channel;
375 	efx_for_each_channel(channel, efx) {
379 		irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
385 	struct efx_channel *channel;
387 	efx_for_each_channel(channel, efx)
388 		irq_set_affinity_hint(channel->irq, NULL);
404 	struct efx_channel *channel;
407 	efx_for_each_channel(channel, efx)
408 		channel->irq = 0;
421  * Event queue memory allocations are done only once.  If the channel
423  * errors during channel reset and also simplifies interrupt handling.
425 static int efx_probe_eventq(struct efx_channel *channel)
427 	struct efx_nic *efx = channel->efx;
431 		  "chan %d create event queue\n", channel->channel);
438 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
440 	return efx_nic_probe_eventq(channel);
443 /* Prepare channel's event queue */
444 static int efx_init_eventq(struct efx_channel *channel)
446 	struct efx_nic *efx = channel->efx;
449 	EFX_WARN_ON_PARANOID(channel->eventq_init);
452 		  "chan %d init event queue\n", channel->channel);
454 	rc = efx_nic_init_eventq(channel);
456 		efx->type->push_irq_moderation(channel);
457 		channel->eventq_read_ptr = 0;
458 		channel->eventq_init = true;
464 void efx_siena_start_eventq(struct efx_channel *channel)
466 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
467 		  "chan %d start event queue\n", channel->channel);
470 	channel->enabled = true;
473 	napi_enable(&channel->napi_str);
474 	efx_nic_eventq_read_ack(channel);
478 void efx_siena_stop_eventq(struct efx_channel *channel)
480 	if (!channel->enabled)
483 	napi_disable(&channel->napi_str);
484 	channel->enabled = false;
487 static void efx_fini_eventq(struct efx_channel *channel)
489 	if (!channel->eventq_init)
492 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
493 		  "chan %d fini event queue\n", channel->channel);
495 	efx_nic_fini_eventq(channel);
496 	channel->eventq_init = false;
499 static void efx_remove_eventq(struct efx_channel *channel)
501 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
502 		  "chan %d remove event queue\n", channel->channel);
504 	efx_nic_remove_eventq(channel);
517 	struct efx_channel *channel;
520 	channel = container_of(dwork, struct efx_channel, filter_work);
521 	time = jiffies - channel->rfs_last_expiry;
522 	quota = channel->rfs_filter_count * time / (30 * HZ);
523 	if (quota >= 20 && __efx_siena_filter_rfs_expire(channel,
524 					min(channel->rfs_filter_count, quota)))
525 		channel->rfs_last_expiry += time;
531 /* Allocate and initialise a channel structure. */
536 	struct efx_channel *channel;
539 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
540 	if (!channel)
543 	channel->efx = efx;
544 	channel->channel = i;
545 	channel->type = &efx_default_channel_type;
548 		tx_queue = &channel->tx_queue[j];
552 		tx_queue->channel = channel;
556 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
559 	rx_queue = &channel->rx_queue;
563 	return channel;
571 		efx->channel[i] = efx_alloc_channel(efx, i);
572 		if (!efx->channel[i])
593 		if (efx->channel[i]) {
594 			kfree(efx->channel[i]);
595 			efx->channel[i] = NULL;
599 /* Allocate and initialise a channel structure, copying parameters
600  * (but not resources) from an old channel structure.
607 	struct efx_channel *channel;
610 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
611 	if (!channel)
614 	*channel = *old_channel;
616 	channel->napi_dev = NULL;
617 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
618 	channel->napi_str.napi_id = 0;
619 	channel->napi_str.state = 0;
620 	memset(&channel->eventq, 0, sizeof(channel->eventq));
623 		tx_queue = &channel->tx_queue[j];
624 		if (tx_queue->channel)
625 			tx_queue->channel = channel;
631 	rx_queue = &channel->rx_queue;
636 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
639 	return channel;
642 static int efx_probe_channel(struct efx_channel *channel)
648 	netif_dbg(channel->efx, probe, channel->efx->net_dev,
649 		  "creating channel %d\n", channel->channel);
651 	rc = channel->type->pre_probe(channel);
655 	rc = efx_probe_eventq(channel);
659 	efx_for_each_channel_tx_queue(tx_queue, channel) {
665 	efx_for_each_channel_rx_queue(rx_queue, channel) {
671 	channel->rx_list = NULL;
676 	efx_siena_remove_channel(channel);
680 static void efx_get_channel_name(struct efx_channel *channel, char *buf,
683 	struct efx_nic *efx = channel->efx;
687 	number = channel->channel;
706 	struct efx_channel *channel;
708 	efx_for_each_channel(channel, efx)
709 		channel->type->get_name(channel,
710 					efx->msi_context[channel->channel].name,
716 	struct efx_channel *channel;
727 	efx_for_each_channel_rev(channel, efx) {
728 		rc = efx_probe_channel(channel);
731 				  "failed to create channel %d\n",
732 				  channel->channel);
745 void efx_siena_remove_channel(struct efx_channel *channel)
750 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
751 		  "destroy chan %d\n", channel->channel);
753 	efx_for_each_channel_rx_queue(rx_queue, channel)
755 	efx_for_each_channel_tx_queue(tx_queue, channel)
757 	efx_remove_eventq(channel);
758 	channel->type->post_remove(channel);
763 	struct efx_channel *channel;
765 	efx_for_each_channel(channel, efx)
766 		efx_siena_remove_channel(channel);
779 		  tx_queue->channel->channel, tx_queue->label,
788 	struct efx_channel *channel;
797 	efx_for_each_channel(channel, efx) {
798 		if (channel->channel < efx->tx_channel_offset)
801 		if (efx_channel_is_xdp_tx(channel)) {
802 			efx_for_each_channel_tx_queue(tx_queue, channel) {
810 			efx_for_each_channel_tx_queue(tx_queue, channel) {
814 					  channel->channel, tx_queue->label,
820 			 * first one of the channel
825 				tx_queue = &channel->tx_queue[0];
852 static void efx_init_napi_channel(struct efx_channel *channel);
853 static void efx_fini_napi_channel(struct efx_channel *channel);
858 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
870 	efx_for_each_channel(channel, efx) {
874 		if (channel->type->copy)
877 					channel->eventq.index +
878 					channel->eventq.entries);
879 		efx_for_each_channel_rx_queue(rx_queue, channel)
883 		efx_for_each_channel_tx_queue(tx_queue, channel)
896 		channel = efx->channel[i];
897 		if (channel->type->copy)
898 			channel = channel->type->copy(channel);
899 		if (!channel) {
903 		other_channel[i] = channel;
906 	/* Swap entry counts and channel pointers */
912 		swap(efx->channel[i], other_channel[i]);
918 		channel = efx->channel[i];
919 		if (!channel->type->copy)
921 		rc = efx_probe_channel(channel);
924 		efx_init_napi_channel(efx->channel[i]);
929 	/* Destroy unused channel structures */
931 		channel = other_channel[i];
932 		if (channel && channel->type->copy) {
933 			efx_fini_napi_channel(channel);
934 			efx_siena_remove_channel(channel);
935 			kfree(channel);
943 			  "unable to restart interrupts on channel reallocation\n");
956 		swap(efx->channel[i], other_channel[i]);
962 	struct efx_channel *channel;
976 	efx_for_each_channel(channel, efx) {
977 		if (channel->channel < efx->n_rx_channels)
978 			channel->rx_queue.core_index = channel->channel;
980 			channel->rx_queue.core_index = -1;
991 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
993 	return channel->channel - channel->efx->tx_channel_offset <
994 		channel->efx->n_tx_channels;
1003 	struct efx_channel *channel, *end_channel;
1011 	efx_for_each_channel(channel, efx) {
1012 		if (!channel->type->keep_eventq) {
1013 			rc = efx_init_eventq(channel);
1017 		efx_siena_start_eventq(channel);
1024 	end_channel = channel;
1025 	efx_for_each_channel(channel, efx) {
1026 		if (channel == end_channel)
1028 		efx_siena_stop_eventq(channel);
1029 		if (!channel->type->keep_eventq)
1030 			efx_fini_eventq(channel);
1038 	struct efx_channel *channel;
1051 	efx_for_each_channel(channel, efx) {
1052 		if (channel->irq)
1053 			synchronize_irq(channel->irq);
1055 		efx_siena_stop_eventq(channel);
1056 		if (!channel->type->keep_eventq)
1057 			efx_fini_eventq(channel);
1066 	struct efx_channel *channel, *end_channel;
1079 	efx_for_each_channel(channel, efx) {
1080 		if (channel->type->keep_eventq) {
1081 			rc = efx_init_eventq(channel);
1094 	end_channel = channel;
1095 	efx_for_each_channel(channel, efx) {
1096 		if (channel == end_channel)
1098 		if (channel->type->keep_eventq)
1099 			efx_fini_eventq(channel);
1109 	struct efx_channel *channel;
1113 	efx_for_each_channel(channel, efx) {
1114 		if (channel->type->keep_eventq)
1115 			efx_fini_eventq(channel);
1125 	struct efx_channel *channel;
1127 	efx_for_each_channel_rev(channel, efx) {
1128 		efx_for_each_channel_tx_queue(tx_queue, channel) {
1133 		efx_for_each_channel_rx_queue(rx_queue, channel) {
1136 			efx_siena_stop_eventq(channel);
1138 			efx_siena_start_eventq(channel);
1141 		WARN_ON(channel->rx_pkt_n_frags);
1149 	struct efx_channel *channel;
1153 	efx_for_each_channel(channel, efx) {
1154 		efx_for_each_channel_rx_queue(rx_queue, channel)
1158 	efx_for_each_channel(channel, efx) {
1165 		if (efx_channel_has_rx_queue(channel)) {
1166 			efx_siena_stop_eventq(channel);
1167 			efx_siena_start_eventq(channel);
1181 	efx_for_each_channel(channel, efx) {
1182 		efx_for_each_channel_rx_queue(rx_queue, channel)
1184 		efx_for_each_channel_tx_queue(tx_queue, channel)
1195 /* Process channel's event queue
1198  * single channel.  The caller must guarantee that this function will
1199  * never be concurrently called more than once on the same channel,
1202 static int efx_process_channel(struct efx_channel *channel, int budget)
1208 	if (unlikely(!channel->enabled))
1212 	EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1214 	channel->rx_list = &rx_list;
1216 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1221 	spent = efx_nic_process_eventq(channel, budget);
1222 	if (spent && efx_channel_has_rx_queue(channel)) {
1224 			efx_channel_get_rx_queue(channel);
1226 		efx_rx_flush_packet(channel);
1231 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1240 	netif_receive_skb_list(channel->rx_list);
1241 	channel->rx_list = NULL;
1246 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1250 	if (channel->irq_mod_score < irq_adapt_low_thresh) {
1251 		if (channel->irq_moderation_us > step) {
1252 			channel->irq_moderation_us -= step;
1253 			efx->type->push_irq_moderation(channel);
1255 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1256 		if (channel->irq_moderation_us <
1258 			channel->irq_moderation_us += step;
1259 			efx->type->push_irq_moderation(channel);
1263 	channel->irq_count = 0;
1264 	channel->irq_mod_score = 0;
1274 	struct efx_channel *channel =
1276 	struct efx_nic *efx = channel->efx;
1283 		   "channel %d NAPI poll executing on CPU %d\n",
1284 		   channel->channel, raw_smp_processor_id());
1286 	spent = efx_process_channel(channel, budget);
1291 		if (efx_channel_has_rx_queue(channel) &&
1293 		    unlikely(++channel->irq_count == 1000)) {
1294 			efx_update_irq_mod(efx, channel);
1299 		time = jiffies - channel->rfs_last_expiry;
1301 		if (channel->rfs_filter_count * time >= 600 * HZ)
1302 			mod_delayed_work(system_wq, &channel->filter_work, 0);
1311 			efx_nic_eventq_read_ack(channel);
1317 static void efx_init_napi_channel(struct efx_channel *channel)
1319 	struct efx_nic *efx = channel->efx;
1321 	channel->napi_dev = efx->net_dev;
1322 	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);
1327 	struct efx_channel *channel;
1329 	efx_for_each_channel(channel, efx)
1330 		efx_init_napi_channel(channel);
1333 static void efx_fini_napi_channel(struct efx_channel *channel)
1335 	if (channel->napi_dev)
1336 		netif_napi_del(&channel->napi_str);
1338 	channel->napi_dev = NULL;
1343 	struct efx_channel *channel;
1345 	efx_for_each_channel(channel, efx)
1346 		efx_fini_napi_channel(channel);
1353 static int efx_channel_dummy_op_int(struct efx_channel *channel)
1358 void efx_siena_channel_dummy_op_void(struct efx_channel *channel)