1/*
2 * Copyright (c) 2019-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <skywalk/os_skywalk_private.h>
30#include <skywalk/nexus/netif/nx_netif.h>
31#include <net/if_vlan_var.h>
32#include <sys/sdt.h>
33
34#define NETIF_DEMUX_ALLOC_SLOTS 128
35
36#define OUTBOUND_CHECK_OFF 0
37#define OUTBOUND_CHECK_ON 1
38#define OUTBOUND_CHECK_FORCED 2
39
40/* Turning this off allows packets to be spoofed for testing purposes */
41static uint32_t outbound_check = OUTBOUND_CHECK_ON;
42
43/* This controls the per-NA pool size of custom ether and llw NAs */
44static uint32_t vp_pool_size = 2048;
45
46/* This enables zerocopy on llw NAs */
47static uint32_t vp_zerocopy = 0;
48
49/* TX Ring size */
50static uint32_t vp_tx_slots = 0;
51
52/* RX Ring size */
53static uint32_t vp_rx_slots = 0;
54
55/*
56 * Disable all packet validation
57 */
58uint32_t nx_netif_vp_accept_all = 0;
59
60static uint16_t nx_netif_vpna_gencnt = 0;
61
62#if (DEVELOPMENT || DEBUG)
63SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, outbound_check,
64 CTLFLAG_RW | CTLFLAG_LOCKED, &outbound_check, 0,
65 "netif outbound packet validation");
66SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_pool_size,
67 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_pool_size, 0,
68 "netif virtual port pool size");
69SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_zerocopy,
70 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_zerocopy, 0,
71 "netif virtual port zero copy");
72SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_tx_slots,
73 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_tx_slots, 0,
74 "netif virtual port tx slots");
75SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_rx_slots,
76 CTLFLAG_RW | CTLFLAG_LOCKED, &vp_rx_slots, 0,
77 "netif virtual port rx slots");
78SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_accept_all,
79 CTLFLAG_RW | CTLFLAG_LOCKED, &nx_netif_vp_accept_all, 0,
80 "netif accept all");
81#endif /* (DEVELOPMENT || DEBUG) */
82
83static int
84netif_vp_na_channel_event_notify(struct nexus_adapter *,
85 struct __kern_channel_event *, uint16_t);
86
87static void
88netif_vp_dump_packet(struct __kern_packet *pkt)
89{
90 uint8_t *baddr;
91
92 MD_BUFLET_ADDR_ABS(pkt, baddr);
93 ASSERT(baddr != NULL);
94 baddr += pkt->pkt_headroom;
95
96 DTRACE_SKYWALK2(dump__packet, struct __kern_packet *,
97 pkt, uint8_t *, baddr);
98}
99
100static int
101netif_copy_or_attach_pkt(struct __kern_channel_ring *ring,
102 kern_channel_slot_t slot, struct __kern_packet *pkt)
103{
104 kern_packet_t ph;
105 struct __kern_packet *dpkt;
106 errno_t err;
107
108 if (pkt->pkt_qum.qum_pp == ring->ckr_pp) {
109 DTRACE_SKYWALK2(attach__pkt, struct __kern_channel_ring *, ring,
110 struct __kern_packet *, pkt);
111 ph = SK_PKT2PH(pkt);
112 err = kern_packet_finalize(ph);
113 VERIFY(err == 0);
114 } else {
115 DTRACE_SKYWALK2(copy__pkt, struct __kern_channel_ring *, ring,
116 struct __kern_packet *, pkt);
117 dpkt = nx_netif_pkt_to_pkt(NIFNA(ring->ckr_na), pkt,
118 ring->ckr_na->na_type == NA_NETIF_VP ? NETIF_CONVERT_RX :
119 NETIF_CONVERT_TX);
120 if (__improbable(dpkt == NULL)) {
121 return ENOMEM;
122 }
123 ph = SK_PKT2PH(dpkt);
124 }
125 err = kern_channel_slot_attach_packet(ring, slot, packet: ph);
126 VERIFY(err == 0);
127 return 0;
128}
129
130static errno_t
131netif_deliver_pkt(struct nexus_adapter *na, struct __kern_packet *pkt_chain,
132 uint32_t flags)
133{
134#pragma unused(flags)
135 struct __kern_channel_ring *ring = &na->na_rx_rings[0];
136 struct __kern_packet *pkt = pkt_chain, *next;
137 kern_channel_slot_t last_slot = NULL, slot = NULL;
138 struct nexus_netif_adapter *nifna = NIFNA(na);
139 struct nx_netif *nif = nifna->nifna_netif;
140 struct netif_stats *nifs = &nif->nif_stats;
141 sk_protect_t protect;
142 int cnt = 0, dropcnt = 0, err;
143
144 (void) kr_enter(ring, TRUE);
145 protect = sk_sync_protect();
146
147 if (__improbable(KR_DROP(ring))) {
148 nx_netif_free_packet_chain(pkt, &dropcnt);
149 STATS_ADD(nifs,
150 NETIF_STATS_VP_DROP_USER_RING_DISABLED, dropcnt);
151 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
152 DTRACE_SKYWALK2(ring__drop, struct __kern_channel_ring *, ring,
153 int, dropcnt);
154 sk_sync_unprotect(protect);
155 kr_exit(ring);
156 return ENXIO;
157 }
158 while (pkt != NULL) {
159 slot = kern_channel_get_next_slot(kring: ring, slot: last_slot, NULL);
160 if (slot == NULL) {
161 break;
162 }
163 next = pkt->pkt_nextpkt;
164 pkt->pkt_nextpkt = NULL;
165 netif_vp_dump_packet(pkt);
166 err = netif_copy_or_attach_pkt(ring, slot, pkt);
167 if (__probable(err == 0)) {
168 last_slot = slot;
169 }
170 pkt = next;
171 cnt++;
172 }
173 if (NETIF_IS_LOW_LATENCY(nif)) {
174 STATS_ADD(nifs, NETIF_STATS_VP_LL_DELIVERED, cnt);
175 } else {
176 STATS_ADD(nifs, NETIF_STATS_VP_DELIVERED, cnt);
177 }
178 DTRACE_SKYWALK4(delivered, struct nexus_adapter *, na,
179 struct __kern_channel_ring *, ring, struct __kern_packet *, pkt,
180 int, cnt);
181
182 if (pkt != NULL) {
183 nx_netif_free_packet_chain(pkt, &dropcnt);
184 STATS_ADD(nifs,
185 NETIF_STATS_VP_DROP_USER_RING_NO_SPACE, dropcnt);
186 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
187 DTRACE_SKYWALK2(deliver__drop, struct nexus_adapter *, na,
188 int, dropcnt);
189 }
190 if (last_slot != NULL) {
191 kern_channel_advance_slot(kring: ring, slot: last_slot);
192 }
193 sk_sync_unprotect(protect);
194 kr_exit(ring);
195 if (cnt > 0) {
196 (void) kern_channel_notify(ring, flags: 0);
197 }
198 return 0;
199}
200
201static errno_t
202netif_deliver_cb(void *arg, void *chain, uint32_t flags)
203{
204 return netif_deliver_pkt(na: arg, pkt_chain: chain, flags);
205}
206
207static int
208netif_hwna_rx_get_pkts(struct __kern_channel_ring *ring, struct proc *p,
209 uint32_t flags, struct __kern_packet **chain)
210{
211 int err, cnt = 0;
212 sk_protect_t protect;
213 slot_idx_t ktail, idx;
214 struct __kern_packet *pkt_chain = NULL, **tailp = &pkt_chain;
215 struct netif_stats *nifs = &NIFNA(KRNA(ring))->nifna_netif->nif_stats;
216
217 err = kr_enter(ring, ((flags & NA_NOTEF_CAN_SLEEP) != 0 ||
218 (ring->ckr_flags & CKRF_HOST) != 0));
219 if (err != 0) {
220 SK_DF(SK_VERB_VP,
221 "hwna \"%s\" (0x%llx) kr \"%s\" (0x%llx) krflags 0x%b "
222 "(%d)", KRNA(ring)->na_name, SK_KVA(KRNA(ring)),
223 ring->ckr_name, SK_KVA(ring), ring->ckr_flags,
224 CKRF_BITS, err);
225 STATS_INC(nifs, NETIF_STATS_VP_KR_ENTER_FAIL);
226 return err;
227 }
228 if (__improbable(KR_DROP(ring))) {
229 kr_exit(ring);
230 STATS_INC(nifs, NETIF_STATS_VP_DEV_RING_DISABLED);
231 return ENODEV;
232 }
233 protect = sk_sync_protect();
234
235 err = ring->ckr_na_sync(ring, p, 0);
236 if (err != 0 && err != EAGAIN) {
237 STATS_INC(nifs, NETIF_STATS_VP_SYNC_UNKNOWN_ERR);
238 goto out;
239 }
240 ktail = ring->ckr_ktail;
241 if (__improbable(ring->ckr_khead == ktail)) {
242 SK_DF(SK_VERB_VP,
243 "spurious wakeup on hwna %s (0x%llx)", KRNA(ring)->na_name,
244 SK_KVA(KRNA(ring)));
245 STATS_INC(nifs, NETIF_STATS_VP_SPURIOUS_NOTIFY);
246 err = ENOENT;
247 goto out;
248 }
249 /* get all packets from the ring */
250 idx = ring->ckr_rhead;
251 while (idx != ktail) {
252 struct __kern_slot_desc *ksd = KR_KSD(ring, idx);
253 struct __kern_packet *pkt = ksd->sd_pkt;
254
255 ASSERT(pkt->pkt_nextpkt == NULL);
256 KR_SLOT_DETACH_METADATA(kring: ring, ksd);
257 cnt++;
258 *tailp = pkt;
259 tailp = &pkt->pkt_nextpkt;
260 idx = SLOT_NEXT(i: idx, lim: ring->ckr_lim);
261 }
262 ring->ckr_rhead = ktail;
263 ring->ckr_rtail = ring->ckr_ktail;
264
265 DTRACE_SKYWALK2(rx__notify, struct __kern_channel_ring *, ring,
266 int, cnt);
267 *chain = pkt_chain;
268out:
269 sk_sync_unprotect(protect);
270 kr_exit(ring);
271 return err;
272}
273
274int
275netif_llw_rx_notify_fast(struct __kern_channel_ring *ring, struct proc *p,
276 uint32_t flags)
277{
278#pragma unused (p, flags)
279 struct nexus_adapter *hwna;
280 uint32_t count;
281 int i, err;
282
283 hwna = KRNA(ring);
284 count = na_get_nslots(na: hwna, t: NR_RX);
285 err = nx_rx_sync_packets(kring: ring, packets: ring->ckr_scratch, count: &count);
286 if (__improbable(err != 0)) {
287 SK_ERR("nx_rx_sync_packets failed: %d", err);
288 DTRACE_SKYWALK2(rx__sync__packets__failed,
289 struct __kern_channel_ring *, ring, int, err);
290 return err;
291 }
292 DTRACE_SKYWALK1(chain__count, uint32_t, count);
293 for (i = 0; i < count; i++) {
294 struct __kern_packet *pkt_chain;
295
296 pkt_chain = SK_PTR_ADDR_KPKT(ring->ckr_scratch[i]);
297 ASSERT(pkt_chain != NULL);
298 (void) nx_netif_demux(NIFNA(KRNA(ring)), pkt_chain, NULL,
299 NETIF_FLOW_SOURCE);
300 }
301 return 0;
302}
303
304int
305netif_llw_rx_notify_default(struct __kern_channel_ring *ring, struct proc *p,
306 uint32_t flags)
307{
308 int err;
309 struct __kern_packet *pkt_chain = NULL;
310
311 err = netif_hwna_rx_get_pkts(ring, p, flags, chain: &pkt_chain);
312 if (err != 0) {
313 return err;
314 }
315 return nx_netif_demux(NIFNA(KRNA(ring)), pkt_chain, NULL,
316 NETIF_FLOW_SOURCE);
317}
318
319static errno_t
320netif_hwna_setup(struct nx_netif *nif)
321{
322 struct kern_channel *ch;
323 struct kern_nexus *nx = nif->nif_nx;
324 struct chreq chr;
325 int err;
326
327 SK_LOCK_ASSERT_HELD();
328 ASSERT(NETIF_IS_LOW_LATENCY(nif));
329 if (nif->nif_hw_ch != NULL) {
330 nif->nif_hw_ch_refcnt++;
331 SK_DF(SK_VERB_VP, "%s: hw channel already open, refcnt %d",
332 if_name(nif->nif_ifp), nif->nif_hw_ch_refcnt);
333 return 0;
334 }
335 ASSERT(nif->nif_hw_ch_refcnt == 0);
336 bzero(s: &chr, n: sizeof(chr));
337 uuid_copy(dst: chr.cr_spec_uuid, src: nx->nx_uuid);
338 chr.cr_ring_id = 0;
339 chr.cr_port = NEXUS_PORT_NET_IF_DEV;
340 chr.cr_mode |= CHMODE_CONFIG;
341
342 err = 0;
343 ch = ch_open_special(nx, &chr, FALSE, &err);
344 if (ch == NULL) {
345 SK_ERR("%s: failed to open nx 0x%llx (err %d)",
346 if_name(nif->nif_ifp), SK_KVA(nx), err);
347 return err;
348 }
349 netif_hwna_set_mode(ch->ch_na, NETIF_MODE_LLW, NULL);
350 na_start_spec(nx, ch);
351 nif->nif_hw_ch_refcnt = 1;
352 nif->nif_hw_ch = ch;
353 SK_DF(SK_VERB_VP, "%s: hw channel opened 0x%llx, %s:%s",
354 if_name(nif->nif_ifp), SK_KVA(ch), NX_DOM(nx)->nxdom_name,
355 NX_DOM_PROV(nx)->nxdom_prov_name);
356 return 0;
357}
358
359static void
360netif_hwna_teardown(struct nx_netif *nif)
361{
362 struct kern_nexus *nx = nif->nif_nx;
363 struct kern_channel *ch = nif->nif_hw_ch;
364
365 SK_LOCK_ASSERT_HELD();
366 ASSERT(NETIF_IS_LOW_LATENCY(nif));
367 ASSERT(ch != NULL);
368 if (--nif->nif_hw_ch_refcnt > 0) {
369 SK_DF(SK_VERB_VP, "%s: hw channel still open, refcnt %d",
370 if_name(nif->nif_ifp), nif->nif_hw_ch_refcnt);
371 return;
372 }
373 SK_DF(SK_VERB_VP, "%s: hw channel closing 0x%llx, %s:%s",
374 if_name(nif->nif_ifp), SK_KVA(ch), NX_DOM(nx)->nxdom_name,
375 NX_DOM_PROV(nx)->nxdom_prov_name);
376
377 na_stop_spec(nx, ch);
378 netif_hwna_clear_mode(ch->ch_na);
379 ch_close_special(ch);
380 (void) ch_release_locked(ch);
381 nif->nif_hw_ch = NULL;
382 SK_DF(SK_VERB_VP, "%s: hw channel closed, %s:%s",
383 if_name(nif->nif_ifp), NX_DOM(nx)->nxdom_name,
384 NX_DOM_PROV(nx)->nxdom_prov_name);
385}
386
387static int
388netif_vp_na_activate_on(struct nexus_adapter *na)
389{
390 errno_t err;
391 struct netif_flow *nf;
392 struct netif_port_info npi;
393 struct nexus_netif_adapter *nifna;
394 struct nx_netif *nif;
395 boolean_t hwna_setup = FALSE;
396
397 nifna = NIFNA(na);
398 nif = nifna->nifna_netif;
399
400 /* lock needed to protect against nxdom_unbind_port */
401 NETIF_WLOCK(nif);
402 err = nx_port_get_info(nif->nif_nx, na->na_nx_port,
403 NX_PORT_INFO_TYPE_NETIF, &npi, sizeof(npi));
404 NETIF_WUNLOCK(nif);
405 if (err != 0) {
406 SK_ERR("port info not found: %d", err);
407 return err;
408 }
409 if (NETIF_IS_LOW_LATENCY(nif)) {
410 err = netif_hwna_setup(nif);
411 if (err != 0) {
412 return err;
413 }
414 hwna_setup = TRUE;
415 }
416 err = nx_netif_flow_add(nif, na->na_nx_port, &npi.npi_fd, na,
417 netif_deliver_cb, &nf);
418 if (err != 0) {
419 if (hwna_setup) {
420 netif_hwna_teardown(nif);
421 }
422 return err;
423 }
424 nifna->nifna_flow = nf;
425 os_atomic_inc(&nx_netif_vpna_gencnt, relaxed);
426 nifna->nifna_gencnt = nx_netif_vpna_gencnt;
427 os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed);
428 return 0;
429}
430
431static int
432netif_vp_na_activate_off(struct nexus_adapter *na)
433{
434 errno_t err;
435 struct nexus_netif_adapter *nifna;
436 struct nx_netif *nif;
437
438 if (!NA_IS_ACTIVE(na)) {
439 DTRACE_SKYWALK1(already__off, struct nexus_adapter *, na);
440 return 0;
441 }
442 nifna = NIFNA(na);
443 nif = nifna->nifna_netif;
444 err = nx_netif_flow_remove(nif, nifna->nifna_flow);
445 VERIFY(err == 0);
446
447 nifna->nifna_flow = NULL;
448 if (NETIF_IS_LOW_LATENCY(nif)) {
449 netif_hwna_teardown(nif);
450 }
451 os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed);
452 return 0;
453}
454
455static int
456netif_vp_na_activate(struct nexus_adapter *na, na_activate_mode_t mode)
457{
458 errno_t err;
459
460 ASSERT(na->na_type == NA_NETIF_VP);
461 if (mode == NA_ACTIVATE_MODE_ON) {
462 err = netif_vp_na_activate_on(na);
463 } else {
464 err = netif_vp_na_activate_off(na);
465 }
466 SK_DF(SK_VERB_VP, "na \"%s\" (0x%llx) %s err %d", na->na_name,
467 SK_KVA(na), na_activate_mode2str(mode), err);
468 return err;
469}
470
471/*
472 * XXX
473 * The native path sends to the dev ring directly, bypassing aqm.
474 * This is ok since this is only used by llw now. This will need to
475 * change when we add native support for filters.
476 */
477static int
478netif_vp_send_pkt_chain_low_latency(struct nexus_netif_adapter *dev_nifna,
479 struct __kern_packet *pkt_chain, struct proc *p)
480{
481 struct __kern_packet *pkt = pkt_chain, *next;
482 struct nexus_adapter *na = &dev_nifna->nifna_up;
483 struct __kern_channel_ring *ring = &na->na_tx_rings[0];
484 struct netif_stats *nifs = &dev_nifna->nifna_netif->nif_stats;
485 sk_protect_t protect;
486 slot_idx_t ktail, idx;
487 uint32_t cnt;
488 int err_stat = -1;
489 errno_t err;
490
491 (void) kr_enter(ring, TRUE);
492 protect = sk_sync_protect();
493 if (__improbable(KR_DROP(ring))) {
494 SK_ERR("ring is not ready");
495 DTRACE_SKYWALK1(ring__drop, struct __kern_channel_ring *, ring);
496 err_stat = NETIF_STATS_VP_DROP_DEV_RING_DISABLED;
497 err = ENXIO;
498 goto done;
499 }
500 idx = ring->ckr_rhead;
501 ktail = ring->ckr_ktail;
502 if (idx == ktail) {
503 SK_ERR("no space to send");
504 DTRACE_SKYWALK1(no__space, struct __kern_channel_ring *, ring);
505 err_stat = NETIF_STATS_VP_DROP_DEV_RING_NO_SPACE;
506 goto sync;
507 }
508 cnt = 0;
509 while (pkt != NULL && idx != ktail) {
510 struct __slot_desc *slot = &ring->ckr_ksds[idx];
511
512 next = pkt->pkt_nextpkt;
513 pkt->pkt_nextpkt = NULL;
514 netif_vp_dump_packet(pkt);
515 err = netif_copy_or_attach_pkt(ring, slot, pkt);
516 if (__probable(err == 0)) {
517 cnt++;
518 idx = SLOT_NEXT(i: idx, lim: ring->ckr_lim);
519 }
520 pkt = next;
521 }
522 ring->ckr_rhead = idx;
523 STATS_ADD(nifs, NETIF_STATS_VP_LL_ENQUEUED, cnt);
524 DTRACE_SKYWALK2(ll__enqueued, struct __kern_channel_ring *, ring,
525 uint32_t, cnt);
526sync:
527 ring->ckr_khead_pre = ring->ckr_khead;
528 err = ring->ckr_na_sync(ring, p, NA_SYNCF_SYNC_ONLY);
529 if (err != 0 && err != EAGAIN) {
530 SK_ERR("unexpected sync err %d", err);
531 DTRACE_SKYWALK1(sync__failed, struct __kern_channel_ring *,
532 ring);
533 err_stat = NETIF_STATS_VP_DROP_UNEXPECTED_ERR;
534 goto done;
535 }
536 /*
537 * Verify that the driver has detached packets from the consumed slots.
538 */
539 idx = ring->ckr_khead_pre;
540 cnt = 0;
541 while (idx != ring->ckr_khead) {
542 struct __kern_slot_desc *ksd = KR_KSD(ring, idx);
543
544 cnt++;
545 VERIFY(!KSD_VALID_METADATA(ksd));
546 idx = SLOT_NEXT(i: idx, lim: ring->ckr_lim);
547 }
548 ring->ckr_khead_pre = ring->ckr_khead;
549 STATS_ADD(nifs, NETIF_STATS_VP_LL_SENT, cnt);
550 DTRACE_SKYWALK2(ll__sent, struct __kern_channel_ring *, ring,
551 uint32_t, cnt);
552 err = 0;
553
554done:
555 sk_sync_unprotect(protect);
556 kr_exit(ring);
557
558 /*
559 * Free all unsent packets.
560 */
561 if (pkt != NULL) {
562 int dropcnt;
563
564 nx_netif_free_packet_chain(pkt, &dropcnt);
565 if (err_stat != -1) {
566 STATS_ADD(nifs, err_stat, dropcnt);
567 }
568 STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt);
569 }
570 return err;
571}
572
573static int
574netif_vp_send_pkt_chain_common(struct nexus_netif_adapter *dev_nifna,
575 struct __kern_packet *pkt_chain, boolean_t compat)
576{
577 struct __kern_packet *pkt = pkt_chain, *next, *p;
578 struct nx_netif *nif = dev_nifna->nifna_netif;
579 struct netif_stats *nifs = &nif->nif_stats;
580 ifnet_t ifp = nif->nif_ifp;
581 struct mbuf *m;
582 boolean_t drop;
583 int cnt = 0;
584 errno_t err;
585
586 while (pkt != NULL) {
587 next = pkt->pkt_nextpkt;
588 pkt->pkt_nextpkt = NULL;
589 drop = FALSE;
590
591 if (compat) {
592 m = nx_netif_pkt_to_mbuf(dev_nifna, pkt, NETIF_CONVERT_TX);
593 if (m == NULL) {
594 pkt = next;
595 continue;
596 }
597 err = ifnet_enqueue_mbuf(ifp, m, FALSE, &drop);
598 } else {
599 p = nx_netif_pkt_to_pkt(dev_nifna, pkt, NETIF_CONVERT_TX);
600 if (p == NULL) {
601 pkt = next;
602 continue;
603 }
604 err = ifnet_enqueue_pkt(ifp, p, FALSE, &drop);
605 }
606 if (err != 0) {
607 SK_ERR("enqueue failed: %d", err);
608 STATS_INC(nifs, NETIF_STATS_VP_ENQUEUE_FAILED);
609 if (drop) {
610 STATS_INC(nifs, NETIF_STATS_DROP);
611 }
612 DTRACE_SKYWALK2(enqueue__failed,
613 struct nexus_netif_adapter *, dev_nifna,
614 boolean_t, drop);
615 } else {
616 STATS_INC(nifs, NETIF_STATS_VP_ENQUEUED);
617 cnt++;
618 }
619 pkt = next;
620 }
621 if (cnt > 0) {
622 ifnet_start(interface: ifp);
623 }
624 return 0;
625}
626
627static int
628netif_vp_send_pkt_chain(struct nexus_netif_adapter *dev_nifna,
629 struct __kern_packet *pkt_chain, struct proc *p)
630{
631 struct nexus_adapter *na = &dev_nifna->nifna_up;
632
633 if (NETIF_IS_LOW_LATENCY(dev_nifna->nifna_netif)) {
634 return netif_vp_send_pkt_chain_low_latency(dev_nifna,
635 pkt_chain, p);
636 }
637 if (na->na_type == NA_NETIF_DEV) {
638 return netif_vp_send_pkt_chain_common(dev_nifna, pkt_chain, FALSE);
639 }
640 ASSERT(na->na_type == NA_NETIF_COMPAT_DEV);
641 return netif_vp_send_pkt_chain_common(dev_nifna, pkt_chain, TRUE);
642}
643
644SK_NO_INLINE_ATTRIBUTE
645static boolean_t
646validate_packet(struct nexus_netif_adapter *nifna, struct __kern_packet *pkt)
647{
648 struct nx_netif *nif = nifna->nifna_netif;
649
650 VERIFY(pkt->pkt_nextpkt == NULL);
651
652 if (nx_netif_vp_accept_all != 0) {
653 return TRUE;
654 }
655 if (outbound_check == 0 ||
656 (NETIF_IS_LOW_LATENCY(nif) &&
657 outbound_check != OUTBOUND_CHECK_FORCED)) {
658 return TRUE;
659 }
660 if (!nx_netif_validate_macaddr(nif, pkt, NETIF_FLOW_OUTBOUND)) {
661 return FALSE;
662 }
663 if (!nx_netif_flow_match(nif, pkt, nifna->nifna_flow,
664 NETIF_FLOW_OUTBOUND)) {
665 return FALSE;
666 }
667 return TRUE;
668}
669
670static int
671netif_vp_na_txsync(struct __kern_channel_ring *kring, struct proc *p,
672 uint32_t flags)
673{
674#pragma unused(flags)
675 kern_channel_slot_t last_slot = NULL, slot = NULL;
676 struct __kern_packet *head = NULL, **tailp = &head, *pkt;
677 struct nexus_netif_adapter *nifna, *dev_nifna;
678 struct nx_netif *nif;
679 struct netif_stats *nifs;
680 kern_packet_t ph;
681 errno_t err;
682 int cnt = 0;
683
684 nifna = NIFNA(KRNA(kring));
685 nif = nifna->nifna_netif;
686 nifs = &nif->nif_stats;
687 for (;;) {
688 slot = kern_channel_get_next_slot(kring, slot, NULL);
689 if (slot == NULL) {
690 break;
691 }
692 ph = kern_channel_slot_get_packet(ring: kring, slot);
693 if (__improbable(ph == 0)) {
694 SK_ERR("packet got dropped by internalize");
695 STATS_INC(nifs, NETIF_STATS_VP_DROP_INTERNALIZE_FAIL);
696 DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *,
697 kring, kern_channel_slot_t, slot);
698 last_slot = slot;
699 continue;
700 }
701 pkt = SK_PTR_ADDR_KPKT(ph);
702 if (__improbable(pkt->pkt_length == 0)) {
703 SK_ERR("dropped zero length packet");
704 STATS_INC(nifs, NETIF_STATS_VP_BAD_PKT_LEN);
705 DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *,
706 kring, kern_channel_slot_t, slot);
707 last_slot = slot;
708 continue;
709 }
710 err = kern_channel_slot_detach_packet(ring: kring, slot, packet: ph);
711 VERIFY(err == 0);
712
713 /* packet needs to be finalized after detach */
714 err = kern_packet_finalize(ph);
715 VERIFY(err == 0);
716 last_slot = slot;
717
718 if (NA_CHANNEL_EVENT_ATTACHED(KRNA(kring))) {
719 __packet_set_tx_nx_port(SK_PKT2PH(pkt),
720 KRNA(kring)->na_nx_port, vpna_gencnt: nifna->nifna_gencnt);
721 }
722
723 if (validate_packet(nifna, pkt)) {
724 nx_netif_snoop(nif, pkt, FALSE);
725 cnt++;
726 *tailp = pkt;
727 tailp = &pkt->pkt_nextpkt;
728 } else {
729 nx_netif_free_packet(pkt);
730 }
731 }
732 if (cnt == 0) {
733 STATS_INC(nifs, NETIF_STATS_VP_SYNC_NO_PKTS);
734 DTRACE_SKYWALK2(no__data, struct nexus_netif_adapter *, nifna,
735 struct __kern_channel_ring *, kring);
736 return 0;
737 }
738 DTRACE_SKYWALK4(injected, struct nexus_netif_adapter *, nifna,
739 struct __kern_channel_ring *, kring, struct __kern_packet *, head,
740 int, cnt);
741 if (last_slot != NULL) {
742 kern_channel_advance_slot(kring, slot: last_slot);
743 }
744
745 dev_nifna = NIFNA(nx_port_get_na(KRNA(kring)->na_nx,
746 NEXUS_PORT_NET_IF_DEV));
747
748 err = netif_vp_send_pkt_chain(dev_nifna, pkt_chain: head, p);
749 if (err != 0) {
750 SK_ERR("send failed: %d\n", err);
751 }
752 return 0;
753}
754
755static int
756netif_vp_na_rxsync(struct __kern_channel_ring *kring, struct proc *p,
757 uint32_t flags)
758{
759#pragma unused(p, flags)
760 (void) kr_reclaim(kr: kring);
761 return 0;
762}
763
764static int
765netif_vp_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch)
766{
767 ASSERT(na->na_type == NA_NETIF_VP);
768 return na_rings_mem_setup(na, FALSE, ch);
769}
770
771
772/* na_krings_delete callback for flow switch ports. */
773static void
774netif_vp_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch,
775 boolean_t defunct)
776{
777 ASSERT(na->na_type == NA_NETIF_VP);
778 na_rings_mem_teardown(na, ch, defunct);
779}
780
781static int
782netif_vp_region_params_setup(struct nexus_adapter *na,
783 struct skmem_region_params *srp, struct kern_pbufpool **tx_pp)
784{
785#pragma unused (tx_pp)
786 uint32_t max_mtu;
787 uint32_t buf_sz, buf_cnt, nslots, afslots, evslots, totalrings;
788 struct nexus_adapter *devna;
789 struct kern_nexus *nx;
790 struct nx_netif *nif;
791 int err, i;
792
793 for (i = 0; i < SKMEM_REGIONS; i++) {
794 srp[i] = *skmem_get_default(i);
795 }
796 totalrings = na_get_nrings(na, t: NR_TX) + na_get_nrings(na, t: NR_RX) +
797 na_get_nrings(na, t: NR_A) + na_get_nrings(na, t: NR_F) +
798 na_get_nrings(na, t: NR_EV);
799
800 srp[SKMEM_REGION_SCHEMA].srp_r_obj_size =
801 (uint32_t)CHANNEL_SCHEMA_SIZE(totalrings);
802 srp[SKMEM_REGION_SCHEMA].srp_r_obj_cnt = totalrings;
803 skmem_region_params_config(&srp[SKMEM_REGION_SCHEMA]);
804
805 srp[SKMEM_REGION_RING].srp_r_obj_size =
806 sizeof(struct __user_channel_ring);
807 srp[SKMEM_REGION_RING].srp_r_obj_cnt = totalrings;
808 skmem_region_params_config(&srp[SKMEM_REGION_RING]);
809
810 /* USD regions need to be writable to support user packet pool */
811 srp[SKMEM_REGION_TXAUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY;
812 srp[SKMEM_REGION_RXFUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY;
813
814 nslots = na_get_nslots(na, t: NR_TX);
815 afslots = na_get_nslots(na, t: NR_A);
816 evslots = na_get_nslots(na, t: NR_EV);
817 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size =
818 MAX(MAX(nslots, afslots), evslots) * SLOT_DESC_SZ;
819 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt =
820 na_get_nrings(na, t: NR_TX) + na_get_nrings(na, t: NR_A) +
821 na_get_nrings(na, t: NR_EV);
822 skmem_region_params_config(&srp[SKMEM_REGION_TXAKSD]);
823
824 /* USD and KSD objects share the same size and count */
825 srp[SKMEM_REGION_TXAUSD].srp_r_obj_size =
826 srp[SKMEM_REGION_TXAKSD].srp_r_obj_size;
827 srp[SKMEM_REGION_TXAUSD].srp_r_obj_cnt =
828 srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt;
829 skmem_region_params_config(&srp[SKMEM_REGION_TXAUSD]);
830
831 /*
832 * Since the rx/free slots share the same region and cache,
833 * we will use the same object size for both types of slots.
834 */
835 nslots = na_get_nslots(na, t: NR_RX);
836 afslots = na_get_nslots(na, t: NR_F);
837 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size =
838 MAX(nslots, afslots) * SLOT_DESC_SZ;
839 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt =
840 na_get_nrings(na, t: NR_RX) + na_get_nrings(na, t: NR_F);
841 skmem_region_params_config(&srp[SKMEM_REGION_RXFKSD]);
842
843 /* USD and KSD objects share the same size and count */
844 srp[SKMEM_REGION_RXFUSD].srp_r_obj_size =
845 srp[SKMEM_REGION_RXFKSD].srp_r_obj_size;
846 srp[SKMEM_REGION_RXFUSD].srp_r_obj_cnt =
847 srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt;
848 skmem_region_params_config(&srp[SKMEM_REGION_RXFUSD]);
849
850 /*
851 * No need to create our own buffer pool if we can share the device's
852 * pool. We don't support sharing split pools to user space.
853 */
854 nx = na->na_nx;
855 nif = nx->nx_arg;
856 if (vp_zerocopy != 0 && NETIF_IS_LOW_LATENCY(nif) &&
857 nx->nx_tx_pp != NULL && (nx->nx_rx_pp == NULL ||
858 nx->nx_tx_pp == nx->nx_rx_pp) && !PP_KERNEL_ONLY(nx->nx_tx_pp)) {
859 struct kern_pbufpool *pp = nx->nx_tx_pp;
860
861 if (nif->nif_hw_ch_refcnt != 0) {
862 SK_ERR("only one channel is supported for zero copy");
863 return ENOTSUP;
864 }
865 SK_DF(SK_VERB_VP, "sharing %s's pool", if_name(na->na_ifp));
866
867 /*
868 * These types need to be initialized otherwise some assertions
869 * skmem_arena_create_for_nexus() will fail.
870 */
871 srp[SKMEM_REGION_UMD].srp_md_type = pp->pp_md_type;
872 srp[SKMEM_REGION_UMD].srp_md_subtype = pp->pp_md_subtype;
873 srp[SKMEM_REGION_KMD].srp_md_type = pp->pp_md_type;
874 srp[SKMEM_REGION_KMD].srp_md_subtype = pp->pp_md_subtype;
875 *tx_pp = nx->nx_tx_pp;
876 return 0;
877 }
878
879 devna = nx_port_get_na(nx, NEXUS_PORT_NET_IF_DEV);
880 ASSERT(devna != NULL);
881 if (devna->na_type == NA_NETIF_DEV) {
882 /*
883 * For native devices, use the driver's buffer size
884 */
885 ASSERT(nx->nx_rx_pp != NULL);
886 ASSERT(nx->nx_tx_pp != NULL);
887 buf_sz = PP_BUF_SIZE_DEF(nx->nx_tx_pp);
888 } else {
889 if ((err = nx_netif_get_max_mtu(na->na_ifp, &max_mtu)) != 0) {
890 /*
891 * If the driver doesn't support SIOCGIFDEVMTU, use the
892 * default MTU size.
893 */
894 max_mtu = ifnet_mtu(interface: na->na_ifp);
895 err = 0;
896 }
897 /* max_mtu does not include the L2 header */
898 buf_sz = MAX(max_mtu + sizeof(struct ether_vlan_header), 2048);
899 }
900 buf_cnt = vp_pool_size;
901 pp_regions_params_adjust(srp, NEXUS_META_TYPE_PACKET,
902 NEXUS_META_SUBTYPE_RAW, buf_cnt, 1, buf_sz, 0, buf_cnt, 0,
903 PP_REGION_CONFIG_BUF_IODIR_BIDIR |
904 PP_REGION_CONFIG_MD_MAGAZINE_ENABLE);
905
906 nx_netif_vp_region_params_adjust(na, srp);
907 return 0;
908}
909
910static int
911netif_vp_na_mem_new(struct kern_nexus *nx, struct nexus_adapter *na)
912{
913#pragma unused(nx)
914 struct skmem_region_params srp[SKMEM_REGIONS];
915 struct kern_pbufpool *tx_pp = NULL;
916 int err;
917
918 err = netif_vp_region_params_setup(na, srp, tx_pp: &tx_pp);
919 if (err != 0) {
920 return err;
921 }
922 na->na_arena = skmem_arena_create_for_nexus(na, srp,
923 tx_pp != NULL ? &tx_pp : NULL, NULL,
924 FALSE, FALSE, &nx->nx_adv, &err);
925 ASSERT(na->na_arena != NULL || err != 0);
926 return err;
927}
928
929static void
930netif_vp_na_dtor(struct nexus_adapter *na)
931{
932 struct kern_nexus *nx = na->na_nx;
933 struct nx_netif *nif = NX_NETIF_PRIVATE(nx);
934 struct nexus_netif_adapter *nifna = NIFNA(na);
935
936 NETIF_WLOCK(nif);
937 (void) nx_port_unbind(nx, na->na_nx_port);
938 nx_port_free(nx, na->na_nx_port);
939 nif->nif_vp_cnt--;
940 if (na->na_ifp != NULL) {
941 ifnet_decr_iorefcnt(na->na_ifp);
942 na->na_ifp = NULL;
943 }
944 if (nifna->nifna_netif != NULL) {
945 nx_netif_release(nifna->nifna_netif);
946 nifna->nifna_netif = NULL;
947 }
948 NETIF_WUNLOCK(nif);
949 SK_DF(SK_VERB_VP, "na \"%s\" (0x%llx)", na->na_name, SK_KVA(na));
950}
951
952int
953netif_vp_na_create(struct kern_nexus *nx, struct chreq *chr,
954 struct nexus_adapter **nap)
955{
956 struct nx_netif *nif = NX_NETIF_PRIVATE(nx);
957 struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params;
958 struct nexus_adapter *na = NULL;
959 struct nexus_netif_adapter *nifna;
960 uint32_t slots;
961 int err;
962
963 NETIF_WLOCK_ASSERT_HELD(nif);
964 if (nif->nif_ifp == NULL) {
965 SK_ERR("ifnet not yet attached");
966 return ENXIO;
967 }
968 ASSERT((chr->cr_mode & CHMODE_KERNEL) == 0);
969 if ((chr->cr_mode & CHMODE_USER_PACKET_POOL) == 0) {
970 SK_ERR("user packet pool required");
971 return EINVAL;
972 }
973 /*
974 * No locking needed while checking for the initialized bit because
975 * if this were not set, no other codepaths would modify the flags.
976 */
977 if ((nif->nif_flow_flags & NETIF_FLOW_FLAG_INITIALIZED) == 0) {
978 SK_ERR("demux vp not supported");
979 return ENOTSUP;
980 }
981 na = (struct nexus_adapter *)na_netif_alloc(Z_WAITOK);
982 nifna = NIFNA(na);
983 nifna->nifna_netif = nif;
984 nx_netif_retain(nif);
985 nifna->nifna_flow = NULL;
986
987 (void) snprintf(na->na_name, count: sizeof(na->na_name),
988 "netif_vp:%d", chr->cr_port);
989 uuid_generate_random(out: na->na_uuid);
990
991 na_set_nrings(na, t: NR_TX, v: nxp->nxp_tx_rings);
992 na_set_nrings(na, t: NR_RX, v: nxp->nxp_rx_rings);
993 /*
994 * If the packet pool is configured to be multi-buflet, then we
995 * need 2 pairs of alloc/free rings(for packet and buflet).
996 */
997 na_set_nrings(na, t: NR_A, v: ((nxp->nxp_max_frags > 1) &&
998 (sk_channel_buflet_alloc != 0)) ? 2 : 1);
999
1000 slots = vp_tx_slots != 0 ? vp_tx_slots :
1001 NX_DOM(nx)->nxdom_tx_slots.nb_def;
1002 na_set_nslots(na, t: NR_TX, v: slots);
1003
1004 slots = vp_rx_slots != 0 ? vp_rx_slots :
1005 NX_DOM(nx)->nxdom_rx_slots.nb_def;
1006 na_set_nslots(na, t: NR_RX, v: slots);
1007
1008 na_set_nslots(na, t: NR_A, NETIF_DEMUX_ALLOC_SLOTS);
1009 ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max);
1010 ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max);
1011 ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max);
1012 ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max);
1013
1014 os_atomic_or(&na->na_flags, NAF_USER_PKT_POOL, relaxed);
1015
1016 if (chr->cr_mode & CHMODE_EVENT_RING) {
1017 na_set_nrings(na, t: NR_EV, NX_NETIF_EVENT_RING_NUM);
1018 na_set_nslots(na, t: NR_EV, NX_NETIF_EVENT_RING_SIZE);
1019 os_atomic_or(&na->na_flags, NAF_EVENT_RING, relaxed);
1020 na->na_channel_event_notify = netif_vp_na_channel_event_notify;
1021 }
1022
1023 na->na_nx_port = chr->cr_port;
1024 na->na_type = NA_NETIF_VP;
1025 na->na_free = na_netif_free;
1026 na->na_dtor = netif_vp_na_dtor;
1027 na->na_activate = netif_vp_na_activate;
1028 na->na_txsync = netif_vp_na_txsync;
1029 na->na_rxsync = netif_vp_na_rxsync;
1030 na->na_krings_create = netif_vp_na_krings_create;
1031 na->na_krings_delete = netif_vp_na_krings_delete;
1032 na->na_special = NULL;
1033 na->na_ifp = nif->nif_ifp;
1034 ifnet_incr_iorefcnt(na->na_ifp);
1035
1036 *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type =
1037 NEXUS_STATS_TYPE_INVALID;
1038
1039 /* other fields are set in the common routine */
1040 na_attach_common(na, nx, &nx_netif_prov_s);
1041
1042 err = netif_vp_na_mem_new(nx, na);
1043 if (err != 0) {
1044 ASSERT(na->na_arena == NULL);
1045 goto err;
1046 }
1047
1048 *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max;
1049 ASSERT(na->na_flowadv_max == 0 ||
1050 skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL);
1051
1052 nif->nif_vp_cnt++;
1053 *nap = na;
1054 return 0;
1055
1056err:
1057 if (na != NULL) {
1058 if (na->na_ifp != NULL) {
1059 ifnet_decr_iorefcnt(na->na_ifp);
1060 na->na_ifp = NULL;
1061 }
1062 if (na->na_arena != NULL) {
1063 skmem_arena_release(na->na_arena);
1064 na->na_arena = NULL;
1065 }
1066 if (nifna->nifna_netif != NULL) {
1067 nx_netif_release(nifna->nifna_netif);
1068 nifna->nifna_netif = NULL;
1069 }
1070 NA_FREE(na);
1071 }
1072 SK_ERR("VP NA creation failed, err(%d)", err);
1073 return err;
1074}
1075
1076static int
1077netif_vp_na_channel_event_notify(struct nexus_adapter *vpna,
1078 struct __kern_channel_event *ev, uint16_t ev_len)
1079{
1080 int err;
1081 char *baddr;
1082 kern_packet_t ph;
1083 kern_buflet_t buf;
1084 sk_protect_t protect;
1085 kern_channel_slot_t slot;
1086 struct __kern_packet *vpna_pkt = NULL;
1087 struct __kern_channel_event_metadata *emd;
1088 struct __kern_channel_ring *ring = &vpna->na_event_rings[0];
1089 struct netif_stats *nifs = &NIFNA(vpna)->nifna_netif->nif_stats;
1090
1091 if (__probable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_STATUS)) {
1092 STATS_INC(nifs, NETIF_STATS_EV_RECV_TX_STATUS);
1093 }
1094 if (__improbable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_EXPIRED)) {
1095 STATS_INC(nifs, NETIF_STATS_EV_RECV_TX_EXPIRED);
1096 }
1097 STATS_INC(nifs, NETIF_STATS_EV_RECV);
1098
1099 if (__improbable(!NA_IS_ACTIVE(vpna))) {
1100 STATS_INC(nifs, NETIF_STATS_EV_DROP_NA_INACTIVE);
1101 err = ENXIO;
1102 goto error;
1103 }
1104 if (__improbable(NA_IS_DEFUNCT(vpna))) {
1105 STATS_INC(nifs, NETIF_STATS_EV_DROP_NA_DEFUNCT);
1106 err = ENXIO;
1107 goto error;
1108 }
1109 if (!NA_CHANNEL_EVENT_ATTACHED(vpna)) {
1110 STATS_INC(nifs, NETIF_STATS_EV_DROP_KEVENT_INACTIVE);
1111 err = ENXIO;
1112 goto error;
1113 }
1114 if (__improbable(KR_DROP(ring))) {
1115 STATS_INC(nifs, NETIF_STATS_EV_DROP_KRDROP_MODE);
1116 err = ENXIO;
1117 goto error;
1118 }
1119 vpna_pkt = nx_netif_alloc_packet(ring->ckr_pp, ev_len, &ph);
1120 if (__improbable(vpna_pkt == NULL)) {
1121 STATS_INC(nifs, NETIF_STATS_EV_DROP_NOMEM_PKT);
1122 err = ENOMEM;
1123 goto error;
1124 }
1125 buf = __packet_get_next_buflet(ph, NULL);
1126 baddr = __buflet_get_data_address(buf);
1127 emd = (struct __kern_channel_event_metadata *)(void *)baddr;
1128 emd->emd_etype = ev->ev_type;
1129 emd->emd_nevents = 1;
1130 bcopy(src: ev, dst: (baddr + __KERN_CHANNEL_EVENT_OFFSET), n: ev_len);
1131 err = __buflet_set_data_length(buf,
1132 dlen: (ev_len + __KERN_CHANNEL_EVENT_OFFSET));
1133 VERIFY(err == 0);
1134 err = __packet_finalize(ph);
1135 VERIFY(err == 0);
1136 kr_enter(ring, TRUE);
1137 protect = sk_sync_protect();
1138 slot = kern_channel_get_next_slot(kring: ring, NULL, NULL);
1139 if (slot == NULL) {
1140 sk_sync_unprotect(protect);
1141 kr_exit(ring);
1142 STATS_INC(nifs, NETIF_STATS_EV_DROP_KRSPACE);
1143 err = ENOSPC;
1144 goto error;
1145 }
1146 err = kern_channel_slot_attach_packet(ring, slot, packet: ph);
1147 VERIFY(err == 0);
1148 vpna_pkt = NULL;
1149 kern_channel_advance_slot(kring: ring, slot);
1150 sk_sync_unprotect(protect);
1151 kr_exit(ring);
1152 kern_channel_event_notify(&vpna->na_tx_rings[0]);
1153 STATS_INC(nifs, NETIF_STATS_EV_SENT);
1154 return 0;
1155
1156error:
1157 ASSERT(err != 0);
1158 if (vpna_pkt != NULL) {
1159 nx_netif_free_packet(vpna_pkt);
1160 }
1161 STATS_INC(nifs, NETIF_STATS_EV_DROP);
1162 return err;
1163}
1164
1165static inline struct nexus_adapter *
1166nx_netif_find_port_vpna(struct nx_netif *netif, uint32_t nx_port_id)
1167{
1168 struct kern_nexus *nx = netif->nif_nx;
1169 struct nexus_adapter *na = NULL;
1170 nexus_port_t port;
1171 uint16_t gencnt;
1172
1173 PKT_DECOMPOSE_NX_PORT_ID(nx_port_id, port, gencnt);
1174 if (port < NEXUS_PORT_NET_IF_CLIENT) {
1175 SK_ERR("non VPNA port");
1176 return NULL;
1177 }
1178 if (__improbable(!nx_port_is_valid(nx, port))) {
1179 SK_ERR("%s[%d] port no longer valid",
1180 if_name(netif->nif_ifp), port);
1181 return NULL;
1182 }
1183 na = nx_port_get_na(nx, port);
1184 if (na != NULL && NIFNA(na)->nifna_gencnt != gencnt) {
1185 return NULL;
1186 }
1187 return na;
1188}
1189
1190errno_t
1191netif_vp_na_channel_event(struct nx_netif *nif, uint32_t nx_port_id,
1192 struct __kern_channel_event *event, uint16_t event_len)
1193{
1194 int err = 0;
1195 struct nexus_adapter *netif_vpna;
1196 struct netif_stats *nifs = &nif->nif_stats;
1197
1198 SK_DF(SK_VERB_EVENTS, "%s[%d] ev: %p ev_len: %hu "
1199 "ev_type: %u ev_flags: %u _reserved: %hu ev_dlen: %hu",
1200 if_name(nif->nif_ifp), nx_port_id, event, event_len,
1201 event->ev_type, event->ev_flags, event->_reserved, event->ev_dlen);
1202
1203 NETIF_RLOCK(nif);
1204 if (!NETIF_IS_LOW_LATENCY(nif)) {
1205 err = ENOTSUP;
1206 goto error;
1207 }
1208 if (__improbable(nif->nif_vp_cnt == 0)) {
1209 STATS_INC(nifs, NETIF_STATS_EV_DROP_NO_VPNA);
1210 err = ENXIO;
1211 goto error;
1212 }
1213 netif_vpna = nx_netif_find_port_vpna(netif: nif, nx_port_id);
1214 if (__improbable(netif_vpna == NULL)) {
1215 err = ENXIO;
1216 STATS_INC(nifs, NETIF_STATS_EV_DROP_DEMUX_ERR);
1217 goto error;
1218 }
1219 if (__improbable(netif_vpna->na_channel_event_notify == NULL)) {
1220 err = ENOTSUP;
1221 STATS_INC(nifs, NETIF_STATS_EV_DROP_EV_VPNA_NOTSUP);
1222 goto error;
1223 }
1224 err = netif_vpna->na_channel_event_notify(netif_vpna, event, event_len);
1225 NETIF_RUNLOCK(nif);
1226 return err;
1227
1228error:
1229 STATS_INC(nifs, NETIF_STATS_EV_DROP);
1230 NETIF_RUNLOCK(nif);
1231 return err;
1232}
1233