1 | /* |
2 | * Copyright (c) 2019-2022 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <skywalk/os_skywalk_private.h> |
30 | #include <skywalk/nexus/netif/nx_netif.h> |
31 | #include <net/if_vlan_var.h> |
32 | #include <sys/sdt.h> |
33 | |
34 | #define NETIF_DEMUX_ALLOC_SLOTS 128 |
35 | |
36 | #define OUTBOUND_CHECK_OFF 0 |
37 | #define OUTBOUND_CHECK_ON 1 |
38 | #define OUTBOUND_CHECK_FORCED 2 |
39 | |
40 | /* Turning this off allows packets to be spoofed for testing purposes */ |
41 | static uint32_t outbound_check = OUTBOUND_CHECK_ON; |
42 | |
43 | /* This controls the per-NA pool size of custom ether and llw NAs */ |
44 | static uint32_t vp_pool_size = 2048; |
45 | |
46 | /* This enables zerocopy on llw NAs */ |
47 | static uint32_t vp_zerocopy = 0; |
48 | |
49 | /* TX Ring size */ |
50 | static uint32_t vp_tx_slots = 0; |
51 | |
52 | /* RX Ring size */ |
53 | static uint32_t vp_rx_slots = 0; |
54 | |
55 | /* |
56 | * Disable all packet validation |
57 | */ |
58 | uint32_t nx_netif_vp_accept_all = 0; |
59 | |
60 | static uint16_t nx_netif_vpna_gencnt = 0; |
61 | |
62 | #if (DEVELOPMENT || DEBUG) |
63 | SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, outbound_check, |
64 | CTLFLAG_RW | CTLFLAG_LOCKED, &outbound_check, 0, |
65 | "netif outbound packet validation" ); |
66 | SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_pool_size, |
67 | CTLFLAG_RW | CTLFLAG_LOCKED, &vp_pool_size, 0, |
68 | "netif virtual port pool size" ); |
69 | SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_zerocopy, |
70 | CTLFLAG_RW | CTLFLAG_LOCKED, &vp_zerocopy, 0, |
71 | "netif virtual port zero copy" ); |
72 | SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_tx_slots, |
73 | CTLFLAG_RW | CTLFLAG_LOCKED, &vp_tx_slots, 0, |
74 | "netif virtual port tx slots" ); |
75 | SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_rx_slots, |
76 | CTLFLAG_RW | CTLFLAG_LOCKED, &vp_rx_slots, 0, |
77 | "netif virtual port rx slots" ); |
78 | SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, vp_accept_all, |
79 | CTLFLAG_RW | CTLFLAG_LOCKED, &nx_netif_vp_accept_all, 0, |
80 | "netif accept all" ); |
81 | #endif /* (DEVELOPMENT || DEBUG) */ |
82 | |
83 | static int |
84 | netif_vp_na_channel_event_notify(struct nexus_adapter *, |
85 | struct __kern_channel_event *, uint16_t); |
86 | |
87 | static void |
88 | netif_vp_dump_packet(struct __kern_packet *pkt) |
89 | { |
90 | uint8_t *baddr; |
91 | |
92 | MD_BUFLET_ADDR_ABS(pkt, baddr); |
93 | ASSERT(baddr != NULL); |
94 | baddr += pkt->pkt_headroom; |
95 | |
96 | DTRACE_SKYWALK2(dump__packet, struct __kern_packet *, |
97 | pkt, uint8_t *, baddr); |
98 | } |
99 | |
100 | static int |
101 | netif_copy_or_attach_pkt(struct __kern_channel_ring *ring, |
102 | kern_channel_slot_t slot, struct __kern_packet *pkt) |
103 | { |
104 | kern_packet_t ph; |
105 | struct __kern_packet *dpkt; |
106 | errno_t err; |
107 | |
108 | if (pkt->pkt_qum.qum_pp == ring->ckr_pp) { |
109 | DTRACE_SKYWALK2(attach__pkt, struct __kern_channel_ring *, ring, |
110 | struct __kern_packet *, pkt); |
111 | ph = SK_PKT2PH(pkt); |
112 | err = kern_packet_finalize(ph); |
113 | VERIFY(err == 0); |
114 | } else { |
115 | DTRACE_SKYWALK2(copy__pkt, struct __kern_channel_ring *, ring, |
116 | struct __kern_packet *, pkt); |
117 | dpkt = nx_netif_pkt_to_pkt(NIFNA(ring->ckr_na), pkt, |
118 | ring->ckr_na->na_type == NA_NETIF_VP ? NETIF_CONVERT_RX : |
119 | NETIF_CONVERT_TX); |
120 | if (__improbable(dpkt == NULL)) { |
121 | return ENOMEM; |
122 | } |
123 | ph = SK_PKT2PH(dpkt); |
124 | } |
125 | err = kern_channel_slot_attach_packet(ring, slot, packet: ph); |
126 | VERIFY(err == 0); |
127 | return 0; |
128 | } |
129 | |
130 | static errno_t |
131 | netif_deliver_pkt(struct nexus_adapter *na, struct __kern_packet *pkt_chain, |
132 | uint32_t flags) |
133 | { |
134 | #pragma unused(flags) |
135 | struct __kern_channel_ring *ring = &na->na_rx_rings[0]; |
136 | struct __kern_packet *pkt = pkt_chain, *next; |
137 | kern_channel_slot_t last_slot = NULL, slot = NULL; |
138 | struct nexus_netif_adapter *nifna = NIFNA(na); |
139 | struct nx_netif *nif = nifna->nifna_netif; |
140 | struct netif_stats *nifs = &nif->nif_stats; |
141 | sk_protect_t protect; |
142 | int cnt = 0, dropcnt = 0, err; |
143 | |
144 | (void) kr_enter(ring, TRUE); |
145 | protect = sk_sync_protect(); |
146 | |
147 | if (__improbable(KR_DROP(ring))) { |
148 | nx_netif_free_packet_chain(pkt, &dropcnt); |
149 | STATS_ADD(nifs, |
150 | NETIF_STATS_VP_DROP_USER_RING_DISABLED, dropcnt); |
151 | STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt); |
152 | DTRACE_SKYWALK2(ring__drop, struct __kern_channel_ring *, ring, |
153 | int, dropcnt); |
154 | sk_sync_unprotect(protect); |
155 | kr_exit(ring); |
156 | return ENXIO; |
157 | } |
158 | while (pkt != NULL) { |
159 | slot = kern_channel_get_next_slot(kring: ring, slot: last_slot, NULL); |
160 | if (slot == NULL) { |
161 | break; |
162 | } |
163 | next = pkt->pkt_nextpkt; |
164 | pkt->pkt_nextpkt = NULL; |
165 | netif_vp_dump_packet(pkt); |
166 | err = netif_copy_or_attach_pkt(ring, slot, pkt); |
167 | if (__probable(err == 0)) { |
168 | last_slot = slot; |
169 | } |
170 | pkt = next; |
171 | cnt++; |
172 | } |
173 | if (NETIF_IS_LOW_LATENCY(nif)) { |
174 | STATS_ADD(nifs, NETIF_STATS_VP_LL_DELIVERED, cnt); |
175 | } else { |
176 | STATS_ADD(nifs, NETIF_STATS_VP_DELIVERED, cnt); |
177 | } |
178 | DTRACE_SKYWALK4(delivered, struct nexus_adapter *, na, |
179 | struct __kern_channel_ring *, ring, struct __kern_packet *, pkt, |
180 | int, cnt); |
181 | |
182 | if (pkt != NULL) { |
183 | nx_netif_free_packet_chain(pkt, &dropcnt); |
184 | STATS_ADD(nifs, |
185 | NETIF_STATS_VP_DROP_USER_RING_NO_SPACE, dropcnt); |
186 | STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt); |
187 | DTRACE_SKYWALK2(deliver__drop, struct nexus_adapter *, na, |
188 | int, dropcnt); |
189 | } |
190 | if (last_slot != NULL) { |
191 | kern_channel_advance_slot(kring: ring, slot: last_slot); |
192 | } |
193 | sk_sync_unprotect(protect); |
194 | kr_exit(ring); |
195 | if (cnt > 0) { |
196 | (void) kern_channel_notify(ring, flags: 0); |
197 | } |
198 | return 0; |
199 | } |
200 | |
201 | static errno_t |
202 | netif_deliver_cb(void *arg, void *chain, uint32_t flags) |
203 | { |
204 | return netif_deliver_pkt(na: arg, pkt_chain: chain, flags); |
205 | } |
206 | |
207 | static int |
208 | netif_hwna_rx_get_pkts(struct __kern_channel_ring *ring, struct proc *p, |
209 | uint32_t flags, struct __kern_packet **chain) |
210 | { |
211 | int err, cnt = 0; |
212 | sk_protect_t protect; |
213 | slot_idx_t ktail, idx; |
214 | struct __kern_packet *pkt_chain = NULL, **tailp = &pkt_chain; |
215 | struct netif_stats *nifs = &NIFNA(KRNA(ring))->nifna_netif->nif_stats; |
216 | |
217 | err = kr_enter(ring, ((flags & NA_NOTEF_CAN_SLEEP) != 0 || |
218 | (ring->ckr_flags & CKRF_HOST) != 0)); |
219 | if (err != 0) { |
220 | SK_DF(SK_VERB_VP, |
221 | "hwna \"%s\" (0x%llx) kr \"%s\" (0x%llx) krflags 0x%b " |
222 | "(%d)" , KRNA(ring)->na_name, SK_KVA(KRNA(ring)), |
223 | ring->ckr_name, SK_KVA(ring), ring->ckr_flags, |
224 | CKRF_BITS, err); |
225 | STATS_INC(nifs, NETIF_STATS_VP_KR_ENTER_FAIL); |
226 | return err; |
227 | } |
228 | if (__improbable(KR_DROP(ring))) { |
229 | kr_exit(ring); |
230 | STATS_INC(nifs, NETIF_STATS_VP_DEV_RING_DISABLED); |
231 | return ENODEV; |
232 | } |
233 | protect = sk_sync_protect(); |
234 | |
235 | err = ring->ckr_na_sync(ring, p, 0); |
236 | if (err != 0 && err != EAGAIN) { |
237 | STATS_INC(nifs, NETIF_STATS_VP_SYNC_UNKNOWN_ERR); |
238 | goto out; |
239 | } |
240 | ktail = ring->ckr_ktail; |
241 | if (__improbable(ring->ckr_khead == ktail)) { |
242 | SK_DF(SK_VERB_VP, |
243 | "spurious wakeup on hwna %s (0x%llx)" , KRNA(ring)->na_name, |
244 | SK_KVA(KRNA(ring))); |
245 | STATS_INC(nifs, NETIF_STATS_VP_SPURIOUS_NOTIFY); |
246 | err = ENOENT; |
247 | goto out; |
248 | } |
249 | /* get all packets from the ring */ |
250 | idx = ring->ckr_rhead; |
251 | while (idx != ktail) { |
252 | struct __kern_slot_desc *ksd = KR_KSD(ring, idx); |
253 | struct __kern_packet *pkt = ksd->sd_pkt; |
254 | |
255 | ASSERT(pkt->pkt_nextpkt == NULL); |
256 | KR_SLOT_DETACH_METADATA(kring: ring, ksd); |
257 | cnt++; |
258 | *tailp = pkt; |
259 | tailp = &pkt->pkt_nextpkt; |
260 | idx = SLOT_NEXT(i: idx, lim: ring->ckr_lim); |
261 | } |
262 | ring->ckr_rhead = ktail; |
263 | ring->ckr_rtail = ring->ckr_ktail; |
264 | |
265 | DTRACE_SKYWALK2(rx__notify, struct __kern_channel_ring *, ring, |
266 | int, cnt); |
267 | *chain = pkt_chain; |
268 | out: |
269 | sk_sync_unprotect(protect); |
270 | kr_exit(ring); |
271 | return err; |
272 | } |
273 | |
274 | int |
275 | netif_llw_rx_notify_fast(struct __kern_channel_ring *ring, struct proc *p, |
276 | uint32_t flags) |
277 | { |
278 | #pragma unused (p, flags) |
279 | struct nexus_adapter *hwna; |
280 | uint32_t count; |
281 | int i, err; |
282 | |
283 | hwna = KRNA(ring); |
284 | count = na_get_nslots(na: hwna, t: NR_RX); |
285 | err = nx_rx_sync_packets(kring: ring, packets: ring->ckr_scratch, count: &count); |
286 | if (__improbable(err != 0)) { |
287 | SK_ERR("nx_rx_sync_packets failed: %d" , err); |
288 | DTRACE_SKYWALK2(rx__sync__packets__failed, |
289 | struct __kern_channel_ring *, ring, int, err); |
290 | return err; |
291 | } |
292 | DTRACE_SKYWALK1(chain__count, uint32_t, count); |
293 | for (i = 0; i < count; i++) { |
294 | struct __kern_packet *pkt_chain; |
295 | |
296 | pkt_chain = SK_PTR_ADDR_KPKT(ring->ckr_scratch[i]); |
297 | ASSERT(pkt_chain != NULL); |
298 | (void) nx_netif_demux(NIFNA(KRNA(ring)), pkt_chain, NULL, |
299 | NETIF_FLOW_SOURCE); |
300 | } |
301 | return 0; |
302 | } |
303 | |
304 | int |
305 | netif_llw_rx_notify_default(struct __kern_channel_ring *ring, struct proc *p, |
306 | uint32_t flags) |
307 | { |
308 | int err; |
309 | struct __kern_packet *pkt_chain = NULL; |
310 | |
311 | err = netif_hwna_rx_get_pkts(ring, p, flags, chain: &pkt_chain); |
312 | if (err != 0) { |
313 | return err; |
314 | } |
315 | return nx_netif_demux(NIFNA(KRNA(ring)), pkt_chain, NULL, |
316 | NETIF_FLOW_SOURCE); |
317 | } |
318 | |
319 | static errno_t |
320 | netif_hwna_setup(struct nx_netif *nif) |
321 | { |
322 | struct kern_channel *ch; |
323 | struct kern_nexus *nx = nif->nif_nx; |
324 | struct chreq chr; |
325 | int err; |
326 | |
327 | SK_LOCK_ASSERT_HELD(); |
328 | ASSERT(NETIF_IS_LOW_LATENCY(nif)); |
329 | if (nif->nif_hw_ch != NULL) { |
330 | nif->nif_hw_ch_refcnt++; |
331 | SK_DF(SK_VERB_VP, "%s: hw channel already open, refcnt %d" , |
332 | if_name(nif->nif_ifp), nif->nif_hw_ch_refcnt); |
333 | return 0; |
334 | } |
335 | ASSERT(nif->nif_hw_ch_refcnt == 0); |
336 | bzero(s: &chr, n: sizeof(chr)); |
337 | uuid_copy(dst: chr.cr_spec_uuid, src: nx->nx_uuid); |
338 | chr.cr_ring_id = 0; |
339 | chr.cr_port = NEXUS_PORT_NET_IF_DEV; |
340 | chr.cr_mode |= CHMODE_CONFIG; |
341 | |
342 | err = 0; |
343 | ch = ch_open_special(nx, &chr, FALSE, &err); |
344 | if (ch == NULL) { |
345 | SK_ERR("%s: failed to open nx 0x%llx (err %d)" , |
346 | if_name(nif->nif_ifp), SK_KVA(nx), err); |
347 | return err; |
348 | } |
349 | netif_hwna_set_mode(ch->ch_na, NETIF_MODE_LLW, NULL); |
350 | na_start_spec(nx, ch); |
351 | nif->nif_hw_ch_refcnt = 1; |
352 | nif->nif_hw_ch = ch; |
353 | SK_DF(SK_VERB_VP, "%s: hw channel opened 0x%llx, %s:%s" , |
354 | if_name(nif->nif_ifp), SK_KVA(ch), NX_DOM(nx)->nxdom_name, |
355 | NX_DOM_PROV(nx)->nxdom_prov_name); |
356 | return 0; |
357 | } |
358 | |
359 | static void |
360 | netif_hwna_teardown(struct nx_netif *nif) |
361 | { |
362 | struct kern_nexus *nx = nif->nif_nx; |
363 | struct kern_channel *ch = nif->nif_hw_ch; |
364 | |
365 | SK_LOCK_ASSERT_HELD(); |
366 | ASSERT(NETIF_IS_LOW_LATENCY(nif)); |
367 | ASSERT(ch != NULL); |
368 | if (--nif->nif_hw_ch_refcnt > 0) { |
369 | SK_DF(SK_VERB_VP, "%s: hw channel still open, refcnt %d" , |
370 | if_name(nif->nif_ifp), nif->nif_hw_ch_refcnt); |
371 | return; |
372 | } |
373 | SK_DF(SK_VERB_VP, "%s: hw channel closing 0x%llx, %s:%s" , |
374 | if_name(nif->nif_ifp), SK_KVA(ch), NX_DOM(nx)->nxdom_name, |
375 | NX_DOM_PROV(nx)->nxdom_prov_name); |
376 | |
377 | na_stop_spec(nx, ch); |
378 | netif_hwna_clear_mode(ch->ch_na); |
379 | ch_close_special(ch); |
380 | (void) ch_release_locked(ch); |
381 | nif->nif_hw_ch = NULL; |
382 | SK_DF(SK_VERB_VP, "%s: hw channel closed, %s:%s" , |
383 | if_name(nif->nif_ifp), NX_DOM(nx)->nxdom_name, |
384 | NX_DOM_PROV(nx)->nxdom_prov_name); |
385 | } |
386 | |
387 | static int |
388 | netif_vp_na_activate_on(struct nexus_adapter *na) |
389 | { |
390 | errno_t err; |
391 | struct netif_flow *nf; |
392 | struct netif_port_info npi; |
393 | struct nexus_netif_adapter *nifna; |
394 | struct nx_netif *nif; |
395 | boolean_t hwna_setup = FALSE; |
396 | |
397 | nifna = NIFNA(na); |
398 | nif = nifna->nifna_netif; |
399 | |
400 | /* lock needed to protect against nxdom_unbind_port */ |
401 | NETIF_WLOCK(nif); |
402 | err = nx_port_get_info(nif->nif_nx, na->na_nx_port, |
403 | NX_PORT_INFO_TYPE_NETIF, &npi, sizeof(npi)); |
404 | NETIF_WUNLOCK(nif); |
405 | if (err != 0) { |
406 | SK_ERR("port info not found: %d" , err); |
407 | return err; |
408 | } |
409 | if (NETIF_IS_LOW_LATENCY(nif)) { |
410 | err = netif_hwna_setup(nif); |
411 | if (err != 0) { |
412 | return err; |
413 | } |
414 | hwna_setup = TRUE; |
415 | } |
416 | err = nx_netif_flow_add(nif, na->na_nx_port, &npi.npi_fd, na, |
417 | netif_deliver_cb, &nf); |
418 | if (err != 0) { |
419 | if (hwna_setup) { |
420 | netif_hwna_teardown(nif); |
421 | } |
422 | return err; |
423 | } |
424 | nifna->nifna_flow = nf; |
425 | os_atomic_inc(&nx_netif_vpna_gencnt, relaxed); |
426 | nifna->nifna_gencnt = nx_netif_vpna_gencnt; |
427 | os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed); |
428 | return 0; |
429 | } |
430 | |
431 | static int |
432 | netif_vp_na_activate_off(struct nexus_adapter *na) |
433 | { |
434 | errno_t err; |
435 | struct nexus_netif_adapter *nifna; |
436 | struct nx_netif *nif; |
437 | |
438 | if (!NA_IS_ACTIVE(na)) { |
439 | DTRACE_SKYWALK1(already__off, struct nexus_adapter *, na); |
440 | return 0; |
441 | } |
442 | nifna = NIFNA(na); |
443 | nif = nifna->nifna_netif; |
444 | err = nx_netif_flow_remove(nif, nifna->nifna_flow); |
445 | VERIFY(err == 0); |
446 | |
447 | nifna->nifna_flow = NULL; |
448 | if (NETIF_IS_LOW_LATENCY(nif)) { |
449 | netif_hwna_teardown(nif); |
450 | } |
451 | os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed); |
452 | return 0; |
453 | } |
454 | |
455 | static int |
456 | netif_vp_na_activate(struct nexus_adapter *na, na_activate_mode_t mode) |
457 | { |
458 | errno_t err; |
459 | |
460 | ASSERT(na->na_type == NA_NETIF_VP); |
461 | if (mode == NA_ACTIVATE_MODE_ON) { |
462 | err = netif_vp_na_activate_on(na); |
463 | } else { |
464 | err = netif_vp_na_activate_off(na); |
465 | } |
466 | SK_DF(SK_VERB_VP, "na \"%s\" (0x%llx) %s err %d" , na->na_name, |
467 | SK_KVA(na), na_activate_mode2str(mode), err); |
468 | return err; |
469 | } |
470 | |
471 | /* |
472 | * XXX |
473 | * The native path sends to the dev ring directly, bypassing aqm. |
474 | * This is ok since this is only used by llw now. This will need to |
475 | * change when we add native support for filters. |
476 | */ |
477 | static int |
478 | netif_vp_send_pkt_chain_low_latency(struct nexus_netif_adapter *dev_nifna, |
479 | struct __kern_packet *pkt_chain, struct proc *p) |
480 | { |
481 | struct __kern_packet *pkt = pkt_chain, *next; |
482 | struct nexus_adapter *na = &dev_nifna->nifna_up; |
483 | struct __kern_channel_ring *ring = &na->na_tx_rings[0]; |
484 | struct netif_stats *nifs = &dev_nifna->nifna_netif->nif_stats; |
485 | sk_protect_t protect; |
486 | slot_idx_t ktail, idx; |
487 | uint32_t cnt; |
488 | int err_stat = -1; |
489 | errno_t err; |
490 | |
491 | (void) kr_enter(ring, TRUE); |
492 | protect = sk_sync_protect(); |
493 | if (__improbable(KR_DROP(ring))) { |
494 | SK_ERR("ring is not ready" ); |
495 | DTRACE_SKYWALK1(ring__drop, struct __kern_channel_ring *, ring); |
496 | err_stat = NETIF_STATS_VP_DROP_DEV_RING_DISABLED; |
497 | err = ENXIO; |
498 | goto done; |
499 | } |
500 | idx = ring->ckr_rhead; |
501 | ktail = ring->ckr_ktail; |
502 | if (idx == ktail) { |
503 | SK_ERR("no space to send" ); |
504 | DTRACE_SKYWALK1(no__space, struct __kern_channel_ring *, ring); |
505 | err_stat = NETIF_STATS_VP_DROP_DEV_RING_NO_SPACE; |
506 | goto sync; |
507 | } |
508 | cnt = 0; |
509 | while (pkt != NULL && idx != ktail) { |
510 | struct __slot_desc *slot = &ring->ckr_ksds[idx]; |
511 | |
512 | next = pkt->pkt_nextpkt; |
513 | pkt->pkt_nextpkt = NULL; |
514 | netif_vp_dump_packet(pkt); |
515 | err = netif_copy_or_attach_pkt(ring, slot, pkt); |
516 | if (__probable(err == 0)) { |
517 | cnt++; |
518 | idx = SLOT_NEXT(i: idx, lim: ring->ckr_lim); |
519 | } |
520 | pkt = next; |
521 | } |
522 | ring->ckr_rhead = idx; |
523 | STATS_ADD(nifs, NETIF_STATS_VP_LL_ENQUEUED, cnt); |
524 | DTRACE_SKYWALK2(ll__enqueued, struct __kern_channel_ring *, ring, |
525 | uint32_t, cnt); |
526 | sync: |
527 | ring->ckr_khead_pre = ring->ckr_khead; |
528 | err = ring->ckr_na_sync(ring, p, NA_SYNCF_SYNC_ONLY); |
529 | if (err != 0 && err != EAGAIN) { |
530 | SK_ERR("unexpected sync err %d" , err); |
531 | DTRACE_SKYWALK1(sync__failed, struct __kern_channel_ring *, |
532 | ring); |
533 | err_stat = NETIF_STATS_VP_DROP_UNEXPECTED_ERR; |
534 | goto done; |
535 | } |
536 | /* |
537 | * Verify that the driver has detached packets from the consumed slots. |
538 | */ |
539 | idx = ring->ckr_khead_pre; |
540 | cnt = 0; |
541 | while (idx != ring->ckr_khead) { |
542 | struct __kern_slot_desc *ksd = KR_KSD(ring, idx); |
543 | |
544 | cnt++; |
545 | VERIFY(!KSD_VALID_METADATA(ksd)); |
546 | idx = SLOT_NEXT(i: idx, lim: ring->ckr_lim); |
547 | } |
548 | ring->ckr_khead_pre = ring->ckr_khead; |
549 | STATS_ADD(nifs, NETIF_STATS_VP_LL_SENT, cnt); |
550 | DTRACE_SKYWALK2(ll__sent, struct __kern_channel_ring *, ring, |
551 | uint32_t, cnt); |
552 | err = 0; |
553 | |
554 | done: |
555 | sk_sync_unprotect(protect); |
556 | kr_exit(ring); |
557 | |
558 | /* |
559 | * Free all unsent packets. |
560 | */ |
561 | if (pkt != NULL) { |
562 | int dropcnt; |
563 | |
564 | nx_netif_free_packet_chain(pkt, &dropcnt); |
565 | if (err_stat != -1) { |
566 | STATS_ADD(nifs, err_stat, dropcnt); |
567 | } |
568 | STATS_ADD(nifs, NETIF_STATS_DROP, dropcnt); |
569 | } |
570 | return err; |
571 | } |
572 | |
573 | static int |
574 | netif_vp_send_pkt_chain_common(struct nexus_netif_adapter *dev_nifna, |
575 | struct __kern_packet *pkt_chain, boolean_t compat) |
576 | { |
577 | struct __kern_packet *pkt = pkt_chain, *next, *p; |
578 | struct nx_netif *nif = dev_nifna->nifna_netif; |
579 | struct netif_stats *nifs = &nif->nif_stats; |
580 | ifnet_t ifp = nif->nif_ifp; |
581 | struct mbuf *m; |
582 | boolean_t drop; |
583 | int cnt = 0; |
584 | errno_t err; |
585 | |
586 | while (pkt != NULL) { |
587 | next = pkt->pkt_nextpkt; |
588 | pkt->pkt_nextpkt = NULL; |
589 | drop = FALSE; |
590 | |
591 | if (compat) { |
592 | m = nx_netif_pkt_to_mbuf(dev_nifna, pkt, NETIF_CONVERT_TX); |
593 | if (m == NULL) { |
594 | pkt = next; |
595 | continue; |
596 | } |
597 | err = ifnet_enqueue_mbuf(ifp, m, FALSE, &drop); |
598 | } else { |
599 | p = nx_netif_pkt_to_pkt(dev_nifna, pkt, NETIF_CONVERT_TX); |
600 | if (p == NULL) { |
601 | pkt = next; |
602 | continue; |
603 | } |
604 | err = ifnet_enqueue_pkt(ifp, p, FALSE, &drop); |
605 | } |
606 | if (err != 0) { |
607 | SK_ERR("enqueue failed: %d" , err); |
608 | STATS_INC(nifs, NETIF_STATS_VP_ENQUEUE_FAILED); |
609 | if (drop) { |
610 | STATS_INC(nifs, NETIF_STATS_DROP); |
611 | } |
612 | DTRACE_SKYWALK2(enqueue__failed, |
613 | struct nexus_netif_adapter *, dev_nifna, |
614 | boolean_t, drop); |
615 | } else { |
616 | STATS_INC(nifs, NETIF_STATS_VP_ENQUEUED); |
617 | cnt++; |
618 | } |
619 | pkt = next; |
620 | } |
621 | if (cnt > 0) { |
622 | ifnet_start(interface: ifp); |
623 | } |
624 | return 0; |
625 | } |
626 | |
627 | static int |
628 | netif_vp_send_pkt_chain(struct nexus_netif_adapter *dev_nifna, |
629 | struct __kern_packet *pkt_chain, struct proc *p) |
630 | { |
631 | struct nexus_adapter *na = &dev_nifna->nifna_up; |
632 | |
633 | if (NETIF_IS_LOW_LATENCY(dev_nifna->nifna_netif)) { |
634 | return netif_vp_send_pkt_chain_low_latency(dev_nifna, |
635 | pkt_chain, p); |
636 | } |
637 | if (na->na_type == NA_NETIF_DEV) { |
638 | return netif_vp_send_pkt_chain_common(dev_nifna, pkt_chain, FALSE); |
639 | } |
640 | ASSERT(na->na_type == NA_NETIF_COMPAT_DEV); |
641 | return netif_vp_send_pkt_chain_common(dev_nifna, pkt_chain, TRUE); |
642 | } |
643 | |
644 | SK_NO_INLINE_ATTRIBUTE |
645 | static boolean_t |
646 | validate_packet(struct nexus_netif_adapter *nifna, struct __kern_packet *pkt) |
647 | { |
648 | struct nx_netif *nif = nifna->nifna_netif; |
649 | |
650 | VERIFY(pkt->pkt_nextpkt == NULL); |
651 | |
652 | if (nx_netif_vp_accept_all != 0) { |
653 | return TRUE; |
654 | } |
655 | if (outbound_check == 0 || |
656 | (NETIF_IS_LOW_LATENCY(nif) && |
657 | outbound_check != OUTBOUND_CHECK_FORCED)) { |
658 | return TRUE; |
659 | } |
660 | if (!nx_netif_validate_macaddr(nif, pkt, NETIF_FLOW_OUTBOUND)) { |
661 | return FALSE; |
662 | } |
663 | if (!nx_netif_flow_match(nif, pkt, nifna->nifna_flow, |
664 | NETIF_FLOW_OUTBOUND)) { |
665 | return FALSE; |
666 | } |
667 | return TRUE; |
668 | } |
669 | |
670 | static int |
671 | netif_vp_na_txsync(struct __kern_channel_ring *kring, struct proc *p, |
672 | uint32_t flags) |
673 | { |
674 | #pragma unused(flags) |
675 | kern_channel_slot_t last_slot = NULL, slot = NULL; |
676 | struct __kern_packet *head = NULL, **tailp = &head, *pkt; |
677 | struct nexus_netif_adapter *nifna, *dev_nifna; |
678 | struct nx_netif *nif; |
679 | struct netif_stats *nifs; |
680 | kern_packet_t ph; |
681 | errno_t err; |
682 | int cnt = 0; |
683 | |
684 | nifna = NIFNA(KRNA(kring)); |
685 | nif = nifna->nifna_netif; |
686 | nifs = &nif->nif_stats; |
687 | for (;;) { |
688 | slot = kern_channel_get_next_slot(kring, slot, NULL); |
689 | if (slot == NULL) { |
690 | break; |
691 | } |
692 | ph = kern_channel_slot_get_packet(ring: kring, slot); |
693 | if (__improbable(ph == 0)) { |
694 | SK_ERR("packet got dropped by internalize" ); |
695 | STATS_INC(nifs, NETIF_STATS_VP_DROP_INTERNALIZE_FAIL); |
696 | DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *, |
697 | kring, kern_channel_slot_t, slot); |
698 | last_slot = slot; |
699 | continue; |
700 | } |
701 | pkt = SK_PTR_ADDR_KPKT(ph); |
702 | if (__improbable(pkt->pkt_length == 0)) { |
703 | SK_ERR("dropped zero length packet" ); |
704 | STATS_INC(nifs, NETIF_STATS_VP_BAD_PKT_LEN); |
705 | DTRACE_SKYWALK2(bad__slot, struct __kern_channel_ring *, |
706 | kring, kern_channel_slot_t, slot); |
707 | last_slot = slot; |
708 | continue; |
709 | } |
710 | err = kern_channel_slot_detach_packet(ring: kring, slot, packet: ph); |
711 | VERIFY(err == 0); |
712 | |
713 | /* packet needs to be finalized after detach */ |
714 | err = kern_packet_finalize(ph); |
715 | VERIFY(err == 0); |
716 | last_slot = slot; |
717 | |
718 | if (NA_CHANNEL_EVENT_ATTACHED(KRNA(kring))) { |
719 | __packet_set_tx_nx_port(SK_PKT2PH(pkt), |
720 | KRNA(kring)->na_nx_port, vpna_gencnt: nifna->nifna_gencnt); |
721 | } |
722 | |
723 | if (validate_packet(nifna, pkt)) { |
724 | nx_netif_snoop(nif, pkt, FALSE); |
725 | cnt++; |
726 | *tailp = pkt; |
727 | tailp = &pkt->pkt_nextpkt; |
728 | } else { |
729 | nx_netif_free_packet(pkt); |
730 | } |
731 | } |
732 | if (cnt == 0) { |
733 | STATS_INC(nifs, NETIF_STATS_VP_SYNC_NO_PKTS); |
734 | DTRACE_SKYWALK2(no__data, struct nexus_netif_adapter *, nifna, |
735 | struct __kern_channel_ring *, kring); |
736 | return 0; |
737 | } |
738 | DTRACE_SKYWALK4(injected, struct nexus_netif_adapter *, nifna, |
739 | struct __kern_channel_ring *, kring, struct __kern_packet *, head, |
740 | int, cnt); |
741 | if (last_slot != NULL) { |
742 | kern_channel_advance_slot(kring, slot: last_slot); |
743 | } |
744 | |
745 | dev_nifna = NIFNA(nx_port_get_na(KRNA(kring)->na_nx, |
746 | NEXUS_PORT_NET_IF_DEV)); |
747 | |
748 | err = netif_vp_send_pkt_chain(dev_nifna, pkt_chain: head, p); |
749 | if (err != 0) { |
750 | SK_ERR("send failed: %d\n" , err); |
751 | } |
752 | return 0; |
753 | } |
754 | |
755 | static int |
756 | netif_vp_na_rxsync(struct __kern_channel_ring *kring, struct proc *p, |
757 | uint32_t flags) |
758 | { |
759 | #pragma unused(p, flags) |
760 | (void) kr_reclaim(kr: kring); |
761 | return 0; |
762 | } |
763 | |
764 | static int |
765 | netif_vp_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch) |
766 | { |
767 | ASSERT(na->na_type == NA_NETIF_VP); |
768 | return na_rings_mem_setup(na, FALSE, ch); |
769 | } |
770 | |
771 | |
772 | /* na_krings_delete callback for flow switch ports. */ |
773 | static void |
774 | netif_vp_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch, |
775 | boolean_t defunct) |
776 | { |
777 | ASSERT(na->na_type == NA_NETIF_VP); |
778 | na_rings_mem_teardown(na, ch, defunct); |
779 | } |
780 | |
781 | static int |
782 | netif_vp_region_params_setup(struct nexus_adapter *na, |
783 | struct skmem_region_params *srp, struct kern_pbufpool **tx_pp) |
784 | { |
785 | #pragma unused (tx_pp) |
786 | uint32_t max_mtu; |
787 | uint32_t buf_sz, buf_cnt, nslots, afslots, evslots, totalrings; |
788 | struct nexus_adapter *devna; |
789 | struct kern_nexus *nx; |
790 | struct nx_netif *nif; |
791 | int err, i; |
792 | |
793 | for (i = 0; i < SKMEM_REGIONS; i++) { |
794 | srp[i] = *skmem_get_default(i); |
795 | } |
796 | totalrings = na_get_nrings(na, t: NR_TX) + na_get_nrings(na, t: NR_RX) + |
797 | na_get_nrings(na, t: NR_A) + na_get_nrings(na, t: NR_F) + |
798 | na_get_nrings(na, t: NR_EV); |
799 | |
800 | srp[SKMEM_REGION_SCHEMA].srp_r_obj_size = |
801 | (uint32_t)CHANNEL_SCHEMA_SIZE(totalrings); |
802 | srp[SKMEM_REGION_SCHEMA].srp_r_obj_cnt = totalrings; |
803 | skmem_region_params_config(&srp[SKMEM_REGION_SCHEMA]); |
804 | |
805 | srp[SKMEM_REGION_RING].srp_r_obj_size = |
806 | sizeof(struct __user_channel_ring); |
807 | srp[SKMEM_REGION_RING].srp_r_obj_cnt = totalrings; |
808 | skmem_region_params_config(&srp[SKMEM_REGION_RING]); |
809 | |
810 | /* USD regions need to be writable to support user packet pool */ |
811 | srp[SKMEM_REGION_TXAUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY; |
812 | srp[SKMEM_REGION_RXFUSD].srp_cflags &= ~SKMEM_REGION_CR_UREADONLY; |
813 | |
814 | nslots = na_get_nslots(na, t: NR_TX); |
815 | afslots = na_get_nslots(na, t: NR_A); |
816 | evslots = na_get_nslots(na, t: NR_EV); |
817 | srp[SKMEM_REGION_TXAKSD].srp_r_obj_size = |
818 | MAX(MAX(nslots, afslots), evslots) * SLOT_DESC_SZ; |
819 | srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt = |
820 | na_get_nrings(na, t: NR_TX) + na_get_nrings(na, t: NR_A) + |
821 | na_get_nrings(na, t: NR_EV); |
822 | skmem_region_params_config(&srp[SKMEM_REGION_TXAKSD]); |
823 | |
824 | /* USD and KSD objects share the same size and count */ |
825 | srp[SKMEM_REGION_TXAUSD].srp_r_obj_size = |
826 | srp[SKMEM_REGION_TXAKSD].srp_r_obj_size; |
827 | srp[SKMEM_REGION_TXAUSD].srp_r_obj_cnt = |
828 | srp[SKMEM_REGION_TXAKSD].srp_r_obj_cnt; |
829 | skmem_region_params_config(&srp[SKMEM_REGION_TXAUSD]); |
830 | |
831 | /* |
832 | * Since the rx/free slots share the same region and cache, |
833 | * we will use the same object size for both types of slots. |
834 | */ |
835 | nslots = na_get_nslots(na, t: NR_RX); |
836 | afslots = na_get_nslots(na, t: NR_F); |
837 | srp[SKMEM_REGION_RXFKSD].srp_r_obj_size = |
838 | MAX(nslots, afslots) * SLOT_DESC_SZ; |
839 | srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt = |
840 | na_get_nrings(na, t: NR_RX) + na_get_nrings(na, t: NR_F); |
841 | skmem_region_params_config(&srp[SKMEM_REGION_RXFKSD]); |
842 | |
843 | /* USD and KSD objects share the same size and count */ |
844 | srp[SKMEM_REGION_RXFUSD].srp_r_obj_size = |
845 | srp[SKMEM_REGION_RXFKSD].srp_r_obj_size; |
846 | srp[SKMEM_REGION_RXFUSD].srp_r_obj_cnt = |
847 | srp[SKMEM_REGION_RXFKSD].srp_r_obj_cnt; |
848 | skmem_region_params_config(&srp[SKMEM_REGION_RXFUSD]); |
849 | |
850 | /* |
851 | * No need to create our own buffer pool if we can share the device's |
852 | * pool. We don't support sharing split pools to user space. |
853 | */ |
854 | nx = na->na_nx; |
855 | nif = nx->nx_arg; |
856 | if (vp_zerocopy != 0 && NETIF_IS_LOW_LATENCY(nif) && |
857 | nx->nx_tx_pp != NULL && (nx->nx_rx_pp == NULL || |
858 | nx->nx_tx_pp == nx->nx_rx_pp) && !PP_KERNEL_ONLY(nx->nx_tx_pp)) { |
859 | struct kern_pbufpool *pp = nx->nx_tx_pp; |
860 | |
861 | if (nif->nif_hw_ch_refcnt != 0) { |
862 | SK_ERR("only one channel is supported for zero copy" ); |
863 | return ENOTSUP; |
864 | } |
865 | SK_DF(SK_VERB_VP, "sharing %s's pool" , if_name(na->na_ifp)); |
866 | |
867 | /* |
868 | * These types need to be initialized otherwise some assertions |
869 | * skmem_arena_create_for_nexus() will fail. |
870 | */ |
871 | srp[SKMEM_REGION_UMD].srp_md_type = pp->pp_md_type; |
872 | srp[SKMEM_REGION_UMD].srp_md_subtype = pp->pp_md_subtype; |
873 | srp[SKMEM_REGION_KMD].srp_md_type = pp->pp_md_type; |
874 | srp[SKMEM_REGION_KMD].srp_md_subtype = pp->pp_md_subtype; |
875 | *tx_pp = nx->nx_tx_pp; |
876 | return 0; |
877 | } |
878 | |
879 | devna = nx_port_get_na(nx, NEXUS_PORT_NET_IF_DEV); |
880 | ASSERT(devna != NULL); |
881 | if (devna->na_type == NA_NETIF_DEV) { |
882 | /* |
883 | * For native devices, use the driver's buffer size |
884 | */ |
885 | ASSERT(nx->nx_rx_pp != NULL); |
886 | ASSERT(nx->nx_tx_pp != NULL); |
887 | buf_sz = PP_BUF_SIZE_DEF(nx->nx_tx_pp); |
888 | } else { |
889 | if ((err = nx_netif_get_max_mtu(na->na_ifp, &max_mtu)) != 0) { |
890 | /* |
891 | * If the driver doesn't support SIOCGIFDEVMTU, use the |
892 | * default MTU size. |
893 | */ |
894 | max_mtu = ifnet_mtu(interface: na->na_ifp); |
895 | err = 0; |
896 | } |
897 | /* max_mtu does not include the L2 header */ |
898 | buf_sz = MAX(max_mtu + sizeof(struct ether_vlan_header), 2048); |
899 | } |
900 | buf_cnt = vp_pool_size; |
901 | pp_regions_params_adjust(srp, NEXUS_META_TYPE_PACKET, |
902 | NEXUS_META_SUBTYPE_RAW, buf_cnt, 1, buf_sz, 0, buf_cnt, 0, |
903 | PP_REGION_CONFIG_BUF_IODIR_BIDIR | |
904 | PP_REGION_CONFIG_MD_MAGAZINE_ENABLE); |
905 | |
906 | nx_netif_vp_region_params_adjust(na, srp); |
907 | return 0; |
908 | } |
909 | |
910 | static int |
911 | netif_vp_na_mem_new(struct kern_nexus *nx, struct nexus_adapter *na) |
912 | { |
913 | #pragma unused(nx) |
914 | struct skmem_region_params srp[SKMEM_REGIONS]; |
915 | struct kern_pbufpool *tx_pp = NULL; |
916 | int err; |
917 | |
918 | err = netif_vp_region_params_setup(na, srp, tx_pp: &tx_pp); |
919 | if (err != 0) { |
920 | return err; |
921 | } |
922 | na->na_arena = skmem_arena_create_for_nexus(na, srp, |
923 | tx_pp != NULL ? &tx_pp : NULL, NULL, |
924 | FALSE, FALSE, &nx->nx_adv, &err); |
925 | ASSERT(na->na_arena != NULL || err != 0); |
926 | return err; |
927 | } |
928 | |
929 | static void |
930 | netif_vp_na_dtor(struct nexus_adapter *na) |
931 | { |
932 | struct kern_nexus *nx = na->na_nx; |
933 | struct nx_netif *nif = NX_NETIF_PRIVATE(nx); |
934 | struct nexus_netif_adapter *nifna = NIFNA(na); |
935 | |
936 | NETIF_WLOCK(nif); |
937 | (void) nx_port_unbind(nx, na->na_nx_port); |
938 | nx_port_free(nx, na->na_nx_port); |
939 | nif->nif_vp_cnt--; |
940 | if (na->na_ifp != NULL) { |
941 | ifnet_decr_iorefcnt(na->na_ifp); |
942 | na->na_ifp = NULL; |
943 | } |
944 | if (nifna->nifna_netif != NULL) { |
945 | nx_netif_release(nifna->nifna_netif); |
946 | nifna->nifna_netif = NULL; |
947 | } |
948 | NETIF_WUNLOCK(nif); |
949 | SK_DF(SK_VERB_VP, "na \"%s\" (0x%llx)" , na->na_name, SK_KVA(na)); |
950 | } |
951 | |
952 | int |
953 | netif_vp_na_create(struct kern_nexus *nx, struct chreq *chr, |
954 | struct nexus_adapter **nap) |
955 | { |
956 | struct nx_netif *nif = NX_NETIF_PRIVATE(nx); |
957 | struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params; |
958 | struct nexus_adapter *na = NULL; |
959 | struct nexus_netif_adapter *nifna; |
960 | uint32_t slots; |
961 | int err; |
962 | |
963 | NETIF_WLOCK_ASSERT_HELD(nif); |
964 | if (nif->nif_ifp == NULL) { |
965 | SK_ERR("ifnet not yet attached" ); |
966 | return ENXIO; |
967 | } |
968 | ASSERT((chr->cr_mode & CHMODE_KERNEL) == 0); |
969 | if ((chr->cr_mode & CHMODE_USER_PACKET_POOL) == 0) { |
970 | SK_ERR("user packet pool required" ); |
971 | return EINVAL; |
972 | } |
973 | /* |
974 | * No locking needed while checking for the initialized bit because |
975 | * if this were not set, no other codepaths would modify the flags. |
976 | */ |
977 | if ((nif->nif_flow_flags & NETIF_FLOW_FLAG_INITIALIZED) == 0) { |
978 | SK_ERR("demux vp not supported" ); |
979 | return ENOTSUP; |
980 | } |
981 | na = (struct nexus_adapter *)na_netif_alloc(Z_WAITOK); |
982 | nifna = NIFNA(na); |
983 | nifna->nifna_netif = nif; |
984 | nx_netif_retain(nif); |
985 | nifna->nifna_flow = NULL; |
986 | |
987 | (void) snprintf(na->na_name, count: sizeof(na->na_name), |
988 | "netif_vp:%d" , chr->cr_port); |
989 | uuid_generate_random(out: na->na_uuid); |
990 | |
991 | na_set_nrings(na, t: NR_TX, v: nxp->nxp_tx_rings); |
992 | na_set_nrings(na, t: NR_RX, v: nxp->nxp_rx_rings); |
993 | /* |
994 | * If the packet pool is configured to be multi-buflet, then we |
995 | * need 2 pairs of alloc/free rings(for packet and buflet). |
996 | */ |
997 | na_set_nrings(na, t: NR_A, v: ((nxp->nxp_max_frags > 1) && |
998 | (sk_channel_buflet_alloc != 0)) ? 2 : 1); |
999 | |
1000 | slots = vp_tx_slots != 0 ? vp_tx_slots : |
1001 | NX_DOM(nx)->nxdom_tx_slots.nb_def; |
1002 | na_set_nslots(na, t: NR_TX, v: slots); |
1003 | |
1004 | slots = vp_rx_slots != 0 ? vp_rx_slots : |
1005 | NX_DOM(nx)->nxdom_rx_slots.nb_def; |
1006 | na_set_nslots(na, t: NR_RX, v: slots); |
1007 | |
1008 | na_set_nslots(na, t: NR_A, NETIF_DEMUX_ALLOC_SLOTS); |
1009 | ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max); |
1010 | ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max); |
1011 | ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max); |
1012 | ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max); |
1013 | |
1014 | os_atomic_or(&na->na_flags, NAF_USER_PKT_POOL, relaxed); |
1015 | |
1016 | if (chr->cr_mode & CHMODE_EVENT_RING) { |
1017 | na_set_nrings(na, t: NR_EV, NX_NETIF_EVENT_RING_NUM); |
1018 | na_set_nslots(na, t: NR_EV, NX_NETIF_EVENT_RING_SIZE); |
1019 | os_atomic_or(&na->na_flags, NAF_EVENT_RING, relaxed); |
1020 | na->na_channel_event_notify = netif_vp_na_channel_event_notify; |
1021 | } |
1022 | |
1023 | na->na_nx_port = chr->cr_port; |
1024 | na->na_type = NA_NETIF_VP; |
1025 | na->na_free = na_netif_free; |
1026 | na->na_dtor = netif_vp_na_dtor; |
1027 | na->na_activate = netif_vp_na_activate; |
1028 | na->na_txsync = netif_vp_na_txsync; |
1029 | na->na_rxsync = netif_vp_na_rxsync; |
1030 | na->na_krings_create = netif_vp_na_krings_create; |
1031 | na->na_krings_delete = netif_vp_na_krings_delete; |
1032 | na->na_special = NULL; |
1033 | na->na_ifp = nif->nif_ifp; |
1034 | ifnet_incr_iorefcnt(na->na_ifp); |
1035 | |
1036 | *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type = |
1037 | NEXUS_STATS_TYPE_INVALID; |
1038 | |
1039 | /* other fields are set in the common routine */ |
1040 | na_attach_common(na, nx, &nx_netif_prov_s); |
1041 | |
1042 | err = netif_vp_na_mem_new(nx, na); |
1043 | if (err != 0) { |
1044 | ASSERT(na->na_arena == NULL); |
1045 | goto err; |
1046 | } |
1047 | |
1048 | *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max; |
1049 | ASSERT(na->na_flowadv_max == 0 || |
1050 | skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL); |
1051 | |
1052 | nif->nif_vp_cnt++; |
1053 | *nap = na; |
1054 | return 0; |
1055 | |
1056 | err: |
1057 | if (na != NULL) { |
1058 | if (na->na_ifp != NULL) { |
1059 | ifnet_decr_iorefcnt(na->na_ifp); |
1060 | na->na_ifp = NULL; |
1061 | } |
1062 | if (na->na_arena != NULL) { |
1063 | skmem_arena_release(na->na_arena); |
1064 | na->na_arena = NULL; |
1065 | } |
1066 | if (nifna->nifna_netif != NULL) { |
1067 | nx_netif_release(nifna->nifna_netif); |
1068 | nifna->nifna_netif = NULL; |
1069 | } |
1070 | NA_FREE(na); |
1071 | } |
1072 | SK_ERR("VP NA creation failed, err(%d)" , err); |
1073 | return err; |
1074 | } |
1075 | |
1076 | static int |
1077 | netif_vp_na_channel_event_notify(struct nexus_adapter *vpna, |
1078 | struct __kern_channel_event *ev, uint16_t ev_len) |
1079 | { |
1080 | int err; |
1081 | char *baddr; |
1082 | kern_packet_t ph; |
1083 | kern_buflet_t buf; |
1084 | sk_protect_t protect; |
1085 | kern_channel_slot_t slot; |
1086 | struct __kern_packet *vpna_pkt = NULL; |
1087 | struct __kern_channel_event_metadata *emd; |
1088 | struct __kern_channel_ring *ring = &vpna->na_event_rings[0]; |
1089 | struct netif_stats *nifs = &NIFNA(vpna)->nifna_netif->nif_stats; |
1090 | |
1091 | if (__probable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_STATUS)) { |
1092 | STATS_INC(nifs, NETIF_STATS_EV_RECV_TX_STATUS); |
1093 | } |
1094 | if (__improbable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_EXPIRED)) { |
1095 | STATS_INC(nifs, NETIF_STATS_EV_RECV_TX_EXPIRED); |
1096 | } |
1097 | STATS_INC(nifs, NETIF_STATS_EV_RECV); |
1098 | |
1099 | if (__improbable(!NA_IS_ACTIVE(vpna))) { |
1100 | STATS_INC(nifs, NETIF_STATS_EV_DROP_NA_INACTIVE); |
1101 | err = ENXIO; |
1102 | goto error; |
1103 | } |
1104 | if (__improbable(NA_IS_DEFUNCT(vpna))) { |
1105 | STATS_INC(nifs, NETIF_STATS_EV_DROP_NA_DEFUNCT); |
1106 | err = ENXIO; |
1107 | goto error; |
1108 | } |
1109 | if (!NA_CHANNEL_EVENT_ATTACHED(vpna)) { |
1110 | STATS_INC(nifs, NETIF_STATS_EV_DROP_KEVENT_INACTIVE); |
1111 | err = ENXIO; |
1112 | goto error; |
1113 | } |
1114 | if (__improbable(KR_DROP(ring))) { |
1115 | STATS_INC(nifs, NETIF_STATS_EV_DROP_KRDROP_MODE); |
1116 | err = ENXIO; |
1117 | goto error; |
1118 | } |
1119 | vpna_pkt = nx_netif_alloc_packet(ring->ckr_pp, ev_len, &ph); |
1120 | if (__improbable(vpna_pkt == NULL)) { |
1121 | STATS_INC(nifs, NETIF_STATS_EV_DROP_NOMEM_PKT); |
1122 | err = ENOMEM; |
1123 | goto error; |
1124 | } |
1125 | buf = __packet_get_next_buflet(ph, NULL); |
1126 | baddr = __buflet_get_data_address(buf); |
1127 | emd = (struct __kern_channel_event_metadata *)(void *)baddr; |
1128 | emd->emd_etype = ev->ev_type; |
1129 | emd->emd_nevents = 1; |
1130 | bcopy(src: ev, dst: (baddr + __KERN_CHANNEL_EVENT_OFFSET), n: ev_len); |
1131 | err = __buflet_set_data_length(buf, |
1132 | dlen: (ev_len + __KERN_CHANNEL_EVENT_OFFSET)); |
1133 | VERIFY(err == 0); |
1134 | err = __packet_finalize(ph); |
1135 | VERIFY(err == 0); |
1136 | kr_enter(ring, TRUE); |
1137 | protect = sk_sync_protect(); |
1138 | slot = kern_channel_get_next_slot(kring: ring, NULL, NULL); |
1139 | if (slot == NULL) { |
1140 | sk_sync_unprotect(protect); |
1141 | kr_exit(ring); |
1142 | STATS_INC(nifs, NETIF_STATS_EV_DROP_KRSPACE); |
1143 | err = ENOSPC; |
1144 | goto error; |
1145 | } |
1146 | err = kern_channel_slot_attach_packet(ring, slot, packet: ph); |
1147 | VERIFY(err == 0); |
1148 | vpna_pkt = NULL; |
1149 | kern_channel_advance_slot(kring: ring, slot); |
1150 | sk_sync_unprotect(protect); |
1151 | kr_exit(ring); |
1152 | kern_channel_event_notify(&vpna->na_tx_rings[0]); |
1153 | STATS_INC(nifs, NETIF_STATS_EV_SENT); |
1154 | return 0; |
1155 | |
1156 | error: |
1157 | ASSERT(err != 0); |
1158 | if (vpna_pkt != NULL) { |
1159 | nx_netif_free_packet(vpna_pkt); |
1160 | } |
1161 | STATS_INC(nifs, NETIF_STATS_EV_DROP); |
1162 | return err; |
1163 | } |
1164 | |
1165 | static inline struct nexus_adapter * |
1166 | nx_netif_find_port_vpna(struct nx_netif *netif, uint32_t nx_port_id) |
1167 | { |
1168 | struct kern_nexus *nx = netif->nif_nx; |
1169 | struct nexus_adapter *na = NULL; |
1170 | nexus_port_t port; |
1171 | uint16_t gencnt; |
1172 | |
1173 | PKT_DECOMPOSE_NX_PORT_ID(nx_port_id, port, gencnt); |
1174 | if (port < NEXUS_PORT_NET_IF_CLIENT) { |
1175 | SK_ERR("non VPNA port" ); |
1176 | return NULL; |
1177 | } |
1178 | if (__improbable(!nx_port_is_valid(nx, port))) { |
1179 | SK_ERR("%s[%d] port no longer valid" , |
1180 | if_name(netif->nif_ifp), port); |
1181 | return NULL; |
1182 | } |
1183 | na = nx_port_get_na(nx, port); |
1184 | if (na != NULL && NIFNA(na)->nifna_gencnt != gencnt) { |
1185 | return NULL; |
1186 | } |
1187 | return na; |
1188 | } |
1189 | |
1190 | errno_t |
1191 | netif_vp_na_channel_event(struct nx_netif *nif, uint32_t nx_port_id, |
1192 | struct __kern_channel_event *event, uint16_t event_len) |
1193 | { |
1194 | int err = 0; |
1195 | struct nexus_adapter *netif_vpna; |
1196 | struct netif_stats *nifs = &nif->nif_stats; |
1197 | |
1198 | SK_DF(SK_VERB_EVENTS, "%s[%d] ev: %p ev_len: %hu " |
1199 | "ev_type: %u ev_flags: %u _reserved: %hu ev_dlen: %hu" , |
1200 | if_name(nif->nif_ifp), nx_port_id, event, event_len, |
1201 | event->ev_type, event->ev_flags, event->_reserved, event->ev_dlen); |
1202 | |
1203 | NETIF_RLOCK(nif); |
1204 | if (!NETIF_IS_LOW_LATENCY(nif)) { |
1205 | err = ENOTSUP; |
1206 | goto error; |
1207 | } |
1208 | if (__improbable(nif->nif_vp_cnt == 0)) { |
1209 | STATS_INC(nifs, NETIF_STATS_EV_DROP_NO_VPNA); |
1210 | err = ENXIO; |
1211 | goto error; |
1212 | } |
1213 | netif_vpna = nx_netif_find_port_vpna(netif: nif, nx_port_id); |
1214 | if (__improbable(netif_vpna == NULL)) { |
1215 | err = ENXIO; |
1216 | STATS_INC(nifs, NETIF_STATS_EV_DROP_DEMUX_ERR); |
1217 | goto error; |
1218 | } |
1219 | if (__improbable(netif_vpna->na_channel_event_notify == NULL)) { |
1220 | err = ENOTSUP; |
1221 | STATS_INC(nifs, NETIF_STATS_EV_DROP_EV_VPNA_NOTSUP); |
1222 | goto error; |
1223 | } |
1224 | err = netif_vpna->na_channel_event_notify(netif_vpna, event, event_len); |
1225 | NETIF_RUNLOCK(nif); |
1226 | return err; |
1227 | |
1228 | error: |
1229 | STATS_INC(nifs, NETIF_STATS_EV_DROP); |
1230 | NETIF_RUNLOCK(nif); |
1231 | return err; |
1232 | } |
1233 | |