1 | /* |
2 | * Copyright (c) 2020-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #include <skywalk/os_skywalk_private.h> |
29 | #include <skywalk/nexus/netif/nx_netif.h> |
30 | #include <pexpert/pexpert.h> /* for PE_parse_boot_argn */ |
31 | #include <os/refcnt.h> |
32 | #include <sys/sdt.h> |
33 | |
34 | #define NX_NETIF_TAG_QSET "com.apple.skywalk.netif.qset" |
35 | static SKMEM_TAG_DEFINE(nx_netif_tag_qset, NX_NETIF_TAG_QSET); |
36 | |
37 | #define NX_NETIF_TAG_LLINK_CFG "com.apple.skywalk.netif.llink.cfg" |
38 | static SKMEM_TAG_DEFINE(nx_netif_tag_llink_cfg, NX_NETIF_TAG_LLINK_CFG); |
39 | |
40 | LCK_ATTR_DECLARE(netif_llink_lock_attr, 0, 0); |
41 | static LCK_GRP_DECLARE(netif_llink_lock_group, "netif llink locks" ); |
42 | |
43 | #if (DEVELOPMENT || DEBUG) |
44 | static TUNABLE(uint32_t, nx_netif_disable_llink, "sk_disable_llink" , 0); |
45 | #endif /* (DEVELOPMENT || DEBUG) */ |
46 | |
47 | static struct netif_llink *nx_netif_llink_alloc(void); |
48 | static void nx_netif_llink_free(struct netif_llink **); |
49 | static struct netif_qset *nx_netif_qset_alloc(uint8_t, uint8_t); |
50 | static void nx_netif_qset_free(struct netif_qset **); |
51 | static void nx_netif_qset_setup_ifclassq(struct netif_llink *, |
52 | struct netif_qset *); |
53 | static void nx_netif_qset_teardown_ifclassq(struct netif_qset *); |
54 | static void nx_netif_qset_init(struct netif_qset *, struct netif_llink *, |
55 | uint8_t idx, struct kern_nexus_netif_llink_qset_init *); |
56 | static struct netif_qset *nx_netif_qset_create(struct netif_llink *, |
57 | uint8_t, struct kern_nexus_netif_llink_qset_init *); |
58 | static void nx_netif_qset_destroy(struct netif_qset *); |
59 | static void nx_netif_llink_initialize(struct netif_llink *, struct nx_netif *, |
60 | struct kern_nexus_netif_llink_init *); |
61 | static void nx_netif_driver_queue_destroy(struct netif_queue *); |
62 | static void nx_netif_driver_queue_init(struct netif_qset *, |
63 | struct netif_queue *, kern_packet_svc_class_t, bool); |
64 | static struct netif_llink *nx_netif_llink_create_locked(struct nx_netif *, |
65 | struct kern_nexus_netif_llink_init *); |
66 | static void nx_netif_default_llink_add(struct nx_netif *); |
67 | static int netif_qset_enqueue_single(struct netif_qset *, |
68 | struct __kern_packet *, uint32_t *, uint32_t *); |
69 | static int nx_netif_llink_ext_init_queues(struct kern_nexus *, |
70 | struct netif_llink *); |
71 | static void nx_netif_llink_ext_fini_queues(struct kern_nexus *, |
72 | struct netif_llink *); |
73 | |
74 | static uint32_t nx_netif_random_qset = 0; |
75 | #if (DEVELOPMENT || DEBUG) |
76 | SYSCTL_UINT(_kern_skywalk_netif, OID_AUTO, random_qset, |
77 | CTLFLAG_RW | CTLFLAG_LOCKED, &nx_netif_random_qset, 0, |
78 | "pick a random qset" ); |
79 | #endif /* DEVELOPMENT || DEBUG */ |
80 | |
81 | /* retains a reference for the callee */ |
82 | static struct netif_llink * |
83 | nx_netif_llink_alloc(void) |
84 | { |
85 | struct netif_llink *llink; |
86 | |
87 | llink = sk_alloc_type(struct netif_llink, Z_WAITOK | Z_NOFAIL, |
88 | skmem_tag_netif_llink); |
89 | os_ref_init(&llink->nll_refcnt, NULL); |
90 | return llink; |
91 | } |
92 | |
93 | SK_NO_INLINE_ATTRIBUTE |
94 | void |
95 | nx_netif_llink_retain(struct netif_llink *llink) |
96 | { |
97 | os_ref_retain(rc: &llink->nll_refcnt); |
98 | } |
99 | |
100 | SK_NO_INLINE_ATTRIBUTE |
101 | static void |
102 | nx_netif_llink_free(struct netif_llink **pllink) |
103 | { |
104 | struct netif_llink *llink = *pllink; |
105 | struct netif_qset *qset, *tqset; |
106 | |
107 | VERIFY(llink->nll_state == NETIF_LLINK_STATE_DESTROYED); |
108 | *pllink = NULL; |
109 | SLIST_FOREACH_SAFE(qset, &llink->nll_qset_list, nqs_list, tqset) { |
110 | SLIST_REMOVE(&llink->nll_qset_list, qset, netif_qset, |
111 | nqs_list); |
112 | nx_netif_qset_destroy(qset); |
113 | } |
114 | if (llink->nll_ifcq != NULL) { |
115 | ifclassq_release(&llink->nll_ifcq); |
116 | } |
117 | |
118 | sk_free_type(struct netif_llink, llink); |
119 | } |
120 | |
121 | SK_NO_INLINE_ATTRIBUTE |
122 | void |
123 | nx_netif_llink_release(struct netif_llink **pllink) |
124 | { |
125 | struct netif_llink *llink = *pllink; |
126 | |
127 | *pllink = NULL; |
128 | if (os_ref_release(rc: &llink->nll_refcnt) == 0) { |
129 | nx_netif_llink_free(pllink: &llink); |
130 | } |
131 | } |
132 | |
133 | /* retains a reference for the callee */ |
134 | static struct netif_qset * |
135 | nx_netif_qset_alloc(uint8_t nrxqs, uint8_t ntxqs) |
136 | { |
137 | struct netif_qset *qset; |
138 | |
139 | _CASSERT(sizeof(struct netif_queue) % sizeof(uint64_t) == 0); |
140 | |
141 | qset = sk_alloc_type_header_array(struct netif_qset, struct netif_queue, |
142 | nrxqs + ntxqs, Z_WAITOK | Z_NOFAIL, nx_netif_tag_qset); |
143 | |
144 | qset->nqs_num_rx_queues = nrxqs; |
145 | qset->nqs_num_tx_queues = ntxqs; |
146 | return qset; |
147 | } |
148 | |
149 | SK_NO_INLINE_ATTRIBUTE |
150 | void |
151 | nx_netif_qset_retain(struct netif_qset *qset) |
152 | { |
153 | /* |
154 | * Logical link is immutable, i.e. Queue Sets can't added/removed |
155 | * from it. We will rely on this property to simply acquire a refcnt |
156 | * on the logical link, which is the parent structure of a qset. |
157 | */ |
158 | nx_netif_llink_retain(llink: qset->nqs_llink); |
159 | } |
160 | |
161 | SK_NO_INLINE_ATTRIBUTE |
162 | void |
163 | nx_netif_qset_release(struct netif_qset **pqset) |
164 | { |
165 | struct netif_qset *qset = *pqset; |
166 | struct netif_llink *llink = qset->nqs_llink; |
167 | |
168 | *pqset = NULL; |
169 | nx_netif_llink_release(pllink: &llink); |
170 | } |
171 | |
172 | SK_NO_INLINE_ATTRIBUTE |
173 | static void |
174 | nx_netif_qset_free(struct netif_qset **pqset) |
175 | { |
176 | struct netif_qset *qset = *pqset; |
177 | uint8_t i; |
178 | |
179 | VERIFY(qset->nqs_llink->nll_state == NETIF_LLINK_STATE_DESTROYED); |
180 | |
181 | for (i = 0; i < qset->nqs_num_rx_queues; i++) { |
182 | nx_netif_driver_queue_destroy(NETIF_QSET_RX_QUEUE(qset, i)); |
183 | } |
184 | for (i = 0; i < qset->nqs_num_tx_queues; i++) { |
185 | nx_netif_driver_queue_destroy(NETIF_QSET_TX_QUEUE(qset, i)); |
186 | } |
187 | if (qset->nqs_flags & NETIF_QSET_FLAG_AQM) { |
188 | nx_netif_qset_teardown_ifclassq(qset); |
189 | } |
190 | qset->nqs_llink = NULL; |
191 | sk_free_type_header_array(struct netif_qset, struct netif_queue, |
192 | qset->nqs_num_rx_queues + qset->nqs_num_tx_queues, qset); |
193 | } |
194 | |
195 | SK_NO_INLINE_ATTRIBUTE |
196 | static void |
197 | nx_netif_qset_destroy(struct netif_qset *qset) |
198 | { |
199 | VERIFY(qset->nqs_llink->nll_state == NETIF_LLINK_STATE_DESTROYED); |
200 | nx_netif_qset_free(pqset: &qset); |
201 | } |
202 | |
203 | SK_NO_INLINE_ATTRIBUTE |
204 | static void |
205 | nx_netif_qset_setup_ifclassq(struct netif_llink *llink, |
206 | struct netif_qset *qset) |
207 | { |
208 | uint8_t flags = 0; |
209 | |
210 | ASSERT((qset->nqs_flags & NETIF_QSET_FLAG_AQM) != 0); |
211 | ASSERT(llink->nll_ifcq != NULL); |
212 | |
213 | ifclassq_retain(llink->nll_ifcq); |
214 | qset->nqs_ifcq = llink->nll_ifcq; |
215 | |
216 | if ((qset->nqs_flags & NETIF_QSET_FLAG_LOW_LATENCY) != 0) { |
217 | flags |= IF_CLASSQ_LOW_LATENCY; |
218 | } |
219 | if ((qset->nqs_flags & NETIF_QSET_FLAG_DEFAULT) != 0) { |
220 | flags |= IF_DEFAULT_GRP; |
221 | } |
222 | |
223 | ifclassq_setup_group(ifcq: qset->nqs_ifcq, grp_idx: qset->nqs_idx, flags); |
224 | } |
225 | |
226 | SK_NO_INLINE_ATTRIBUTE |
227 | static void |
228 | nx_netif_qset_teardown_ifclassq(struct netif_qset *qset) |
229 | { |
230 | ASSERT((qset->nqs_flags & NETIF_QSET_FLAG_AQM) != 0); |
231 | ASSERT(qset->nqs_ifcq != NULL); |
232 | |
233 | qset->nqs_flags &= ~NETIF_QSET_FLAG_AQM; |
234 | ifclassq_release(&qset->nqs_ifcq); |
235 | } |
236 | |
237 | SK_NO_INLINE_ATTRIBUTE |
238 | static void |
239 | nx_netif_qset_init(struct netif_qset *qset, struct netif_llink *llink, |
240 | uint8_t idx, struct kern_nexus_netif_llink_qset_init *qset_init) |
241 | { |
242 | #define _NETIF_QSET_MAX_TXQS 4 |
243 | kern_packet_svc_class_t svc[_NETIF_QSET_MAX_TXQS] = |
244 | {KPKT_SC_BE, KPKT_SC_BK, KPKT_SC_VI, KPKT_SC_VO}; |
245 | struct ifnet *ifp = llink->nll_nif->nif_ifp; |
246 | uint8_t i; |
247 | |
248 | /* |
249 | * no need to retain a reference for llink, as the logical link is |
250 | * immutable and qsets are created and destroyed along with logical |
251 | * link. |
252 | */ |
253 | qset->nqs_llink = llink; |
254 | qset->nqs_id = NETIF_QSET_ID_ENCODE(llink->nll_link_id_internal, idx); |
255 | qset->nqs_idx = idx; |
256 | |
257 | if (qset_init->nlqi_flags & KERN_NEXUS_NET_LLINK_QSET_DEFAULT) { |
258 | qset->nqs_flags |= NETIF_QSET_FLAG_DEFAULT; |
259 | } |
260 | if (qset_init->nlqi_flags & KERN_NEXUS_NET_LLINK_QSET_LOW_LATENCY) { |
261 | qset->nqs_flags |= NETIF_QSET_FLAG_LOW_LATENCY; |
262 | } |
263 | if (qset_init->nlqi_flags & KERN_NEXUS_NET_LLINK_QSET_AQM) { |
264 | qset->nqs_flags |= NETIF_QSET_FLAG_AQM; |
265 | nx_netif_qset_setup_ifclassq(llink, qset); |
266 | } |
267 | |
268 | |
269 | for (i = 0; i < qset->nqs_num_rx_queues; i++) { |
270 | nx_netif_driver_queue_init(qset, NETIF_QSET_RX_QUEUE(qset, i), |
271 | KPKT_SC_UNSPEC, true); |
272 | } |
273 | |
274 | if (ifp->if_output_sched_model == IFNET_SCHED_MODEL_DRIVER_MANAGED) { |
275 | VERIFY(qset->nqs_num_tx_queues == _NETIF_QSET_MAX_TXQS); |
276 | for (i = 0; i < qset->nqs_num_tx_queues; i++) { |
277 | nx_netif_driver_queue_init(qset, |
278 | NETIF_QSET_TX_QUEUE(qset, i), svc[i], false); |
279 | } |
280 | } else { |
281 | for (i = 0; i < qset->nqs_num_tx_queues; i++) { |
282 | nx_netif_driver_queue_init(qset, |
283 | NETIF_QSET_TX_QUEUE(qset, i), KPKT_SC_UNSPEC, false); |
284 | } |
285 | } |
286 | } |
287 | |
288 | SK_NO_INLINE_ATTRIBUTE |
289 | static struct netif_qset * |
290 | nx_netif_qset_create(struct netif_llink *llink, uint8_t idx, |
291 | struct kern_nexus_netif_llink_qset_init *qset_init) |
292 | { |
293 | struct netif_qset *qset; |
294 | |
295 | qset = nx_netif_qset_alloc(nrxqs: qset_init->nlqi_num_rxqs, |
296 | ntxqs: qset_init->nlqi_num_txqs); |
297 | nx_netif_qset_init(qset, llink, idx, qset_init); |
298 | return qset; |
299 | } |
300 | |
301 | static uint16_t |
302 | nx_netif_generate_internal_llink_id(struct nx_netif *nif) |
303 | { |
304 | struct netif_llink *llink; |
305 | struct netif_stats *nifs = &nif->nif_stats; |
306 | uint16_t id; |
307 | |
308 | again: |
309 | id = (uint16_t)(random() % 65536); |
310 | STAILQ_FOREACH(llink, &nif->nif_llink_list, nll_link) { |
311 | if (__improbable(llink->nll_link_id_internal == id)) { |
312 | break; |
313 | } |
314 | } |
315 | if (__probable(llink == NULL && id != 0)) { |
316 | return id; |
317 | } else { |
318 | STATS_INC(nifs, NETIF_STATS_LLINK_DUP_INT_ID_GENERATED); |
319 | DTRACE_SKYWALK1(dup__llink__id__internal, uint16_t, id); |
320 | goto again; |
321 | } |
322 | } |
323 | |
324 | static void |
325 | nx_netif_llink_initialize(struct netif_llink *llink, struct nx_netif *nif, |
326 | struct kern_nexus_netif_llink_init *llink_init) |
327 | { |
328 | uint8_t i; |
329 | struct ifnet *ifp = nif->nif_ifp; |
330 | |
331 | LCK_RW_ASSERT(&nif->nif_llink_lock, LCK_RW_ASSERT_EXCLUSIVE); |
332 | |
333 | llink->nll_nif = nif; |
334 | llink->nll_link_id = llink_init->nli_link_id; |
335 | if (llink_init->nli_flags & KERN_NEXUS_NET_LLINK_DEFAULT) { |
336 | llink->nll_flags |= NETIF_LLINK_FLAG_DEFAULT; |
337 | } |
338 | llink->nll_link_id_internal = nx_netif_generate_internal_llink_id(nif); |
339 | llink->nll_ctx = llink_init->nli_ctx; |
340 | SLIST_INIT(&llink->nll_qset_list); |
341 | |
342 | for (i = 0; i < llink_init->nli_num_qsets; i++) { |
343 | if (llink->nll_ifcq == NULL && |
344 | (llink_init->nli_qsets[i].nlqi_flags & |
345 | KERN_NEXUS_NET_LLINK_QSET_AQM)) { |
346 | if (NETIF_DEFAULT_LLINK(llink)) { |
347 | /* use the default AQM queues from ifnet */ |
348 | ifclassq_retain(ifp->if_snd); |
349 | llink->nll_ifcq = ifp->if_snd; |
350 | } else { |
351 | llink->nll_ifcq = ifclassq_alloc(); |
352 | dlil_ifclassq_setup(ifp, llink->nll_ifcq); |
353 | } |
354 | } |
355 | |
356 | struct netif_qset *qset = nx_netif_qset_create(llink, idx: i, |
357 | qset_init: &llink_init->nli_qsets[i]); |
358 | /* nx_netif_qset_create retains a reference for the callee */ |
359 | SLIST_INSERT_HEAD(&llink->nll_qset_list, qset, nqs_list); |
360 | if (NETIF_DEFAULT_QSET(qset)) { |
361 | /* there can only be one default queue set */ |
362 | VERIFY(llink->nll_default_qset == NULL); |
363 | llink->nll_default_qset = qset; |
364 | } |
365 | } |
366 | llink->nll_qset_cnt = llink_init->nli_num_qsets; |
367 | /* there should be a default queue set */ |
368 | VERIFY(llink->nll_default_qset != NULL); |
369 | llink->nll_state = NETIF_LLINK_STATE_INIT; |
370 | } |
371 | |
372 | static void |
373 | nx_netif_driver_queue_destroy(struct netif_queue *drvq) |
374 | { |
375 | VERIFY(drvq->nq_qset->nqs_llink->nll_state == |
376 | NETIF_LLINK_STATE_DESTROYED); |
377 | |
378 | lck_mtx_lock(lck: &drvq->nq_lock); |
379 | VERIFY(KPKTQ_EMPTY(&drvq->nq_pktq)); |
380 | lck_mtx_unlock(lck: &drvq->nq_lock); |
381 | |
382 | drvq->nq_qset = NULL; |
383 | lck_mtx_destroy(lck: &drvq->nq_lock, grp: &netif_llink_lock_group); |
384 | } |
385 | |
386 | static void |
387 | nx_netif_driver_queue_init(struct netif_qset *qset, |
388 | struct netif_queue *drvq, kern_packet_svc_class_t svc, bool is_rx) |
389 | { |
390 | lck_mtx_init(lck: &drvq->nq_lock, grp: &netif_llink_lock_group, |
391 | attr: &netif_llink_lock_attr); |
392 | |
393 | lck_mtx_lock(lck: &drvq->nq_lock); |
394 | KPKTQ_INIT(&drvq->nq_pktq); |
395 | lck_mtx_unlock(lck: &drvq->nq_lock); |
396 | |
397 | /* |
398 | * no need to retain a reference for qset, as queue set is |
399 | * immutable and driver queue is part of the queue set data structure. |
400 | */ |
401 | drvq->nq_qset = qset; |
402 | drvq->nq_svc = svc; |
403 | if (is_rx) { |
404 | drvq->nq_flags |= NETIF_QUEUE_IS_RX; |
405 | } |
406 | } |
407 | |
408 | SK_NO_INLINE_ATTRIBUTE |
409 | static struct netif_llink * |
410 | nx_netif_llink_create_locked(struct nx_netif *nif, |
411 | struct kern_nexus_netif_llink_init *llink_init) |
412 | { |
413 | struct netif_llink *llink; |
414 | struct netif_stats *nifs = &nif->nif_stats; |
415 | |
416 | LCK_RW_ASSERT(&nif->nif_llink_lock, LCK_RW_ASSERT_EXCLUSIVE); |
417 | llink = nx_netif_llink_alloc(); |
418 | nx_netif_llink_initialize(llink, nif, llink_init); |
419 | /* nx_netif_llink_alloc retains a reference for the caller */ |
420 | STAILQ_INSERT_TAIL(&nif->nif_llink_list, llink, nll_link); |
421 | nif->nif_llink_cnt++; |
422 | STATS_INC(nifs, NETIF_STATS_LLINK_ADD); |
423 | if (NETIF_DEFAULT_LLINK(llink)) { |
424 | /* there can only be one default logical link */ |
425 | VERIFY(nif->nif_default_llink == NULL); |
426 | } |
427 | return llink; |
428 | } |
429 | |
430 | SK_NO_INLINE_ATTRIBUTE |
431 | static void |
432 | nx_netif_llink_destroy_locked(struct nx_netif *nif, struct netif_llink **pllink) |
433 | { |
434 | struct netif_stats *nifs = &nif->nif_stats; |
435 | |
436 | LCK_RW_ASSERT(&nif->nif_llink_lock, LCK_RW_ASSERT_EXCLUSIVE); |
437 | (*pllink)->nll_state = NETIF_LLINK_STATE_DESTROYED; |
438 | STAILQ_REMOVE(&nif->nif_llink_list, *pllink, netif_llink, nll_link); |
439 | nif->nif_llink_cnt--; |
440 | STATS_INC(nifs, NETIF_STATS_LLINK_REMOVE); |
441 | nx_netif_llink_release(pllink); |
442 | } |
443 | |
444 | int |
445 | nx_netif_llink_add(struct nx_netif *nif, |
446 | struct kern_nexus_netif_llink_init *llink_init, struct netif_llink **pllink) |
447 | { |
448 | int err; |
449 | struct netif_llink *llink; |
450 | struct netif_stats *nifs = &nif->nif_stats; |
451 | |
452 | *pllink = NULL; |
453 | lck_rw_lock_exclusive(lck: &nif->nif_llink_lock); |
454 | /* ensure logical_link_id is unique */ |
455 | STAILQ_FOREACH(llink, &nif->nif_llink_list, nll_link) { |
456 | if (llink->nll_link_id == llink_init->nli_link_id) { |
457 | SK_ERR("duplicate llink_id 0x%llu" , |
458 | llink_init->nli_link_id); |
459 | STATS_INC(nifs, NETIF_STATS_LLINK_DUP_ID_GIVEN); |
460 | DTRACE_SKYWALK1(dup__id__given, uint64_t, |
461 | llink_init->nli_link_id); |
462 | lck_rw_unlock_exclusive(lck: &nif->nif_llink_lock); |
463 | return EINVAL; |
464 | } |
465 | } |
466 | llink = nx_netif_llink_create_locked(nif, llink_init); |
467 | lck_rw_unlock_exclusive(lck: &nif->nif_llink_lock); |
468 | VERIFY(llink != NULL); |
469 | err = nx_netif_llink_ext_init_queues(nif->nif_nx, llink); |
470 | if (err != 0) { |
471 | lck_rw_lock_exclusive(lck: &nif->nif_llink_lock); |
472 | nx_netif_llink_destroy_locked(nif, pllink: &llink); |
473 | lck_rw_unlock_exclusive(lck: &nif->nif_llink_lock); |
474 | } else { |
475 | /* increment reference for the caller */ |
476 | nx_netif_llink_retain(llink); |
477 | *pllink = llink; |
478 | } |
479 | return err; |
480 | } |
481 | |
482 | int |
483 | nx_netif_llink_remove(struct nx_netif *nif, |
484 | kern_nexus_netif_llink_id_t llink_id) |
485 | { |
486 | bool llink_found = false; |
487 | struct netif_llink *llink; |
488 | struct netif_stats *nifs = &nif->nif_stats; |
489 | |
490 | lck_rw_lock_exclusive(lck: &nif->nif_llink_lock); |
491 | STAILQ_FOREACH(llink, &nif->nif_llink_list, nll_link) { |
492 | if (llink->nll_link_id == llink_id) { |
493 | llink_found = true; |
494 | break; |
495 | } |
496 | } |
497 | lck_rw_unlock_exclusive(lck: &nif->nif_llink_lock); |
498 | if (!llink_found) { |
499 | STATS_INC(nifs, NETIF_STATS_LLINK_NOT_FOUND_REMOVE); |
500 | DTRACE_SKYWALK1(not__found, uint64_t, llink_id); |
501 | return ENOENT; |
502 | } |
503 | nx_netif_llink_ext_fini_queues(nif->nif_nx, llink); |
504 | lck_rw_lock_exclusive(lck: &nif->nif_llink_lock); |
505 | nx_netif_llink_destroy_locked(nif, pllink: &llink); |
506 | lck_rw_unlock_exclusive(lck: &nif->nif_llink_lock); |
507 | return 0; |
508 | } |
509 | |
510 | static void |
511 | nx_netif_default_llink_add(struct nx_netif *nif) |
512 | { |
513 | struct kern_nexus_netif_llink_init llink_init, *pllink_init; |
514 | struct kern_nexus_netif_llink_qset_init qset; |
515 | struct ifnet *ifp = nif->nif_ifp; |
516 | struct netif_llink *llink; |
517 | |
518 | LCK_RW_ASSERT(&nif->nif_llink_lock, LCK_RW_ASSERT_EXCLUSIVE); |
519 | VERIFY(SKYWALK_NATIVE(ifp)); |
520 | |
521 | llink_init.nli_flags = KERN_NEXUS_NET_LLINK_DEFAULT; |
522 | |
523 | if (NX_LLINK_PROV(nif->nif_nx)) { |
524 | VERIFY(nif->nif_default_llink_params != NULL); |
525 | pllink_init = nif->nif_default_llink_params; |
526 | } else { |
527 | struct nexus_adapter *devna = |
528 | nx_port_get_na(nif->nif_nx, NEXUS_PORT_NET_IF_DEV); |
529 | |
530 | llink_init.nli_link_id = NETIF_LLINK_ID_DEFAULT; |
531 | qset.nlqi_flags = KERN_NEXUS_NET_LLINK_QSET_DEFAULT; |
532 | /* |
533 | * For the legacy mode of operation we will assume that |
534 | * AQM is not needed on low-latency interface. |
535 | */ |
536 | if (NETIF_IS_LOW_LATENCY(nif)) { |
537 | qset.nlqi_flags |= |
538 | KERN_NEXUS_NET_LLINK_QSET_LOW_LATENCY; |
539 | } else { |
540 | qset.nlqi_flags |= KERN_NEXUS_NET_LLINK_QSET_AQM; |
541 | } |
542 | qset.nlqi_num_rxqs = |
543 | (uint8_t)na_get_nrings(na: devna, t: NR_RX); |
544 | qset.nlqi_num_txqs = |
545 | (uint8_t)na_get_nrings(na: devna, t: NR_TX); |
546 | llink_init.nli_num_qsets = 1; |
547 | llink_init.nli_qsets = &qset; |
548 | llink_init.nli_ctx = NULL; |
549 | pllink_init = &llink_init; |
550 | } |
551 | llink = nx_netif_llink_create_locked(nif, llink_init: pllink_init); |
552 | /* there can only be one default logical link */ |
553 | VERIFY(nif->nif_default_llink == NULL); |
554 | nx_netif_llink_retain(llink); |
555 | /* obtain a reference for the default logical link pointer */ |
556 | nif->nif_default_llink = llink; |
557 | } |
558 | |
559 | static void |
560 | nx_netif_default_llink_remove(struct nx_netif *nif) |
561 | { |
562 | struct netif_llink *llink; |
563 | |
564 | LCK_RW_ASSERT(&nif->nif_llink_lock, LCK_RW_ASSERT_EXCLUSIVE); |
565 | ASSERT(nif->nif_default_llink != NULL); |
566 | ASSERT(nif->nif_llink_cnt == 1); |
567 | llink = nif->nif_default_llink; |
568 | nx_netif_llink_release(pllink: &nif->nif_default_llink); |
569 | ASSERT(nif->nif_default_llink == NULL); |
570 | nx_netif_llink_destroy_locked(nif, pllink: &llink); |
571 | } |
572 | |
573 | static int |
574 | netif_qset_enqueue_single(struct netif_qset *qset, struct __kern_packet *pkt, |
575 | uint32_t *flowctl, uint32_t *dropped) |
576 | { |
577 | struct ifnet *ifp = qset->nqs_ifcq->ifcq_ifp; |
578 | boolean_t pkt_drop = FALSE; |
579 | int err; |
580 | |
581 | /* |
582 | * we are using the first 4 bytes of flow_id as the AQM flow |
583 | * identifier. |
584 | */ |
585 | ASSERT(!uuid_is_null(pkt->pkt_flow_id)); |
586 | netif_ifp_inc_traffic_class_out_pkt(ifp, pkt->pkt_svc_class, |
587 | 1, pkt->pkt_length); |
588 | |
589 | if (__improbable(pkt->pkt_trace_id != 0)) { |
590 | KDBG(SK_KTRACE_PKT_TX_FSW | DBG_FUNC_END, pkt->pkt_trace_id); |
591 | KDBG(SK_KTRACE_PKT_TX_AQM | DBG_FUNC_START, pkt->pkt_trace_id); |
592 | } |
593 | |
594 | /* Only native path is supported */ |
595 | ASSERT((pkt->pkt_pflags & PKT_F_MBUF_DATA) == 0); |
596 | ASSERT(pkt->pkt_mbuf == NULL); |
597 | |
598 | err = ifnet_enqueue_ifcq_pkt(ifp, qset->nqs_ifcq, pkt, false, |
599 | &pkt_drop); |
600 | if (__improbable(err != 0)) { |
601 | if ((err == EQFULL || err == EQSUSPENDED) && flowctl != NULL) { |
602 | (*flowctl)++; |
603 | } |
604 | if (pkt_drop && dropped != NULL) { |
605 | (*dropped)++; |
606 | } |
607 | } |
608 | return err; |
609 | } |
610 | |
611 | int |
612 | netif_qset_enqueue(struct netif_qset *qset, struct __kern_packet *pkt_chain, |
613 | struct __kern_packet *tail, uint32_t cnt, uint32_t bytes, uint32_t *flowctl, |
614 | uint32_t *dropped) |
615 | { |
616 | #pragma unused(tail) |
617 | struct __kern_packet *pkt = pkt_chain; |
618 | struct __kern_packet *next; |
619 | struct netif_stats *nifs = &qset->nqs_llink->nll_nif->nif_stats; |
620 | uint32_t c = 0, b = 0, drop_cnt = 0, flowctl_cnt = 0; |
621 | int err = 0; |
622 | |
623 | /* drop packets if logical link state is destroyed */ |
624 | if (qset->nqs_llink->nll_state == NETIF_LLINK_STATE_DESTROYED) { |
625 | pp_free_packet_chain(pkt_chain, (int *)&drop_cnt); |
626 | STATS_ADD(nifs, NETIF_STATS_LLINK_TX_DROP_BAD_STATE, drop_cnt); |
627 | if (dropped != NULL) { |
628 | *dropped = drop_cnt; |
629 | } |
630 | return ENXIO; |
631 | } |
632 | |
633 | /* We don't support chains for now */ |
634 | while (pkt != NULL) { |
635 | next = pkt->pkt_nextpkt; |
636 | pkt->pkt_nextpkt = NULL; |
637 | c++; |
638 | b += pkt->pkt_length; |
639 | |
640 | (void) netif_qset_enqueue_single(qset, pkt, flowctl: &flowctl_cnt, |
641 | dropped: &drop_cnt); |
642 | pkt = next; |
643 | } |
644 | VERIFY(c == cnt); |
645 | VERIFY(b == bytes); |
646 | if (flowctl != NULL && flowctl_cnt > 0) { |
647 | *flowctl = flowctl_cnt; |
648 | STATS_ADD(nifs, NETIF_STATS_LLINK_AQM_QFULL, flowctl_cnt); |
649 | err = EIO; |
650 | } |
651 | if (dropped != NULL && drop_cnt > 0) { |
652 | *dropped = drop_cnt; |
653 | STATS_ADD(nifs, NETIF_STATS_LLINK_AQM_DROPPED, drop_cnt); |
654 | err = EIO; |
655 | } |
656 | return err; |
657 | } |
658 | |
659 | struct netif_qset * |
660 | nx_netif_get_default_qset_noref(struct nx_netif *nif) |
661 | { |
662 | struct netif_qset *qset; |
663 | struct netif_stats *nifs = &nif->nif_stats; |
664 | |
665 | ASSERT(NETIF_LLINK_ENABLED(nif)); |
666 | if (__improbable(nif->nif_default_llink->nll_state != |
667 | NETIF_LLINK_STATE_INIT)) { |
668 | STATS_INC(nifs, NETIF_STATS_LLINK_QSET_BAD_STATE); |
669 | DTRACE_SKYWALK1(llink__bad__state, struct nx_netif *, nif); |
670 | return NULL; |
671 | } |
672 | qset = nif->nif_default_llink->nll_default_qset; |
673 | return qset; |
674 | } |
675 | |
676 | static void |
677 | nx_netif_qset_hint_decode(uint64_t hint, |
678 | uint16_t *link_id_internal, uint16_t *qset_idx) |
679 | { |
680 | /* The top 32 bits are unused for now */ |
681 | *link_id_internal = (uint16_t)((0xffff0000 & hint) >> 16); |
682 | *qset_idx = (uint16_t)((0x0000ffff & hint)); |
683 | } |
684 | |
685 | /* retains a reference for the caller */ |
686 | static struct netif_qset * |
687 | nx_netif_get_default_qset(struct nx_netif *nif) |
688 | { |
689 | struct netif_qset *qset; |
690 | |
691 | qset = nif->nif_default_llink->nll_default_qset; |
692 | nx_netif_qset_retain(qset); |
693 | return qset; |
694 | } |
695 | |
696 | /* |
697 | * Find the qset based on the qset hint. Fall back to the default qset |
698 | * if not found. The random qset is used for experimentation. |
699 | */ |
700 | struct netif_qset * |
701 | nx_netif_find_qset(struct nx_netif *nif, uint64_t hint) |
702 | { |
703 | uint16_t ll_id_internal, qset_idx; |
704 | struct netif_llink *llink; |
705 | struct netif_qset *qset; |
706 | struct netif_stats *nifs = &nif->nif_stats; |
707 | int i, j, random_id; |
708 | |
709 | ASSERT(NETIF_LLINK_ENABLED(nif)); |
710 | if (__improbable(nif->nif_default_llink->nll_state != |
711 | NETIF_LLINK_STATE_INIT)) { |
712 | STATS_INC(nifs, NETIF_STATS_LLINK_QSET_BAD_STATE); |
713 | DTRACE_SKYWALK1(llink__bad__state, struct nx_netif *, nif); |
714 | return NULL; |
715 | } |
716 | if (!NX_LLINK_PROV(nif->nif_nx) || |
717 | (nx_netif_random_qset == 0 && hint == 0)) { |
718 | goto def_qset; |
719 | } |
720 | if (nx_netif_random_qset == 0) { |
721 | nx_netif_qset_hint_decode(hint, link_id_internal: &ll_id_internal, qset_idx: &qset_idx); |
722 | } else { |
723 | ll_id_internal = 0; |
724 | qset_idx = 0; |
725 | } |
726 | lck_rw_lock_shared(lck: &nif->nif_llink_lock); |
727 | i = 0; |
728 | random_id = random(); |
729 | STAILQ_FOREACH(llink, &nif->nif_llink_list, nll_link) { |
730 | if (nx_netif_random_qset != 0 && |
731 | (random_id % nif->nif_llink_cnt) == i) { |
732 | break; |
733 | } else if (llink->nll_link_id_internal == ll_id_internal) { |
734 | break; |
735 | } |
736 | i++; |
737 | } |
738 | if (llink == NULL) { |
739 | STATS_INC(nifs, NETIF_STATS_LLINK_HINT_NOT_USEFUL); |
740 | lck_rw_unlock_shared(lck: &nif->nif_llink_lock); |
741 | goto def_qset; |
742 | } |
743 | j = 0; |
744 | random_id = random(); |
745 | SLIST_FOREACH(qset, &llink->nll_qset_list, nqs_list) { |
746 | if (nx_netif_random_qset != 0 && |
747 | (random_id % llink->nll_qset_cnt) == j) { |
748 | break; |
749 | } else if (qset->nqs_idx == qset_idx) { |
750 | break; |
751 | } |
752 | j++; |
753 | } |
754 | if (qset == NULL) { |
755 | STATS_INC(nifs, NETIF_STATS_LLINK_HINT_NOT_USEFUL); |
756 | lck_rw_unlock_shared(lck: &nif->nif_llink_lock); |
757 | goto def_qset; |
758 | } |
759 | nx_netif_qset_retain(qset); |
760 | STATS_INC(nifs, NETIF_STATS_LLINK_NONDEF_QSET_USED); |
761 | lck_rw_unlock_shared(lck: &nif->nif_llink_lock); |
762 | if (nx_netif_random_qset != 0) { |
763 | SK_DF(SK_VERB_LLINK, "%s: random qset: qset %p, ifcq %p, " |
764 | "llink_idx %d, qset_idx %d" , if_name(nif->nif_ifp), |
765 | qset, qset->nqs_ifcq, i, j); |
766 | |
767 | DTRACE_SKYWALK5(random__qset, struct nx_netif *, nif, |
768 | struct netif_qset *, qset, struct ifclassq *, |
769 | qset->nqs_ifcq, int, i, int, j); |
770 | } else { |
771 | SK_DF(SK_VERB_LLINK, "%s: non-default qset: qset %p, ifcq %p, " |
772 | " ll_id_internal 0x%x, qset_idx %d" , if_name(nif->nif_ifp), |
773 | qset, qset->nqs_ifcq, ll_id_internal, qset_idx); |
774 | |
775 | DTRACE_SKYWALK5(nondef__qset, struct nx_netif *, nif, |
776 | struct netif_qset *, qset, struct ifclassq *, |
777 | qset->nqs_ifcq, uint16_t, ll_id_internal, |
778 | uint16_t, qset_idx); |
779 | } |
780 | return qset; |
781 | |
782 | def_qset: |
783 | STATS_INC(nifs, NETIF_STATS_LLINK_DEF_QSET_USED); |
784 | qset = nx_netif_get_default_qset(nif); |
785 | ASSERT(qset != NULL); |
786 | |
787 | SK_DF(SK_VERB_LLINK, "%s: default qset: qset %p, ifcq %p, hint %llx" , |
788 | if_name(nif->nif_ifp), qset, qset->nqs_ifcq, hint); |
789 | |
790 | DTRACE_SKYWALK4(def__qset, struct nx_netif *, nif, struct netif_qset *, |
791 | qset, struct ifclassq *, qset->nqs_ifcq, uint64_t, hint); |
792 | return qset; |
793 | } |
794 | |
795 | void |
796 | nx_netif_llink_init(struct nx_netif *nif) |
797 | { |
798 | ifnet_t ifp = nif->nif_ifp; |
799 | |
800 | #if (DEVELOPMENT || DEBUG) |
801 | if (__improbable(nx_netif_disable_llink != 0)) { |
802 | SK_DF(SK_VERB_LLINK, "%s: llink is disabled" , |
803 | if_name(nif->nif_ifp)); |
804 | return; |
805 | } |
806 | #endif /* (DEVELOPMENT || DEBUG) */ |
807 | |
808 | if (!SKYWALK_NATIVE(ifp)) { |
809 | SK_DF(SK_VERB_LLINK, |
810 | "%s: llink is supported on native devices only" , |
811 | if_name(ifp)); |
812 | return; |
813 | } |
814 | ASSERT(!NETIF_LLINK_ENABLED(nif)); |
815 | lck_rw_init(lck: &nif->nif_llink_lock, grp: &netif_llink_lock_group, |
816 | attr: &netif_llink_lock_attr); |
817 | |
818 | lck_rw_lock_exclusive(lck: &nif->nif_llink_lock); |
819 | |
820 | STAILQ_INIT(&nif->nif_llink_list); |
821 | nif->nif_llink_cnt = 0; |
822 | nx_netif_default_llink_add(nif); |
823 | nif->nif_flags |= NETIF_FLAG_LLINK_INITIALIZED; |
824 | |
825 | lck_rw_unlock_exclusive(lck: &nif->nif_llink_lock); |
826 | |
827 | SK_DF(SK_VERB_LLINK, "%s: llink initialized" , if_name(ifp)); |
828 | } |
829 | |
830 | void |
831 | nx_netif_llink_fini(struct nx_netif *nif) |
832 | { |
833 | if (!NETIF_LLINK_ENABLED(nif)) { |
834 | SK_DF(SK_VERB_LLINK, "%s: llink not initialized" , |
835 | if_name(nif->nif_ifp)); |
836 | return; |
837 | } |
838 | |
839 | lck_rw_lock_exclusive(lck: &nif->nif_llink_lock); |
840 | |
841 | nif->nif_flags &= ~NETIF_FLAG_LLINK_INITIALIZED; |
842 | nx_netif_default_llink_remove(nif); |
843 | ASSERT(nif->nif_llink_cnt == 0); |
844 | ASSERT(STAILQ_EMPTY(&nif->nif_llink_list)); |
845 | |
846 | lck_rw_unlock_exclusive(lck: &nif->nif_llink_lock); |
847 | |
848 | nx_netif_llink_config_free(nif); |
849 | lck_rw_destroy(lck: &nif->nif_llink_lock, grp: &netif_llink_lock_group); |
850 | SK_DF(SK_VERB_LLINK, "%s: llink uninitialization done" , |
851 | if_name(nif->nif_ifp)); |
852 | } |
853 | |
854 | int |
855 | nx_netif_validate_llink_config(struct kern_nexus_netif_llink_init *init, |
856 | bool default_llink) |
857 | { |
858 | struct kern_nexus_netif_llink_qset_init *qsinit; |
859 | bool has_default_qset = false; |
860 | bool default_llink_flag; |
861 | uint8_t i; |
862 | |
863 | default_llink_flag = |
864 | ((init->nli_flags & KERN_NEXUS_NET_LLINK_DEFAULT) != 0); |
865 | |
866 | if (default_llink != default_llink_flag) { |
867 | SK_ERR("default llink flag incompatible: default_llink(%s), " |
868 | "default_llink_flag(%s)" , |
869 | default_llink ? "true" : "false" , |
870 | default_llink_flag ? "true" : "false" ); |
871 | return EINVAL; |
872 | } |
873 | if (init->nli_num_qsets == 0) { |
874 | SK_ERR("num qsets is zero" ); |
875 | return EINVAL; |
876 | } |
877 | if ((qsinit = init->nli_qsets) == NULL) { |
878 | SK_ERR("qsets is NULL" ); |
879 | return EINVAL; |
880 | } |
881 | for (i = 0; i < init->nli_num_qsets; i++) { |
882 | if (qsinit[i].nlqi_flags & |
883 | KERN_NEXUS_NET_LLINK_QSET_DEFAULT) { |
884 | if (has_default_qset) { |
885 | SK_ERR("has more than one default qset" ); |
886 | return EINVAL; |
887 | } |
888 | if (qsinit[i].nlqi_num_rxqs == 0) { |
889 | SK_ERR("num_rxqs == 0" ); |
890 | return EINVAL; |
891 | } |
892 | has_default_qset = true; |
893 | } |
894 | if (qsinit[i].nlqi_num_txqs == 0) { |
895 | SK_ERR("num_txqs == 0" ); |
896 | return EINVAL; |
897 | } |
898 | if ((qsinit[i].nlqi_flags & |
899 | KERN_NEXUS_NET_LLINK_QSET_WMM_MODE) && |
900 | (qsinit[i].nlqi_num_txqs != NEXUS_NUM_WMM_QUEUES)) { |
901 | SK_ERR("invalid wmm mode" ); |
902 | return EINVAL; |
903 | } |
904 | } |
905 | return 0; |
906 | } |
907 | |
908 | int |
909 | nx_netif_default_llink_config(struct nx_netif *nif, |
910 | struct kern_nexus_netif_llink_init *init) |
911 | { |
912 | struct kern_nexus_netif_llink_qset_init *qsinit; |
913 | int i, err; |
914 | |
915 | err = nx_netif_validate_llink_config(init, true); |
916 | if (err != 0) { |
917 | return err; |
918 | } |
919 | nif->nif_default_llink_params = sk_alloc_type( |
920 | struct kern_nexus_netif_llink_init, |
921 | Z_WAITOK | Z_NOFAIL, nx_netif_tag_llink_cfg); |
922 | |
923 | qsinit = sk_alloc_type_array(struct kern_nexus_netif_llink_qset_init, |
924 | init->nli_num_qsets, Z_WAITOK, nx_netif_tag_llink_cfg); |
925 | if (qsinit == NULL) { |
926 | SK_ERR("failed to alloc kern_nexus_netif_llink_qset_init" ); |
927 | sk_free_type(struct kern_nexus_netif_llink_init, |
928 | nif->nif_default_llink_params); |
929 | nif->nif_default_llink_params = NULL; |
930 | return ENOMEM; |
931 | } |
932 | memcpy(dst: nif->nif_default_llink_params, src: init, |
933 | n: __builtin_offsetof(struct kern_nexus_netif_llink_init, |
934 | nli_qsets)); |
935 | for (i = 0; i < init->nli_num_qsets; i++) { |
936 | *(&qsinit[i]) = *(&init->nli_qsets[i]); |
937 | } |
938 | nif->nif_default_llink_params->nli_qsets = qsinit; |
939 | return 0; |
940 | } |
941 | |
942 | void |
943 | nx_netif_llink_config_free(struct nx_netif *nif) |
944 | { |
945 | if (nif->nif_default_llink_params == NULL) { |
946 | return; |
947 | } |
948 | sk_free_type_array(struct kern_nexus_netif_llink_qset_init, |
949 | nif->nif_default_llink_params->nli_num_qsets, |
950 | nif->nif_default_llink_params->nli_qsets); |
951 | nif->nif_default_llink_params->nli_qsets = NULL; |
952 | |
953 | sk_free_type(struct kern_nexus_netif_llink_init, |
954 | nif->nif_default_llink_params); |
955 | nif->nif_default_llink_params = NULL; |
956 | } |
957 | |
958 | static int |
959 | nx_netif_llink_ext_init_queues(struct kern_nexus *nx, struct netif_llink *llink) |
960 | { |
961 | struct kern_nexus_provider *nxprov = NX_PROV(nx); |
962 | struct kern_nexus_netif_provider_init *nxnpi; |
963 | struct netif_qset *qset; |
964 | struct netif_stats *nifs = &NX_NETIF_PRIVATE(nx)->nif_stats; |
965 | int err = 0; |
966 | uint8_t i; |
967 | |
968 | nxnpi = &nxprov->nxprov_netif_ext; |
969 | ASSERT(nxprov->nxprov_netif_ext.nxnpi_qset_init != NULL); |
970 | ASSERT(nxprov->nxprov_netif_ext.nxnpi_queue_init != NULL); |
971 | |
972 | SLIST_FOREACH(qset, &llink->nll_qset_list, nqs_list) { |
973 | struct netif_queue *drvq; |
974 | |
975 | ASSERT((qset->nqs_flags & NETIF_QSET_FLAG_EXT_INITED) == 0); |
976 | err = nxnpi->nxnpi_qset_init(nxprov, nx, llink->nll_ctx, |
977 | qset->nqs_idx, qset->nqs_id, qset, &qset->nqs_ctx); |
978 | if (err != 0) { |
979 | STATS_INC(nifs, NETIF_STATS_LLINK_QSET_INIT_FAIL); |
980 | SK_ERR("nx: 0x%llx, qset: %d, qset init err %d" , |
981 | SK_KVA(nx), qset->nqs_idx, err); |
982 | goto out; |
983 | } |
984 | qset->nqs_flags |= NETIF_QSET_FLAG_EXT_INITED; |
985 | |
986 | for (i = 0; i < qset->nqs_num_rx_queues; i++) { |
987 | drvq = NETIF_QSET_RX_QUEUE(qset, i); |
988 | |
989 | ASSERT((drvq->nq_flags & NETIF_QUEUE_EXT_INITED) == 0); |
990 | err = nxnpi->nxnpi_queue_init(nxprov, nx, qset->nqs_ctx, |
991 | i, false, drvq, &drvq->nq_ctx); |
992 | if (err != 0) { |
993 | STATS_INC(nifs, NETIF_STATS_LLINK_RXQ_INIT_FAIL); |
994 | SK_ERR("nx: 0x%llx qset: %d queue_init err %d" , |
995 | SK_KVA(nx), qset->nqs_idx, err); |
996 | goto out; |
997 | } |
998 | drvq->nq_flags |= NETIF_QUEUE_EXT_INITED; |
999 | } |
1000 | for (i = 0; i < qset->nqs_num_tx_queues; i++) { |
1001 | drvq = NETIF_QSET_TX_QUEUE(qset, i); |
1002 | |
1003 | ASSERT((drvq->nq_flags & NETIF_QUEUE_EXT_INITED) == 0); |
1004 | err = nxnpi->nxnpi_queue_init(nxprov, nx, qset->nqs_ctx, |
1005 | i, true, drvq, &drvq->nq_ctx); |
1006 | if (err != 0) { |
1007 | STATS_INC(nifs, NETIF_STATS_LLINK_TXQ_INIT_FAIL); |
1008 | SK_ERR("nx: 0x%llx qset: %d queue_init err %d" , |
1009 | SK_KVA(nx), qset->nqs_idx, err); |
1010 | goto out; |
1011 | } |
1012 | drvq->nq_flags |= NETIF_QUEUE_EXT_INITED; |
1013 | } |
1014 | } |
1015 | out: |
1016 | if (err != 0) { |
1017 | nx_netif_llink_ext_fini_queues(nx, llink); |
1018 | } |
1019 | return err; |
1020 | } |
1021 | |
1022 | static void |
1023 | nx_netif_llink_ext_fini_queues(struct kern_nexus *nx, struct netif_llink *llink) |
1024 | { |
1025 | struct kern_nexus_provider *nxprov = NX_PROV(nx); |
1026 | struct kern_nexus_netif_provider_init *nxnpi; |
1027 | struct netif_qset *qset; |
1028 | uint8_t i; |
1029 | |
1030 | nxnpi = &nxprov->nxprov_netif_ext; |
1031 | ASSERT(nxprov->nxprov_netif_ext.nxnpi_qset_fini != NULL); |
1032 | ASSERT(nxprov->nxprov_netif_ext.nxnpi_queue_fini != NULL); |
1033 | |
1034 | SLIST_FOREACH(qset, &llink->nll_qset_list, nqs_list) { |
1035 | struct netif_queue *drvq; |
1036 | |
1037 | for (i = 0; i < qset->nqs_num_rx_queues; i++) { |
1038 | drvq = NETIF_QSET_RX_QUEUE(qset, i); |
1039 | if ((drvq->nq_flags & NETIF_QUEUE_EXT_INITED) == 0) { |
1040 | continue; |
1041 | } |
1042 | nxnpi->nxnpi_queue_fini(nxprov, nx, drvq->nq_ctx); |
1043 | drvq->nq_flags &= ~NETIF_QUEUE_EXT_INITED; |
1044 | } |
1045 | for (i = 0; i < qset->nqs_num_tx_queues; i++) { |
1046 | drvq = NETIF_QSET_TX_QUEUE(qset, i); |
1047 | if ((drvq->nq_flags & NETIF_QUEUE_EXT_INITED) == 0) { |
1048 | continue; |
1049 | } |
1050 | nxnpi->nxnpi_queue_fini(nxprov, nx, drvq->nq_ctx); |
1051 | drvq->nq_flags &= ~NETIF_QUEUE_EXT_INITED; |
1052 | } |
1053 | if ((qset->nqs_flags & NETIF_QSET_FLAG_EXT_INITED) == 0) { |
1054 | continue; |
1055 | } |
1056 | nxnpi->nxnpi_qset_fini(nxprov, nx, qset->nqs_ctx); |
1057 | qset->nqs_flags &= ~NETIF_QSET_FLAG_EXT_INITED; |
1058 | } |
1059 | } |
1060 | |
1061 | int |
1062 | nx_netif_llink_ext_init_default_queues(struct kern_nexus *nx) |
1063 | { |
1064 | struct nx_netif *nif = NX_NETIF_PRIVATE(nx); |
1065 | return nx_netif_llink_ext_init_queues(nx, llink: nif->nif_default_llink); |
1066 | } |
1067 | |
1068 | void |
1069 | nx_netif_llink_ext_fini_default_queues(struct kern_nexus *nx) |
1070 | { |
1071 | struct nx_netif *nif = NX_NETIF_PRIVATE(nx); |
1072 | nx_netif_llink_ext_fini_queues(nx, llink: nif->nif_default_llink); |
1073 | } |
1074 | |