1 | /* |
2 | * Copyright (c) 2015-2023 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | /* |
30 | * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved. |
31 | * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. |
32 | * |
33 | * Redistribution and use in source and binary forms, with or without |
34 | * modification, are permitted provided that the following conditions |
35 | * are met: |
36 | * 1. Redistributions of source code must retain the above copyright |
37 | * notice, this list of conditions and the following disclaimer. |
38 | * 2. Redistributions in binary form must reproduce the above copyright |
39 | * notice, this list of conditions and the following disclaimer in the |
40 | * documentation and/or other materials provided with the distribution. |
41 | * |
42 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
43 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
44 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
45 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
46 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
47 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
48 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
49 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
50 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
51 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
52 | * SUCH DAMAGE. |
53 | */ |
54 | |
55 | #ifndef _SKYWALK_NEXUS_FLOWSWITCH_H_ |
56 | #define _SKYWALK_NEXUS_FLOWSWITCH_H_ |
57 | |
58 | #include <skywalk/os_skywalk_private.h> |
59 | #include <net/ethernet.h> |
60 | #include <net/if_vlan_var.h> |
61 | #include <netinet/ip6.h> |
62 | |
63 | #include <skywalk/nexus/flowswitch/flow/flow_var.h> |
64 | |
65 | #if CONFIG_NEXUS_FLOWSWITCH |
66 | /* Shared declarations for the flow switch. */ |
67 | #define NX_FSW_NAME "fsw" /* prefix for flow switch port name */ |
68 | |
69 | #define NX_FSW_MAXRINGS NX_MAX_NUM_RING_PAIR |
70 | #define NX_FSW_TXRINGSIZE 256 /* default TX ring size */ |
71 | #define NX_FSW_RXRINGSIZE 1024 /* default RX ring size */ |
72 | #define NX_FSW_AFRINGSIZE 256 /* default Alloc/Free ring size */ |
73 | |
74 | #define NX_FSW_CHUNK 64 /* port chunk */ |
75 | #define NX_FSW_CHUNK_FREE 0xffffffffffffffff /* entire chunk is free */ |
76 | |
77 | #define NX_FSW_VP_MIN NX_FSW_CHUNK |
78 | #define NX_FSW_VP_MAX 4096 /* up to 4k ports */ |
79 | #define NX_FSW_VP_NOPORT (NX_FSW_VP_MAX+1) |
80 | #define NX_FSW_VP_BROADCAST NX_FSW_VP_MAX |
81 | |
82 | #define NX_FSW_MINSLOTS 2 /* XXX unclear how many */ |
83 | #define NX_FSW_MAXSLOTS NX_MAX_NUM_SLOT_PER_RING /* max # of slots */ |
84 | |
85 | #define NX_FSW_TXBATCH 64 /* default TX batch size */ |
86 | #define NX_FSW_RXBATCH 32 /* default RX batch size */ |
87 | |
88 | #define NX_FSW_BUFSIZE (2 * 1024) /* default buffer size */ |
89 | |
90 | #define NX_FSW_MINBUFSIZE 512 /* min buffer size */ |
91 | #define NX_FSW_MAXBUFSIZE (32 * 1024) /* max buffer size */ |
92 | #define NX_FSW_MAXBUFFERS (4 * 1024) /* max number of buffers */ |
93 | /* max number of buffers for memory constrained device */ |
94 | #define NX_FSW_MAXBUFFERS_MEM_CONSTRAINED (2 * 1024) |
95 | /* default user buffer segment size for non memory-constrained device */ |
96 | #define NX_FSW_BUF_SEG_SIZE (32 * 1024) |
97 | |
98 | /* {min, def, max} values for large buffer size */ |
99 | #define NX_FSW_MIN_LARGE_BUFSIZE 0 |
100 | #define NX_FSW_DEF_LARGE_BUFSIZE (16 * 1024) |
101 | #define NX_FSW_MAX_LARGE_BUFSIZE (32 * 1024) |
102 | |
103 | /* |
104 | * TODO: adi@apple.com -- minimum buflets for now; we will need to |
105 | * have a way to adjust this based on the underlying interface's |
106 | * parameters, e.g. jumbo MTU, large segment offload, etc. |
107 | */ |
108 | #define NX_FSW_UMD_SIZE _USER_PACKET_SIZE(BUFLETS_MIN) |
109 | #define NX_FSW_KMD_SIZE _KERN_PACKET_SIZE(BUFLETS_MIN) |
110 | |
111 | #define NX_FSW_EVENT_RING_NUM 1 /* number of event rings */ |
112 | #define NX_FSW_EVENT_RING_SIZE 32 /* default event ring size */ |
113 | |
114 | |
115 | /* |
116 | * macro to check if TCP RX aggregation is enabled in flowswitch. |
117 | */ |
118 | #define NX_FSW_TCP_RX_AGG_ENABLED() (sk_fsw_rx_agg_tcp != 0) |
119 | |
120 | struct nx_flowswitch; |
121 | |
122 | /* |
123 | * Virtual port nexus adapter |
124 | */ |
125 | struct nexus_vp_adapter { |
126 | /* |
127 | * This is an overlay structure on nexus_adapter; |
128 | * make sure it contains 'up' as the first member. |
129 | */ |
130 | struct nexus_adapter vpna_up; |
131 | |
132 | /* |
133 | * Flow switch support: |
134 | * |
135 | * If the adapter is associated with a nexus port, vpna_fsw points |
136 | * to the flow switch this NA is attached to; vpna_nx_port is the |
137 | * port number used in the flow switch. Otherwise, vpna_fsw would |
138 | * be NULL and vpna_nx_port would be NEXUS_PORT_ANY. |
139 | */ |
140 | struct nx_flowswitch *vpna_fsw; |
141 | nexus_port_t vpna_nx_port; |
142 | uint16_t vpna_gencnt; |
143 | boolean_t vpna_retry; |
144 | boolean_t vpna_pid_bound; |
145 | boolean_t vpna_defunct; |
146 | pid_t vpna_pid; |
147 | }; |
148 | |
149 | #define VPNA(_na) ((struct nexus_vp_adapter *)(_na)) |
150 | |
151 | #define NEXUS_PROVIDER_FLOW_SWITCH "com.apple.nexus.flowswitch" |
152 | |
153 | /* fsw_state_flags */ |
154 | #define FSW_STATEF_QUIESCED 0x0001 |
155 | #define FSW_STATEF_NETAGENT_ADDED 0x0002 |
156 | #define FSW_STATEF_NETAGENT_ENABLED 0x0004 |
157 | |
158 | #define FSW_QUIESCED(_fsw) \ |
159 | (((_fsw)->fsw_state_flags & FSW_STATEF_QUIESCED) != 0) |
160 | |
161 | #define FSW_NETAGENT_ADDED(_fsw) \ |
162 | (((_fsw)->fsw_state_flags & FSW_STATEF_NETAGENT_ADDED) != 0) |
163 | |
164 | #define FSW_NETAGENT_ENABLED(_fsw) \ |
165 | (((_fsw)->fsw_state_flags & FSW_STATEF_NETAGENT_ENABLED) != 0) |
166 | |
167 | #if (DEVELOPMENT || DEBUG) |
168 | #define FSW_RPS_MAX_NTHREADS 64 |
169 | struct fsw_rps_thread { |
170 | lck_mtx_t frt_lock; |
171 | struct nx_flowswitch *frt_fsw; |
172 | struct thread *frt_thread; |
173 | struct pktq frt_pktq; |
174 | |
175 | uint32_t frt_idx; |
176 | uint32_t frt_flags; |
177 | uint32_t frt_requests; |
178 | }; |
179 | |
180 | #define FRT_RUNNING 0x00000001 |
181 | #define FRT_TERMINATEBLOCK 0x10000000 |
182 | #define FRT_TERMINATING 0x20000000 |
183 | #define FRT_TERMINATED 0x40000000 |
184 | #endif /* !DEVELOPMENT && !DEBUG */ |
185 | |
186 | typedef enum { |
187 | FSW_TSO_MODE_NONE, |
188 | FSW_TSO_MODE_HW, |
189 | FSW_TSO_MODE_SW |
190 | } fsw_tso_mode_t; |
191 | |
192 | /* |
193 | * nx_flowswitch is a descriptor for a flow switch instance. |
194 | * Interfaces for a flow switch are all in fsw_ports[]. |
195 | * The array has fixed size, an empty entry does not terminate |
196 | * the search, but lookups only occur on attach/detach so we |
197 | * don't mind if they are slow. |
198 | * |
199 | * The flow switch is non blocking on the transmit ports: excess |
200 | * packets are dropped if there is no room on the output port. |
201 | * |
202 | * fsw_lock protects accesses to the fsw_ports array. |
203 | * This is a rw lock (or equivalent). |
204 | */ |
205 | struct nx_flowswitch { |
206 | decl_lck_rw_data(, fsw_lock); |
207 | uint32_t fsw_tx_rings; |
208 | uint32_t fsw_rx_rings; |
209 | |
210 | struct kern_nexus *fsw_nx; |
211 | |
212 | /* packet type enqueued by the class queues */ |
213 | classq_pkt_type_t fsw_classq_enq_ptype; |
214 | boolean_t fsw_classq_enabled; |
215 | |
216 | /* packet copy routines */ |
217 | pkt_copy_from_pkt_t *fsw_pkt_copy_from_pkt; |
218 | pkt_copy_from_mbuf_t *fsw_pkt_copy_from_mbuf; |
219 | pkt_copy_to_mbuf_t *fsw_pkt_copy_to_mbuf; |
220 | |
221 | uint8_t fsw_frame_headroom; |
222 | uint32_t fsw_src_lla_gencnt; |
223 | uint32_t fsw_pending_nonviable; |
224 | uint32_t fsw_low_power_gencnt; |
225 | |
226 | /* The following are protected by fsw_lock. */ |
227 | struct flow_mgr *fsw_flow_mgr; |
228 | netagent_session_t fsw_agent_session; |
229 | uuid_t fsw_agent_uuid; |
230 | struct ifnet *fsw_ifp; /* host interface */ |
231 | struct nexus_adapter *fsw_nifna; /* netif adapter */ |
232 | uint32_t fsw_state_flags; /* FSW_STATEF_* */ |
233 | |
234 | |
235 | union { |
236 | uint64_t _buf[1]; |
237 | uint8_t _eth_src[ETHER_ADDR_LEN]; |
238 | } __fsw_slladdr __attribute((aligned(sizeof(uint64_t)))); |
239 | |
240 | #define fsw_slla __fsw_slladdr._buf |
241 | #define fsw_ether_shost __fsw_slladdr._eth_src |
242 | |
243 | int (*fsw_resolve)(struct nx_flowswitch *, struct flow_route *, |
244 | struct __kern_packet *); |
245 | void (*fsw_frame)(struct nx_flowswitch *, struct flow_route *, |
246 | struct __kern_packet *); |
247 | sa_family_t (*fsw_demux)(struct nx_flowswitch *, |
248 | struct __kern_packet *); |
249 | |
250 | struct fsw_stats fsw_stats; |
251 | |
252 | /* |
253 | * The host interface attachment to the flowswitch (fsw_ifp), as well |
254 | * as the netagent registration, are guarded by the flowswitch's RW |
255 | * lock. During fsw_flow_bind() time, we need to make sure they are |
256 | * valid before proceeding forward, but holding that RW lock across |
257 | * the routine is not possible since the thread may block if there |
258 | * are other threads performing fsw_flow_{bind,unbind} on the same |
259 | * flow owner bucket. To prevent fsw_dtor() from happening while |
260 | * fsw_flow_bind() is in progress, we need to have it wait until all |
261 | * pending flow binds are done. To do this we add a busy counter |
262 | * incremented at flow bind time, and use the lock for synchronization. |
263 | */ |
264 | decl_lck_mtx_data(, fsw_detach_barrier_lock); |
265 | uint32_t fsw_detach_flags; /* see fsw_DETACHF_* */ |
266 | uint32_t fsw_detach_barriers; |
267 | uint32_t fsw_detach_waiters; |
268 | |
269 | uint32_t fsw_ifp_dlt; |
270 | |
271 | /* |
272 | * largest allocated packet size. |
273 | * used by: mbuf batch allocation logic during netif copy. |
274 | */ |
275 | uint32_t fsw_rx_largest_size; |
276 | |
277 | void (*fsw_ctor)(struct nx_flowswitch *, struct flow_route *); |
278 | |
279 | /* store stats from na that is going to be deactivated */ |
280 | struct __nx_stats_fsw *fsw_closed_na_stats; |
281 | |
282 | /* ip fragments manager */ |
283 | struct fsw_ip_frag_mgr *fsw_ipfm; |
284 | |
285 | struct skoid fsw_skoid; |
286 | |
287 | /* input network emulator */ |
288 | struct netem *fsw_input_netem; |
289 | |
290 | struct kern_channel *fsw_dev_ch; |
291 | struct kern_channel *fsw_host_ch; |
292 | |
293 | /* |
294 | * The reaper thread gets scheduled on-demand, whenever there |
295 | * is any lingering flow entry needing to be freed or becoming |
296 | * nonviable. Access is protected by fsw_reap_lock. |
297 | */ |
298 | decl_lck_mtx_data(, fsw_reap_lock); |
299 | uint32_t fsw_reap_flags; /* see fsw_REAPF_* */ |
300 | uint32_t fsw_reap_requests; |
301 | struct thread *fsw_reap_thread; |
302 | char fsw_reap_name[MAXTHREADNAMESIZE]; |
303 | |
304 | uint64_t fsw_reap_last; |
305 | uint64_t fsw_drain_channel_chk_last; |
306 | uint64_t fsw_drain_netif_chk_last; |
307 | |
308 | decl_lck_mtx_data(, fsw_linger_lock); |
309 | struct flow_entry_linger_head fsw_linger_head; |
310 | uint32_t fsw_linger_cnt; |
311 | |
312 | fsw_tso_mode_t fsw_tso_mode; |
313 | uint32_t fsw_tso_sw_mtu; |
314 | |
315 | #if (DEVELOPMENT || DEBUG) |
316 | uint32_t fsw_rps_nthreads; |
317 | struct fsw_rps_thread *fsw_rps_threads; |
318 | #endif /* !DEVELOPMENT && !DEBUG */ |
319 | }; |
320 | |
321 | #define NX_FSW_PRIVATE(_nx) ((struct nx_flowswitch *)(_nx)->nx_arg) |
322 | |
323 | #define FSW_RWINIT(_fsw) \ |
324 | lck_rw_init(&(_fsw)->fsw_lock, &nexus_lock_group, &nexus_lock_attr) |
325 | #define FSW_WLOCK(_fsw) \ |
326 | lck_rw_lock_exclusive(&(_fsw)->fsw_lock) |
327 | #define FSW_WUNLOCK(_fsw) \ |
328 | lck_rw_unlock_exclusive(&(_fsw)->fsw_lock) |
329 | #define FSW_WLOCKTORLOCK(_fsw) \ |
330 | lck_rw_lock_exclusive_to_shared(&(_fsw)->fsw_lock) |
331 | #define FSW_RLOCK(_fsw) \ |
332 | lck_rw_lock_shared(&(_fsw)->fsw_lock) |
333 | #define FSW_RLOCKTOWLOCK(_fsw) \ |
334 | lck_rw_lock_shared_to_exclusive(&(_fsw)->fsw_lock) |
335 | #define FSW_RTRYLOCK(_fsw) \ |
336 | lck_rw_try_lock(&(_fsw)->fsw_lock, LCK_RW_TYPE_SHARED) |
337 | #define FSW_RUNLOCK(_fsw) \ |
338 | lck_rw_unlock_shared(&(_fsw)->fsw_lock) |
339 | #define FSW_UNLOCK(_fsw) \ |
340 | lck_rw_done(&(_fsw)->fsw_lock) |
341 | #define FSW_RWDESTROY(_fsw) \ |
342 | lck_rw_destroy(&(_fsw)->fsw_lock, &nexus_lock_group) |
343 | #define FSW_WLOCK_ASSERT_HELD(_fsw) \ |
344 | LCK_RW_ASSERT(&(_fsw)->fsw_lock, LCK_RW_ASSERT_EXCLUSIVE) |
345 | #define FSW_RLOCK_ASSERT_HELD(_fsw) \ |
346 | LCK_RW_ASSERT(&(_fsw)->fsw_lock, LCK_RW_ASSERT_SHARED) |
347 | #define FSW_LOCK_ASSERT_HELD(_fsw) \ |
348 | LCK_RW_ASSERT(&(_fsw)->fsw_lock, LCK_RW_ASSERT_HELD) |
349 | |
350 | extern struct nxdom nx_flowswitch_dom_s; |
351 | extern struct kern_nexus_domain_provider nx_fsw_prov_s; |
352 | |
353 | SYSCTL_DECL(_kern_skywalk_flowswitch); |
354 | |
355 | /* functions used by external modules to interface with flow switch */ |
356 | __BEGIN_DECLS |
357 | extern int nx_fsw_na_find(struct kern_nexus *, struct kern_channel *, |
358 | struct chreq *, struct nxbind *, struct proc *, struct nexus_adapter **, |
359 | boolean_t); |
360 | extern boolean_t nx_fsw_dom_port_is_reserved(struct kern_nexus *nx, |
361 | nexus_port_t nx_port); |
362 | extern int nx_fsw_netagent_add(struct kern_nexus *nx); |
363 | extern int nx_fsw_netagent_remove(struct kern_nexus *nx); |
364 | extern void nx_fsw_netagent_update(struct kern_nexus *nx); |
365 | extern void fsw_devna_rx(struct nexus_adapter *, struct __kern_packet *, |
366 | struct nexus_pkt_stats *); |
367 | extern struct nx_flowswitch *fsw_ifp_to_fsw(struct ifnet *); |
368 | extern void fsw_get_tso_capabilities(struct ifnet *, uint32_t *, uint32_t *); |
369 | __END_DECLS |
370 | #endif /* CONFIG_NEXUS_FLOWSWITCH */ |
371 | #endif /* _SKYWALK_NEXUS_FLOWSWITCH_H_ */ |
372 | |