1 | /* |
2 | * Copyright (c) 2015-2023 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _SKYWALK_NEXUS_FLOWSWITCH_FSWVAR_H_ |
30 | #define _SKYWALK_NEXUS_FLOWSWITCH_FSWVAR_H_ |
31 | |
32 | #include <skywalk/os_skywalk_private.h> |
33 | #include <skywalk/nexus/flowswitch/nx_flowswitch.h> |
34 | #include <skywalk/nexus/flowswitch/flow/flow_var.h> |
35 | |
36 | #include <net/if_var.h> |
37 | #include <net/network_agent.h> |
38 | #include <net/necp.h> |
39 | #include <net/pktap.h> |
40 | |
41 | #define FSW_VP_DEV 0 /* device port */ |
42 | #define FSW_VP_HOST 1 /* host port (MS) */ |
43 | #define FSW_VP_USER_MIN 2 /* start of user vp port num */ |
44 | #define FSW_VP_USER_MAX NX_FSW_VP_MAX /* end of user vp port num */ |
45 | /* max flush batch size for device port */ |
46 | #if !XNU_TARGET_OS_OSX |
47 | #define FSW_VP_DEV_BATCH_MAX 16 |
48 | #else /* XNU_TARGET_OS_OSX */ |
49 | #define FSW_VP_DEV_BATCH_MAX 32 |
50 | #endif /* XNU_TARGET_OS_OSX */ |
51 | |
52 | #define FSW_REAP_THREADNAME "skywalk_fsw_reap_%s%s" |
53 | |
54 | struct fsw_ip_frag_mgr; /* forward declaration */ |
55 | extern uint32_t fsw_ip_reass; |
56 | struct pktq; |
57 | |
58 | #define FSW_DETACHF_DETACHING 0x10000000 /* detach in progress */ |
59 | #define FSW_DETACHF_DETACHED 0x20000000 /* detached */ |
60 | |
61 | #define FSW_REAPF_RUNNING 0x00000001 /* thread is running */ |
62 | #define FSW_REAPF_TERMINATEBLOCK 0x20000000 /* blocked waiting terminate */ |
63 | #define FSW_REAPF_TERMINATING 0x40000000 /* thread is terminating */ |
64 | #define FSW_REAPF_TERMINATED 0x80000000 /* thread is terminated */ |
65 | |
66 | extern kern_allocation_name_t skmem_tag_fsw_ports; |
67 | extern kern_allocation_name_t skmem_tag_fsw_ft; |
68 | extern kern_allocation_name_t skmem_tag_fsw_fb_hash; |
69 | extern kern_allocation_name_t skmem_tag_fsw_fob_hash; |
70 | extern kern_allocation_name_t skmem_tag_fsw_frb_hash; |
71 | extern kern_allocation_name_t skmem_tag_fsw_frib_hash; |
72 | extern kern_allocation_name_t skmem_tag_fsw_frag_mgr; |
73 | |
74 | __BEGIN_DECLS |
75 | |
76 | // generic |
77 | extern void fsw_init(void); |
78 | extern void fsw_uninit(void); |
79 | extern struct nx_flowswitch * fsw_alloc(zalloc_flags_t); |
80 | extern void fsw_free(struct nx_flowswitch *fsw); |
81 | extern int fsw_grow(struct nx_flowswitch *fsw, uint32_t grow); |
82 | extern int fsw_port_find(struct nx_flowswitch *fsw, nexus_port_t first, |
83 | nexus_port_t last, nexus_port_t *nx_port); |
84 | extern int fsw_port_bind(struct nx_flowswitch *fsw, nexus_port_t nx_port, |
85 | struct nxbind *nxb0); |
86 | extern int fsw_port_unbind(struct nx_flowswitch *fsw, nexus_port_t nx_port); |
87 | extern int fsw_port_na_defunct(struct nx_flowswitch *fsw, |
88 | struct nexus_vp_adapter *vpna); |
89 | extern size_t fsw_mib_get(struct nx_flowswitch *fsw, |
90 | struct nexus_mib_filter *filter, void *out, size_t len, struct proc *p); |
91 | extern int fsw_attach_vp(struct kern_nexus *nx, struct kern_channel *ch, |
92 | struct chreq *chr, struct nxbind *nxb, struct proc *p, |
93 | struct nexus_vp_adapter **vpna); |
94 | extern int fsw_ctl(struct kern_nexus *nx, nxcfg_cmd_t nc_cmd, struct proc *p, |
95 | void *data); |
96 | extern int fsw_ctl_detach(struct kern_nexus *nx, struct proc *p, |
97 | struct nx_spec_req *nsr); |
98 | extern boolean_t fsw_should_drop_packet(boolean_t is_input, sa_family_t af, |
99 | uint8_t proto, const char *ifname); |
100 | extern int fsw_port_alloc(struct nx_flowswitch *fsw, struct nxbind *nxb, |
101 | struct nexus_vp_adapter **vpna, nexus_port_t nx_port, struct proc *p, |
102 | boolean_t ifattach, boolean_t host); |
103 | extern void fsw_port_free(struct nx_flowswitch *fsw, |
104 | struct nexus_vp_adapter *vpna, nexus_port_t nx_port, boolean_t defunct); |
105 | extern int fsw_port_grow(struct nx_flowswitch *fsw, uint32_t num_ports); |
106 | extern int fsw_port_na_activate(struct nx_flowswitch *fsw, |
107 | struct nexus_vp_adapter *vpna, na_activate_mode_t mode); |
108 | extern boolean_t fsw_detach_barrier_add(struct nx_flowswitch *fsw); |
109 | extern void fsw_detach_barrier_remove(struct nx_flowswitch *fsw); |
110 | |
111 | // vp related |
112 | extern int fsw_vp_na_activate(struct nexus_adapter *na, |
113 | na_activate_mode_t mode); |
114 | extern int fsw_vp_na_krings_create(struct nexus_adapter *na, |
115 | struct kern_channel *ch); |
116 | extern void fsw_vp_na_krings_delete(struct nexus_adapter *na, |
117 | struct kern_channel *ch, boolean_t defunct); |
118 | extern int fsw_vp_na_txsync(struct __kern_channel_ring *kring, |
119 | struct proc *p, uint32_t flags); |
120 | extern int fsw_vp_na_rxsync(struct __kern_channel_ring *kring, |
121 | struct proc *p, uint32_t flags); |
122 | extern int fsw_vp_na_create(struct kern_nexus *nx, struct chreq *chr, |
123 | struct proc *p, struct nexus_vp_adapter **ret); |
124 | extern void fsw_vp_channel_error_stats_fold(struct fsw_stats *fs, |
125 | struct __nx_stats_channel_errors *es); |
126 | extern errno_t fsw_vp_na_channel_event(struct nx_flowswitch *fsw, |
127 | uint32_t nx_port_id, struct __kern_channel_event *event, |
128 | uint16_t event_len); |
129 | |
130 | // classq related |
131 | extern void fsw_classq_setup(struct nx_flowswitch *fsw, |
132 | struct nexus_adapter *hostna); |
133 | extern void fsw_classq_teardown(struct nx_flowswitch *fsw, |
134 | struct nexus_adapter *hostna); |
135 | extern struct mbuf * fsw_classq_kpkt_to_mbuf(struct nx_flowswitch *fsw, |
136 | struct __kern_packet *pkt); |
137 | |
138 | // routing related |
139 | extern int fsw_generic_resolve(struct nx_flowswitch *fsw, struct flow_route *fr, |
140 | struct __kern_packet *pkt); |
141 | |
142 | // data plane related |
143 | extern int fsw_dp_init(void); |
144 | extern void fsw_dp_uninit(void); |
145 | extern int fsw_dp_ctor(struct nx_flowswitch *fsw); |
146 | extern void fsw_dp_dtor(struct nx_flowswitch *fsw); |
147 | extern void fsw_ring_flush(struct nx_flowswitch *fsw, |
148 | struct __kern_channel_ring *skring, struct proc *p); |
149 | extern void fsw_ring_enqueue_tail_drop(struct nx_flowswitch *fsw, |
150 | struct __kern_channel_ring *ring, struct pktq *pktq); |
151 | extern boolean_t fsw_detach_barrier_add(struct nx_flowswitch *fsw); |
152 | extern void fsw_detach_barrier_remove(struct nx_flowswitch *fsw); |
153 | extern void fsw_linger_insert(struct flow_entry *fsw); |
154 | extern void fsw_linger_purge(struct nx_flowswitch *fsw); |
155 | extern void fsw_reap_sched(struct nx_flowswitch *fsw); |
156 | |
157 | extern int fsw_dev_input_netem_dequeue(void *handle, pktsched_pkt_t *pkts, |
158 | uint32_t n_pkts); |
159 | extern void fsw_snoop(struct nx_flowswitch *fsw, struct flow_entry *fe, |
160 | bool input); |
161 | |
162 | extern void fsw_receive(struct nx_flowswitch *fsw, struct pktq *pktq); |
163 | extern void dp_flow_tx_process(struct nx_flowswitch *fsw, |
164 | struct flow_entry *fe, uint32_t flags); |
165 | extern void dp_flow_rx_process(struct nx_flowswitch *fsw, |
166 | struct flow_entry *fe, uint32_t flags); |
167 | |
168 | #if (DEVELOPMENT || DEBUG) |
169 | extern int fsw_rps_set_nthreads(struct nx_flowswitch* fsw, uint32_t n); |
170 | #endif /* !DEVELOPMENT && !DEBUG */ |
171 | |
172 | extern uint32_t fsw_tx_batch; |
173 | extern uint32_t fsw_rx_batch; |
174 | extern uint32_t fsw_chain_enqueue; |
175 | extern uint32_t fsw_use_dual_sized_pool; |
176 | |
177 | // flow related |
178 | extern struct flow_owner * fsw_flow_add(struct nx_flowswitch *fsw, |
179 | struct nx_flow_req *req0, int *error); |
180 | extern int fsw_flow_del(struct nx_flowswitch *fsw, struct nx_flow_req *req, |
181 | bool nolinger, void *params); |
182 | extern int fsw_flow_config(struct nx_flowswitch *fsw, struct nx_flow_req *req); |
183 | extern void fsw_flow_abort_tcp(struct nx_flowswitch *fsw, struct flow_entry *fe, |
184 | struct __kern_packet *pkt); |
185 | extern void fsw_flow_abort_quic(struct flow_entry *fe, uint8_t *token); |
186 | extern struct __kern_channel_ring * fsw_flow_get_rx_ring(struct nx_flowswitch *fsw, |
187 | struct flow_entry *fe); |
188 | extern bool dp_flow_rx_route_process(struct nx_flowswitch *fsw, |
189 | struct flow_entry *fe); |
190 | // stats related |
191 | extern void fsw_fold_stats(struct nx_flowswitch *fsw, void *data, |
192 | nexus_stats_type_t type); |
193 | |
194 | // netagent related |
195 | extern int fsw_netagent_add_remove(struct kern_nexus *nx, boolean_t add); |
196 | extern void fsw_netagent_update(struct kern_nexus *nx); |
197 | extern int fsw_netagent_register(struct nx_flowswitch *fsw, struct ifnet *ifp); |
198 | extern void fsw_netagent_unregister(struct nx_flowswitch *fsw, |
199 | struct ifnet *ifp); |
200 | |
201 | // interface related |
202 | extern int fsw_ip_setup(struct nx_flowswitch *fsw, struct ifnet *ifp); |
203 | extern int fsw_cellular_setup(struct nx_flowswitch *fsw, struct ifnet *ifp); |
204 | extern int fsw_ethernet_setup(struct nx_flowswitch *fsw, struct ifnet *ifp); |
205 | extern void fsw_classq_setup(struct nx_flowswitch *fsw, |
206 | struct nexus_adapter *hostna); |
207 | extern void fsw_classq_teardown(struct nx_flowswitch *fsw, |
208 | struct nexus_adapter *hostna); |
209 | extern void fsw_qos_mark(struct nx_flowswitch *fsw, struct flow_entry *fe, |
210 | struct __kern_packet *pkt); |
211 | extern boolean_t fsw_qos_default_restricted(void); |
212 | extern struct mbuf * fsw_classq_kpkt_to_mbuf(struct nx_flowswitch *fsw, |
213 | struct __kern_packet *pkt); |
214 | extern sa_family_t fsw_ip_demux(struct nx_flowswitch *, struct __kern_packet *); |
215 | |
216 | // fragment reassembly related |
217 | extern struct fsw_ip_frag_mgr * fsw_ip_frag_mgr_create( |
218 | struct nx_flowswitch *fsw, struct ifnet *ifp, size_t f_limit); |
219 | extern void fsw_ip_frag_mgr_destroy(struct fsw_ip_frag_mgr *mgr); |
220 | extern int fsw_ip_frag_reass_v4(struct fsw_ip_frag_mgr *mgr, |
221 | struct __kern_packet **pkt, struct ip *ip4, uint16_t *nfrags, |
222 | uint16_t *tlen); |
223 | extern int fsw_ip_frag_reass_v6(struct fsw_ip_frag_mgr *mgr, |
224 | struct __kern_packet **pkt, struct ip6_hdr *ip6, struct ip6_frag *ip6f, |
225 | uint16_t *nfrags, uint16_t *tlen); |
226 | |
227 | __END_DECLS |
228 | |
229 | __attribute__((always_inline)) |
230 | static inline void |
231 | fsw_snoop_and_dequeue(struct flow_entry *fe, struct pktq *target, bool input) |
232 | { |
233 | if (pktap_total_tap_count != 0) { |
234 | fsw_snoop(fsw: fe->fe_fsw, fe, input); |
235 | } |
236 | KPKTQ_CONCAT(target, input ? &fe->fe_rx_pktq : &fe->fe_tx_pktq); |
237 | } |
238 | |
239 | #define FSW_STATS_VAL(x) STATS_VAL(&fsw->fsw_stats, x) |
240 | #define FSW_STATS_INC(x) STATS_INC(&fsw->fsw_stats, x) |
241 | #define FSW_STATS_ADD(x, n) STATS_ADD(&fsw->fsw_stats, x, n) |
242 | |
243 | #endif /* _SKYWALK_NEXUS_FLOWSWITCH_FSWVAR_H_ */ |
244 | |