1 | /* |
2 | * Copyright (c) 2011-2017 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _NET_CLASSQ_IF_CLASSQ_H_ |
30 | #define _NET_CLASSQ_IF_CLASSQ_H_ |
31 | |
32 | #ifdef PRIVATE |
33 | #define IFCQ_SC_MAX 10 /* max number of queues */ |
34 | |
35 | #ifdef BSD_KERNEL_PRIVATE |
36 | #include <net/classq/classq.h> |
37 | |
38 | /* maximum number of packets stored across all queues */ |
39 | #define IFCQ_DEFAULT_PKT_DROP_LIMIT 2048 |
40 | |
41 | /* classq request types */ |
42 | typedef enum cqrq { |
43 | CLASSQRQ_PURGE = 1, /* purge all packets */ |
44 | CLASSQRQ_PURGE_SC = 2, /* purge service class (and flow) */ |
45 | CLASSQRQ_EVENT = 3, /* interface events */ |
46 | CLASSQRQ_THROTTLE = 4, /* throttle packets */ |
47 | CLASSQRQ_STAT_SC = 5, /* get service class queue stats */ |
48 | } cqrq_t; |
49 | |
50 | /* classq purge_sc request argument */ |
51 | typedef struct cqrq_purge_sc { |
52 | mbuf_svc_class_t sc; /* (in) service class */ |
53 | u_int32_t flow; /* (in) 0 means all flows */ |
54 | u_int32_t packets; /* (out) purged packets */ |
55 | u_int32_t bytes; /* (out) purged bytes */ |
56 | } cqrq_purge_sc_t; |
57 | |
58 | /* classq throttle request argument */ |
59 | typedef struct cqrq_throttle { |
60 | u_int32_t set; /* set or get */ |
61 | u_int32_t level; /* (in/out) throttling level */ |
62 | } cqrq_throttle_t; |
63 | |
64 | /* classq service class stats request argument */ |
65 | typedef struct cqrq_stat_sc { |
66 | mbuf_svc_class_t sc; /* (in) service class */ |
67 | u_int32_t packets; /* (out) packets enqueued */ |
68 | u_int32_t bytes; /* (out) bytes enqueued */ |
69 | } cqrq_stat_sc_t; |
70 | |
71 | /* |
72 | * A token-bucket regulator limits the rate that a network driver can |
73 | * dequeue packets from the output queue. Modern cards are able to buffer |
74 | * a large amount of packets and dequeue too many packets at a time. This |
75 | * bursty dequeue behavior makes it impossible to schedule packets by |
76 | * queueing disciplines. A token-bucket is used to control the burst size |
77 | * in a device independent manner. |
78 | */ |
79 | struct tb_regulator { |
80 | u_int64_t tbr_rate_raw; /* (unscaled) token bucket rate */ |
81 | u_int32_t tbr_percent; /* token bucket rate in percentage */ |
82 | int64_t tbr_rate; /* (scaled) token bucket rate */ |
83 | int64_t tbr_depth; /* (scaled) token bucket depth */ |
84 | |
85 | int64_t tbr_token; /* (scaled) current token */ |
86 | int64_t tbr_filluptime; /* (scaled) time to fill up bucket */ |
87 | u_int64_t tbr_last; /* last time token was updated */ |
88 | |
89 | /* needed for poll-and-dequeue */ |
90 | }; |
91 | |
92 | /* simple token bucket meter profile */ |
93 | struct tb_profile { |
94 | u_int64_t rate; /* rate in bit-per-sec */ |
95 | u_int32_t percent; /* rate in percentage */ |
96 | u_int32_t depth; /* depth in bytes */ |
97 | }; |
98 | |
99 | struct ifclassq; |
100 | enum cqdq_op; |
101 | enum cqrq; |
102 | |
103 | typedef int (*ifclassq_enq_func)(struct ifclassq *, void *, classq_pkt_type_t, |
104 | boolean_t *); |
105 | typedef void *(*ifclassq_deq_func)(struct ifclassq *, classq_pkt_type_t *); |
106 | typedef void *(*ifclassq_deq_sc_func)(struct ifclassq *, |
107 | mbuf_svc_class_t, classq_pkt_type_t *); |
108 | typedef int (*ifclassq_deq_multi_func)(struct ifclassq *, u_int32_t, |
109 | u_int32_t, void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); |
110 | typedef int (*ifclassq_deq_sc_multi_func)(struct ifclassq *, |
111 | mbuf_svc_class_t, u_int32_t, u_int32_t, void **, void **, |
112 | u_int32_t *, u_int32_t *, classq_pkt_type_t *); |
113 | typedef int (*ifclassq_req_func)(struct ifclassq *, enum cqrq, void *); |
114 | |
115 | /* |
116 | * Structure defining a queue for a network interface. |
117 | */ |
118 | struct ifclassq { |
119 | decl_lck_mtx_data(, ifcq_lock); |
120 | |
121 | struct ifnet *ifcq_ifp; /* back pointer to interface */ |
122 | u_int32_t ifcq_len; /* packet count */ |
123 | u_int32_t ifcq_maxlen; |
124 | struct pktcntr ifcq_xmitcnt; |
125 | struct pktcntr ifcq_dropcnt; |
126 | |
127 | u_int32_t ifcq_type; /* scheduler type */ |
128 | u_int32_t ifcq_flags; /* flags */ |
129 | u_int32_t ifcq_sflags; /* scheduler flags */ |
130 | u_int32_t ifcq_target_qdelay; /* target queue delay */ |
131 | u_int32_t ifcq_bytes; /* bytes count */ |
132 | u_int32_t ifcq_pkt_drop_limit; |
133 | void *ifcq_disc; /* for scheduler-specific use */ |
134 | /* |
135 | * ifcq_disc_slots[] represents the leaf classes configured for the |
136 | * corresponding discpline/scheduler, ordered by their corresponding |
137 | * service class index. Each slot holds the queue ID used to identify |
138 | * the class instance, as well as the class instance pointer itself. |
139 | * The latter is used during enqueue and dequeue in order to avoid the |
140 | * costs associated with looking up the class pointer based on the |
141 | * queue ID. The queue ID is used when querying the statistics from |
142 | * user space. |
143 | * |
144 | * Avoiding the use of queue ID during enqueue and dequeue is made |
145 | * possible by virtue of knowing the particular mbuf service class |
146 | * associated with the packets. The service class index of the |
147 | * packet is used as the index to ifcq_disc_slots[]. |
148 | * |
149 | * ifcq_disc_slots[] therefore also acts as a lookup table which |
150 | * provides for the mapping between MBUF_SC values and the actual |
151 | * scheduler classes. |
152 | */ |
153 | struct ifclassq_disc_slot { |
154 | u_int32_t qid; |
155 | void *cl; |
156 | } ifcq_disc_slots[IFCQ_SC_MAX]; /* for discipline use */ |
157 | |
158 | ifclassq_enq_func ifcq_enqueue; |
159 | ifclassq_deq_func ifcq_dequeue; |
160 | ifclassq_deq_sc_func ifcq_dequeue_sc; |
161 | ifclassq_deq_multi_func ifcq_dequeue_multi; |
162 | ifclassq_deq_sc_multi_func ifcq_dequeue_sc_multi; |
163 | ifclassq_req_func ifcq_request; |
164 | |
165 | /* token bucket regulator */ |
166 | struct tb_regulator ifcq_tbr; /* TBR */ |
167 | }; |
168 | |
169 | /* ifcq_flags */ |
170 | #define IFCQF_READY 0x01 /* ifclassq supports discipline */ |
171 | #define IFCQF_ENABLED 0x02 /* ifclassq is in use */ |
172 | #define IFCQF_TBR 0x04 /* Token Bucket Regulator is in use */ |
173 | |
174 | #define IFCQ_IS_READY(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_READY) |
175 | #define IFCQ_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_ENABLED) |
176 | #define IFCQ_TBR_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_TBR) |
177 | |
178 | /* classq enqueue return value */ |
179 | /* packet has to be dropped */ |
180 | #define CLASSQEQ_DROP (-1) |
181 | /* packet successfully enqueued */ |
182 | #define CLASSQEQ_SUCCESS 0 |
183 | /* packet enqueued; give flow control feedback */ |
184 | #define CLASSQEQ_SUCCESS_FC 1 |
185 | /* packet needs to be dropped due to flowcontrol; give flow control feedback */ |
186 | #define CLASSQEQ_DROP_FC 2 |
187 | /* packet needs to be dropped due to suspension; give flow control feedback */ |
188 | #define CLASSQEQ_DROP_SP 3 |
189 | |
190 | /* interface event argument for CLASSQRQ_EVENT */ |
191 | typedef enum cqev { |
192 | CLASSQ_EV_INIT = 0, |
193 | CLASSQ_EV_LINK_BANDWIDTH = 1, /* link bandwidth has changed */ |
194 | CLASSQ_EV_LINK_LATENCY = 2, /* link latency has changed */ |
195 | CLASSQ_EV_LINK_MTU = 3, /* link MTU has changed */ |
196 | CLASSQ_EV_LINK_UP = 4, /* link is now up */ |
197 | CLASSQ_EV_LINK_DOWN = 5, /* link is now down */ |
198 | } cqev_t; |
199 | #endif /* BSD_KERNEL_PRIVATE */ |
200 | |
201 | #include <net/pktsched/pktsched_tcq.h> |
202 | #include <net/pktsched/pktsched_qfq.h> |
203 | #include <net/pktsched/pktsched_fq_codel.h> |
204 | |
205 | #ifdef __cplusplus |
206 | extern "C" { |
207 | #endif |
208 | struct if_ifclassq_stats { |
209 | u_int32_t ifqs_len; |
210 | u_int32_t ifqs_maxlen; |
211 | struct pktcntr ifqs_xmitcnt; |
212 | struct pktcntr ifqs_dropcnt; |
213 | u_int32_t ifqs_scheduler; |
214 | union { |
215 | struct tcq_classstats ifqs_tcq_stats; |
216 | struct qfq_classstats ifqs_qfq_stats; |
217 | struct fq_codel_classstats ifqs_fq_codel_stats; |
218 | }; |
219 | } __attribute__((aligned(8))); |
220 | |
221 | #ifdef __cplusplus |
222 | } |
223 | #endif |
224 | |
225 | #ifdef BSD_KERNEL_PRIVATE |
226 | /* |
227 | * For ifclassq lock |
228 | */ |
229 | #define IFCQ_LOCK_ASSERT_HELD(_ifcq) \ |
230 | LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_OWNED) |
231 | |
232 | #define IFCQ_LOCK_ASSERT_NOTHELD(_ifcq) \ |
233 | LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_NOTOWNED) |
234 | |
235 | #define IFCQ_LOCK(_ifcq) \ |
236 | lck_mtx_lock(&(_ifcq)->ifcq_lock) |
237 | |
238 | #define IFCQ_LOCK_SPIN(_ifcq) \ |
239 | lck_mtx_lock_spin(&(_ifcq)->ifcq_lock) |
240 | |
241 | #define IFCQ_CONVERT_LOCK(_ifcq) do { \ |
242 | IFCQ_LOCK_ASSERT_HELD(_ifcq); \ |
243 | lck_mtx_convert_spin(&(_ifcq)->ifcq_lock); \ |
244 | } while (0) |
245 | |
246 | #define IFCQ_UNLOCK(_ifcq) \ |
247 | lck_mtx_unlock(&(_ifcq)->ifcq_lock) |
248 | |
249 | /* |
250 | * For ifclassq operations |
251 | */ |
252 | #define IFCQ_ENQUEUE(_ifq, _p, _t, _err, _drop) do { \ |
253 | (_err) = (*(_ifq)->ifcq_enqueue)(_ifq, _p, _t, _drop); \ |
254 | } while (0) |
255 | |
256 | #define IFCQ_DEQUEUE(_ifq, _p, _t) do { \ |
257 | (_p) = (*(_ifq)->ifcq_dequeue)(_ifq, _t); \ |
258 | } while (0) |
259 | |
260 | #define IFCQ_DEQUEUE_SC(_ifq, _sc, _p, _t) do { \ |
261 | (_p) = (*(_ifq)->ifcq_dequeue_sc)(_ifq, _sc, _t); \ |
262 | } while (0) |
263 | |
264 | #define IFCQ_TBR_DEQUEUE(_ifcq, _p, _t) do { \ |
265 | (_p) = ifclassq_tbr_dequeue(_ifcq, _t); \ |
266 | } while (0) |
267 | |
268 | #define IFCQ_TBR_DEQUEUE_SC(_ifcq, _sc, _p, _t) do { \ |
269 | (_p) = ifclassq_tbr_dequeue_sc(_ifcq, _sc, _t); \ |
270 | } while (0) |
271 | |
272 | #define IFCQ_PURGE(_ifq) do { \ |
273 | (void) (*(_ifq)->ifcq_request)(_ifq, CLASSQRQ_PURGE, NULL); \ |
274 | } while (0) |
275 | |
276 | #define IFCQ_PURGE_SC(_ifq, _sc, _flow, _packets, _bytes) do { \ |
277 | cqrq_purge_sc_t _req = { _sc, _flow, 0, 0 }; \ |
278 | (void) (*(_ifq)->ifcq_request)(_ifq, CLASSQRQ_PURGE_SC, &_req); \ |
279 | (_packets) = _req.packets; \ |
280 | (_bytes) = _req.bytes; \ |
281 | } while (0) |
282 | |
283 | #define IFCQ_UPDATE(_ifq, _ev) do { \ |
284 | (void) (*(_ifq)->ifcq_request)(_ifq, CLASSQRQ_EVENT, \ |
285 | (void *)(_ev)); \ |
286 | } while (0) |
287 | |
288 | #define IFCQ_SET_THROTTLE(_ifq, _level, _err) do { \ |
289 | cqrq_throttle_t _req = { 1, _level }; \ |
290 | (_err) = (*(_ifq)->ifcq_request) \ |
291 | (_ifq, CLASSQRQ_THROTTLE, &_req); \ |
292 | } while (0) |
293 | |
294 | #define IFCQ_GET_THROTTLE(_ifq, _level, _err) do { \ |
295 | cqrq_throttle_t _req = { 0, IFNET_THROTTLE_OFF }; \ |
296 | (_err) = (*(_ifq)->ifcq_request) \ |
297 | (_ifq, CLASSQRQ_THROTTLE, &_req); \ |
298 | (_level) = _req.level; \ |
299 | } while (0) |
300 | |
301 | #define IFCQ_LEN_SC(_ifq, _sc, _packets, _bytes, _err) do { \ |
302 | cqrq_stat_sc_t _req = { _sc, 0, 0 }; \ |
303 | (_err) = (*(ifq)->ifcq_request)(_ifq, CLASSQRQ_STAT_SC, &_req); \ |
304 | if ((_packets) != NULL) \ |
305 | (*(_packets)) = _req.packets; \ |
306 | if ((_bytes) != NULL) \ |
307 | (*(_bytes)) = _req.bytes; \ |
308 | } while (0) |
309 | |
310 | #define IFCQ_LEN(_ifcq) ((_ifcq)->ifcq_len) |
311 | #define IFCQ_QFULL(_ifcq) (IFCQ_LEN(_ifcq) >= (_ifcq)->ifcq_maxlen) |
312 | #define IFCQ_IS_EMPTY(_ifcq) (IFCQ_LEN(_ifcq) == 0) |
313 | #define IFCQ_INC_LEN(_ifcq) (IFCQ_LEN(_ifcq)++) |
314 | #define IFCQ_DEC_LEN(_ifcq) (IFCQ_LEN(_ifcq)--) |
315 | #define IFCQ_MAXLEN(_ifcq) ((_ifcq)->ifcq_maxlen) |
316 | #define IFCQ_SET_MAXLEN(_ifcq, _len) ((_ifcq)->ifcq_maxlen = (_len)) |
317 | #define IFCQ_TARGET_QDELAY(_ifcq) ((_ifcq)->ifcq_target_qdelay) |
318 | #define IFCQ_BYTES(_ifcq) ((_ifcq)->ifcq_bytes) |
319 | #define IFCQ_INC_BYTES(_ifcq, _len) \ |
320 | ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes + (_len)) |
321 | #define IFCQ_DEC_BYTES(_ifcq, _len) \ |
322 | ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes - (_len)) |
323 | |
324 | #define IFCQ_XMIT_ADD(_ifcq, _pkt, _len) do { \ |
325 | PKTCNTR_ADD(&(_ifcq)->ifcq_xmitcnt, _pkt, _len); \ |
326 | } while (0) |
327 | |
328 | #define IFCQ_DROP_ADD(_ifcq, _pkt, _len) do { \ |
329 | PKTCNTR_ADD(&(_ifcq)->ifcq_dropcnt, _pkt, _len); \ |
330 | } while (0) |
331 | |
332 | #define IFCQ_PKT_DROP_LIMIT(_ifcq) ((_ifcq)->ifcq_pkt_drop_limit) |
333 | |
334 | extern int ifclassq_setup(struct ifnet *, u_int32_t, boolean_t); |
335 | extern void ifclassq_teardown(struct ifnet *); |
336 | extern int ifclassq_pktsched_setup(struct ifclassq *); |
337 | extern void ifclassq_set_maxlen(struct ifclassq *, u_int32_t); |
338 | extern u_int32_t ifclassq_get_maxlen(struct ifclassq *); |
339 | extern int ifclassq_get_len(struct ifclassq *, mbuf_svc_class_t, |
340 | u_int32_t *, u_int32_t *); |
341 | extern errno_t ifclassq_enqueue(struct ifclassq *, void *, classq_pkt_type_t, |
342 | boolean_t *); |
343 | extern errno_t ifclassq_dequeue(struct ifclassq *, u_int32_t, u_int32_t, |
344 | void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); |
345 | extern errno_t ifclassq_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, |
346 | u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *, |
347 | classq_pkt_type_t *); |
348 | extern void *ifclassq_poll(struct ifclassq *, classq_pkt_type_t *); |
349 | extern void *ifclassq_poll_sc(struct ifclassq *, mbuf_svc_class_t, |
350 | classq_pkt_type_t *); |
351 | extern void ifclassq_update(struct ifclassq *, cqev_t); |
352 | extern int ifclassq_attach(struct ifclassq *, u_int32_t, void *, |
353 | ifclassq_enq_func, ifclassq_deq_func, ifclassq_deq_sc_func, |
354 | ifclassq_deq_multi_func, ifclassq_deq_sc_multi_func, ifclassq_req_func); |
355 | extern int ifclassq_detach(struct ifclassq *); |
356 | extern int ifclassq_getqstats(struct ifclassq *, u_int32_t, |
357 | void *, u_int32_t *); |
358 | extern const char *ifclassq_ev2str(cqev_t); |
359 | extern int ifclassq_tbr_set(struct ifclassq *, struct tb_profile *, boolean_t); |
360 | extern void *ifclassq_tbr_dequeue(struct ifclassq *, classq_pkt_type_t *); |
361 | extern void *ifclassq_tbr_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, |
362 | classq_pkt_type_t *); |
363 | extern void ifclassq_calc_target_qdelay(struct ifnet *ifp, |
364 | u_int64_t *if_target_qdelay); |
365 | extern void ifclassq_calc_update_interval(u_int64_t *update_interval); |
366 | extern void ifclassq_set_packet_metadata(struct ifclassq *ifq, |
367 | struct ifnet *ifp, void *p, classq_pkt_type_t ptype); |
368 | extern void ifclassq_reap_caches(boolean_t); |
369 | |
370 | #endif /* BSD_KERNEL_PRIVATE */ |
371 | #endif /* PRIVATE */ |
372 | #endif /* _NET_CLASSQ_IF_CLASSQ_H_ */ |
373 | |