1 | /* |
2 | * Copyright (c) 2011-2017 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <sys/cdefs.h> |
30 | |
31 | #include <sys/param.h> |
32 | #include <sys/malloc.h> |
33 | #include <sys/mbuf.h> |
34 | #include <sys/systm.h> |
35 | #include <sys/kernel.h> |
36 | #include <sys/errno.h> |
37 | #include <sys/mcache.h> |
38 | #include <sys/sysctl.h> |
39 | |
40 | #include <net/if.h> |
41 | #include <net/if_var.h> |
42 | #include <net/if_dl.h> |
43 | #include <net/if_types.h> |
44 | #include <net/net_osdep.h> |
45 | #include <net/pktsched/pktsched.h> |
46 | #include <net/pktsched/pktsched_tcq.h> |
47 | #include <net/pktsched/pktsched_qfq.h> |
48 | #include <net/pktsched/pktsched_fq_codel.h> |
49 | |
50 | #include <pexpert/pexpert.h> |
51 | |
52 | |
53 | u_int32_t machclk_freq = 0; |
54 | u_int64_t machclk_per_sec = 0; |
55 | u_int32_t pktsched_verbose; /* more noise if greater than 1 */ |
56 | |
57 | static void init_machclk(void); |
58 | |
59 | SYSCTL_NODE(_net, OID_AUTO, pktsched, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "pktsched" ); |
60 | |
61 | SYSCTL_UINT(_net_pktsched, OID_AUTO, verbose, CTLFLAG_RW|CTLFLAG_LOCKED, |
62 | &pktsched_verbose, 0, "Packet scheduler verbosity level" ); |
63 | |
64 | void |
65 | pktsched_init(void) |
66 | { |
67 | init_machclk(); |
68 | if (machclk_freq == 0) { |
69 | panic("%s: no CPU clock available!\n" , __func__); |
70 | /* NOTREACHED */ |
71 | } |
72 | |
73 | tcq_init(); |
74 | qfq_init(); |
75 | } |
76 | |
77 | static void |
78 | init_machclk(void) |
79 | { |
80 | /* |
81 | * Initialize machclk_freq using the timerbase frequency |
82 | * value from device specific info. |
83 | */ |
84 | machclk_freq = gPEClockFrequencyInfo.timebase_frequency_hz; |
85 | |
86 | clock_interval_to_absolutetime_interval(1, NSEC_PER_SEC, |
87 | &machclk_per_sec); |
88 | } |
89 | |
90 | u_int64_t |
91 | pktsched_abs_to_nsecs(u_int64_t abstime) |
92 | { |
93 | u_int64_t nsecs; |
94 | |
95 | absolutetime_to_nanoseconds(abstime, &nsecs); |
96 | return (nsecs); |
97 | } |
98 | |
99 | u_int64_t |
100 | pktsched_nsecs_to_abstime(u_int64_t nsecs) |
101 | { |
102 | u_int64_t abstime; |
103 | |
104 | nanoseconds_to_absolutetime(nsecs, &abstime); |
105 | return (abstime); |
106 | } |
107 | |
108 | int |
109 | pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags, |
110 | classq_pkt_type_t ptype) |
111 | { |
112 | int error = 0; |
113 | u_int32_t rflags; |
114 | |
115 | IFCQ_LOCK_ASSERT_HELD(ifq); |
116 | |
117 | VERIFY(machclk_freq != 0); |
118 | |
119 | /* Nothing to do unless the scheduler type changes */ |
120 | if (ifq->ifcq_type == scheduler) |
121 | return (0); |
122 | |
123 | /* |
124 | * Remember the flags that need to be restored upon success, as |
125 | * they may be cleared when we tear down existing scheduler. |
126 | */ |
127 | rflags = (ifq->ifcq_flags & IFCQF_ENABLED); |
128 | |
129 | if (ifq->ifcq_type != PKTSCHEDT_NONE) { |
130 | (void) pktsched_teardown(ifq); |
131 | |
132 | /* Teardown should have succeeded */ |
133 | VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); |
134 | VERIFY(ifq->ifcq_disc == NULL); |
135 | VERIFY(ifq->ifcq_enqueue == NULL); |
136 | VERIFY(ifq->ifcq_dequeue == NULL); |
137 | VERIFY(ifq->ifcq_dequeue_sc == NULL); |
138 | VERIFY(ifq->ifcq_request == NULL); |
139 | } |
140 | |
141 | switch (scheduler) { |
142 | case PKTSCHEDT_TCQ: |
143 | error = tcq_setup_ifclassq(ifq, sflags, ptype); |
144 | break; |
145 | |
146 | case PKTSCHEDT_QFQ: |
147 | error = qfq_setup_ifclassq(ifq, sflags, ptype); |
148 | break; |
149 | case PKTSCHEDT_FQ_CODEL: |
150 | error = fq_if_setup_ifclassq(ifq, sflags, ptype); |
151 | break; |
152 | default: |
153 | error = ENXIO; |
154 | break; |
155 | } |
156 | |
157 | if (error == 0) |
158 | ifq->ifcq_flags |= rflags; |
159 | |
160 | return (error); |
161 | } |
162 | |
163 | int |
164 | pktsched_teardown(struct ifclassq *ifq) |
165 | { |
166 | int error = 0; |
167 | |
168 | IFCQ_LOCK_ASSERT_HELD(ifq); |
169 | |
170 | if_qflush(ifq->ifcq_ifp, 1); |
171 | VERIFY(IFCQ_IS_EMPTY(ifq)); |
172 | |
173 | ifq->ifcq_flags &= ~IFCQF_ENABLED; |
174 | |
175 | switch (ifq->ifcq_type) { |
176 | case PKTSCHEDT_NONE: |
177 | break; |
178 | |
179 | case PKTSCHEDT_TCQ: |
180 | error = tcq_teardown_ifclassq(ifq); |
181 | break; |
182 | |
183 | case PKTSCHEDT_QFQ: |
184 | error = qfq_teardown_ifclassq(ifq); |
185 | break; |
186 | |
187 | case PKTSCHEDT_FQ_CODEL: |
188 | error = fq_if_teardown_ifclassq(ifq); |
189 | break; |
190 | default: |
191 | error = ENXIO; |
192 | break; |
193 | } |
194 | return (error); |
195 | } |
196 | |
197 | int |
198 | pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid, |
199 | struct if_ifclassq_stats *ifqs) |
200 | { |
201 | int error; |
202 | |
203 | IFCQ_LOCK_ASSERT_HELD(ifq); |
204 | |
205 | switch (ifq->ifcq_type) { |
206 | case PKTSCHEDT_TCQ: |
207 | error = tcq_getqstats_ifclassq(ifq, qid, ifqs); |
208 | break; |
209 | |
210 | case PKTSCHEDT_QFQ: |
211 | error = qfq_getqstats_ifclassq(ifq, qid, ifqs); |
212 | break; |
213 | |
214 | case PKTSCHEDT_FQ_CODEL: |
215 | error = fq_if_getqstats_ifclassq(ifq, qid, ifqs); |
216 | break; |
217 | default: |
218 | error = ENXIO; |
219 | break; |
220 | } |
221 | |
222 | return (error); |
223 | } |
224 | |
225 | void |
226 | pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_type_t ptype, void *pp) |
227 | { |
228 | pkt->pktsched_ptype = ptype; |
229 | pkt->pktsched_pkt = pp; |
230 | |
231 | switch (ptype) { |
232 | case QP_MBUF: |
233 | pkt->pktsched_plen = |
234 | (uint32_t)m_pktlen((struct mbuf *)pkt->pktsched_pkt); |
235 | break; |
236 | |
237 | |
238 | default: |
239 | VERIFY(0); |
240 | /* NOTREACHED */ |
241 | } |
242 | } |
243 | |
244 | void |
245 | pktsched_free_pkt(pktsched_pkt_t *pkt) |
246 | { |
247 | switch (pkt->pktsched_ptype) { |
248 | case QP_MBUF: |
249 | m_freem(pkt->pktsched_pkt); |
250 | break; |
251 | |
252 | |
253 | default: |
254 | VERIFY(0); |
255 | /* NOTREACHED */ |
256 | } |
257 | |
258 | pkt->pktsched_pkt = NULL; |
259 | pkt->pktsched_plen = 0; |
260 | pkt->pktsched_ptype = 0; |
261 | } |
262 | |
263 | uint32_t |
264 | pktsched_get_pkt_len(pktsched_pkt_t *pkt) |
265 | { |
266 | return (pkt->pktsched_plen); |
267 | } |
268 | |
269 | mbuf_svc_class_t |
270 | pktsched_get_pkt_svc(pktsched_pkt_t *pkt) |
271 | { |
272 | mbuf_svc_class_t svc = MBUF_SC_UNSPEC; |
273 | |
274 | switch (pkt->pktsched_ptype) { |
275 | case QP_MBUF: |
276 | svc = m_get_service_class((mbuf_t)pkt->pktsched_pkt); |
277 | break; |
278 | |
279 | |
280 | default: |
281 | VERIFY(0); |
282 | /* NOTREACHED */ |
283 | } |
284 | |
285 | return (svc); |
286 | } |
287 | |
288 | void |
289 | pktsched_get_pkt_vars(pktsched_pkt_t *pkt, uint32_t **flags, |
290 | uint64_t **timestamp, uint32_t *flowid, uint8_t *flowsrc, uint8_t *proto, |
291 | uint32_t *tcp_start_seq) |
292 | { |
293 | switch (pkt->pktsched_ptype) { |
294 | case QP_MBUF: { |
295 | struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; |
296 | struct pkthdr *pkth = &m->m_pkthdr; |
297 | |
298 | if (flags != NULL) |
299 | *flags = &pkth->pkt_flags; |
300 | if (timestamp != NULL) |
301 | *timestamp = &pkth->pkt_timestamp; |
302 | if (flowid != NULL) |
303 | *flowid = pkth->pkt_flowid; |
304 | if (flowsrc != NULL) |
305 | *flowsrc = pkth->pkt_flowsrc; |
306 | if (proto != NULL) |
307 | *proto = pkth->pkt_proto; |
308 | /* |
309 | * caller should use this value only if PKTF_START_SEQ |
310 | * is set in the mbuf packet flags |
311 | */ |
312 | if (tcp_start_seq != NULL) |
313 | *tcp_start_seq = pkth->tx_start_seq; |
314 | |
315 | break; |
316 | } |
317 | |
318 | |
319 | default: |
320 | VERIFY(0); |
321 | /* NOTREACHED */ |
322 | } |
323 | } |
324 | |
325 | struct flowadv_fcentry * |
326 | pktsched_alloc_fcentry(pktsched_pkt_t *pkt, struct ifnet *ifp, int how) |
327 | { |
328 | #pragma unused(ifp) |
329 | struct flowadv_fcentry *fce = NULL; |
330 | |
331 | switch (pkt->pktsched_ptype) { |
332 | case QP_MBUF: { |
333 | struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; |
334 | |
335 | fce = flowadv_alloc_entry(how); |
336 | if (fce == NULL) |
337 | break; |
338 | |
339 | _CASSERT(sizeof (m->m_pkthdr.pkt_flowid) == |
340 | sizeof (fce->fce_flowid)); |
341 | |
342 | fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc; |
343 | fce->fce_flowid = m->m_pkthdr.pkt_flowid; |
344 | break; |
345 | } |
346 | |
347 | |
348 | default: |
349 | VERIFY(0); |
350 | /* NOTREACHED */ |
351 | } |
352 | |
353 | return (fce); |
354 | } |
355 | |
356 | uint32_t * |
357 | pktsched_get_pkt_sfb_vars(pktsched_pkt_t *pkt, uint32_t **sfb_flags) |
358 | { |
359 | uint32_t *hashp = NULL; |
360 | |
361 | switch (pkt->pktsched_ptype) { |
362 | case QP_MBUF: { |
363 | struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; |
364 | struct pkthdr *pkth = &m->m_pkthdr; |
365 | |
366 | _CASSERT(sizeof (pkth->pkt_mpriv_hash) == sizeof (uint32_t)); |
367 | _CASSERT(sizeof (pkth->pkt_mpriv_flags) == sizeof (uint32_t)); |
368 | |
369 | *sfb_flags = &pkth->pkt_mpriv_flags; |
370 | hashp = &pkth->pkt_mpriv_hash; |
371 | break; |
372 | } |
373 | |
374 | |
375 | default: |
376 | VERIFY(0); |
377 | /* NOTREACHED */ |
378 | } |
379 | |
380 | return (hashp); |
381 | } |
382 | |