1 | /* |
2 | * Copyright (c) 2019-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <skywalk/os_skywalk.h> |
30 | #include <skywalk/os_skywalk_private.h> |
31 | #include <skywalk/namespace/protons.h> |
32 | |
33 | #include <kern/bits.h> |
34 | #include <netinet/in_var.h> |
35 | #include <netinet6/in6_var.h> |
36 | #include <sys/domain.h> |
37 | |
38 | static int __protons_inited = 0; |
39 | |
40 | decl_lck_mtx_data(static, protons_lock); |
41 | static LCK_GRP_DECLARE(protons_lock_group, "protons_lock" ); |
42 | static LCK_MTX_DECLARE(protons_lock, &protons_lock_group); |
43 | |
44 | #define PROTONS_LOCK() \ |
45 | lck_mtx_lock(&protons_lock) |
46 | #define PROTONS_UNLOCK() \ |
47 | lck_mtx_unlock(&protons_lock) |
48 | #define PROTONS_LOCK_ASSERT_HELD() \ |
49 | LCK_MTX_ASSERT(&protons_lock, LCK_MTX_ASSERT_OWNED) |
50 | #define PROTONS_LOCK_ASSERT_NOTHELD() \ |
51 | LCK_MTX_ASSERT(&protons_lock, LCK_MTX_ASSERT_NOTOWNED) |
52 | |
53 | os_refgrp_decl(static, protons_token_refgrp, "protons_token" , NULL); |
54 | |
55 | struct protons_token { |
56 | RB_ENTRY(protons_token) pt_link; |
57 | os_refcnt_t pt_refcnt; |
58 | pid_t pt_pid; |
59 | pid_t pt_epid; |
60 | uint8_t pt_protocol; |
61 | uint8_t pt_flags; |
62 | }; |
63 | |
64 | enum { |
65 | PROTONSF_VALID = (((uint8_t)1) << 0), |
66 | }; |
67 | |
68 | __attribute__((always_inline)) |
69 | static inline int |
70 | pt_cmp(const struct protons_token *pt1, const struct protons_token *pt2) |
71 | { |
72 | return (int)pt1->pt_protocol - (int)pt2->pt_protocol; |
73 | } |
74 | RB_HEAD(protons_token_tree, protons_token); |
75 | RB_PROTOTYPE_PREV(protons_token_tree, protons_token, pt_link, pt_cmp); |
76 | RB_GENERATE_PREV(protons_token_tree, protons_token, pt_link, pt_cmp); |
77 | static struct protons_token_tree protons_tokens; |
78 | |
79 | static SKMEM_TYPE_DEFINE(protons_token_zone, struct protons_token); |
80 | |
81 | static struct protons_token * |
82 | protons_token_alloc(bool can_block) |
83 | { |
84 | PROTONS_LOCK_ASSERT_HELD(); |
85 | |
86 | struct protons_token *pt = NULL; |
87 | pt = can_block ? zalloc(kt_view: protons_token_zone) : |
88 | zalloc_noblock(kt_view: protons_token_zone); |
89 | if (pt == NULL) { |
90 | return NULL; |
91 | } |
92 | |
93 | memset(s: pt, c: 0, n: sizeof(*pt)); |
94 | os_ref_init(&pt->pt_refcnt, &protons_token_refgrp); |
95 | |
96 | SK_DF(SK_VERB_NS_PROTO, "token %p alloc" , (void *)SK_KVA(pt)); |
97 | |
98 | return pt; |
99 | } |
100 | |
101 | static void |
102 | protons_token_free(struct protons_token *pt) |
103 | { |
104 | PROTONS_LOCK_ASSERT_HELD(); |
105 | |
106 | SK_DF(SK_VERB_NS_PROTO, "token %p free" , (void *)SK_KVA(pt)); |
107 | ASSERT(os_ref_get_count(&pt->pt_refcnt) == 0); |
108 | zfree(protons_token_zone, pt); |
109 | } |
110 | |
111 | bool |
112 | protons_token_is_valid(struct protons_token *pt) |
113 | { |
114 | if (__improbable(pt == NULL)) { |
115 | return false; |
116 | } |
117 | return pt->pt_flags & PROTONSF_VALID; |
118 | } |
119 | |
120 | bool |
121 | protons_token_has_matching_pid(struct protons_token *pt, pid_t pid, pid_t epid) |
122 | { |
123 | ASSERT(pt != NULL); |
124 | return pt->pt_pid == pid && pt->pt_epid == epid; |
125 | } |
126 | |
127 | static struct protons_token * |
128 | protons_find_token_with_protocol(uint8_t proto) |
129 | { |
130 | struct protons_token find = { .pt_protocol = proto }; |
131 | |
132 | PROTONS_LOCK_ASSERT_HELD(); |
133 | struct protons_token *pt = NULL; |
134 | pt = RB_FIND(protons_token_tree, &protons_tokens, &find); |
135 | if (pt) { |
136 | os_ref_retain(rc: &pt->pt_refcnt); |
137 | } |
138 | return pt; |
139 | } |
140 | |
141 | int |
142 | protons_token_get_use_count(struct protons_token *pt) |
143 | { |
144 | /* minus one refcnt in RB tree*/ |
145 | return os_ref_get_count(rc: &pt->pt_refcnt) - 1; |
146 | } |
147 | |
148 | static void |
149 | protons_token_release(struct protons_token *pt) |
150 | { |
151 | os_ref_count_t refcnt = os_ref_release(rc: &pt->pt_refcnt); |
152 | |
153 | SK_DF(SK_VERB_NS_PROTO, |
154 | "token %p proto %u released by pid %d epid %d, curr use %u" , |
155 | (void *)SK_KVA(pt), pt->pt_protocol, pt->pt_pid, pt->pt_epid, |
156 | protons_token_get_use_count(pt)); |
157 | |
158 | if (refcnt == 1) { |
159 | RB_REMOVE(protons_token_tree, &protons_tokens, pt); |
160 | (void) os_ref_release(rc: &pt->pt_refcnt); |
161 | pt->pt_flags &= ~PROTONSF_VALID; |
162 | pt->pt_protocol = 0; |
163 | pt->pt_pid = 0; |
164 | pt->pt_epid = 0; |
165 | protons_token_free(pt); |
166 | } |
167 | } |
168 | |
169 | static int |
170 | protons_reserve_locked(struct protons_token **ptp, pid_t pid, pid_t epid, |
171 | uint8_t proto) |
172 | { |
173 | struct protons_token *pt = NULL, *dup = NULL; |
174 | *ptp = NULL; |
175 | |
176 | pt = protons_find_token_with_protocol(proto); |
177 | if (pt != NULL) { |
178 | /* use previously reserved token with same process */ |
179 | ASSERT(pt->pt_flags & PROTONSF_VALID); |
180 | if (pt->pt_pid != pid || pt->pt_epid != epid) { |
181 | SK_ERR("proto %u existed with pid %d epid %d" , |
182 | proto, pt->pt_pid, pt->pt_epid); |
183 | (void) os_ref_release(rc: &pt->pt_refcnt); |
184 | pt = NULL; |
185 | return EEXIST; |
186 | } |
187 | } else { |
188 | /* start with new token */ |
189 | pt = protons_token_alloc(true); |
190 | if (pt == NULL) { |
191 | return ENOMEM; |
192 | } |
193 | |
194 | os_ref_retain(rc: &pt->pt_refcnt); |
195 | pt->pt_flags |= PROTONSF_VALID; |
196 | pt->pt_pid = pid; |
197 | pt->pt_epid = (epid != -1) ? epid : pid; |
198 | pt->pt_protocol = proto; |
199 | dup = RB_INSERT(protons_token_tree, &protons_tokens, pt); |
200 | ASSERT(dup == NULL); |
201 | } |
202 | |
203 | SK_DF(SK_VERB_NS_PROTO, |
204 | "token %p proto %u reserved by pid %d epid %d, curr use %u" , |
205 | (void *)SK_KVA(pt), proto, pid, epid, protons_token_get_use_count(pt)); |
206 | *ptp = pt; |
207 | |
208 | return 0; |
209 | } |
210 | |
211 | int |
212 | protons_reserve(struct protons_token **ptp, pid_t pid, pid_t epid, |
213 | uint8_t proto) |
214 | { |
215 | int err = 0; |
216 | PROTONS_LOCK(); |
217 | err = protons_reserve_locked(ptp, pid, epid, proto); |
218 | PROTONS_UNLOCK(); |
219 | return err; |
220 | } |
221 | |
222 | void |
223 | protons_release(struct protons_token **ptp) |
224 | { |
225 | struct protons_token *pt = *ptp; |
226 | ASSERT(pt != NULL); |
227 | |
228 | PROTONS_LOCK(); |
229 | protons_token_release(pt); |
230 | PROTONS_UNLOCK(); |
231 | *ptp = NULL; |
232 | } |
233 | |
234 | /* Reserved all protocol used by BSD stack. */ |
235 | static void |
236 | protons_init_netinet_protocol(void) |
237 | { |
238 | PROTONS_LOCK(); |
239 | |
240 | uint8_t proto = 0; |
241 | struct protons_token *pt = NULL; |
242 | int error = 0; |
243 | struct protosw *pp = NULL; |
244 | TAILQ_FOREACH(pp, &inetdomain->dom_protosw, pr_entry) { |
245 | pt = NULL; |
246 | proto = (uint8_t)pp->pr_protocol; |
247 | error = protons_reserve_locked(ptp: &pt, pid: 0, epid: 0, proto); |
248 | VERIFY(error == 0 || error == EEXIST); |
249 | VERIFY(pt != NULL); |
250 | } |
251 | |
252 | TAILQ_FOREACH(pp, &inet6domain->dom_protosw, pr_entry) { |
253 | pt = NULL; |
254 | proto = (uint8_t)pp->pr_protocol; |
255 | error = protons_reserve_locked(ptp: &pt, pid: 0, epid: 0, proto); |
256 | VERIFY(error == 0 || error == EEXIST); |
257 | VERIFY(pt != NULL); |
258 | } |
259 | |
260 | PROTONS_UNLOCK(); |
261 | } |
262 | |
263 | int |
264 | protons_init(void) |
265 | { |
266 | VERIFY(__protons_inited == 0); |
267 | |
268 | RB_INIT(&protons_tokens); |
269 | |
270 | protons_init_netinet_protocol(); |
271 | |
272 | __protons_inited = 1; |
273 | sk_features |= SK_FEATURE_PROTONS; |
274 | |
275 | SK_D("initialized protons" ); |
276 | |
277 | return 0; |
278 | } |
279 | |
280 | void |
281 | protons_fini(void) |
282 | { |
283 | if (__protons_inited == 1) { |
284 | __protons_inited = 0; |
285 | sk_features &= ~SK_FEATURE_PROTONS; |
286 | } |
287 | } |
288 | |
289 | static int protons_stats_sysctl SYSCTL_HANDLER_ARGS; |
290 | SYSCTL_PROC(_kern_skywalk_stats, OID_AUTO, protons, |
291 | CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, |
292 | 0, 0, protons_stats_sysctl, "" , "" ); |
293 | |
294 | static int |
295 | protons_stats_sysctl SYSCTL_HANDLER_ARGS |
296 | { |
297 | #pragma unused(arg1, arg2, oidp) |
298 | int error = 0; |
299 | size_t actual_space; |
300 | caddr_t buffer = NULL; |
301 | size_t buffer_space; |
302 | size_t allocated_space; |
303 | int out_error; |
304 | caddr_t scan; |
305 | |
306 | if (!kauth_cred_issuser(cred: kauth_cred_get())) { |
307 | return EPERM; |
308 | } |
309 | |
310 | net_update_uptime(); |
311 | buffer_space = req->oldlen; |
312 | if (req->oldptr != USER_ADDR_NULL && buffer_space != 0) { |
313 | if (buffer_space > SK_SYSCTL_ALLOC_MAX) { |
314 | buffer_space = SK_SYSCTL_ALLOC_MAX; |
315 | } |
316 | allocated_space = buffer_space; |
317 | buffer = sk_alloc_data(allocated_space, Z_WAITOK, skmem_tag_sysctl_buf); |
318 | if (__improbable(buffer == NULL)) { |
319 | return ENOBUFS; |
320 | } |
321 | } else if (req->oldptr == USER_ADDR_NULL) { |
322 | buffer_space = 0; |
323 | } |
324 | actual_space = 0; |
325 | scan = buffer; |
326 | |
327 | struct sk_stats_protons_token *spt = (void *)scan; |
328 | size_t spt_size = sizeof(*spt); |
329 | struct protons_token *pt = NULL; |
330 | PROTONS_LOCK(); |
331 | RB_FOREACH(pt, protons_token_tree, &protons_tokens) { |
332 | if (scan != NULL) { |
333 | if (buffer_space < spt_size) { |
334 | /* supplied buffer too small, stop copying */ |
335 | error = ENOMEM; |
336 | break; |
337 | } |
338 | spt->spt_protocol = pt->pt_protocol; |
339 | spt->spt_refcnt = protons_token_get_use_count(pt); |
340 | spt->spt_pid = pt->pt_pid; |
341 | spt->spt_epid = pt->pt_epid; |
342 | spt++; |
343 | buffer_space -= spt_size; |
344 | } |
345 | actual_space += spt_size; |
346 | } |
347 | PROTONS_UNLOCK(); |
348 | |
349 | if (actual_space != 0) { |
350 | out_error = SYSCTL_OUT(req, buffer, actual_space); |
351 | if (out_error != 0) { |
352 | error = out_error; |
353 | } |
354 | } |
355 | if (buffer != NULL) { |
356 | sk_free_data(buffer, allocated_space); |
357 | } |
358 | return error; |
359 | } |
360 | |