1 | /* |
2 | * Copyright (c) 2017 Apple Inc. All rights reserved. |
3 | */ |
4 | |
5 | #pragma once |
6 | |
7 | #ifdef KERNEL_PRIVATE |
8 | #ifdef __cplusplus |
9 | |
10 | #include <IOKit/IOService.h> |
11 | #include <stdatomic.h> |
12 | #include <kern/bits.h> |
13 | #include <libkern/c++/OSPtr.h> |
14 | |
15 | struct thread_group; |
16 | |
17 | enum{ |
18 | kIOPerfControlClientWorkUntracked = 0, |
19 | }; |
20 | |
21 | /*! |
22 | * @class IOPerfControlClient : public OSObject |
23 | * @abstract Class which implements an interface allowing device drivers to participate in performance control. |
24 | * @discussion TODO |
25 | */ |
26 | class IOPerfControlClient final : public OSObject |
27 | { |
28 | OSDeclareDefaultStructors(IOPerfControlClient); |
29 | |
30 | protected: |
31 | virtual bool init(IOService *driver, uint64_t maxWorkCapacity); |
32 | virtual void free() APPLE_KEXT_OVERRIDE; |
33 | |
34 | public: |
35 | /*! |
36 | * @function copyClient |
37 | * @abstract Return a retained reference to a client object, to be released by the driver. It may be |
38 | * shared with other drivers in the system. |
39 | * @param driver The device driver that will be using this interface. |
40 | * @param maxWorkCapacity The maximum number of concurrent work items supported by the device driver. |
41 | * @returns An instance of IOPerfControlClient. |
42 | */ |
43 | static IOPerfControlClient *copyClient(IOService *driver, uint64_t maxWorkCapacity); |
44 | |
45 | /*! |
46 | * @function registerDevice |
47 | * @abstract Inform the system that work will be dispatched to a device in the future. |
48 | * @discussion The system will do some one-time setup work associated with the device, and may block the |
49 | * current thread during the setup. Devices should not be passed to work workSubmit, workSubmitAndBegin, |
50 | * workBegin, or workEnd until they have been successfully registered. The unregistration process happens |
51 | * automatically when the device object is deallocated. |
52 | * @param device The device object. Some platforms require device to be a specific subclass of IOService. |
53 | * @returns kIOReturnSuccess or an IOReturn error code |
54 | */ |
55 | virtual IOReturn registerDevice(IOService *driver, IOService *device); |
56 | |
57 | /*! |
58 | * @function unregisterDevice |
59 | * @abstract Inform the system that work will be no longer be dispatched to a device in the future. |
60 | * @discussion This call is optional as the unregistration process happens automatically when the device |
61 | * object is deallocated. This call may block the current thread and/or acquire locks. It should not be |
62 | * called until after all submitted work has been ended using workEnd. |
63 | * @param device The device object. Some platforms require device to be a specific subclass of IOService. |
64 | */ |
65 | virtual void unregisterDevice(IOService *driver, IOService *device); |
66 | |
67 | /*! |
68 | * @struct WorkSubmitArgs |
69 | * @discussion Drivers may submit additional device-specific arguments related to the submission of a work item |
70 | * by passing a struct with WorkSubmitArgs as its first member. Note: Drivers are responsible for publishing |
71 | * a header file describing these arguments. |
72 | */ |
73 | struct WorkSubmitArgs { |
74 | uint32_t version; |
75 | uint32_t size; |
76 | uint64_t submit_time; |
77 | uint64_t reserved[4]; |
78 | void *driver_data; |
79 | }; |
80 | |
81 | /*! |
82 | * @function workSubmit |
83 | * @abstract Tell the performance controller that work was submitted. |
84 | * @param device The device that will execute the work. Some platforms require device to be a |
85 | * specific subclass of IOService. |
86 | * @param args Optional device-specific arguments related to the submission of this work item. |
87 | * @returns A token representing this work item, which must be passed to workEnd when the work is finished |
88 | * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks |
89 | * and a degradation of system performance. |
90 | */ |
91 | virtual uint64_t workSubmit(IOService *device, WorkSubmitArgs *args = nullptr); |
92 | |
93 | /*! |
94 | * @struct WorkBeginArgs |
95 | * @discussion Drivers may submit additional device-specific arguments related to the start of a work item |
96 | * by passing a struct with WorkBeginArgs as its first member. Note: Drivers are responsible for publishing |
97 | * a header file describing these arguments. |
98 | */ |
99 | struct WorkBeginArgs { |
100 | uint32_t version; |
101 | uint32_t size; |
102 | uint64_t begin_time; |
103 | uint64_t reserved[4]; |
104 | void *driver_data; |
105 | }; |
106 | |
107 | /*! |
108 | * @function workSubmitAndBegin |
109 | * @abstract Tell the performance controller that work was submitted and immediately began executing. |
110 | * @param device The device that is executing the work. Some platforms require device to be a |
111 | * specific subclass of IOService. |
112 | * @param submitArgs Optional device-specific arguments related to the submission of this work item. |
113 | * @param beginArgs Optional device-specific arguments related to the start of this work item. |
114 | * @returns A token representing this work item, which must be passed to workEnd when the work is finished |
115 | * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks |
116 | * and a degradation of system performance. |
117 | */ |
118 | virtual uint64_t workSubmitAndBegin(IOService *device, WorkSubmitArgs *submitArgs = nullptr, |
119 | WorkBeginArgs *beginArgs = nullptr); |
120 | |
121 | /*! |
122 | * @function workBegin |
123 | * @abstract Tell the performance controller that previously submitted work began executing. |
124 | * @param device The device that is executing the work. Some platforms require device to be a |
125 | * specific subclass of IOService. |
126 | * @param args Optional device-specific arguments related to the start of this work item. |
127 | */ |
128 | virtual void workBegin(IOService *device, uint64_t token, WorkBeginArgs *args = nullptr); |
129 | |
130 | /*! |
131 | * @struct WorkEndArgs |
132 | * @discussion Drivers may submit additional device-specific arguments related to the end of a work item |
133 | * by passing a struct with WorkEndArgs as its first member. Note: Drivers are responsible for publishing |
134 | * a header file describing these arguments. |
135 | */ |
136 | struct WorkEndArgs { |
137 | uint32_t version; |
138 | uint32_t size; |
139 | uint64_t end_time; |
140 | uint64_t reserved[4]; |
141 | void *driver_data; |
142 | }; |
143 | |
144 | /*! |
145 | * @function workEnd |
146 | * @abstract Tell the performance controller that previously started work finished executing. |
147 | * @param device The device that executed the work. Some platforms require device to be a |
148 | * specific subclass of IOService. |
149 | * @param args Optional device-specific arguments related to the end of this work item. |
150 | * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for |
151 | * calling workBegin when the work resumes and workEnd with done set to True when it has completed. A workEnd() call |
152 | * without a corresponding workBegin() call is a way to cancel a work item and return token to IOPerfControl. |
153 | */ |
154 | virtual void workEnd(IOService *device, uint64_t token, WorkEndArgs *args = nullptr, bool done = true); |
155 | |
156 | /*! |
157 | * @function copyWorkContext |
158 | * @abstract Return a retained reference to an opaque OSObject, to be released by the driver. This object can |
159 | * be used by IOPerfControl to track a work item. This may perform dynamic memory allocation. |
160 | * @returns A pointer to an OSObject |
161 | */ |
162 | OSPtr<OSObject> copyWorkContext(); |
163 | |
164 | /*! |
165 | * @function workSubmitAndBeginWithContext |
166 | * @abstract Tell the performance controller that work was submitted and immediately began executing |
167 | * @param device The device that is executing the work. Some platforms require device to be a |
168 | * specific subclass of IOService. |
169 | * @param context An OSObject returned by copyWorkContext(). The context object will be used by IOPerfControl to track |
170 | * this work item. |
171 | * @param submitArgs Optional device-specific arguments related to the submission of this work item. |
172 | * @param beginArgs Optional device-specific arguments related to the start of this work item. |
173 | * @returns true if IOPerfControl is tracking this work item, else false. |
174 | * @note The workEndWithContext() call is optional if the corresponding workSubmitWithContext() call returned false. |
175 | */ |
176 | bool workSubmitAndBeginWithContext(IOService *device, OSObject *context, WorkSubmitArgs *submitArgs = nullptr, |
177 | WorkBeginArgs *beginArgs = nullptr); |
178 | |
179 | /*! |
180 | * @function workSubmitWithContext |
181 | * @abstract Tell the performance controller that work was submitted. |
182 | * @param device The device that will execute the work. Some platforms require device to be a |
183 | * specific subclass of IOService. |
184 | * @param context An OSObject returned by copyWorkContext(). The context object will be used by IOPerfControl to track |
185 | * this work item. |
186 | * @param args Optional device-specific arguments related to the submission of this work item. |
187 | * @returns true if IOPerfControl is tracking this work item, else false. |
188 | */ |
189 | bool workSubmitWithContext(IOService *device, OSObject *context, WorkSubmitArgs *args = nullptr); |
190 | |
191 | /*! |
192 | * @function workBeginWithContext |
193 | * @abstract Tell the performance controller that previously submitted work began executing. |
194 | * @param device The device that is executing the work. Some platforms require device to be a |
195 | * specific subclass of IOService. |
196 | * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). |
197 | * @param args Optional device-specific arguments related to the start of this work item. |
198 | * @note The workBeginWithContext() and workEndWithContext() calls are optional if the corresponding workSubmitWithContext() call returned false. |
199 | */ |
200 | void workBeginWithContext(IOService *device, OSObject *context, WorkBeginArgs *args = nullptr); |
201 | |
202 | /*! |
203 | * @function workEndWithContext |
204 | * @abstract Tell the performance controller that previously started work finished executing. |
205 | * @param device The device that executed the work. Some platforms require device to be a |
206 | * specific subclass of IOService. |
207 | * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). |
208 | * @param args Optional device-specific arguments related to the end of this work item. |
209 | * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for |
210 | * calling workBegin when the work resumes and workEnd with done set to True when it has completed. |
211 | * @note The workEndWithContext() call is optional if the corresponding workSubmitWithContext() call returned false. A workEndWithContext() |
212 | * call without a corresponding workBeginWithContext() call is a way to cancel a work item. |
213 | */ |
214 | void workEndWithContext(IOService *device, OSObject *context, WorkEndArgs *args = nullptr, bool done = true); |
215 | |
216 | /*! |
217 | * @struct WorkUpdateArgs |
218 | * @discussion Drivers may submit additional device-specific arguments related to a work item by passing a |
219 | * struct with WorkUpdateArgs as its first member. Note: Drivers are responsible for publishing |
220 | * a header file describing these arguments. |
221 | */ |
222 | struct WorkUpdateArgs { |
223 | uint32_t version; |
224 | uint32_t size; |
225 | uint64_t update_time; |
226 | uint64_t reserved[4]; |
227 | void *driver_data; |
228 | }; |
229 | |
230 | /*! |
231 | * @function workUpdateWithContext |
232 | * @abstract Provide and receive additional information from the performance controller. If this call is |
233 | * made at all, it should be between workSubmit and workEnd. The purpose and implementation of this call are |
234 | * device specific, and may do nothing on some devices. |
235 | * @param device The device that submitted the work. Some platforms require device to be a |
236 | * specific subclass of IOService. |
237 | * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). |
238 | * @param args Optional device-specific arguments. |
239 | */ |
240 | void workUpdateWithContext(IOService *device, OSObject *context, WorkUpdateArgs *args = nullptr); |
241 | |
242 | /* |
243 | * Callers should always use the CURRENT version so that the kernel can detect both older |
244 | * and newer structure layouts. New callbacks should always be added at the end of the |
245 | * structure, and xnu should expect existing source recompiled against newer headers |
246 | * to pass NULL for unimplemented callbacks. |
247 | */ |
248 | |
249 | #define PERFCONTROL_INTERFACE_VERSION_NONE (0) /* no interface */ |
250 | #define PERFCONTROL_INTERFACE_VERSION_1 (1) /* up-to workEnd */ |
251 | #define PERFCONTROL_INTERFACE_VERSION_2 (2) /* up-to workUpdate */ |
252 | #define PERFCONTROL_INTERFACE_VERSION_3 (3) /* up-to (un)registerDriverDevice */ |
253 | #define PERFCONTROL_INTERFACE_VERSION_CURRENT PERFCONTROL_INTERFACE_VERSION_3 |
254 | |
255 | /*! |
256 | * @struct PerfControllerInterface |
257 | * @discussion Function pointers necessary to register a performance controller. Not for general driver use. |
258 | */ |
259 | struct PerfControllerInterface { |
260 | struct DriverState { |
261 | uint32_t has_target_thread_group : 1; |
262 | uint32_t has_device_info : 1; |
263 | uint32_t reserved : 30; |
264 | |
265 | uint64_t target_thread_group_id; |
266 | void *target_thread_group_data; |
267 | |
268 | uint32_t device_type; |
269 | uint32_t instance_id; |
270 | }; |
271 | |
272 | struct WorkState { |
273 | uint64_t thread_group_id; |
274 | void *thread_group_data; |
275 | void *work_data; |
276 | uint32_t work_data_size; |
277 | uint32_t started : 1; |
278 | uint32_t reserved : 31; |
279 | const DriverState* driver_state; |
280 | }; |
281 | |
282 | using RegisterDeviceFunction = IOReturn (*)(IOService *); |
283 | using RegisterDriverDeviceFunction = IOReturn (*)(IOService *, IOService *, DriverState *); |
284 | using WorkCanSubmitFunction = bool (*)(IOService *, WorkState *, WorkSubmitArgs *); |
285 | using WorkSubmitFunction = void (*)(IOService *, uint64_t, WorkState *, WorkSubmitArgs *); |
286 | using WorkBeginFunction = void (*)(IOService *, uint64_t, WorkState *, WorkBeginArgs *); |
287 | using WorkEndFunction = void (*)(IOService *, uint64_t, WorkState *, WorkEndArgs *, bool); |
288 | using WorkUpdateFunction = void (*)(IOService *, uint64_t, WorkState *, WorkUpdateArgs *); |
289 | |
290 | uint64_t version; |
291 | RegisterDeviceFunction registerDevice; |
292 | RegisterDeviceFunction unregisterDevice; |
293 | WorkCanSubmitFunction workCanSubmit; |
294 | WorkSubmitFunction workSubmit; |
295 | WorkBeginFunction workBegin; |
296 | WorkEndFunction workEnd; |
297 | WorkUpdateFunction workUpdate; |
298 | RegisterDriverDeviceFunction registerDriverDevice; |
299 | RegisterDriverDeviceFunction unregisterDriverDevice; |
300 | }; |
301 | |
302 | struct IOPerfControlClientShared { |
303 | atomic_uint_fast8_t maxDriverIndex; |
304 | PerfControllerInterface interface; |
305 | IOLock *interfaceLock; |
306 | OSSet *deviceRegistrationList; |
307 | }; |
308 | |
309 | struct IOPerfControlClientData { |
310 | struct thread_group *target_thread_group; |
311 | PerfControllerInterface::DriverState driverState; |
312 | IOService* device; |
313 | }; |
314 | /*! |
315 | * @function registerPerformanceController |
316 | * @abstract Register a performance controller to receive callbacks. Not for general driver use. |
317 | * @param interface Struct containing callback functions implemented by the performance controller. |
318 | * @returns kIOReturnSuccess or kIOReturnError if the interface was already registered. |
319 | */ |
320 | virtual IOReturn registerPerformanceController(PerfControllerInterface *interface); |
321 | |
322 | /*! |
323 | * @function getClientData |
324 | * @abstract Not for general driver use. Only used by registerPerformanceController(). Allows performanceController to register existing IOPerfControlClient. |
325 | * @returns IOPerfControlData associated with a IOPerfControlClient |
326 | */ |
327 | IOPerfControlClientData * |
328 | getClientData() |
329 | { |
330 | return &clientData; |
331 | } |
332 | |
333 | private: |
334 | struct WorkTableEntry { |
335 | struct thread_group *thread_group; |
336 | bool started; |
337 | uint8_t perfcontrol_data[32]; |
338 | }; |
339 | |
340 | static constexpr size_t kMaxWorkTableNumEntries = 1024; |
341 | static constexpr size_t kWorkTableIndexBits = 24; |
342 | static constexpr size_t kWorkTableMaxSize = (1 << kWorkTableIndexBits) - 1; // - 1 since |
343 | // kIOPerfControlClientWorkUntracked takes number 0 |
344 | static constexpr size_t kWorkTableIndexMask = (const size_t)mask(kWorkTableIndexBits); |
345 | |
346 | uint64_t allocateToken(thread_group *thread_group); |
347 | void deallocateToken(uint64_t token); |
348 | WorkTableEntry *getEntryForToken(uint64_t token); |
349 | void markEntryStarted(uint64_t token, bool started); |
350 | inline uint64_t tokenToGlobalUniqueToken(uint64_t token); |
351 | |
352 | uint8_t driverIndex; |
353 | IOPerfControlClientShared *shared; |
354 | WorkTableEntry *workTable; |
355 | size_t workTableLength; |
356 | size_t workTableNextIndex; |
357 | IOSimpleLock *workTableLock; |
358 | |
359 | IOPerfControlClientData clientData; |
360 | }; |
361 | |
362 | #endif /* __cplusplus */ |
363 | #endif /* KERNEL_PRIVATE */ |
364 | |