1/*
2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#define IOKIT_ENABLE_SHARED_PTR
30
31#include <IOKit/IOLib.h>
32#include <IOKit/IOMapper.h>
33#include <IOKit/IODMACommand.h>
34#include <libkern/c++/OSData.h>
35#include <libkern/OSDebug.h>
36#include <mach_debug/zone_info.h>
37#include "IOKitKernelInternal.h"
38
39__BEGIN_DECLS
40extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
41__END_DECLS
42
43#define super IOService
44OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);
45
46OSMetaClassDefineReservedUnused(IOMapper, 0);
47OSMetaClassDefineReservedUnused(IOMapper, 1);
48OSMetaClassDefineReservedUnused(IOMapper, 2);
49OSMetaClassDefineReservedUnused(IOMapper, 3);
50OSMetaClassDefineReservedUnused(IOMapper, 4);
51OSMetaClassDefineReservedUnused(IOMapper, 5);
52OSMetaClassDefineReservedUnused(IOMapper, 6);
53OSMetaClassDefineReservedUnused(IOMapper, 7);
54OSMetaClassDefineReservedUnused(IOMapper, 8);
55OSMetaClassDefineReservedUnused(IOMapper, 9);
56OSMetaClassDefineReservedUnused(IOMapper, 10);
57OSMetaClassDefineReservedUnused(IOMapper, 11);
58OSMetaClassDefineReservedUnused(IOMapper, 12);
59OSMetaClassDefineReservedUnused(IOMapper, 13);
60OSMetaClassDefineReservedUnused(IOMapper, 14);
61OSMetaClassDefineReservedUnused(IOMapper, 15);
62
63IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown;
64
65class IOMapperLock {
66 IOLock *fWaitLock;
67public:
68 IOMapperLock()
69 {
70 fWaitLock = IOLockAlloc();
71 }
72 ~IOMapperLock()
73 {
74 IOLockFree(lock: fWaitLock);
75 }
76
77 void
78 lock()
79 {
80 IOLockLock(fWaitLock);
81 }
82 void
83 unlock()
84 {
85 IOLockUnlock(fWaitLock);
86 }
87 void
88 sleep(void *event)
89 {
90 IOLockSleep(lock: fWaitLock, event, THREAD_UNINT);
91 }
92 void
93 wakeup(void *event)
94 {
95 IOLockWakeup(lock: fWaitLock, event, oneThread: false);
96 }
97};
98
99static IOMapperLock sMapperLock;
100
101bool
102IOMapper::start(IOService *provider)
103{
104 OSSharedPtr<OSObject> obj;
105 if (!super::start(provider)) {
106 return false;
107 }
108
109 if (!initHardware(provider)) {
110 return false;
111 }
112
113 uint64_t pageSize = getPageSize();
114 assert(pageSize <= UINT_MAX);
115 fPageSize = (uint32_t) pageSize;
116
117 if (fIsSystem) {
118 sMapperLock.lock();
119 IOMapper::gSystem = this;
120 sMapperLock.wakeup(event: &IOMapper::gSystem);
121 sMapperLock.unlock();
122 }
123
124 if (provider) {
125 obj = provider->copyProperty(aKey: "iommu-id");
126 if (!obj) {
127 obj = provider->copyProperty(aKey: "AAPL,phandle");
128 }
129 if (obj) {
130 setProperty(aKey: gIOMapperIDKey, anObject: obj.get());
131 }
132 }
133 return true;
134}
135
136void
137IOMapper::free()
138{
139 super::free();
140}
141
142void
143IOMapper::setMapperRequired(bool hasMapper)
144{
145 if (hasMapper) {
146 IOMapper::gSystem = (IOMapper *) kHasMapper;
147 } else {
148 sMapperLock.lock();
149 IOMapper::gSystem = (IOMapper *) kNoMapper;
150 sMapperLock.unlock();
151 sMapperLock.wakeup(event: &IOMapper::gSystem);
152 }
153}
154
155void
156IOMapper::waitForSystemMapper()
157{
158 sMapperLock.lock();
159 while ((uintptr_t) IOMapper::gSystem & kWaitMask) {
160 OSReportWithBacktrace(str: "waitForSystemMapper");
161 sMapperLock.sleep(event: &IOMapper::gSystem);
162 }
163 sMapperLock.unlock();
164}
165
166OSSharedPtr<IOMapper>
167IOMapper::copyMapperForDevice(IOService * device)
168{
169 return copyMapperForDeviceWithIndex(device, index: 0);
170}
171
172OSSharedPtr<IOMapper>
173IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index)
174{
175 OSSharedPtr<OSData> data;
176 OSSharedPtr<OSObject> obj;
177 OSSharedPtr<IOMapper> mapper;
178 OSSharedPtr<OSDictionary> matching;
179
180 obj = device->copyProperty(aKey: "iommu-parent");
181 if (!obj) {
182 return NULL;
183 }
184
185 if ((mapper = OSDynamicPtrCast<IOMapper>(source: obj))) {
186 goto found;
187 }
188
189 if ((data = OSDynamicPtrCast<OSData>(source: obj))) {
190 if (index >= data->getLength() / sizeof(UInt32)) {
191 goto found;
192 }
193
194 data = OSData::withValueNoCopy(value&: *((UInt32 *)data->getBytesNoCopy() + index));
195 if (!data) {
196 goto found;
197 }
198
199 matching = IOService::propertyMatching(key: gIOMapperIDKey, value: data.get());
200 } else {
201 matching = IOService::propertyMatching(key: gIOMapperIDKey, value: obj.get());
202 }
203
204 if (matching) {
205 mapper = OSDynamicPtrCast<IOMapper>(source: IOService::waitForMatchingService(matching: matching.get()));
206 }
207
208found:
209 if (mapper) {
210 if (!mapper->fAllocName) {
211 char name[MACH_ZONE_NAME_MAX_LEN];
212 char kmodname[KMOD_MAX_NAME];
213 vm_tag_t tag;
214 uint32_t kmodid;
215
216 tag = IOMemoryTag(map: kernel_map);
217 if (!(kmodid = vm_tag_get_kext(tag, name: &kmodname[0], KMOD_MAX_NAME))) {
218 snprintf(kmodname, count: sizeof(kmodname), "%d", tag);
219 }
220 snprintf(name, count: sizeof(name), "%s.DMA.%s", kmodname, device->getName());
221 mapper->fAllocName = kern_allocation_name_allocate(name, suballocs: 16);
222 }
223 }
224
225 return mapper;
226}
227
228__BEGIN_DECLS
229
230// These are C accessors to the system mapper for non-IOKit clients
231ppnum_t
232IOMapperIOVMAlloc(unsigned pages)
233{
234 IOReturn ret;
235 uint64_t dmaAddress, dmaLength;
236
237 IOMapper::checkForSystemMapper();
238
239 ret = kIOReturnUnsupported;
240 if (IOMapper::gSystem) {
241 ret = IOMapper::gSystem->iovmMapMemory(
242 NULL, descriptorOffset: 0, ptoa_64(pages),
243 mapOptions: (kIODMAMapReadAccess | kIODMAMapWriteAccess),
244 NULL, NULL, NULL,
245 mapAddress: &dmaAddress, mapLength: &dmaLength);
246 }
247
248 if (kIOReturnSuccess == ret) {
249 uint64_t dmaAddressPage64;
250 dmaAddressPage64 = atop_64(dmaAddress);
251 if (dmaAddressPage64 > UINT_MAX) {
252 return 0;
253 }
254 return (ppnum_t) atop_64(dmaAddress);
255 }
256 return 0;
257}
258
259void
260IOMapperIOVMFree(ppnum_t addr, unsigned pages)
261{
262 if (IOMapper::gSystem) {
263 IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages));
264 }
265}
266
267ppnum_t
268IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page)
269{
270 if (!IOMapper::gSystem) {
271 return page;
272 }
273 if (!addr) {
274 panic("!addr");
275 }
276 IOMapper::gSystem->iovmInsert(options: (kIODMAMapReadAccess | kIODMAMapWriteAccess),
277 ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1));
278 return addr + offset;
279}
280
281/////////////////////////////////////////////////////////////////////////////
282//
283//
284// IOLib.h APIs
285//
286//
287/////////////////////////////////////////////////////////////////////////////
288
289#include <machine/machine_routines.h>
290
291UInt8
292IOMappedRead8(IOPhysicalAddress address)
293{
294 IOMapper::checkForSystemMapper();
295
296 if (IOMapper::gSystem) {
297 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address);
298 return (UInt8) ml_phys_read_byte_64(paddr: addr);
299 } else {
300 return (UInt8) ml_phys_read_byte(paddr: (vm_offset_t) address);
301 }
302}
303
304UInt16
305IOMappedRead16(IOPhysicalAddress address)
306{
307 IOMapper::checkForSystemMapper();
308
309 if (IOMapper::gSystem) {
310 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address);
311 return (UInt16) ml_phys_read_half_64(paddr: addr);
312 } else {
313 return (UInt16) ml_phys_read_half(paddr: (vm_offset_t) address);
314 }
315}
316
317UInt32
318IOMappedRead32(IOPhysicalAddress address)
319{
320 IOMapper::checkForSystemMapper();
321
322 if (IOMapper::gSystem) {
323 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address);
324 return (UInt32) ml_phys_read_word_64(paddr: addr);
325 } else {
326 return (UInt32) ml_phys_read_word(paddr: (vm_offset_t) address);
327 }
328}
329
330UInt64
331IOMappedRead64(IOPhysicalAddress address)
332{
333 IOMapper::checkForSystemMapper();
334
335 if (IOMapper::gSystem) {
336 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address);
337 return (UInt64) ml_phys_read_double_64(paddr: addr);
338 } else {
339 return (UInt64) ml_phys_read_double(paddr: (vm_offset_t) address);
340 }
341}
342
343void
344IOMappedWrite8(IOPhysicalAddress address, UInt8 value)
345{
346 IOMapper::checkForSystemMapper();
347
348 if (IOMapper::gSystem) {
349 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address);
350 ml_phys_write_byte_64(paddr: addr, data: value);
351 } else {
352 ml_phys_write_byte(paddr: (vm_offset_t) address, data: value);
353 }
354}
355
356void
357IOMappedWrite16(IOPhysicalAddress address, UInt16 value)
358{
359 IOMapper::checkForSystemMapper();
360
361 if (IOMapper::gSystem) {
362 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address);
363 ml_phys_write_half_64(paddr: addr, data: value);
364 } else {
365 ml_phys_write_half(paddr: (vm_offset_t) address, data: value);
366 }
367}
368
369void
370IOMappedWrite32(IOPhysicalAddress address, UInt32 value)
371{
372 IOMapper::checkForSystemMapper();
373
374 if (IOMapper::gSystem) {
375 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address);
376 ml_phys_write_word_64(paddr: addr, data: value);
377 } else {
378 ml_phys_write_word(paddr: (vm_offset_t) address, data: value);
379 }
380}
381
382void
383IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
384{
385 IOMapper::checkForSystemMapper();
386
387 if (IOMapper::gSystem) {
388 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(mappedAddress: address);
389 ml_phys_write_double_64(paddr: addr, data: value);
390 } else {
391 ml_phys_write_double(paddr: (vm_offset_t) address, data: value);
392 }
393}
394
395__END_DECLS
396