// This file is part of AsmJit project // // See asmjit.h or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #ifndef ASMJIT_NO_JIT #include "../core/osutils.h" #include "../core/string.h" #include "../core/support.h" #include "../core/virtmem.h" #if !defined(_WIN32) #include #include #include #include #include #include // Linux has a `memfd_create` syscall that we would like to use, if available. #if defined(__linux__) #include #endif // Apple recently introduced MAP_JIT flag, which we want to use. #if defined(__APPLE__) #include #include #if TARGET_OS_OSX #include #include // sys_icache_invalidate(). #endif // Older SDK doesn't define `MAP_JIT`. #ifndef MAP_JIT #define MAP_JIT 0x800 #endif #endif // BSD/MAC: `MAP_ANONYMOUS` is not defined, `MAP_ANON` is. #if !defined(MAP_ANONYMOUS) #define MAP_ANONYMOUS MAP_ANON #endif #endif #include #if defined(__APPLE__) || defined(__BIONIC__) #define ASMJIT_VM_SHM_DETECT 0 #else #define ASMJIT_VM_SHM_DETECT 1 #endif // Android NDK doesn't provide `shm_open()` and `shm_unlink()`. #if !defined(_WIN32) && !defined(__BIONIC__) #define ASMJIT_HAS_SHM_OPEN_AND_UNLINK #endif #if defined(__APPLE__) && TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64 #define ASMJIT_HAS_PTHREAD_JIT_WRITE_PROTECT_NP #endif ASMJIT_BEGIN_SUB_NAMESPACE(VirtMem) // Virtual Memory Utilities // ======================== static const MemoryFlags dualMappingFilter[2] = { MemoryFlags::kAccessWrite | MemoryFlags::kMMapMaxAccessWrite, MemoryFlags::kAccessExecute | MemoryFlags::kMMapMaxAccessExecute }; // Virtual Memory [Windows] // ======================== #if defined(_WIN32) struct ScopedHandle { inline ScopedHandle() noexcept : value(nullptr) {} inline ~ScopedHandle() noexcept { if (value != nullptr) ::CloseHandle(value); } HANDLE value; }; static void getVMInfo(Info& vmInfo) noexcept { SYSTEM_INFO systemInfo; ::GetSystemInfo(&systemInfo); vmInfo.pageSize = Support::alignUpPowerOf2(systemInfo.dwPageSize); vmInfo.pageGranularity = systemInfo.dwAllocationGranularity; } // Returns windows-specific protectFlags from \ref MemoryFlags. static DWORD protectFlagsFromMemoryFlags(MemoryFlags memoryFlags) noexcept { DWORD protectFlags; // READ|WRITE|EXECUTE. if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; else if (Support::test(memoryFlags, MemoryFlags::kAccessRW)) protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_READWRITE : PAGE_READONLY; else protectFlags = PAGE_NOACCESS; // Any other flags to consider? return protectFlags; } static DWORD desiredAccessFromMemoryFlags(MemoryFlags memoryFlags) noexcept { DWORD access = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? FILE_MAP_WRITE : FILE_MAP_READ; if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) access |= FILE_MAP_EXECUTE; return access; } static HardenedRuntimeFlags getHardenedRuntimeFlags() noexcept { return HardenedRuntimeFlags::kNone; } Error alloc(void** p, size_t size, MemoryFlags memoryFlags) noexcept { *p = nullptr; if (size == 0) return DebugUtils::errored(kErrorInvalidArgument); DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags); void* result = ::VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, protectFlags); if (!result) return DebugUtils::errored(kErrorOutOfMemory); *p = result; return kErrorOk; } Error release(void* p, size_t size) noexcept { DebugUtils::unused(size); if (ASMJIT_UNLIKELY(!::VirtualFree(p, 0, MEM_RELEASE))) return DebugUtils::errored(kErrorInvalidArgument); return kErrorOk; } Error protect(void* p, size_t size, MemoryFlags memoryFlags) noexcept { DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags); DWORD oldFlags; if (::VirtualProtect(p, size, protectFlags, &oldFlags)) return kErrorOk; return DebugUtils::errored(kErrorInvalidArgument); } Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept { dm->rx = nullptr; dm->rw = nullptr; if (size == 0) return DebugUtils::errored(kErrorInvalidArgument); ScopedHandle handle; handle.value = ::CreateFileMappingW( INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, (DWORD)(uint64_t(size) >> 32), (DWORD)(size & 0xFFFFFFFFu), nullptr); if (ASMJIT_UNLIKELY(!handle.value)) return DebugUtils::errored(kErrorOutOfMemory); void* ptr[2]; for (uint32_t i = 0; i < 2; i++) { MemoryFlags accessFlags = memoryFlags & ~dualMappingFilter[i]; DWORD desiredAccess = desiredAccessFromMemoryFlags(accessFlags); ptr[i] = ::MapViewOfFile(handle.value, desiredAccess, 0, 0, size); if (ptr[i] == nullptr) { if (i == 0) ::UnmapViewOfFile(ptr[0]); return DebugUtils::errored(kErrorOutOfMemory); } } dm->rx = ptr[0]; dm->rw = ptr[1]; return kErrorOk; } Error releaseDualMapping(DualMapping* dm, size_t size) noexcept { DebugUtils::unused(size); bool failed = false; if (!::UnmapViewOfFile(dm->rx)) failed = true; if (dm->rx != dm->rw && !UnmapViewOfFile(dm->rw)) failed = true; if (failed) return DebugUtils::errored(kErrorInvalidArgument); dm->rx = nullptr; dm->rw = nullptr; return kErrorOk; } #endif // Virtual Memory [Posix] // ====================== #if !defined(_WIN32) static void getVMInfo(Info& vmInfo) noexcept { uint32_t pageSize = uint32_t(::getpagesize()); vmInfo.pageSize = pageSize; vmInfo.pageGranularity = Support::max(pageSize, 65536); } #if !defined(SHM_ANON) static const char* getTmpDir() noexcept { const char* tmpDir = getenv("TMPDIR"); return tmpDir ? tmpDir : "/tmp"; } #endif // Translates libc errors specific to VirtualMemory mapping to `asmjit::Error`. static Error asmjitErrorFromErrno(int e) noexcept { switch (e) { case EACCES: case EAGAIN: case ENODEV: case EPERM: return kErrorInvalidState; case EFBIG: case ENOMEM: case EOVERFLOW: return kErrorOutOfMemory; case EMFILE: case ENFILE: return kErrorTooManyHandles; default: return kErrorInvalidArgument; } } // Some operating systems don't allow /dev/shm to be executable. On Linux this happens when /dev/shm is mounted with // 'noexec', which is enforced by systemd. Other operating systems like MacOS also restrict executable permissions // regarding /dev/shm, so we use a runtime detection before attempting to allocate executable memory. Sometimes we // don't need the detection as we know it would always result in `ShmStrategy::kTmpDir`. enum class ShmStrategy : uint32_t { kUnknown = 0, kDevShm = 1, kTmpDir = 2 }; class AnonymousMemory { public: enum FileType : uint32_t { kFileTypeNone, kFileTypeShm, kFileTypeTmp }; int _fd; FileType _fileType; StringTmp<128> _tmpName; inline AnonymousMemory() noexcept : _fd(-1), _fileType(kFileTypeNone), _tmpName() {} inline ~AnonymousMemory() noexcept { unlink(); close(); } inline int fd() const noexcept { return _fd; } Error open(bool preferTmpOverDevShm) noexcept { #if defined(__linux__) && defined(__NR_memfd_create) // Linux specific 'memfd_create' - if the syscall returns `ENOSYS` it means // it's not available and we will never call it again (would be pointless). // Zero initialized, if ever changed to '1' that would mean the syscall is not // available and we must use `shm_open()` and `shm_unlink()`. static volatile uint32_t memfd_create_not_supported; if (!memfd_create_not_supported) { _fd = (int)syscall(__NR_memfd_create, "vmem", 0); if (ASMJIT_LIKELY(_fd >= 0)) return kErrorOk; int e = errno; if (e == ENOSYS) memfd_create_not_supported = 1; else return DebugUtils::errored(asmjitErrorFromErrno(e)); } #endif #if defined(SHM_ANON) // Originally FreeBSD extension, apparently works in other BSDs too. DebugUtils::unused(preferTmpOverDevShm); _fd = ::shm_open(SHM_ANON, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); if (ASMJIT_LIKELY(_fd >= 0)) return kErrorOk; else return DebugUtils::errored(asmjitErrorFromErrno(errno)); #else // POSIX API. We have to generate somehow a unique name. This is nothing cryptographic, just using a bit from // the stack address to always have a different base for different threads (as threads have their own stack) // and retries for avoiding collisions. We use `shm_open()` with flags that require creation of the file so we // never open an existing shared memory. static std::atomic internalCounter; const char* kShmFormat = "/shm-id-%016llX"; uint32_t kRetryCount = 100; uint64_t bits = ((uintptr_t)(void*)this) & 0x55555555u; for (uint32_t i = 0; i < kRetryCount; i++) { bits -= uint64_t(OSUtils::getTickCount()) * 773703683; bits = ((bits >> 14) ^ (bits << 6)) + uint64_t(++internalCounter) * 10619863; bool useTmp = !ASMJIT_VM_SHM_DETECT || preferTmpOverDevShm; if (useTmp) { _tmpName.assign(getTmpDir()); _tmpName.appendFormat(kShmFormat, (unsigned long long)bits); _fd = ::open(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, 0); if (ASMJIT_LIKELY(_fd >= 0)) { _fileType = kFileTypeTmp; return kErrorOk; } } #ifdef ASMJIT_HAS_SHM_OPEN_AND_UNLINK else { _tmpName.assignFormat(kShmFormat, (unsigned long long)bits); _fd = ::shm_open(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); if (ASMJIT_LIKELY(_fd >= 0)) { _fileType = kFileTypeShm; return kErrorOk; } } #endif int e = errno; if (e != EEXIST) return DebugUtils::errored(asmjitErrorFromErrno(e)); } return DebugUtils::errored(kErrorFailedToOpenAnonymousMemory); #endif } void unlink() noexcept { FileType type = _fileType; _fileType = kFileTypeNone; #ifdef ASMJIT_HAS_SHM_OPEN_AND_UNLINK if (type == kFileTypeShm) { ::shm_unlink(_tmpName.data()); return; } #endif if (type == kFileTypeTmp) { ::unlink(_tmpName.data()); return; } } void close() noexcept { if (_fd >= 0) { ::close(_fd); _fd = -1; } } Error allocate(size_t size) noexcept { // TODO: Improve this by using `posix_fallocate()` when available. if (ftruncate(_fd, off_t(size)) != 0) return DebugUtils::errored(asmjitErrorFromErrno(errno)); return kErrorOk; } }; // Returns `mmap()` protection flags from \ref MemoryFlags. static int mmProtFromMemoryFlags(MemoryFlags memoryFlags) noexcept { int protection = 0; if (Support::test(memoryFlags, MemoryFlags::kAccessRead)) protection |= PROT_READ; if (Support::test(memoryFlags, MemoryFlags::kAccessWrite)) protection |= PROT_READ | PROT_WRITE; if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) protection |= PROT_READ | PROT_EXEC; return protection; } #if defined(__APPLE__) && TARGET_OS_OSX static int getOSXVersion() noexcept { // MAP_JIT flag required to run unsigned JIT code is only supported by kernel version 10.14+ (Mojave). static std::atomic globalVersion; int ver = globalVersion.load(); if (!ver) { struct utsname osname {}; uname(&osname); ver = atoi(osname.release); globalVersion.store(ver); } return ver; } #endif // __APPLE__ && TARGET_OS_OSX // Detects whether the current process is hardened, which means that pages that have WRITE and EXECUTABLE flags // cannot be normally allocated. On OSX + AArch64 such allocation requires MAP_JIT flag, other platforms don't // support this combination. static bool hasHardenedRuntime() noexcept { #if TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64 // OSX on AArch64 has always hardened runtime enabled. return true; #else static std::atomic globalHardenedFlag; enum HardenedFlag : uint32_t { kHardenedFlagUnknown = 0, kHardenedFlagDisabled = 1, kHardenedFlagEnabled = 2 }; uint32_t flag = globalHardenedFlag.load(); if (flag == kHardenedFlagUnknown) { uint32_t pageSize = uint32_t(::getpagesize()); void* ptr = mmap(nullptr, pageSize, PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (ptr == MAP_FAILED) { flag = kHardenedFlagEnabled; } else { flag = kHardenedFlagDisabled; munmap(ptr, pageSize); } globalHardenedFlag.store(flag); } return flag == kHardenedFlagEnabled; #endif } // Detects whether MAP_JIT is available. static inline bool hasMapJitSupport() noexcept { #if defined(__APPLE__) && TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64 // OSX on AArch64 always uses hardened runtime + MAP_JIT: // - https://developer.apple.com/documentation/apple_silicon/porting_just-in-time_compilers_to_apple_silicon return true; #elif defined(__APPLE__) && TARGET_OS_OSX return getOSXVersion() >= 18; #else // MAP_JIT is not available (it's only available on OSX). return false; #endif } // Returns either MAP_JIT or 0 based on `flags` and the host operating system. static inline int mmMapJitFromMemoryFlags(MemoryFlags memoryFlags) noexcept { #if defined(__APPLE__) // Always use MAP_JIT flag if user asked for it (could be used for testing on non-hardened processes) and detect // whether it must be used when the process is actually hardened (in that case it doesn't make sense to rely on // user `memoryFlags`). bool useMapJit = Support::test(memoryFlags, MemoryFlags::kMMapEnableMapJit) || hasHardenedRuntime(); if (useMapJit) return hasMapJitSupport() ? int(MAP_JIT) : 0; else return 0; #else DebugUtils::unused(memoryFlags); return 0; #endif } // Returns BSD-specific `PROT_MAX()` flags. static inline int mmMaxProtFromMemoryFlags(MemoryFlags memoryFlags) noexcept { #if defined(PROT_MAX) static constexpr uint32_t kMaxProtShift = Support::ConstCTZ::value; if (Support::test(memoryFlags, MemoryFlags::kMMapMaxAccessReadWrite | MemoryFlags::kMMapMaxAccessExecute)) return PROT_MAX(mmProtFromMemoryFlags((MemoryFlags)(uint32_t(memoryFlags) >> kMaxProtShift))); else return 0; #else DebugUtils::unused(memoryFlags); return 0; #endif } #if ASMJIT_VM_SHM_DETECT static Error detectShmStrategy(ShmStrategy* strategyOut) noexcept { AnonymousMemory anonMem; Info vmInfo = info(); ASMJIT_PROPAGATE(anonMem.open(false)); ASMJIT_PROPAGATE(anonMem.allocate(vmInfo.pageSize)); void* ptr = mmap(nullptr, vmInfo.pageSize, PROT_READ | PROT_EXEC, MAP_SHARED, anonMem.fd(), 0); if (ptr == MAP_FAILED) { int e = errno; if (e == EINVAL) { *strategyOut = ShmStrategy::kTmpDir; return kErrorOk; } return DebugUtils::errored(asmjitErrorFromErrno(e)); } else { munmap(ptr, vmInfo.pageSize); *strategyOut = ShmStrategy::kDevShm; return kErrorOk; } } #endif static Error getShmStrategy(ShmStrategy* strategyOut) noexcept { #if ASMJIT_VM_SHM_DETECT // Initially don't assume anything. It has to be tested whether '/dev/shm' was mounted with 'noexec' flag or not. static std::atomic globalShmStrategy; ShmStrategy strategy = static_cast(globalShmStrategy.load()); if (strategy == ShmStrategy::kUnknown) { ASMJIT_PROPAGATE(detectShmStrategy(&strategy)); globalShmStrategy.store(static_cast(strategy)); } *strategyOut = strategy; return kErrorOk; #else *strategyOut = ShmStrategy::kTmpDir; return kErrorOk; #endif } static HardenedRuntimeFlags getHardenedRuntimeFlags() noexcept { HardenedRuntimeFlags flags = HardenedRuntimeFlags::kNone; if (hasHardenedRuntime()) flags |= HardenedRuntimeFlags::kEnabled; if (hasMapJitSupport()) flags |= HardenedRuntimeFlags::kMapJit; return flags; } Error alloc(void** p, size_t size, MemoryFlags memoryFlags) noexcept { *p = nullptr; if (size == 0) return DebugUtils::errored(kErrorInvalidArgument); int protection = mmProtFromMemoryFlags(memoryFlags) | mmMaxProtFromMemoryFlags(memoryFlags); int mmFlags = MAP_PRIVATE | MAP_ANONYMOUS | mmMapJitFromMemoryFlags(memoryFlags); void* ptr = mmap(nullptr, size, protection, mmFlags, -1, 0); if (ptr == MAP_FAILED) return DebugUtils::errored(kErrorOutOfMemory); *p = ptr; return kErrorOk; } Error release(void* p, size_t size) noexcept { if (ASMJIT_UNLIKELY(munmap(p, size) != 0)) return DebugUtils::errored(kErrorInvalidArgument); return kErrorOk; } Error protect(void* p, size_t size, MemoryFlags memoryFlags) noexcept { int protection = mmProtFromMemoryFlags(memoryFlags); if (mprotect(p, size, protection) == 0) return kErrorOk; return DebugUtils::errored(kErrorInvalidArgument); } Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept { dm->rx = nullptr; dm->rw = nullptr; if (off_t(size) <= 0) return DebugUtils::errored(size == 0 ? kErrorInvalidArgument : kErrorTooLarge); bool preferTmpOverDevShm = Support::test(memoryFlags, MemoryFlags::kMappingPreferTmp); if (!preferTmpOverDevShm) { ShmStrategy strategy; ASMJIT_PROPAGATE(getShmStrategy(&strategy)); preferTmpOverDevShm = (strategy == ShmStrategy::kTmpDir); } AnonymousMemory anonMem; ASMJIT_PROPAGATE(anonMem.open(preferTmpOverDevShm)); ASMJIT_PROPAGATE(anonMem.allocate(size)); void* ptr[2]; for (uint32_t i = 0; i < 2; i++) { MemoryFlags accessFlags = memoryFlags & ~dualMappingFilter[i]; int protection = mmProtFromMemoryFlags(accessFlags) | mmMaxProtFromMemoryFlags(accessFlags); ptr[i] = mmap(nullptr, size, protection, MAP_SHARED, anonMem.fd(), 0); if (ptr[i] == MAP_FAILED) { // Get the error now before `munmap()` has a chance to clobber it. int e = errno; if (i == 1) munmap(ptr[0], size); return DebugUtils::errored(asmjitErrorFromErrno(e)); } } dm->rx = ptr[0]; dm->rw = ptr[1]; return kErrorOk; } Error releaseDualMapping(DualMapping* dm, size_t size) noexcept { Error err = release(dm->rx, size); if (dm->rx != dm->rw) err |= release(dm->rw, size); if (err) return DebugUtils::errored(kErrorInvalidArgument); dm->rx = nullptr; dm->rw = nullptr; return kErrorOk; } #endif // Virtual Memory - Flush Instruction Cache // ======================================== void flushInstructionCache(void* p, size_t size) noexcept { #if ASMJIT_ARCH_X86 // X86/X86_64 architecture doesn't require to do anything to flush instruction cache. DebugUtils::unused(p, size); #elif defined(__APPLE__) sys_icache_invalidate(p, size); #elif defined(_WIN32) // Windows has a built-in support in `kernel32.dll`. FlushInstructionCache(GetCurrentProcess(), p, size); #elif defined(__GNUC__) char* start = static_cast(p); char* end = start + size; __builtin___clear_cache(start, end); #else #pragma message("asmjit::VirtMem::flushInstructionCache() doesn't have implementation for the target OS and compiler") DebugUtils::unused(p, size); #endif } // Virtual Memory - Memory Info // ============================ Info info() noexcept { static std::atomic vmInfoInitialized; static Info vmInfo; if (!vmInfoInitialized.load()) { Info localMemInfo; getVMInfo(localMemInfo); vmInfo = localMemInfo; vmInfoInitialized.store(1u); } return vmInfo; } // Virtual Memory - Hardened Runtime Info // ====================================== HardenedRuntimeInfo hardenedRuntimeInfo() noexcept { return HardenedRuntimeInfo { getHardenedRuntimeFlags() }; } // Virtual Memory - Project JIT Memory // =================================== void protectJitMemory(ProtectJitAccess access) noexcept { #if defined(ASMJIT_HAS_PTHREAD_JIT_WRITE_PROTECT_NP) pthread_jit_write_protect_np(static_cast(access)); #else DebugUtils::unused(access); #endif } ASMJIT_END_SUB_NAMESPACE #endif