// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: 2013 Dolphin Emulator Project // SPDX-FileCopyrightText: 2014 Citra Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #include #include #include #include #include "common/adpf.h" #include "common/error.h" #include "common/logging.h" #include "common/assert.h" #include "common/thread.h" #ifdef __APPLE__ #include #elif defined(__HAIKU__) #include #elif defined(_WIN32) #include #include "common/string_util.h" #include "common/windows/timer_resolution.h" #else #if defined(__FreeBSD__) #include #include #include // Compatibility with CPUset #define cpu_set_t cpuset_t #elif defined(__DragonFly__) || defined(__OpenBSD__) || defined(__Bitrig__) #include #endif #include #include #endif #ifndef _WIN32 #include #endif #ifdef __linux__ #include #include namespace { constexpr int NICE_AUDIO = -16; constexpr int NICE_URGENT_DISPLAY = -8; constexpr int NICE_DISPLAY = -4; constexpr int NICE_DEFAULT = 0; constexpr int NICE_BACKGROUND = 10; int LowestAllowedNice() { static const int lowest = [] { rlimit limit{}; if (getrlimit(RLIMIT_NICE, &limit) != 0) { return 0; } if (limit.rlim_cur >= 40) { return -20; } return 20 - static_cast(limit.rlim_cur); }(); return lowest; } int NiceValueForPriority(Common::ThreadPriority priority) { const int wanted = [priority] { switch (priority) { case Common::ThreadPriority::Low: return NICE_BACKGROUND; case Common::ThreadPriority::Normal: return NICE_DEFAULT; case Common::ThreadPriority::High: return NICE_DISPLAY; case Common::ThreadPriority::VeryHigh: return NICE_URGENT_DISPLAY; case Common::ThreadPriority::Critical: return NICE_AUDIO; default: return NICE_DEFAULT; } }(); return (std::max)(wanted, (std::min)(NICE_DEFAULT, LowestAllowedNice())); } } // Anonymous namespace #endif #ifdef __ANDROID__ #include #include #include #include #include #include #include #include #include namespace { constexpr size_t ANDROID_MINIMUM_PERFORMANCE_CORES = 4; // A core counts as a performance core while it is within this much of the fastest one. constexpr s64 ANDROID_PERFORMANCE_CAPACITY_PERCENT = 50; constexpr std::chrono::nanoseconds ANDROID_POLICY_POLL_INTERVAL = std::chrono::milliseconds{500}; enum class CoreGroup { Unrestricted, Performance, Efficiency, }; struct ThreadPolicy { pid_t tid; CoreGroup group; s32 nice_value; bool has_nice; }; struct CoreInfo { s64 weight; u64 midr; s32 cpu; }; struct CpuTopologyState { std::mutex topology_mutex; cpu_set_t allowed{}; cpu_set_t performance{}; cpu_set_t efficiency{}; bool separated = false; bool initialized = false; pid_t canary_tid = 0; cpu_set_t canary_mask{}; bool canary_valid = false; std::atomic next_poll_ns{0}; std::mutex policy_mutex; std::vector policies; }; CpuTopologyState& State() { static CpuTopologyState* const state = new CpuTopologyState(); return *state; } struct PolicyRegistration { ~PolicyRegistration() { const pid_t tid = gettid(); ::Common::ADPF::RemoveCurrentThread(); CpuTopologyState& state = State(); std::scoped_lock lock{state.policy_mutex}; std::erase_if(state.policies, [tid](const ThreadPolicy& policy) { return policy.tid == tid; }); } }; thread_local PolicyRegistration t_policy_registration; s32 PossibleCpuCount() { std::ifstream file("/sys/devices/system/cpu/possible"); std::string list; if (file && std::getline(file, list) && !list.empty()) { s64 highest = -1; const char* cursor = list.c_str(); while (*cursor != '\0') { char* end = nullptr; const s64 value = std::strtol(cursor, &end, 10); if (end == cursor) { break; } highest = (std::max)(highest, value); cursor = end; while (*cursor == '-' || *cursor == ',') { ++cursor; } } if (highest >= 0) { return static_cast((std::min)(highest + 1, CPU_SETSIZE)); } } const s64 configured = sysconf(_SC_NPROCESSORS_CONF); if (configured > 0) { return static_cast((std::min)(configured, CPU_SETSIZE)); } return static_cast((std::min)(std::thread::hardware_concurrency(), CPU_SETSIZE)); } s64 ReadCpuScalar(s32 cpu, const char* node) { s64 value = 0; std::ifstream file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/" + node); if (!file || !(file >> value) || value <= 0) { return 0; } return value; } u64 ReadCpuMidr(s32 cpu) { u64 midr = 0; std::ifstream file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/regs/identification/midr_el1"); if (!file || !(file >> std::hex >> midr)) { return 0; } return midr; } std::vector CollectCores(const cpu_set_t& allowed, s32 total, const char* node, bool require_all) { std::vector cores; for (s32 cpu = 0; cpu < total; ++cpu) { if (!CPU_ISSET(cpu, &allowed)) { continue; } const s64 weight = ReadCpuScalar(cpu, node); if (weight <= 0) { if (require_all) { return {}; } LOG_WARNING(Common, "Could not read {} for CPU {}, treating it as an efficiency core", node, cpu); continue; } cores.push_back(CoreInfo{weight, ReadCpuMidr(cpu), cpu}); } return cores; } bool WeightsAreUniform(const std::vector& cores) { return std::all_of(cores.begin(), cores.end(), [&](const CoreInfo& core) { return core.weight == cores.front().weight; }); } bool MidrsAreDistinct(const std::vector& cores) { return std::none_of(cores.begin(), cores.end(), [](const CoreInfo& core) { return core.midr == 0; }) && std::any_of(cores.begin(), cores.end(), [&](const CoreInfo& core) { return core.midr != cores.front().midr; }); } void ComputeTopologyLocked(CpuTopologyState& state) { state.initialized = true; state.separated = false; CPU_ZERO(&state.allowed); CPU_ZERO(&state.performance); CPU_ZERO(&state.efficiency); if (sched_getaffinity(getpid(), sizeof(state.allowed), &state.allowed) != 0) { LOG_WARNING(Common, "Could not query process CPU affinity: {}", ::Common::GetLastErrorMsg()); return; } const s32 total = PossibleCpuCount(); auto cores = CollectCores(state.allowed, total, "cpu_capacity", true); if (cores.empty() || (WeightsAreUniform(cores) && MidrsAreDistinct(cores))) { auto by_frequency = CollectCores(state.allowed, total, "cpufreq/cpuinfo_max_freq", false); if (!by_frequency.empty()) { cores = std::move(by_frequency); } } if (cores.empty()) { LOG_WARNING(Common, "Could not determine CPU topology, thread placement is disabled"); return; } if (WeightsAreUniform(cores)) { if (MidrsAreDistinct(cores)) { LOG_WARNING(Common, "CPU clusters differ but rank identically, thread placement is " "disabled"); } else { LOG_INFO(Common, "CPU cores are symmetric, thread placement is disabled"); } return; } std::sort(cores.begin(), cores.end(), [](const CoreInfo& lhs, const CoreInfo& rhs) { if (lhs.weight != rhs.weight) { return lhs.weight > rhs.weight; } return lhs.cpu < rhs.cpu; }); const s64 fastest = cores.front().weight; size_t taken = 0; for (const auto& core : cores) { const bool fast_enough = core.weight * 100 >= fastest * ANDROID_PERFORMANCE_CAPACITY_PERCENT; if (!fast_enough && taken >= ANDROID_MINIMUM_PERFORMANCE_CORES) { break; } CPU_SET(core.cpu, &state.performance); ++taken; } if (taken == 0) { return; } for (s32 cpu = 0; cpu < total; ++cpu) { if (CPU_ISSET(cpu, &state.allowed) && !CPU_ISSET(cpu, &state.performance)) { CPU_SET(cpu, &state.efficiency); } } state.separated = CPU_COUNT(&state.efficiency) > 0; LOG_INFO(Common, "CPU topology: {} performance cores, {} efficiency cores, separation {}", CPU_COUNT(&state.performance), CPU_COUNT(&state.efficiency), state.separated ? "enabled" : "unavailable"); } void EnsureTopologyLocked(CpuTopologyState& state) { if (!state.initialized) { ComputeTopologyLocked(state); } } void RefreshTopologyLocked(CpuTopologyState& state) { if (!state.initialized) { ComputeTopologyLocked(state); return; } cpu_set_t current; CPU_ZERO(¤t); if (sched_getaffinity(getpid(), sizeof(current), ¤t) != 0) { return; } if (std::memcmp(¤t, &state.allowed, sizeof(current)) != 0) { ComputeTopologyLocked(state); } } bool ApplyCoreGroupLocked(CpuTopologyState& state, pid_t tid, CoreGroup group, bool* gone = nullptr) { const bool restrict_group = group != CoreGroup::Unrestricted && state.separated; const cpu_set_t* mask = &state.allowed; if (restrict_group) { mask = group == CoreGroup::Performance ? &state.performance : &state.efficiency; } if (CPU_COUNT(mask) == 0) { return false; } if (sched_setaffinity(tid, sizeof(*mask), mask) != 0) { if (gone != nullptr && errno == ESRCH) { *gone = true; return false; } LOG_WARNING(Common, "Could not restrict thread {} to its core group: {}", tid, ::Common::GetLastErrorMsg()); return false; } return true; } bool KernelPreservesRequestedAffinity() { utsname info{}; if (uname(&info) != 0) { return false; } s32 major = 0; s32 minor = 0; if (std::sscanf(info.release, "%d.%d", &major, &minor) != 2) { return false; } return major > 6 || (major == 6 && minor >= 2); } void SnapshotCanaryLocked(CpuTopologyState& state) { state.canary_valid = false; if (!state.separated) { return; } for (const auto& policy : state.policies) { if (policy.group == CoreGroup::Unrestricted) { continue; } cpu_set_t mask; CPU_ZERO(&mask); if (sched_getaffinity(policy.tid, sizeof(mask), &mask) != 0) { continue; } state.canary_tid = policy.tid; state.canary_mask = mask; state.canary_valid = true; return; } } bool DueForPoll(CpuTopologyState& state) { const auto now = std::chrono::steady_clock::now().time_since_epoch(); const s64 now_ns = std::chrono::duration_cast(now).count(); s64 next = state.next_poll_ns.load(std::memory_order_relaxed); if (now_ns < next) { return false; } return state.next_poll_ns.compare_exchange_strong( next, now_ns + ANDROID_POLICY_POLL_INTERVAL.count(), std::memory_order_relaxed); } ThreadPolicy& AcquirePolicyLocked(CpuTopologyState& state, pid_t tid) { for (auto& policy : state.policies) { if (policy.tid == tid) { return policy; } } return state.policies.emplace_back(ThreadPolicy{tid, CoreGroup::Unrestricted, 0, false}); } void SetCurrentThreadCoreGroup(CoreGroup group) { const pid_t tid = gettid(); (void)&t_policy_registration; CoreGroup effective = group; if (tid == getpid() && group != CoreGroup::Unrestricted) { LOG_WARNING(Common, "Refusing to place the main thread: the CPU topology is read from it"); effective = CoreGroup::Unrestricted; } CpuTopologyState& state = State(); std::scoped_lock topology_lock{state.topology_mutex}; EnsureTopologyLocked(state); ApplyCoreGroupLocked(state, tid, effective); std::scoped_lock policy_lock{state.policy_mutex}; AcquirePolicyLocked(state, tid).group = effective; if (!state.canary_valid) { SnapshotCanaryLocked(state); } } void RememberCurrentThreadNice(pid_t tid, s32 nice_value) { (void)&t_policy_registration; CpuTopologyState& state = State(); std::scoped_lock lock{state.policy_mutex}; ThreadPolicy& policy = AcquirePolicyLocked(state, tid); policy.nice_value = nice_value; policy.has_nice = true; } } // Anonymous namespace #endif #include "common/cpu_features.h" #ifdef ARCHITECTURE_x86_64 #ifdef _MSC_VER #include #else #include #endif #include "common/x64/rdtsc.h" #endif namespace Common { void SetCurrentThreadPriority(ThreadPriority new_priority) { #ifdef _WIN32 int windows_priority = [&]() { switch (new_priority) { case ThreadPriority::Low: return THREAD_PRIORITY_BELOW_NORMAL; case ThreadPriority::Normal: return THREAD_PRIORITY_NORMAL; case ThreadPriority::High: return THREAD_PRIORITY_ABOVE_NORMAL; case ThreadPriority::VeryHigh: return THREAD_PRIORITY_HIGHEST; case ThreadPriority::Critical: return THREAD_PRIORITY_TIME_CRITICAL; default: return THREAD_PRIORITY_NORMAL; } }(); SetThreadPriority(GetCurrentThread(), windows_priority); #elif defined(__HAIKU__) // TODO: We have priorities for 3D rendering applications - may help lavapipe? int priority = [&]() { switch (new_priority) { case ThreadPriority::Low: return B_LOW_PRIORITY; case ThreadPriority::Normal: return B_NORMAL_PRIORITY; case ThreadPriority::High: return B_DISPLAY_PRIORITY; case ThreadPriority::VeryHigh: return B_URGENT_DISPLAY_PRIORITY; case ThreadPriority::Critical: return B_URGENT_PRIORITY; default: return B_NORMAL_PRIORITY; } }(); set_thread_priority(find_thread(NULL), priority); #elif defined(__ANDROID__) const int nice_value = NiceValueForPriority(new_priority); const pid_t tid = gettid(); if (setpriority(PRIO_PROCESS, static_cast(tid), nice_value) != 0) { LOG_WARNING(Common, "Could not set thread nice value to {}: {}", nice_value, GetLastErrorMsg()); return; } RememberCurrentThreadNice(tid, nice_value); #elif defined(__linux__) const int nice_value = NiceValueForPriority(new_priority); if (setpriority(PRIO_PROCESS, 0, nice_value) != 0) { LOG_DEBUG(Common, "Could not set thread nice value to {}: {}", nice_value, GetLastErrorMsg()); } #else const s32 max_prio = sched_get_priority_max(SCHED_OTHER); const s32 min_prio = sched_get_priority_min(SCHED_OTHER); if (max_prio > min_prio) { const u32 level = (std::min)(static_cast(new_priority), 4U); sched_param params{}; params.sched_priority = min_prio + static_cast(static_cast(max_prio - min_prio) * level) / 4; pthread_setschedparam(pthread_self(), SCHED_OTHER, ¶ms); } #endif } void SetCurrentThreadName(const char* name) { #ifdef _MSC_VER // Sets the debugger-visible name of the current thread. if (auto pf = (decltype(&SetThreadDescription))(void*)GetProcAddress(GetModuleHandle(TEXT("KernelBase.dll")), "SetThreadDescription"); pf) pf(GetCurrentThread(), UTF8ToUTF16W(name).data()); // Windows 10+ else ; // No-op #elif defined(__APPLE__) pthread_setname_np(name); #elif defined(__HAIKU__) rename_thread(find_thread(NULL), name); #elif defined(__Bitrig__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__) pthread_set_name_np(pthread_self(), name); #elif defined(__NetBSD__) pthread_setname_np(pthread_self(), "%s", (void*)name); #elif defined(__linux__) || defined(__CYGWIN__) || defined(__sun__) || defined(__glibc__) || defined(__managarm__) int ret = pthread_setname_np(pthread_self(), name); if (ret == ERANGE) { // Linux limits thread names to 15 characters and will outright reject any // attempt to set a longer name with ERANGE. char buf[16]; size_t const len = std::min(std::strlen(name), sizeof(buf) - 1); std::memcpy(buf, name, len); buf[len] = '\0'; pthread_setname_np(pthread_self(), buf); } #elif defined(_WIN32) // MinGW with the POSIX threading model does not support pthread_setname_np // See for reference // https://gitlab.freedesktop.org/mesa/mesa/-/blame/main/src/util/u_thread.c?ref_type=heads#L75 (void)name; #else pthread_setname_np(pthread_self(), name); #endif } void SetCurrentThreadToPerformanceCores() { #if defined(__ANDROID__) if (ADPF::AddCurrentThread(ADPF::Session::Render)) { SetCurrentThreadCoreGroup(CoreGroup::Unrestricted); return; } SetCurrentThreadCoreGroup(CoreGroup::Performance); #endif } void SetCurrentThreadToEfficiencyCores() { #if defined(__ANDROID__) if (ADPF::AddCurrentThread(ADPF::Session::Background)) { SetCurrentThreadCoreGroup(CoreGroup::Unrestricted); return; } SetCurrentThreadCoreGroup(CoreGroup::Efficiency); #endif } void SetCurrentThreadToBackgroundWork() { #if defined(__ANDROID__) ADPF::AddCurrentThread(ADPF::Session::Background); SetCurrentThreadCoreGroup(CoreGroup::Unrestricted); #endif } void SetCurrentThreadToAllCores() { #if defined(__ANDROID__) ADPF::RemoveCurrentThread(); SetCurrentThreadCoreGroup(CoreGroup::Unrestricted); #endif } void RefreshThreadPolicies() { #if defined(__ANDROID__) CpuTopologyState& state = State(); std::scoped_lock topology_lock{state.topology_mutex}; RefreshTopologyLocked(state); std::scoped_lock policy_lock{state.policy_mutex}; std::erase_if(state.policies, [&state](const ThreadPolicy& policy) { bool gone = false; if (policy.has_nice && setpriority(PRIO_PROCESS, static_cast(policy.tid), policy.nice_value) != 0 && errno == ESRCH) { gone = true; } if (!gone) { ApplyCoreGroupLocked(state, policy.tid, policy.group, &gone); } return gone; }); SnapshotCanaryLocked(state); #endif } void PollThreadPolicies() { #if defined(__ANDROID__) static const bool needed = !KernelPreservesRequestedAffinity(); if (!needed) { return; } CpuTopologyState& state = State(); if (!DueForPoll(state)) { return; } pid_t tid; cpu_set_t expected; { std::scoped_lock lock{state.topology_mutex}; if (!state.canary_valid) { return; } tid = state.canary_tid; expected = state.canary_mask; } cpu_set_t current; CPU_ZERO(¤t); if (sched_getaffinity(tid, sizeof(current), ¤t) == 0 && std::memcmp(¤t, &expected, sizeof(current)) == 0) { return; } RefreshThreadPolicies(); #endif } #ifdef ARCHITECTURE_x86_64 // On Linux and UNIX systems, a futex would nominally be used to cover the costs // the idea is that it's intuitivelly cheaper to use a direct instruction as opposed to a full futex call // the underlying libc++ implementation uses pthread_cond_timedwait which MAY invoke a futex // Let's pretend the OS is too expensive to jump into, and avoid ANY context switches // this should *IN THEORY* lower CPU usage while just waiting for stuff effectively // For windows the minimal quanta resolution is about 500us, and normal CRT cond var is 1.5ms(?) // so may as well avoid that too // Let's just give ALL platforms the same mechanisms (almost) for when they have umonitor OR waitpkg #ifdef __clang__ __attribute__((target("waitpkg,mwaitx"))) #elif defined(__GNUC__) #pragma GCC target("waitpkg") #pragma GCC target("mwaitx") #endif bool Event::WaitFor(const std::chrono::nanoseconds time) { #ifdef _WIN32 auto const start = Common::X64::FencedRDTSC(); auto const& caps = Common::g_cpu_caps; [[maybe_unused]] auto const end = start + Common::g_wall_clock.NsToTicks(time); if (caps.monitorx) { while (true) { // Armed monitor, as per manual, MWAITX must be conditional if the condition isn't satisfied // to prevent a race condition. _mm_monitorx(reinterpret_cast(std::addressof(is_set)), 0, 0); if (!is_set.load()) { // RDTSC may be fenced here due to atomic load #ifdef _MSC_VER auto const now = __rdtsc(); #else auto const now = _rdtsc(); #endif if (end > now) { u32 const cycles = std::min((std::numeric_limits::max)(), s64(end) - s64(now)); // See here: https://github.com/torvalds/linux/blob/948a64995aca6820abefd17f1a4258f5835c5ad9/arch/x86/lib/delay.c#L93 // MWAITX accepts a 32-bit input timer which determines the total number of cycles to wait for // NOT THE TOTAL ABSOLUTE TSC VALUE, it's just a delta // BIT[1] = use a timer // Hint = 0: Use C1 state when sleepy (means slower wakeup but better savings) _mm_mwaitx(1 << 1, 0u, cycles); if (!is_set.load()) return false; } else return false; //timeout } bool expected = true; if (is_set.compare_exchange_weak(expected, false, std::memory_order_release)) return true; } } else if (caps.waitpkg) { // #UD If CPUID.7.0:ECX.WAITPKG[bit 5]=0. while (true) { _umonitor(std::addressof(is_set)); if (!is_set.load() && !_umwait(0, end)) //umwait is absolute time!!! return false; bool expected = true; if (is_set.compare_exchange_weak(expected, false, std::memory_order_release)) return true; } } else { #ifdef _MSC_VER while (!is_set.load() && end > __rdtsc()) Common::Windows::SleepForOneTick(); #else while (!is_set.load() && end > _rdtsc()) Common::Windows::SleepForOneTick(); #endif if (is_set.load()) Reset(); return true; } #else std::unique_lock lk{mutex}; if (!condvar.wait_for(lk, time, [this] { return is_set.load(); })) return false; is_set = false; return true; #endif } #else bool Event::WaitFor(const std::chrono::nanoseconds time) { #ifdef _WIN32 auto const end = Common::g_wall_clock.GetTimeNS() + time; while (!is_set.load() && end > Common::g_wall_clock.GetTimeNS()) Common::Windows::SleepForOneTick(); if (is_set.load()) Reset(); return true; #else std::unique_lock lk{mutex}; if (!condvar.wait_for(lk, time, [this] { return is_set.load(); })) return false; is_set = false; return true; #endif } #endif } // namespace Common