2026-02-01 01:54:59 +01:00
|
|
|
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
2025-09-21 21:58:59 +02:00
|
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
2026-08-08 04:39:47 +02:00
|
|
|
|
2022-04-28 18:24:11 +02:00
|
|
|
// SPDX-FileCopyrightText: 2013 Dolphin Emulator Project
|
|
|
|
|
// SPDX-FileCopyrightText: 2014 Citra Emulator Project
|
|
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2013-09-04 20:17:46 -04:00
|
|
|
|
2026-05-30 21:59:10 +02:00
|
|
|
#include <chrono>
|
|
|
|
|
#include <limits>
|
2021-09-08 14:36:20 -04:00
|
|
|
#include <string>
|
2026-02-01 01:54:59 +01:00
|
|
|
#include <thread>
|
2021-09-08 14:36:20 -04:00
|
|
|
|
2026-08-08 04:39:47 +02:00
|
|
|
#include "common/adpf.h"
|
2021-09-08 14:36:20 -04:00
|
|
|
#include "common/error.h"
|
2026-03-12 18:29:15 +01:00
|
|
|
#include "common/logging.h"
|
2026-02-01 01:54:59 +01:00
|
|
|
#include "common/assert.h"
|
2014-04-08 20:15:08 -04:00
|
|
|
#include "common/thread.h"
|
2013-09-04 20:17:46 -04:00
|
|
|
#ifdef __APPLE__
|
2016-09-18 09:38:01 +09:00
|
|
|
#include <mach/mach.h>
|
2025-10-22 04:53:40 +02:00
|
|
|
#elif defined(__HAIKU__)
|
|
|
|
|
#include <kernel/OS.h>
|
2014-09-02 22:05:45 -07:00
|
|
|
#elif defined(_WIN32)
|
2016-12-03 21:08:02 +01:00
|
|
|
#include <windows.h>
|
2023-10-20 11:41:29 -04:00
|
|
|
#include "common/string_util.h"
|
2026-05-30 21:59:10 +02:00
|
|
|
#include "common/windows/timer_resolution.h"
|
2015-06-20 22:45:15 +01:00
|
|
|
#else
|
2026-03-08 19:32:24 +01:00
|
|
|
#if defined(__FreeBSD__)
|
2026-02-01 01:54:59 +01:00
|
|
|
#include <sys/cpuset.h>
|
|
|
|
|
#include <sys/_cpuset.h>
|
2016-09-18 09:38:01 +09:00
|
|
|
#include <pthread_np.h>
|
2026-05-30 21:59:10 +02:00
|
|
|
// Compatibility with CPUset
|
|
|
|
|
#define cpu_set_t cpuset_t
|
2026-03-08 19:32:24 +01:00
|
|
|
#elif defined(__DragonFly__) || defined(__OpenBSD__) || defined(__Bitrig__)
|
|
|
|
|
#include <pthread_np.h>
|
2016-09-18 09:38:01 +09:00
|
|
|
#endif
|
2025-10-07 06:38:22 +02:00
|
|
|
#include <pthread.h>
|
2016-09-18 09:38:01 +09:00
|
|
|
#include <sched.h>
|
2015-06-20 22:45:15 +01:00
|
|
|
#endif
|
2026-05-30 21:59:10 +02:00
|
|
|
|
2015-06-20 22:45:15 +01:00
|
|
|
#ifndef _WIN32
|
2016-09-18 09:38:01 +09:00
|
|
|
#include <unistd.h>
|
2013-09-04 20:17:46 -04:00
|
|
|
#endif
|
|
|
|
|
|
2026-08-08 04:39:47 +02:00
|
|
|
#ifdef __linux__
|
|
|
|
|
#include <sys/resource.h>
|
|
|
|
|
#include <algorithm>
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
constexpr int NICE_AUDIO = -16;
|
|
|
|
|
constexpr int NICE_URGENT_DISPLAY = -8;
|
|
|
|
|
constexpr int NICE_DISPLAY = -4;
|
|
|
|
|
constexpr int NICE_DEFAULT = 0;
|
|
|
|
|
constexpr int NICE_BACKGROUND = 10;
|
|
|
|
|
|
|
|
|
|
int LowestAllowedNice() {
|
|
|
|
|
static const int lowest = [] {
|
|
|
|
|
rlimit limit{};
|
|
|
|
|
if (getrlimit(RLIMIT_NICE, &limit) != 0) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
if (limit.rlim_cur >= 40) {
|
|
|
|
|
return -20;
|
|
|
|
|
}
|
|
|
|
|
return 20 - static_cast<int>(limit.rlim_cur);
|
|
|
|
|
}();
|
|
|
|
|
return lowest;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int NiceValueForPriority(Common::ThreadPriority priority) {
|
|
|
|
|
const int wanted = [priority] {
|
|
|
|
|
switch (priority) {
|
|
|
|
|
case Common::ThreadPriority::Low: return NICE_BACKGROUND;
|
|
|
|
|
case Common::ThreadPriority::Normal: return NICE_DEFAULT;
|
|
|
|
|
case Common::ThreadPriority::High: return NICE_DISPLAY;
|
|
|
|
|
case Common::ThreadPriority::VeryHigh: return NICE_URGENT_DISPLAY;
|
|
|
|
|
case Common::ThreadPriority::Critical: return NICE_AUDIO;
|
|
|
|
|
default: return NICE_DEFAULT;
|
|
|
|
|
}
|
|
|
|
|
}();
|
|
|
|
|
return (std::max)(wanted, (std::min)(NICE_DEFAULT, LowestAllowedNice()));
|
|
|
|
|
}
|
|
|
|
|
} // Anonymous namespace
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef __ANDROID__
|
|
|
|
|
#include <sys/utsname.h>
|
|
|
|
|
#include <atomic>
|
|
|
|
|
#include <cerrno>
|
|
|
|
|
#include <cstdio>
|
|
|
|
|
#include <cstdlib>
|
|
|
|
|
#include <cstring>
|
|
|
|
|
#include <fstream>
|
|
|
|
|
#include <mutex>
|
|
|
|
|
#include <vector>
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
constexpr size_t ANDROID_MINIMUM_PERFORMANCE_CORES = 4;
|
|
|
|
|
|
|
|
|
|
// A core counts as a performance core while it is within this much of the fastest one.
|
|
|
|
|
constexpr s64 ANDROID_PERFORMANCE_CAPACITY_PERCENT = 50;
|
|
|
|
|
|
|
|
|
|
constexpr std::chrono::nanoseconds ANDROID_POLICY_POLL_INTERVAL = std::chrono::milliseconds{500};
|
|
|
|
|
|
|
|
|
|
enum class CoreGroup {
|
|
|
|
|
Unrestricted,
|
|
|
|
|
Performance,
|
|
|
|
|
Efficiency,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct ThreadPolicy {
|
|
|
|
|
pid_t tid;
|
|
|
|
|
CoreGroup group;
|
|
|
|
|
s32 nice_value;
|
|
|
|
|
bool has_nice;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct CoreInfo {
|
|
|
|
|
s64 weight;
|
|
|
|
|
u64 midr;
|
|
|
|
|
s32 cpu;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct CpuTopologyState {
|
|
|
|
|
std::mutex topology_mutex;
|
|
|
|
|
cpu_set_t allowed{};
|
|
|
|
|
cpu_set_t performance{};
|
|
|
|
|
cpu_set_t efficiency{};
|
|
|
|
|
bool separated = false;
|
|
|
|
|
bool initialized = false;
|
|
|
|
|
|
|
|
|
|
pid_t canary_tid = 0;
|
|
|
|
|
cpu_set_t canary_mask{};
|
|
|
|
|
bool canary_valid = false;
|
|
|
|
|
|
|
|
|
|
std::atomic<s64> next_poll_ns{0};
|
|
|
|
|
|
|
|
|
|
std::mutex policy_mutex;
|
|
|
|
|
std::vector<ThreadPolicy> policies;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
CpuTopologyState& State() {
|
|
|
|
|
static CpuTopologyState* const state = new CpuTopologyState();
|
|
|
|
|
return *state;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct PolicyRegistration {
|
|
|
|
|
~PolicyRegistration() {
|
|
|
|
|
const pid_t tid = gettid();
|
|
|
|
|
::Common::ADPF::RemoveCurrentThread();
|
|
|
|
|
CpuTopologyState& state = State();
|
|
|
|
|
std::scoped_lock lock{state.policy_mutex};
|
|
|
|
|
std::erase_if(state.policies,
|
|
|
|
|
[tid](const ThreadPolicy& policy) { return policy.tid == tid; });
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
thread_local PolicyRegistration t_policy_registration;
|
|
|
|
|
|
|
|
|
|
s32 PossibleCpuCount() {
|
|
|
|
|
std::ifstream file("/sys/devices/system/cpu/possible");
|
|
|
|
|
std::string list;
|
|
|
|
|
if (file && std::getline(file, list) && !list.empty()) {
|
|
|
|
|
s64 highest = -1;
|
|
|
|
|
const char* cursor = list.c_str();
|
|
|
|
|
while (*cursor != '\0') {
|
|
|
|
|
char* end = nullptr;
|
|
|
|
|
const s64 value = std::strtol(cursor, &end, 10);
|
|
|
|
|
if (end == cursor) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
highest = (std::max)(highest, value);
|
|
|
|
|
cursor = end;
|
|
|
|
|
while (*cursor == '-' || *cursor == ',') {
|
|
|
|
|
++cursor;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (highest >= 0) {
|
|
|
|
|
return static_cast<s32>((std::min<s64>)(highest + 1, CPU_SETSIZE));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const s64 configured = sysconf(_SC_NPROCESSORS_CONF);
|
|
|
|
|
if (configured > 0) {
|
|
|
|
|
return static_cast<s32>((std::min<s64>)(configured, CPU_SETSIZE));
|
|
|
|
|
}
|
|
|
|
|
return static_cast<s32>((std::min<s64>)(std::thread::hardware_concurrency(), CPU_SETSIZE));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
s64 ReadCpuScalar(s32 cpu, const char* node) {
|
|
|
|
|
s64 value = 0;
|
|
|
|
|
std::ifstream file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/" + node);
|
|
|
|
|
if (!file || !(file >> value) || value <= 0) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
return value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
u64 ReadCpuMidr(s32 cpu) {
|
|
|
|
|
u64 midr = 0;
|
|
|
|
|
std::ifstream file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) +
|
|
|
|
|
"/regs/identification/midr_el1");
|
|
|
|
|
if (!file || !(file >> std::hex >> midr)) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
return midr;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::vector<CoreInfo> CollectCores(const cpu_set_t& allowed, s32 total, const char* node,
|
|
|
|
|
bool require_all) {
|
|
|
|
|
std::vector<CoreInfo> cores;
|
|
|
|
|
for (s32 cpu = 0; cpu < total; ++cpu) {
|
|
|
|
|
if (!CPU_ISSET(cpu, &allowed)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
const s64 weight = ReadCpuScalar(cpu, node);
|
|
|
|
|
if (weight <= 0) {
|
|
|
|
|
if (require_all) {
|
|
|
|
|
return {};
|
|
|
|
|
}
|
|
|
|
|
LOG_WARNING(Common, "Could not read {} for CPU {}, treating it as an efficiency core",
|
|
|
|
|
node, cpu);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
cores.push_back(CoreInfo{weight, ReadCpuMidr(cpu), cpu});
|
|
|
|
|
}
|
|
|
|
|
return cores;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool WeightsAreUniform(const std::vector<CoreInfo>& cores) {
|
|
|
|
|
return std::all_of(cores.begin(), cores.end(),
|
|
|
|
|
[&](const CoreInfo& core) { return core.weight == cores.front().weight; });
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool MidrsAreDistinct(const std::vector<CoreInfo>& cores) {
|
|
|
|
|
return std::none_of(cores.begin(), cores.end(),
|
|
|
|
|
[](const CoreInfo& core) { return core.midr == 0; }) &&
|
|
|
|
|
std::any_of(cores.begin(), cores.end(),
|
|
|
|
|
[&](const CoreInfo& core) { return core.midr != cores.front().midr; });
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void ComputeTopologyLocked(CpuTopologyState& state) {
|
|
|
|
|
state.initialized = true;
|
|
|
|
|
state.separated = false;
|
|
|
|
|
CPU_ZERO(&state.allowed);
|
|
|
|
|
CPU_ZERO(&state.performance);
|
|
|
|
|
CPU_ZERO(&state.efficiency);
|
|
|
|
|
|
|
|
|
|
if (sched_getaffinity(getpid(), sizeof(state.allowed), &state.allowed) != 0) {
|
|
|
|
|
LOG_WARNING(Common, "Could not query process CPU affinity: {}",
|
|
|
|
|
::Common::GetLastErrorMsg());
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const s32 total = PossibleCpuCount();
|
|
|
|
|
auto cores = CollectCores(state.allowed, total, "cpu_capacity", true);
|
|
|
|
|
if (cores.empty() || (WeightsAreUniform(cores) && MidrsAreDistinct(cores))) {
|
|
|
|
|
auto by_frequency = CollectCores(state.allowed, total, "cpufreq/cpuinfo_max_freq", false);
|
|
|
|
|
if (!by_frequency.empty()) {
|
|
|
|
|
cores = std::move(by_frequency);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (cores.empty()) {
|
|
|
|
|
LOG_WARNING(Common, "Could not determine CPU topology, thread placement is disabled");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (WeightsAreUniform(cores)) {
|
|
|
|
|
if (MidrsAreDistinct(cores)) {
|
|
|
|
|
LOG_WARNING(Common, "CPU clusters differ but rank identically, thread placement is "
|
|
|
|
|
"disabled");
|
|
|
|
|
} else {
|
|
|
|
|
LOG_INFO(Common, "CPU cores are symmetric, thread placement is disabled");
|
|
|
|
|
}
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::sort(cores.begin(), cores.end(), [](const CoreInfo& lhs, const CoreInfo& rhs) {
|
|
|
|
|
if (lhs.weight != rhs.weight) {
|
|
|
|
|
return lhs.weight > rhs.weight;
|
|
|
|
|
}
|
|
|
|
|
return lhs.cpu < rhs.cpu;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const s64 fastest = cores.front().weight;
|
|
|
|
|
size_t taken = 0;
|
|
|
|
|
for (const auto& core : cores) {
|
|
|
|
|
const bool fast_enough =
|
|
|
|
|
core.weight * 100 >= fastest * ANDROID_PERFORMANCE_CAPACITY_PERCENT;
|
|
|
|
|
if (!fast_enough && taken >= ANDROID_MINIMUM_PERFORMANCE_CORES) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
CPU_SET(core.cpu, &state.performance);
|
|
|
|
|
++taken;
|
|
|
|
|
}
|
|
|
|
|
if (taken == 0) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (s32 cpu = 0; cpu < total; ++cpu) {
|
|
|
|
|
if (CPU_ISSET(cpu, &state.allowed) && !CPU_ISSET(cpu, &state.performance)) {
|
|
|
|
|
CPU_SET(cpu, &state.efficiency);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
state.separated = CPU_COUNT(&state.efficiency) > 0;
|
|
|
|
|
LOG_INFO(Common, "CPU topology: {} performance cores, {} efficiency cores, separation {}",
|
|
|
|
|
CPU_COUNT(&state.performance), CPU_COUNT(&state.efficiency),
|
|
|
|
|
state.separated ? "enabled" : "unavailable");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void EnsureTopologyLocked(CpuTopologyState& state) {
|
|
|
|
|
if (!state.initialized) {
|
|
|
|
|
ComputeTopologyLocked(state);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void RefreshTopologyLocked(CpuTopologyState& state) {
|
|
|
|
|
if (!state.initialized) {
|
|
|
|
|
ComputeTopologyLocked(state);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
cpu_set_t current;
|
|
|
|
|
CPU_ZERO(¤t);
|
|
|
|
|
if (sched_getaffinity(getpid(), sizeof(current), ¤t) != 0) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (std::memcmp(¤t, &state.allowed, sizeof(current)) != 0) {
|
|
|
|
|
ComputeTopologyLocked(state);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool ApplyCoreGroupLocked(CpuTopologyState& state, pid_t tid, CoreGroup group,
|
|
|
|
|
bool* gone = nullptr) {
|
|
|
|
|
const bool restrict_group = group != CoreGroup::Unrestricted && state.separated;
|
|
|
|
|
const cpu_set_t* mask = &state.allowed;
|
|
|
|
|
if (restrict_group) {
|
|
|
|
|
mask = group == CoreGroup::Performance ? &state.performance : &state.efficiency;
|
|
|
|
|
}
|
|
|
|
|
if (CPU_COUNT(mask) == 0) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
if (sched_setaffinity(tid, sizeof(*mask), mask) != 0) {
|
|
|
|
|
if (gone != nullptr && errno == ESRCH) {
|
|
|
|
|
*gone = true;
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
LOG_WARNING(Common, "Could not restrict thread {} to its core group: {}", tid,
|
|
|
|
|
::Common::GetLastErrorMsg());
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool KernelPreservesRequestedAffinity() {
|
|
|
|
|
utsname info{};
|
|
|
|
|
if (uname(&info) != 0) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
s32 major = 0;
|
|
|
|
|
s32 minor = 0;
|
|
|
|
|
if (std::sscanf(info.release, "%d.%d", &major, &minor) != 2) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
return major > 6 || (major == 6 && minor >= 2);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void SnapshotCanaryLocked(CpuTopologyState& state) {
|
|
|
|
|
state.canary_valid = false;
|
|
|
|
|
if (!state.separated) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
for (const auto& policy : state.policies) {
|
|
|
|
|
if (policy.group == CoreGroup::Unrestricted) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
cpu_set_t mask;
|
|
|
|
|
CPU_ZERO(&mask);
|
|
|
|
|
if (sched_getaffinity(policy.tid, sizeof(mask), &mask) != 0) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
state.canary_tid = policy.tid;
|
|
|
|
|
state.canary_mask = mask;
|
|
|
|
|
state.canary_valid = true;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool DueForPoll(CpuTopologyState& state) {
|
|
|
|
|
const auto now = std::chrono::steady_clock::now().time_since_epoch();
|
|
|
|
|
const s64 now_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(now).count();
|
|
|
|
|
s64 next = state.next_poll_ns.load(std::memory_order_relaxed);
|
|
|
|
|
if (now_ns < next) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
return state.next_poll_ns.compare_exchange_strong(
|
|
|
|
|
next, now_ns + ANDROID_POLICY_POLL_INTERVAL.count(), std::memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ThreadPolicy& AcquirePolicyLocked(CpuTopologyState& state, pid_t tid) {
|
|
|
|
|
for (auto& policy : state.policies) {
|
|
|
|
|
if (policy.tid == tid) {
|
|
|
|
|
return policy;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return state.policies.emplace_back(ThreadPolicy{tid, CoreGroup::Unrestricted, 0, false});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void SetCurrentThreadCoreGroup(CoreGroup group) {
|
|
|
|
|
const pid_t tid = gettid();
|
|
|
|
|
(void)&t_policy_registration;
|
|
|
|
|
|
|
|
|
|
CoreGroup effective = group;
|
|
|
|
|
if (tid == getpid() && group != CoreGroup::Unrestricted) {
|
|
|
|
|
LOG_WARNING(Common, "Refusing to place the main thread: the CPU topology is read from it");
|
|
|
|
|
effective = CoreGroup::Unrestricted;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
CpuTopologyState& state = State();
|
|
|
|
|
std::scoped_lock topology_lock{state.topology_mutex};
|
|
|
|
|
EnsureTopologyLocked(state);
|
|
|
|
|
ApplyCoreGroupLocked(state, tid, effective);
|
|
|
|
|
|
|
|
|
|
std::scoped_lock policy_lock{state.policy_mutex};
|
|
|
|
|
AcquirePolicyLocked(state, tid).group = effective;
|
|
|
|
|
if (!state.canary_valid) {
|
|
|
|
|
SnapshotCanaryLocked(state);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void RememberCurrentThreadNice(pid_t tid, s32 nice_value) {
|
|
|
|
|
(void)&t_policy_registration;
|
|
|
|
|
CpuTopologyState& state = State();
|
|
|
|
|
std::scoped_lock lock{state.policy_mutex};
|
|
|
|
|
ThreadPolicy& policy = AcquirePolicyLocked(state, tid);
|
|
|
|
|
policy.nice_value = nice_value;
|
|
|
|
|
policy.has_nice = true;
|
|
|
|
|
}
|
|
|
|
|
} // Anonymous namespace
|
|
|
|
|
#endif
|
|
|
|
|
|
2026-05-30 21:59:10 +02:00
|
|
|
#include "common/cpu_features.h"
|
|
|
|
|
#ifdef ARCHITECTURE_x86_64
|
|
|
|
|
#ifdef _MSC_VER
|
|
|
|
|
#include <intrin.h>
|
|
|
|
|
#else
|
|
|
|
|
#include <x86intrin.h>
|
2016-10-10 02:33:41 +00:00
|
|
|
#endif
|
2026-05-30 21:59:10 +02:00
|
|
|
#include "common/x64/rdtsc.h"
|
|
|
|
|
#endif
|
2016-10-10 02:33:41 +00:00
|
|
|
|
2016-09-18 09:38:01 +09:00
|
|
|
namespace Common {
|
2013-09-04 20:17:46 -04:00
|
|
|
|
2020-04-05 09:48:53 -04:00
|
|
|
void SetCurrentThreadPriority(ThreadPriority new_priority) {
|
2025-10-22 04:53:40 +02:00
|
|
|
#ifdef _WIN32
|
|
|
|
|
int windows_priority = [&]() {
|
|
|
|
|
switch (new_priority) {
|
|
|
|
|
case ThreadPriority::Low: return THREAD_PRIORITY_BELOW_NORMAL;
|
|
|
|
|
case ThreadPriority::Normal: return THREAD_PRIORITY_NORMAL;
|
|
|
|
|
case ThreadPriority::High: return THREAD_PRIORITY_ABOVE_NORMAL;
|
|
|
|
|
case ThreadPriority::VeryHigh: return THREAD_PRIORITY_HIGHEST;
|
|
|
|
|
case ThreadPriority::Critical: return THREAD_PRIORITY_TIME_CRITICAL;
|
|
|
|
|
default: return THREAD_PRIORITY_NORMAL;
|
|
|
|
|
}
|
|
|
|
|
}();
|
|
|
|
|
SetThreadPriority(GetCurrentThread(), windows_priority);
|
|
|
|
|
#elif defined(__HAIKU__)
|
|
|
|
|
// TODO: We have priorities for 3D rendering applications - may help lavapipe?
|
|
|
|
|
int priority = [&]() {
|
|
|
|
|
switch (new_priority) {
|
|
|
|
|
case ThreadPriority::Low: return B_LOW_PRIORITY;
|
|
|
|
|
case ThreadPriority::Normal: return B_NORMAL_PRIORITY;
|
|
|
|
|
case ThreadPriority::High: return B_DISPLAY_PRIORITY;
|
|
|
|
|
case ThreadPriority::VeryHigh: return B_URGENT_DISPLAY_PRIORITY;
|
|
|
|
|
case ThreadPriority::Critical: return B_URGENT_PRIORITY;
|
|
|
|
|
default: return B_NORMAL_PRIORITY;
|
|
|
|
|
}
|
|
|
|
|
}();
|
|
|
|
|
set_thread_priority(find_thread(NULL), priority);
|
2026-08-08 04:39:47 +02:00
|
|
|
#elif defined(__ANDROID__)
|
|
|
|
|
const int nice_value = NiceValueForPriority(new_priority);
|
|
|
|
|
const pid_t tid = gettid();
|
|
|
|
|
if (setpriority(PRIO_PROCESS, static_cast<id_t>(tid), nice_value) != 0) {
|
|
|
|
|
LOG_WARNING(Common, "Could not set thread nice value to {}: {}", nice_value,
|
|
|
|
|
GetLastErrorMsg());
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
RememberCurrentThreadNice(tid, nice_value);
|
|
|
|
|
#elif defined(__linux__)
|
|
|
|
|
const int nice_value = NiceValueForPriority(new_priority);
|
|
|
|
|
if (setpriority(PRIO_PROCESS, 0, nice_value) != 0) {
|
|
|
|
|
LOG_DEBUG(Common, "Could not set thread nice value to {}: {}", nice_value,
|
|
|
|
|
GetLastErrorMsg());
|
|
|
|
|
}
|
2020-04-05 09:48:53 -04:00
|
|
|
#else
|
2026-08-08 04:39:47 +02:00
|
|
|
const s32 max_prio = sched_get_priority_max(SCHED_OTHER);
|
|
|
|
|
const s32 min_prio = sched_get_priority_min(SCHED_OTHER);
|
2020-04-05 09:48:53 -04:00
|
|
|
if (max_prio > min_prio) {
|
2026-08-08 04:39:47 +02:00
|
|
|
const u32 level = (std::min)(static_cast<u32>(new_priority), 4U);
|
|
|
|
|
sched_param params{};
|
|
|
|
|
params.sched_priority =
|
|
|
|
|
min_prio + static_cast<s32>(static_cast<u32>(max_prio - min_prio) * level) / 4;
|
|
|
|
|
pthread_setschedparam(pthread_self(), SCHED_OTHER, ¶ms);
|
2020-04-05 09:48:53 -04:00
|
|
|
}
|
|
|
|
|
#endif
|
2025-10-22 04:53:40 +02:00
|
|
|
}
|
2020-04-05 09:48:53 -04:00
|
|
|
|
2018-11-21 21:53:35 -05:00
|
|
|
void SetCurrentThreadName(const char* name) {
|
2026-02-01 01:54:59 +01:00
|
|
|
#ifdef _MSC_VER
|
|
|
|
|
// Sets the debugger-visible name of the current thread.
|
|
|
|
|
if (auto pf = (decltype(&SetThreadDescription))(void*)GetProcAddress(GetModuleHandle(TEXT("KernelBase.dll")), "SetThreadDescription"); pf)
|
2025-11-03 21:06:51 +01:00
|
|
|
pf(GetCurrentThread(), UTF8ToUTF16W(name).data()); // Windows 10+
|
2026-02-01 01:54:59 +01:00
|
|
|
else
|
|
|
|
|
; // No-op
|
|
|
|
|
#elif defined(__APPLE__)
|
2018-11-21 21:53:35 -05:00
|
|
|
pthread_setname_np(name);
|
2025-10-07 06:38:22 +02:00
|
|
|
#elif defined(__HAIKU__)
|
|
|
|
|
rename_thread(find_thread(NULL), name);
|
2016-10-10 02:25:19 +00:00
|
|
|
#elif defined(__Bitrig__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__)
|
2018-11-21 21:53:35 -05:00
|
|
|
pthread_set_name_np(pthread_self(), name);
|
2016-10-10 02:25:19 +00:00
|
|
|
#elif defined(__NetBSD__)
|
2018-11-21 21:53:35 -05:00
|
|
|
pthread_setname_np(pthread_self(), "%s", (void*)name);
|
2025-10-07 06:38:22 +02:00
|
|
|
#elif defined(__linux__) || defined(__CYGWIN__) || defined(__sun__) || defined(__glibc__) || defined(__managarm__)
|
|
|
|
|
int ret = pthread_setname_np(pthread_self(), name);
|
|
|
|
|
if (ret == ERANGE) {
|
|
|
|
|
// Linux limits thread names to 15 characters and will outright reject any
|
|
|
|
|
// attempt to set a longer name with ERANGE.
|
|
|
|
|
char buf[16];
|
|
|
|
|
size_t const len = std::min<size_t>(std::strlen(name), sizeof(buf) - 1);
|
|
|
|
|
std::memcpy(buf, name, len);
|
|
|
|
|
buf[len] = '\0';
|
|
|
|
|
pthread_setname_np(pthread_self(), buf);
|
2020-08-02 10:57:08 -07:00
|
|
|
}
|
2025-11-02 08:15:48 +01:00
|
|
|
#elif defined(_WIN32)
|
2026-02-01 01:54:59 +01:00
|
|
|
// MinGW with the POSIX threading model does not support pthread_setname_np
|
|
|
|
|
// See for reference
|
|
|
|
|
// https://gitlab.freedesktop.org/mesa/mesa/-/blame/main/src/util/u_thread.c?ref_type=heads#L75
|
2025-10-07 06:38:22 +02:00
|
|
|
(void)name;
|
2013-09-04 20:17:46 -04:00
|
|
|
#else
|
2018-11-21 21:53:35 -05:00
|
|
|
pthread_setname_np(pthread_self(), name);
|
2013-09-04 20:17:46 -04:00
|
|
|
#endif
|
|
|
|
|
}
|
2020-02-24 22:04:12 -04:00
|
|
|
|
2026-08-08 04:39:47 +02:00
|
|
|
void SetCurrentThreadToPerformanceCores() {
|
2026-02-01 01:54:59 +01:00
|
|
|
#if defined(__ANDROID__)
|
2026-08-08 04:39:47 +02:00
|
|
|
if (ADPF::AddCurrentThread(ADPF::Session::Render)) {
|
|
|
|
|
SetCurrentThreadCoreGroup(CoreGroup::Unrestricted);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
SetCurrentThreadCoreGroup(CoreGroup::Performance);
|
2013-09-04 20:17:46 -04:00
|
|
|
#endif
|
2026-08-08 04:39:47 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void SetCurrentThreadToEfficiencyCores() {
|
|
|
|
|
#if defined(__ANDROID__)
|
|
|
|
|
if (ADPF::AddCurrentThread(ADPF::Session::Background)) {
|
|
|
|
|
SetCurrentThreadCoreGroup(CoreGroup::Unrestricted);
|
|
|
|
|
return;
|
2026-02-01 01:54:59 +01:00
|
|
|
}
|
2026-08-08 04:39:47 +02:00
|
|
|
SetCurrentThreadCoreGroup(CoreGroup::Efficiency);
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void SetCurrentThreadToBackgroundWork() {
|
|
|
|
|
#if defined(__ANDROID__)
|
|
|
|
|
ADPF::AddCurrentThread(ADPF::Session::Background);
|
|
|
|
|
SetCurrentThreadCoreGroup(CoreGroup::Unrestricted);
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void SetCurrentThreadToAllCores() {
|
|
|
|
|
#if defined(__ANDROID__)
|
|
|
|
|
ADPF::RemoveCurrentThread();
|
|
|
|
|
SetCurrentThreadCoreGroup(CoreGroup::Unrestricted);
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void RefreshThreadPolicies() {
|
|
|
|
|
#if defined(__ANDROID__)
|
|
|
|
|
CpuTopologyState& state = State();
|
|
|
|
|
std::scoped_lock topology_lock{state.topology_mutex};
|
|
|
|
|
RefreshTopologyLocked(state);
|
|
|
|
|
|
|
|
|
|
std::scoped_lock policy_lock{state.policy_mutex};
|
|
|
|
|
std::erase_if(state.policies, [&state](const ThreadPolicy& policy) {
|
|
|
|
|
bool gone = false;
|
|
|
|
|
if (policy.has_nice &&
|
|
|
|
|
setpriority(PRIO_PROCESS, static_cast<id_t>(policy.tid), policy.nice_value) != 0 &&
|
|
|
|
|
errno == ESRCH) {
|
|
|
|
|
gone = true;
|
|
|
|
|
}
|
|
|
|
|
if (!gone) {
|
|
|
|
|
ApplyCoreGroupLocked(state, policy.tid, policy.group, &gone);
|
|
|
|
|
}
|
|
|
|
|
return gone;
|
|
|
|
|
});
|
|
|
|
|
SnapshotCanaryLocked(state);
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void PollThreadPolicies() {
|
|
|
|
|
#if defined(__ANDROID__)
|
|
|
|
|
static const bool needed = !KernelPreservesRequestedAffinity();
|
|
|
|
|
if (!needed) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
CpuTopologyState& state = State();
|
|
|
|
|
if (!DueForPoll(state)) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pid_t tid;
|
|
|
|
|
cpu_set_t expected;
|
|
|
|
|
{
|
|
|
|
|
std::scoped_lock lock{state.topology_mutex};
|
|
|
|
|
if (!state.canary_valid) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
tid = state.canary_tid;
|
|
|
|
|
expected = state.canary_mask;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cpu_set_t current;
|
|
|
|
|
CPU_ZERO(¤t);
|
|
|
|
|
if (sched_getaffinity(tid, sizeof(current), ¤t) == 0 &&
|
|
|
|
|
std::memcmp(¤t, &expected, sizeof(current)) == 0) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
RefreshThreadPolicies();
|
|
|
|
|
#endif
|
2026-02-01 01:54:59 +01:00
|
|
|
}
|
2013-09-04 20:17:46 -04:00
|
|
|
|
2026-05-30 21:59:10 +02:00
|
|
|
#ifdef ARCHITECTURE_x86_64
|
|
|
|
|
// On Linux and UNIX systems, a futex would nominally be used to cover the costs
|
|
|
|
|
// the idea is that it's intuitivelly cheaper to use a direct instruction as opposed to a full futex call
|
|
|
|
|
// the underlying libc++ implementation uses pthread_cond_timedwait which MAY invoke a futex
|
|
|
|
|
// Let's pretend the OS is too expensive to jump into, and avoid ANY context switches
|
|
|
|
|
// this should *IN THEORY* lower CPU usage while just waiting for stuff effectively
|
|
|
|
|
// For windows the minimal quanta resolution is about 500us, and normal CRT cond var is 1.5ms(?)
|
|
|
|
|
// so may as well avoid that too
|
|
|
|
|
// Let's just give ALL platforms the same mechanisms (almost) for when they have umonitor OR waitpkg
|
|
|
|
|
#ifdef __clang__
|
|
|
|
|
__attribute__((target("waitpkg,mwaitx")))
|
|
|
|
|
#elif defined(__GNUC__)
|
|
|
|
|
#pragma GCC target("waitpkg")
|
|
|
|
|
#pragma GCC target("mwaitx")
|
|
|
|
|
#endif
|
|
|
|
|
bool Event::WaitFor(const std::chrono::nanoseconds time) {
|
|
|
|
|
#ifdef _WIN32
|
|
|
|
|
auto const start = Common::X64::FencedRDTSC();
|
|
|
|
|
auto const& caps = Common::g_cpu_caps;
|
|
|
|
|
[[maybe_unused]] auto const end = start + Common::g_wall_clock.NsToTicks(time);
|
|
|
|
|
if (caps.monitorx) {
|
|
|
|
|
while (true) {
|
|
|
|
|
// Armed monitor, as per manual, MWAITX must be conditional if the condition isn't satisfied
|
|
|
|
|
// to prevent a race condition.
|
|
|
|
|
_mm_monitorx(reinterpret_cast<u64*>(std::addressof(is_set)), 0, 0);
|
|
|
|
|
if (!is_set.load()) {
|
|
|
|
|
// RDTSC may be fenced here due to atomic load
|
2026-05-31 02:32:21 +02:00
|
|
|
#ifdef _MSC_VER
|
|
|
|
|
auto const now = __rdtsc();
|
|
|
|
|
#else
|
2026-05-30 21:59:10 +02:00
|
|
|
auto const now = _rdtsc();
|
2026-05-31 02:32:21 +02:00
|
|
|
#endif
|
2026-05-30 21:59:10 +02:00
|
|
|
if (end > now) {
|
|
|
|
|
u32 const cycles = std::min<u32>((std::numeric_limits<u32>::max)(), s64(end) - s64(now));
|
|
|
|
|
// See here: https://github.com/torvalds/linux/blob/948a64995aca6820abefd17f1a4258f5835c5ad9/arch/x86/lib/delay.c#L93
|
|
|
|
|
// MWAITX accepts a 32-bit input timer which determines the total number of cycles to wait for
|
|
|
|
|
// NOT THE TOTAL ABSOLUTE TSC VALUE, it's just a delta
|
|
|
|
|
// BIT[1] = use a timer
|
|
|
|
|
// Hint = 0: Use C1 state when sleepy (means slower wakeup but better savings)
|
|
|
|
|
_mm_mwaitx(1 << 1, 0u, cycles);
|
|
|
|
|
if (!is_set.load())
|
|
|
|
|
return false;
|
|
|
|
|
} else
|
|
|
|
|
return false; //timeout
|
|
|
|
|
}
|
|
|
|
|
bool expected = true;
|
|
|
|
|
if (is_set.compare_exchange_weak(expected, false, std::memory_order_release))
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
} else if (caps.waitpkg) {
|
|
|
|
|
// #UD If CPUID.7.0:ECX.WAITPKG[bit 5]=0.
|
|
|
|
|
while (true) {
|
|
|
|
|
_umonitor(std::addressof(is_set));
|
|
|
|
|
if (!is_set.load() && !_umwait(0, end)) //umwait is absolute time!!!
|
|
|
|
|
return false;
|
|
|
|
|
bool expected = true;
|
|
|
|
|
if (is_set.compare_exchange_weak(expected, false, std::memory_order_release))
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
2026-05-31 02:32:21 +02:00
|
|
|
#ifdef _MSC_VER
|
|
|
|
|
while (!is_set.load() && end > __rdtsc())
|
|
|
|
|
Common::Windows::SleepForOneTick();
|
|
|
|
|
#else
|
2026-05-30 21:59:10 +02:00
|
|
|
while (!is_set.load() && end > _rdtsc())
|
|
|
|
|
Common::Windows::SleepForOneTick();
|
2026-05-31 02:32:21 +02:00
|
|
|
#endif
|
2026-05-30 21:59:10 +02:00
|
|
|
if (is_set.load())
|
|
|
|
|
Reset();
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
std::unique_lock lk{mutex};
|
|
|
|
|
if (!condvar.wait_for(lk, time, [this] { return is_set.load(); }))
|
|
|
|
|
return false;
|
|
|
|
|
is_set = false;
|
|
|
|
|
return true;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
bool Event::WaitFor(const std::chrono::nanoseconds time) {
|
|
|
|
|
#ifdef _WIN32
|
|
|
|
|
auto const end = Common::g_wall_clock.GetTimeNS() + time;
|
|
|
|
|
while (!is_set.load() && end > Common::g_wall_clock.GetTimeNS())
|
|
|
|
|
Common::Windows::SleepForOneTick();
|
|
|
|
|
if (is_set.load())
|
|
|
|
|
Reset();
|
|
|
|
|
return true;
|
|
|
|
|
#else
|
|
|
|
|
std::unique_lock lk{mutex};
|
|
|
|
|
if (!condvar.wait_for(lk, time, [this] { return is_set.load(); }))
|
|
|
|
|
return false;
|
|
|
|
|
is_set = false;
|
|
|
|
|
return true;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2013-09-04 20:17:46 -04:00
|
|
|
} // namespace Common
|