From patchwork series 484432

https://patchwork.ozlabs.org//project/linuxppc-dev/list/?series=484432

Author: Fox Snowpatch

arch/powerpc/include/asm/paravirt.h

@@ -145,6 +145,15 @@ static inline bool vcpu_is_preempted(int cpu)
 	if (!is_shared_processor())
 		return false;
 
+#ifdef CONFIG_PPC_SPLPAR
+	/*
+	 * Assume the target CPU to be preempted if it is above soft
+	 * entitlement limit
+	 */
+	if (!is_kvm_guest())
+		return !cpu_active(cpu);
+#endif
+
 	/*
 	 * If the hypervisor has dispatched the target CPU on a physical
 	 * processor, then the target CPU is definitely not preempted.
@@ -159,59 +168,6 @@ static inline bool vcpu_is_preempted(int cpu)
 	if (!is_vcpu_idle(cpu))
 		return true;
 
-#ifdef CONFIG_PPC_SPLPAR
-	if (!is_kvm_guest()) {
-		int first_cpu, i;
-
-		/*
-		 * The result of vcpu_is_preempted() is used in a
-		 * speculative way, and is always subject to invalidation
-		 * by events internal and external to Linux. While we can
-		 * be called in preemptable context (in the Linux sense),
-		 * we're not accessing per-cpu resources in a way that can
-		 * race destructively with Linux scheduler preemption and
-		 * migration, and callers can tolerate the potential for
-		 * error introduced by sampling the CPU index without
-		 * pinning the task to it. So it is permissible to use
-		 * raw_smp_processor_id() here to defeat the preempt debug
-		 * warnings that can arise from using smp_processor_id()
-		 * in arbitrary contexts.
-		 */
-		first_cpu = cpu_first_thread_sibling(raw_smp_processor_id());
-
-		/*
-		 * The PowerVM hypervisor dispatches VMs on a whole core
-		 * basis. So we know that a thread sibling of the executing CPU
-		 * cannot have been preempted by the hypervisor, even if it
-		 * has called H_CONFER, which will set the yield bit.
-		 */
-		if (cpu_first_thread_sibling(cpu) == first_cpu)
-			return false;
-
-		/*
-		 * The specific target CPU was marked by guest OS as idle, but
-		 * then also check all other cpus in the core for PowerVM
-		 * because it does core scheduling and one of the vcpu
-		 * of the core getting preempted by hypervisor implies
-		 * other vcpus can also be considered preempted.
-		 */
-		first_cpu = cpu_first_thread_sibling(cpu);
-		for (i = first_cpu; i < first_cpu + threads_per_core; i++) {
-			if (i == cpu)
-				continue;
-			if (vcpu_is_dispatched(i))
-				return false;
-			if (!is_vcpu_idle(i))
-				return true;
-		}
-	}
-#endif
-
-	/*
-	 * None of the threads in target CPU's core are running but none of
-	 * them were preempted too. Hence assume the target CPU to be
-	 * non-preempted.
-	 */
 	return false;
 }
 

arch/powerpc/include/asm/smp.h

@@ -60,6 +60,9 @@ struct smp_ops_t {
 #ifdef CONFIG_HOTPLUG_CPU
 	void  (*cpu_offline_self)(void);
 #endif
+#ifdef CONFIG_PPC_SPLPAR
+	unsigned int (*num_available_cores)(void);
+#endif
 };
 
 extern struct task_struct *secondary_current;
@@ -266,6 +269,9 @@ extern char __secondary_hold;
 extern unsigned int booting_thread_hwid;
 
 extern void __early_start(void);
+#ifdef CONFIG_PPC_SPLPAR
+int arch_update_cpu_topology(void);
+#endif /* CONFIG_PPC_SPLPAR */
 #endif /* __ASSEMBLER__ */
 
 #endif /* __KERNEL__ */

arch/powerpc/include/asm/topology.h

@@ -177,5 +177,10 @@ static inline bool topology_is_core_online(unsigned int cpu)
 }
 #endif
 
+#ifdef CONFIG_PPC_SPLPAR
+#define arch_scale_cpu_capacity arch_scale_cpu_capacity
+unsigned long arch_scale_cpu_capacity(int cpu);
+#endif
+
 #endif /* __KERNEL__ */
 #endif	/* _ASM_POWERPC_TOPOLOGY_H */

arch/powerpc/kernel/smp.c

@@ -82,6 +82,9 @@ bool has_big_cores __ro_after_init;
 bool coregroup_enabled __ro_after_init;
 bool thread_group_shares_l2 __ro_after_init;
 bool thread_group_shares_l3 __ro_after_init;
+#ifdef CONFIG_PPC_SPLPAR
+bool process_steal_enable __ro_after_init;
+#endif
 
 DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
 DEFINE_PER_CPU(cpumask_var_t, cpu_smallcore_map);
@@ -1755,6 +1758,16 @@ void __init smp_cpus_done(unsigned int max_cpus)
 
 	dump_numa_cpu_topology();
 	build_sched_topology();
+
+#ifdef CONFIG_PPC_SPLPAR
+	if (smp_ops->num_available_cores)
+		smp_ops->num_available_cores();
+
+	if (is_shared_processor() && !is_kvm_guest())
+		process_steal_enable = true;
+	else
+		process_steal_enable = false;
+#endif
 }
 
 /*
@@ -1821,3 +1834,28 @@ void __noreturn arch_cpu_idle_dead(void)
 }
 
 #endif
+
+#ifdef CONFIG_PPC_SPLPAR
+#define MIN_CAPACITY 1
+
+/*
+ * Assume CPU capacity to be low if CPU number happens be above soft
+ * available limit. This forces load balancer to prefer higher capacity CPUs
+ */
+unsigned long arch_scale_cpu_capacity(int cpu)
+{
+	if (is_shared_processor() && !is_kvm_guest()) {
+		if (!cpu_active(cpu))
+			return MIN_CAPACITY;
+	}
+	return SCHED_CAPACITY_SCALE;
+}
+
+int arch_update_cpu_topology(void)
+{
+	if (is_shared_processor() && !is_kvm_guest())
+		return (num_online_cpus() != cpumask_weight(cpu_active_mask));
+
+	return 0;
+}
+#endif /* CONFIG_PPC_SPLPAR */

arch/powerpc/platforms/pseries/hotplug-cpu.c

@@ -284,6 +284,9 @@ static int pseries_add_processor(struct device_node *np)
 
 out:
 	cpu_maps_update_done();
+#ifdef CONFIG_PPC_SPLPAR
+	pseries_num_available_cores();
+#endif
 	free_cpumask_var(cpu_mask);
 	return rc;
 }
@@ -323,6 +326,9 @@ static void pseries_remove_processor(struct device_node *np)
 			       "with physical id 0x%x\n", thread);
 	}
 	cpu_maps_update_done();
+#ifdef CONFIG_PPC_SPLPAR
+	pseries_num_available_cores();
+#endif
 }
 
 static int dlpar_offline_cpu(struct device_node *dn)

arch/powerpc/platforms/pseries/lpar.c

@@ -659,18 +659,78 @@ static int __init vcpudispatch_stats_procfs_init(void)
 machine_device_initcall(pseries, vcpudispatch_stats_procfs_init);
 
 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+#define STEAL_MULTIPLE (STEAL_RATIO * STEAL_RATIO)
+
+static u8 steal_interval = 1;
+
+static bool should_cpu_process_steal(int cpu)
+{
+	if (cpu == cpumask_first(cpu_online_mask))
+		return true;
+
+	return false;
+}
+
+extern bool process_steal_enable;
+static void process_steal(int cpu)
+{
+	unsigned long steal_ratio, delta_tb, interval_tb;
+	static unsigned long next_tb, prev_steal;
+	unsigned long tb = mftb();
+	unsigned long steal = 0;
+	unsigned int i;
+
+	if (!process_steal_enable)
+		return;
+
+	if (!should_cpu_process_steal(cpu))
+		return;
+
+	if (tb < next_tb)
+		return;
+
+	for_each_online_cpu(i) {
+		struct lppaca *lppaca = &lppaca_of(i);
+
+		steal += be64_to_cpu(READ_ONCE(lppaca->ready_enqueue_tb));
+		steal += be64_to_cpu(READ_ONCE(lppaca->enqueue_dispatch_tb));
+	}
+
+	if (!steal_interval)
+		steal_interval = 1;
+
+	interval_tb = steal_interval * tb_ticks_per_sec;
+	if (next_tb && prev_steal) {
+		delta_tb = max(tb - (next_tb - interval_tb), 1);
+		steal_ratio = (steal - prev_steal) * STEAL_MULTIPLE;
+		steal_ratio /= (delta_tb * num_online_cpus());
+		trigger_softoffline(steal_ratio);
+	}
+
+	next_tb = tb + interval_tb;
+	prev_steal = steal;
+}
+
 u64 pseries_paravirt_steal_clock(int cpu)
 {
 	struct lppaca *lppaca = &lppaca_of(cpu);
+	unsigned long steal;
+
+	steal = be64_to_cpu(READ_ONCE(lppaca->ready_enqueue_tb));
+	steal += be64_to_cpu(READ_ONCE(lppaca->enqueue_dispatch_tb));
+
+	if (is_shared_processor() && !is_kvm_guest())
+		process_steal(cpu);
 
 	/*
 	 * VPA steal time counters are reported at TB frequency. Hence do a
-	 * conversion to ns before returning
+	 * conversion to ns before using.
 	 */
-	return tb_to_ns(be64_to_cpu(READ_ONCE(lppaca->enqueue_dispatch_tb)) +
-			be64_to_cpu(READ_ONCE(lppaca->ready_enqueue_tb)));
+	steal = tb_to_ns(steal);
+
+	return steal;
 }
-#endif
+#endif /* CONFIG_PARAVIRT_TIME_ACCOUNTING */
 
 #endif /* CONFIG_PPC_SPLPAR */
 
@@ -2025,6 +2085,9 @@ static int __init vpa_debugfs_init(void)
 		debugfs_create_file(name, 0400, vpa_dir, (void *)i, &vpa_fops);
 	}
 
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	debugfs_create_u8("steal_interval_secs", 0600, arch_debugfs_dir, &steal_interval);
+#endif
 	return 0;
 }
 machine_arch_initcall(pseries, vpa_debugfs_init);

arch/powerpc/platforms/pseries/pseries.h

@@ -26,6 +26,10 @@ void pSeries_machine_check_log_err(void);
 #ifdef CONFIG_SMP
 extern void smp_init_pseries(void);
 
+#ifdef CONFIG_PPC_SPLPAR
+#define STEAL_RATIO 100
+#endif
+
 /* Get state of physical CPU from query_cpu_stopped */
 int smp_query_cpu_stopped(unsigned int pcpu);
 #define QCSS_STOPPED 0
@@ -115,6 +119,10 @@ int dlpar_workqueue_init(void);
 
 extern u32 pseries_security_flavor;
 void pseries_setup_security_mitigations(void);
+#ifdef CONFIG_PPC_SPLPAR
+void trigger_softoffline(unsigned long steal_ratio);
+unsigned int pseries_num_available_cores(void);
+#endif
 
 #ifdef CONFIG_PPC_64S_HASH_MMU
 void pseries_lpar_read_hblkrm_characteristics(void);