1/* 2 * Xen time implementation. 3 * 4 * This is implemented in terms of a clocksource driver which uses 5 * the hypervisor clock as a nanosecond timebase, and a clockevent 6 * driver which uses the hypervisor's timer mechanism. 7 * 8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 9 */ 10#include <linux/kernel.h> 11#include <linux/interrupt.h> 12#include <linux/clocksource.h> 13#include <linux/clockchips.h> 14#include <linux/kernel_stat.h> 15#include <linux/math64.h> 16#include <linux/gfp.h> 17#include <linux/slab.h> 18#include <linux/pvclock_gtod.h> 19 20#include <asm/pvclock.h> 21#include <asm/xen/hypervisor.h> 22#include <asm/xen/hypercall.h> 23 24#include <xen/events.h> 25#include <xen/features.h> 26#include <xen/interface/xen.h> 27#include <xen/interface/vcpu.h> 28 29#include "xen-ops.h" 30 31/* Xen may fire a timer up to this many ns early */ 32#define TIMER_SLOP 100000 33#define NS_PER_TICK (1000000000LL / HZ) 34 35/* runstate info updated by Xen */ 36static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); 37 38/* snapshots of runstate info */ 39static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot); 40 41/* unused ns of stolen time */ 42static DEFINE_PER_CPU(u64, xen_residual_stolen); 43 44/* return an consistent snapshot of 64-bit time/counter value */ 45static u64 get64(const u64 *p) 46{ 47 u64 ret; 48 49 if (BITS_PER_LONG < 64) { 50 u32 *p32 = (u32 *)p; 51 u32 h, l; 52 53 /* 54 * Read high then low, and then make sure high is 55 * still the same; this will only loop if low wraps 56 * and carries into high. 57 * XXX some clean way to make this endian-proof? 58 */ 59 do { 60 h = p32[1]; 61 barrier(); 62 l = p32[0]; 63 barrier(); 64 } while (p32[1] != h); 65 66 ret = (((u64)h) << 32) | l; 67 } else 68 ret = *p; 69 70 return ret; 71} 72 73/* 74 * Runstate accounting 75 */ 76static void get_runstate_snapshot(struct vcpu_runstate_info *res) 77{ 78 u64 state_time; 79 struct vcpu_runstate_info *state; 80 81 BUG_ON(preemptible()); 82 83 state = this_cpu_ptr(&xen_runstate); 84 85 /* 86 * The runstate info is always updated by the hypervisor on 87 * the current CPU, so there's no need to use anything 88 * stronger than a compiler barrier when fetching it. 89 */ 90 do { 91 state_time = get64(&state->state_entry_time); 92 barrier(); 93 *res = *state; 94 barrier(); 95 } while (get64(&state->state_entry_time) != state_time); 96} 97 98/* return true when a vcpu could run but has no real cpu to run on */ 99bool xen_vcpu_stolen(int vcpu) 100{ 101 return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable; 102} 103 104void xen_setup_runstate_info(int cpu) 105{ 106 struct vcpu_register_runstate_memory_area area; 107 108 area.addr.v = &per_cpu(xen_runstate, cpu); 109 110 if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area, 111 cpu, &area)) 112 BUG(); 113} 114 115static void do_stolen_accounting(void) 116{ 117 struct vcpu_runstate_info state; 118 struct vcpu_runstate_info *snap; 119 s64 runnable, offline, stolen; 120 cputime_t ticks; 121 122 get_runstate_snapshot(&state); 123 124 WARN_ON(state.state != RUNSTATE_running); 125 126 snap = this_cpu_ptr(&xen_runstate_snapshot); 127 128 /* work out how much time the VCPU has not been runn*ing* */ 129 runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable]; 130 offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline]; 131 132 *snap = state; 133 134 /* Add the appropriate number of ticks of stolen time, 135 including any left-overs from last time. */ 136 stolen = runnable + offline + __this_cpu_read(xen_residual_stolen); 137 138 if (stolen < 0) 139 stolen = 0; 140 141 ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen); 142 __this_cpu_write(xen_residual_stolen, stolen); 143 account_steal_ticks(ticks); 144} 145 146/* Get the TSC speed from Xen */ 147static unsigned long xen_tsc_khz(void) 148{ 149 struct pvclock_vcpu_time_info *info = 150 &HYPERVISOR_shared_info->vcpu_info[0].time; 151 152 return pvclock_tsc_khz(info); 153} 154 155cycle_t xen_clocksource_read(void) 156{ 157 struct pvclock_vcpu_time_info *src; 158 cycle_t ret; 159 160 preempt_disable_notrace(); 161 src = &__this_cpu_read(xen_vcpu)->time; 162 ret = pvclock_clocksource_read(src); 163 preempt_enable_notrace(); 164 return ret; 165} 166 167static cycle_t xen_clocksource_get_cycles(struct clocksource *cs) 168{ 169 return xen_clocksource_read(); 170} 171 172static void xen_read_wallclock(struct timespec *ts) 173{ 174 struct shared_info *s = HYPERVISOR_shared_info; 175 struct pvclock_wall_clock *wall_clock = &(s->wc); 176 struct pvclock_vcpu_time_info *vcpu_time; 177 178 vcpu_time = &get_cpu_var(xen_vcpu)->time; 179 pvclock_read_wallclock(wall_clock, vcpu_time, ts); 180 put_cpu_var(xen_vcpu); 181} 182 183static void xen_get_wallclock(struct timespec *now) 184{ 185 xen_read_wallclock(now); 186} 187 188static int xen_set_wallclock(const struct timespec *now) 189{ 190 return -1; 191} 192 193static int xen_pvclock_gtod_notify(struct notifier_block *nb, 194 unsigned long was_set, void *priv) 195{ 196 /* Protected by the calling core code serialization */ 197 static struct timespec next_sync; 198 199 struct xen_platform_op op; 200 struct timespec now; 201 202 now = __current_kernel_time(); 203 204 /* 205 * We only take the expensive HV call when the clock was set 206 * or when the 11 minutes RTC synchronization time elapsed. 207 */ 208 if (!was_set && timespec_compare(&now, &next_sync) < 0) 209 return NOTIFY_OK; 210 211 op.cmd = XENPF_settime; 212 op.u.settime.secs = now.tv_sec; 213 op.u.settime.nsecs = now.tv_nsec; 214 op.u.settime.system_time = xen_clocksource_read(); 215 216 (void)HYPERVISOR_dom0_op(&op); 217 218 /* 219 * Move the next drift compensation time 11 minutes 220 * ahead. That's emulating the sync_cmos_clock() update for 221 * the hardware RTC. 222 */ 223 next_sync = now; 224 next_sync.tv_sec += 11 * 60; 225 226 return NOTIFY_OK; 227} 228 229static struct notifier_block xen_pvclock_gtod_notifier = { 230 .notifier_call = xen_pvclock_gtod_notify, 231}; 232 233static struct clocksource xen_clocksource __read_mostly = { 234 .name = "xen", 235 .rating = 400, 236 .read = xen_clocksource_get_cycles, 237 .mask = ~0, 238 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 239}; 240 241/* 242 Xen clockevent implementation 243 244 Xen has two clockevent implementations: 245 246 The old timer_op one works with all released versions of Xen prior 247 to version 3.0.4. This version of the hypervisor provides a 248 single-shot timer with nanosecond resolution. However, sharing the 249 same event channel is a 100Hz tick which is delivered while the 250 vcpu is running. We don't care about or use this tick, but it will 251 cause the core time code to think the timer fired too soon, and 252 will end up resetting it each time. It could be filtered, but 253 doing so has complications when the ktime clocksource is not yet 254 the xen clocksource (ie, at boot time). 255 256 The new vcpu_op-based timer interface allows the tick timer period 257 to be changed or turned off. The tick timer is not useful as a 258 periodic timer because events are only delivered to running vcpus. 259 The one-shot timer can report when a timeout is in the past, so 260 set_next_event is capable of returning -ETIME when appropriate. 261 This interface is used when available. 262*/ 263 264 265/* 266 Get a hypervisor absolute time. In theory we could maintain an 267 offset between the kernel's time and the hypervisor's time, and 268 apply that to a kernel's absolute timeout. Unfortunately the 269 hypervisor and kernel times can drift even if the kernel is using 270 the Xen clocksource, because ntp can warp the kernel's clocksource. 271*/ 272static s64 get_abs_timeout(unsigned long delta) 273{ 274 return xen_clocksource_read() + delta; 275} 276 277static void xen_timerop_set_mode(enum clock_event_mode mode, 278 struct clock_event_device *evt) 279{ 280 switch (mode) { 281 case CLOCK_EVT_MODE_PERIODIC: 282 /* unsupported */ 283 WARN_ON(1); 284 break; 285 286 case CLOCK_EVT_MODE_ONESHOT: 287 case CLOCK_EVT_MODE_RESUME: 288 break; 289 290 case CLOCK_EVT_MODE_UNUSED: 291 case CLOCK_EVT_MODE_SHUTDOWN: 292 HYPERVISOR_set_timer_op(0); /* cancel timeout */ 293 break; 294 } 295} 296 297static int xen_timerop_set_next_event(unsigned long delta, 298 struct clock_event_device *evt) 299{ 300 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT); 301 302 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0) 303 BUG(); 304 305 /* We may have missed the deadline, but there's no real way of 306 knowing for sure. If the event was in the past, then we'll 307 get an immediate interrupt. */ 308 309 return 0; 310} 311 312static const struct clock_event_device xen_timerop_clockevent = { 313 .name = "xen", 314 .features = CLOCK_EVT_FEAT_ONESHOT, 315 316 .max_delta_ns = 0xffffffff, 317 .min_delta_ns = TIMER_SLOP, 318 319 .mult = 1, 320 .shift = 0, 321 .rating = 500, 322 323 .set_mode = xen_timerop_set_mode, 324 .set_next_event = xen_timerop_set_next_event, 325}; 326 327 328 329static void xen_vcpuop_set_mode(enum clock_event_mode mode, 330 struct clock_event_device *evt) 331{ 332 int cpu = smp_processor_id(); 333 334 switch (mode) { 335 case CLOCK_EVT_MODE_PERIODIC: 336 WARN_ON(1); /* unsupported */ 337 break; 338 339 case CLOCK_EVT_MODE_ONESHOT: 340 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) 341 BUG(); 342 break; 343 344 case CLOCK_EVT_MODE_UNUSED: 345 case CLOCK_EVT_MODE_SHUTDOWN: 346 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) || 347 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) 348 BUG(); 349 break; 350 case CLOCK_EVT_MODE_RESUME: 351 break; 352 } 353} 354 355static int xen_vcpuop_set_next_event(unsigned long delta, 356 struct clock_event_device *evt) 357{ 358 int cpu = smp_processor_id(); 359 struct vcpu_set_singleshot_timer single; 360 int ret; 361 362 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT); 363 364 single.timeout_abs_ns = get_abs_timeout(delta); 365 single.flags = VCPU_SSHOTTMR_future; 366 367 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single); 368 369 BUG_ON(ret != 0 && ret != -ETIME); 370 371 return ret; 372} 373 374static const struct clock_event_device xen_vcpuop_clockevent = { 375 .name = "xen", 376 .features = CLOCK_EVT_FEAT_ONESHOT, 377 378 .max_delta_ns = 0xffffffff, 379 .min_delta_ns = TIMER_SLOP, 380 381 .mult = 1, 382 .shift = 0, 383 .rating = 500, 384 385 .set_mode = xen_vcpuop_set_mode, 386 .set_next_event = xen_vcpuop_set_next_event, 387}; 388 389static const struct clock_event_device *xen_clockevent = 390 &xen_timerop_clockevent; 391 392struct xen_clock_event_device { 393 struct clock_event_device evt; 394 char name[16]; 395}; 396static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 }; 397 398static irqreturn_t xen_timer_interrupt(int irq, void *dev_id) 399{ 400 struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt); 401 irqreturn_t ret; 402 403 ret = IRQ_NONE; 404 if (evt->event_handler) { 405 evt->event_handler(evt); 406 ret = IRQ_HANDLED; 407 } 408 409 do_stolen_accounting(); 410 411 return ret; 412} 413 414void xen_teardown_timer(int cpu) 415{ 416 struct clock_event_device *evt; 417 BUG_ON(cpu == 0); 418 evt = &per_cpu(xen_clock_events, cpu).evt; 419 420 if (evt->irq >= 0) { 421 unbind_from_irqhandler(evt->irq, NULL); 422 evt->irq = -1; 423 } 424} 425 426void xen_setup_timer(int cpu) 427{ 428 struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu); 429 struct clock_event_device *evt = &xevt->evt; 430 int irq; 431 432 WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu); 433 if (evt->irq >= 0) 434 xen_teardown_timer(cpu); 435 436 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu); 437 438 snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu); 439 440 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt, 441 IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER| 442 IRQF_FORCE_RESUME|IRQF_EARLY_RESUME, 443 xevt->name, NULL); 444 (void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX); 445 446 memcpy(evt, xen_clockevent, sizeof(*evt)); 447 448 evt->cpumask = cpumask_of(cpu); 449 evt->irq = irq; 450} 451 452 453void xen_setup_cpu_clockevents(void) 454{ 455 clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt)); 456} 457 458void xen_timer_resume(void) 459{ 460 int cpu; 461 462 pvclock_resume(); 463 464 if (xen_clockevent != &xen_vcpuop_clockevent) 465 return; 466 467 for_each_online_cpu(cpu) { 468 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) 469 BUG(); 470 } 471} 472 473static const struct pv_time_ops xen_time_ops __initconst = { 474 .sched_clock = xen_clocksource_read, 475}; 476 477static void __init xen_time_init(void) 478{ 479 int cpu = smp_processor_id(); 480 struct timespec tp; 481 482 /* As Dom0 is never moved, no penalty on using TSC there */ 483 if (xen_initial_domain()) 484 xen_clocksource.rating = 275; 485 486 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC); 487 488 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) { 489 /* Successfully turned off 100Hz tick, so we have the 490 vcpuop-based timer interface */ 491 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n"); 492 xen_clockevent = &xen_vcpuop_clockevent; 493 } 494 495 /* Set initial system time with full resolution */ 496 xen_read_wallclock(&tp); 497 do_settimeofday(&tp); 498 499 setup_force_cpu_cap(X86_FEATURE_TSC); 500 501 xen_setup_runstate_info(cpu); 502 xen_setup_timer(cpu); 503 xen_setup_cpu_clockevents(); 504 505 if (xen_initial_domain()) 506 pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier); 507} 508 509void __init xen_init_time_ops(void) 510{ 511 pv_time_ops = xen_time_ops; 512 513 x86_init.timers.timer_init = xen_time_init; 514 x86_init.timers.setup_percpu_clockev = x86_init_noop; 515 x86_cpuinit.setup_percpu_clockev = x86_init_noop; 516 517 x86_platform.calibrate_tsc = xen_tsc_khz; 518 x86_platform.get_wallclock = xen_get_wallclock; 519 /* Dom0 uses the native method to set the hardware RTC. */ 520 if (!xen_initial_domain()) 521 x86_platform.set_wallclock = xen_set_wallclock; 522} 523 524#ifdef CONFIG_XEN_PVHVM 525static void xen_hvm_setup_cpu_clockevents(void) 526{ 527 int cpu = smp_processor_id(); 528 xen_setup_runstate_info(cpu); 529 /* 530 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence 531 * doing it xen_hvm_cpu_notify (which gets called by smp_init during 532 * early bootup and also during CPU hotplug events). 533 */ 534 xen_setup_cpu_clockevents(); 535} 536 537void __init xen_hvm_init_time_ops(void) 538{ 539 /* vector callback is needed otherwise we cannot receive interrupts 540 * on cpu > 0 and at this point we don't know how many cpus are 541 * available */ 542 if (!xen_have_vector_callback) 543 return; 544 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) { 545 printk(KERN_INFO "Xen doesn't support pvclock on HVM," 546 "disable pv timer\n"); 547 return; 548 } 549 550 pv_time_ops = xen_time_ops; 551 x86_init.timers.setup_percpu_clockev = xen_time_init; 552 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents; 553 554 x86_platform.calibrate_tsc = xen_tsc_khz; 555 x86_platform.get_wallclock = xen_get_wallclock; 556 x86_platform.set_wallclock = xen_set_wallclock; 557} 558#endif 559