// SPDX-License-Identifier: MIT /* * Copyright © 2019 Intel Corporation */ #include #include #include "i915_drv.h" #include "i915_irq.h" #include "i915_params.h" #include "intel_context.h" #include "intel_engine_pm.h" #include "intel_gt.h" #include "intel_gt_clock_utils.h" #include "intel_gt_pm.h" #include "intel_gt_print.h" #include "intel_gt_requests.h" #include "intel_llc.h" #include "intel_rc6.h" #include "intel_rps.h" #include "intel_wakeref.h" #include "pxp/intel_pxp_pm.h" #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2) static void user_forcewake(struct intel_gt *gt, bool suspend) { int count = atomic_read(>->user_wakeref); /* Inside suspend/resume so single threaded, no races to worry about. */ if (likely(!count)) return; intel_gt_pm_get(gt); if (suspend) { GEM_BUG_ON(count > atomic_read(>->wakeref.count)); atomic_sub(count, >->wakeref.count); } else { atomic_add(count, >->wakeref.count); } intel_gt_pm_put(gt); } static void runtime_begin(struct intel_gt *gt) { local_irq_disable(); #ifdef notyet write_seqcount_begin(>->stats.lock); #else write_seqcount_begin((seqcount_t *)>->stats.lock); #endif gt->stats.start = ktime_get(); gt->stats.active = true; #ifdef notyet write_seqcount_end(>->stats.lock); #else write_seqcount_end((seqcount_t *)>->stats.lock); #endif local_irq_enable(); } static void runtime_end(struct intel_gt *gt) { local_irq_disable(); #ifdef notyet write_seqcount_begin(>->stats.lock); #else write_seqcount_begin((seqcount_t *)>->stats.lock); #endif gt->stats.active = false; gt->stats.total = ktime_add(gt->stats.total, ktime_sub(ktime_get(), gt->stats.start)); #ifdef notyet write_seqcount_end(>->stats.lock); #else write_seqcount_end((seqcount_t *)>->stats.lock); #endif local_irq_enable(); } static int __gt_unpark(struct intel_wakeref *wf) { struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); struct drm_i915_private *i915 = gt->i915; GT_TRACE(gt, "\n"); /* * It seems that the DMC likes to transition between the DC states a lot * when there are no connected displays (no active power domains) during * command submission. * * This activity has negative impact on the performance of the chip with * huge latencies observed in the interrupt handler and elsewhere. * * Work around it by grabbing a GT IRQ power domain whilst there is any * GT activity, preventing any DC state transitions. */ gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); GEM_BUG_ON(!gt->awake); intel_rc6_unpark(>->rc6); intel_rps_unpark(>->rps); i915_pmu_gt_unparked(gt); intel_guc_busyness_unpark(gt); intel_gt_unpark_requests(gt); runtime_begin(gt); return 0; } static int __gt_park(struct intel_wakeref *wf) { struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); intel_wakeref_t wakeref = fetch_and_zero(>->awake); struct drm_i915_private *i915 = gt->i915; GT_TRACE(gt, "\n"); runtime_end(gt); intel_gt_park_requests(gt); intel_guc_busyness_park(gt); i915_vma_parked(gt); i915_pmu_gt_parked(gt); intel_rps_park(>->rps); intel_rc6_park(>->rc6); /* Everything switched off, flush any residual interrupt just in case */ intel_synchronize_irq(i915); /* Defer dropping the display power well for 100ms, it's slow! */ GEM_BUG_ON(!wakeref); intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref); return 0; } static const struct intel_wakeref_ops wf_ops = { .get = __gt_unpark, .put = __gt_park, }; void intel_gt_pm_init_early(struct intel_gt *gt) { /* * We access the runtime_pm structure via gt->i915 here rather than * gt->uncore as we do elsewhere in the file because gt->uncore is not * yet initialized for all tiles at this point in the driver startup. * runtime_pm is per-device rather than per-tile, so this is still the * correct structure. */ intel_wakeref_init(>->wakeref, gt->i915, &wf_ops); seqcount_mutex_init(>->stats.lock, >->wakeref.mutex); } void intel_gt_pm_init(struct intel_gt *gt) { /* * Enabling power-management should be "self-healing". If we cannot * enable a feature, simply leave it disabled with a notice to the * user. */ intel_rc6_init(>->rc6); intel_rps_init(>->rps); } static bool reset_engines(struct intel_gt *gt) { if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) return false; return __intel_gt_reset(gt, ALL_ENGINES) == 0; } static void gt_sanitize(struct intel_gt *gt, bool force) { struct intel_engine_cs *engine; enum intel_engine_id id; intel_wakeref_t wakeref; GT_TRACE(gt, "force:%s", str_yes_no(force)); /* Use a raw wakeref to avoid calling intel_display_power_get early */ wakeref = intel_runtime_pm_get(gt->uncore->rpm); intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); intel_gt_check_clock_frequency(gt); /* * As we have just resumed the machine and woken the device up from * deep PCI sleep (presumably D3_cold), assume the HW has been reset * back to defaults, recovering from whatever wedged state we left it * in and so worth trying to use the device once more. */ if (intel_gt_is_wedged(gt)) intel_gt_unset_wedged(gt); /* For GuC mode, ensure submission is disabled before stopping ring */ intel_uc_reset_prepare(>->uc); for_each_engine(engine, gt, id) { if (engine->reset.prepare) engine->reset.prepare(engine); if (engine->sanitize) engine->sanitize(engine); } if (reset_engines(gt) || force) { for_each_engine(engine, gt, id) __intel_engine_reset(engine, false); } intel_uc_reset(>->uc, false); for_each_engine(engine, gt, id) if (engine->reset.finish) engine->reset.finish(engine); intel_rps_sanitize(>->rps); intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); intel_runtime_pm_put(gt->uncore->rpm, wakeref); } void intel_gt_pm_fini(struct intel_gt *gt) { intel_rc6_fini(>->rc6); } int intel_gt_resume(struct intel_gt *gt) { struct intel_engine_cs *engine; enum intel_engine_id id; int err; err = intel_gt_has_unrecoverable_error(gt); if (err) return err; GT_TRACE(gt, "\n"); /* * After resume, we may need to poke into the pinned kernel * contexts to paper over any damage caused by the sudden suspend. * Only the kernel contexts should remain pinned over suspend, * allowing us to fixup the user contexts on their first pin. */ gt_sanitize(gt, true); intel_gt_pm_get(gt); intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); intel_rc6_sanitize(>->rc6); if (intel_gt_is_wedged(gt)) { err = -EIO; goto out_fw; } /* Only when the HW is re-initialised, can we replay the requests */ err = intel_gt_init_hw(gt); if (err) { gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n"); goto err_wedged; } intel_uc_reset_finish(>->uc); intel_rps_enable(>->rps); intel_llc_enable(>->llc); for_each_engine(engine, gt, id) { intel_engine_pm_get(engine); engine->serial++; /* kernel context lost */ err = intel_engine_resume(engine); intel_engine_pm_put(engine); if (err) { gt_err(gt, "Failed to restart %s (%d)\n", engine->name, err); goto err_wedged; } } intel_rc6_enable(>->rc6); intel_uc_resume(>->uc); user_forcewake(gt, false); out_fw: intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); intel_gt_pm_put(gt); intel_gt_bind_context_set_ready(gt); return err; err_wedged: intel_gt_set_wedged(gt); goto out_fw; } static void wait_for_suspend(struct intel_gt *gt) { if (!intel_gt_pm_is_awake(gt)) return; if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) { /* * Forcibly cancel outstanding work and leave * the gpu quiet. */ intel_gt_set_wedged(gt); intel_gt_retire_requests(gt); } intel_gt_pm_wait_for_idle(gt); } void intel_gt_suspend_prepare(struct intel_gt *gt) { intel_gt_bind_context_set_unready(gt); user_forcewake(gt, true); wait_for_suspend(gt); } static suspend_state_t pm_suspend_target(void) { #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP) return pm_suspend_target_state; #else return PM_SUSPEND_TO_IDLE; #endif } void intel_gt_suspend_late(struct intel_gt *gt) { intel_wakeref_t wakeref; /* We expect to be idle already; but also want to be independent */ wait_for_suspend(gt); if (is_mock_gt(gt)) return; GEM_BUG_ON(gt->awake); intel_uc_suspend(>->uc); /* * On disabling the device, we want to turn off HW access to memory * that we no longer own. * * However, not all suspend-states disable the device. S0 (s2idle) * is effectively runtime-suspend, the device is left powered on * but needs to be put into a low power state. We need to keep * powermanagement enabled, but we also retain system state and so * it remains safe to keep on using our allocated memory. */ if (pm_suspend_target() == PM_SUSPEND_TO_IDLE) return; with_intel_runtime_pm(gt->uncore->rpm, wakeref) { intel_rps_disable(>->rps); intel_rc6_disable(>->rc6); intel_llc_disable(>->llc); } gt_sanitize(gt, false); GT_TRACE(gt, "\n"); } void intel_gt_runtime_suspend(struct intel_gt *gt) { intel_gt_bind_context_set_unready(gt); intel_uc_runtime_suspend(>->uc); GT_TRACE(gt, "\n"); } int intel_gt_runtime_resume(struct intel_gt *gt) { int ret; GT_TRACE(gt, "\n"); intel_gt_init_swizzling(gt); intel_ggtt_restore_fences(gt->ggtt); ret = intel_uc_runtime_resume(>->uc); if (ret) return ret; intel_gt_bind_context_set_ready(gt); return 0; } static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt) { ktime_t total = gt->stats.total; if (gt->stats.active) total = ktime_add(total, ktime_sub(ktime_get(), gt->stats.start)); return total; } ktime_t intel_gt_get_awake_time(const struct intel_gt *gt) { unsigned int seq; ktime_t total; #ifdef notyet do { seq = read_seqcount_begin(>->stats.lock); total = __intel_gt_get_awake_time(gt); } while (read_seqcount_retry(>->stats.lock, seq)); #else do { seq = read_seqcount_begin((seqcount_t *)>->stats.lock); total = __intel_gt_get_awake_time(gt); } while (read_seqcount_retry((seqcount_t *)>->stats.lock, seq)); #endif return total; } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftest_gt_pm.c" #endif