// SPDX-License-Identifier: MIT /* * Copyright © 2022-2023 Intel Corporation */ #include "i915_drv.h" #include "i915_reg.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_vblank.h" #include "intel_vrr.h" /* * This timing diagram depicts the video signal in and * around the vertical blanking period. * * Assumptions about the fictitious mode used in this example: * vblank_start >= 3 * vsync_start = vblank_start + 1 * vsync_end = vblank_start + 2 * vtotal = vblank_start + 3 * * start of vblank: * latch double buffered registers * increment frame counter (ctg+) * generate start of vblank interrupt (gen4+) * | * | frame start: * | generate frame start interrupt (aka. vblank interrupt) (gmch) * | may be shifted forward 1-3 extra lines via TRANSCONF * | | * | | start of vsync: * | | generate vsync interrupt * | | | * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx * . \hs/ . \hs/ \hs/ \hs/ . \hs/ * ----va---> <-----------------vb--------------------> <--------va------------- * | | <----vs-----> | * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2) * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+) * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi) * | | | * last visible pixel first visible pixel * | increment frame counter (gen3/4) * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4) * * x = horizontal active * _ = horizontal blanking * hs = horizontal sync * va = vertical active * vb = vertical blanking * vs = vertical sync * vbs = vblank_start (number) * * Summary: * - most events happen at the start of horizontal sync * - frame start happens at the start of horizontal blank, 1-4 lines * (depending on TRANSCONF settings) after the start of vblank * - gen3/4 pixel and frame counter are synchronized with the start * of horizontal active on the first line of vertical active */ /* * Called from drm generic code, passed a 'crtc', which we use as a pipe index. */ u32 i915_get_vblank_counter(struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)]; const struct drm_display_mode *mode = &vblank->hwmode; enum pipe pipe = to_intel_crtc(crtc)->pipe; u32 pixel, vbl_start, hsync_start, htotal; u64 frame; /* * On i965gm TV output the frame counter only works up to * the point when we enable the TV encoder. After that the * frame counter ceases to work and reads zero. We need a * vblank wait before enabling the TV encoder and so we * have to enable vblank interrupts while the frame counter * is still in a working state. However the core vblank code * does not like us returning non-zero frame counter values * when we've told it that we don't have a working frame * counter. Thus we must stop non-zero values leaking out. */ if (!vblank->max_vblank_count) return 0; htotal = mode->crtc_htotal; hsync_start = mode->crtc_hsync_start; vbl_start = mode->crtc_vblank_start; if (mode->flags & DRM_MODE_FLAG_INTERLACE) vbl_start = DIV_ROUND_UP(vbl_start, 2); /* Convert to pixel count */ vbl_start *= htotal; /* Start of vblank event occurs at start of hsync */ vbl_start -= htotal - hsync_start; /* * High & low register fields aren't synchronized, so make sure * we get a low value that's stable across two reads of the high * register. */ frame = intel_de_read64_2x32(dev_priv, PIPEFRAMEPIXEL(pipe), PIPEFRAME(pipe)); pixel = frame & PIPE_PIXEL_MASK; frame = (frame >> PIPE_FRAME_LOW_SHIFT) & 0xffffff; /* * The frame counter increments at beginning of active. * Cook up a vblank counter by also checking the pixel * counter against vblank start. */ return (frame + (pixel >= vbl_start)) & 0xffffff; } u32 g4x_get_vblank_counter(struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)]; enum pipe pipe = to_intel_crtc(crtc)->pipe; if (!vblank->max_vblank_count) return 0; return intel_de_read(dev_priv, PIPE_FRMCOUNT_G4X(pipe)); } static u32 intel_crtc_scanlines_since_frame_timestamp(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct drm_vblank_crtc *vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)]; const struct drm_display_mode *mode = &vblank->hwmode; u32 htotal = mode->crtc_htotal; u32 clock = mode->crtc_clock; u32 scan_prev_time, scan_curr_time, scan_post_time; /* * To avoid the race condition where we might cross into the * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR * during the same frame. */ do { /* * This field provides read back of the display * pipe frame time stamp. The time stamp value * is sampled at every start of vertical blank. */ scan_prev_time = intel_de_read_fw(dev_priv, PIPE_FRMTMSTMP(crtc->pipe)); /* * The TIMESTAMP_CTR register has the current * time stamp value. */ scan_curr_time = intel_de_read_fw(dev_priv, IVB_TIMESTAMP_CTR); scan_post_time = intel_de_read_fw(dev_priv, PIPE_FRMTMSTMP(crtc->pipe)); } while (scan_post_time != scan_prev_time); return div_u64(mul_u32_u32(scan_curr_time - scan_prev_time, clock), 1000 * htotal); } /* * On certain encoders on certain platforms, pipe * scanline register will not work to get the scanline, * since the timings are driven from the PORT or issues * with scanline register updates. * This function will use Framestamp and current * timestamp registers to calculate the scanline. */ static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc) { struct drm_vblank_crtc *vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)]; const struct drm_display_mode *mode = &vblank->hwmode; u32 vblank_start = mode->crtc_vblank_start; u32 vtotal = mode->crtc_vtotal; u32 scanline; scanline = intel_crtc_scanlines_since_frame_timestamp(crtc); scanline = min(scanline, vtotal - 1); scanline = (scanline + vblank_start) % vtotal; return scanline; } /* * intel_de_read_fw(), only for fast reads of display block, no need for * forcewake etc. */ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); const struct drm_display_mode *mode; struct drm_vblank_crtc *vblank; enum pipe pipe = crtc->pipe; int position, vtotal; if (!crtc->active) return 0; vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)]; mode = &vblank->hwmode; if (crtc->mode_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP) return __intel_get_crtc_scanline_from_timestamp(crtc); vtotal = mode->crtc_vtotal; if (mode->flags & DRM_MODE_FLAG_INTERLACE) vtotal /= 2; position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & PIPEDSL_LINE_MASK; /* * On HSW, the DSL reg (0x70000) appears to return 0 if we * read it just before the start of vblank. So try it again * so we don't accidentally end up spanning a vblank frame * increment, causing the pipe_update_end() code to squak at us. * * The nature of this problem means we can't simply check the ISR * bit and return the vblank start value; nor can we use the scanline * debug register in the transcoder as it appears to have the same * problem. We may need to extend this to include other platforms, * but so far testing only shows the problem on HSW. */ if (HAS_DDI(dev_priv) && !position) { int i, temp; for (i = 0; i < 100; i++) { udelay(1); temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & PIPEDSL_LINE_MASK; if (temp != position) { position = temp; break; } } } /* * See update_scanline_offset() for the details on the * scanline_offset adjustment. */ return (position + crtc->scanline_offset) % vtotal; } static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, bool in_vblank_irq, int *vpos, int *hpos, ktime_t *stime, ktime_t *etime, const struct drm_display_mode *mode) { struct drm_device *dev = _crtc->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(_crtc); enum pipe pipe = crtc->pipe; int position; int vbl_start, vbl_end, hsync_start, htotal, vtotal; unsigned long irqflags; bool use_scanline_counter = DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) == 2 || crtc->mode_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER; if (drm_WARN_ON(&dev_priv->drm, !mode->crtc_clock)) { drm_dbg(&dev_priv->drm, "trying to get scanoutpos for disabled pipe %c\n", pipe_name(pipe)); return false; } htotal = mode->crtc_htotal; hsync_start = mode->crtc_hsync_start; vtotal = mode->crtc_vtotal; vbl_start = mode->crtc_vblank_start; vbl_end = mode->crtc_vblank_end; if (mode->flags & DRM_MODE_FLAG_INTERLACE) { vbl_start = DIV_ROUND_UP(vbl_start, 2); vbl_end /= 2; vtotal /= 2; } /* * Lock uncore.lock, as we will do multiple timing critical raw * register reads, potentially with preemption disabled, so the * following code must not block on uncore.lock. */ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ /* Get optional system timestamp before query. */ if (stime) *stime = ktime_get(); if (crtc->mode_flags & I915_MODE_FLAG_VRR) { int scanlines = intel_crtc_scanlines_since_frame_timestamp(crtc); position = __intel_get_crtc_scanline(crtc); /* * Already exiting vblank? If so, shift our position * so it looks like we're already apporaching the full * vblank end. This should make the generated timestamp * more or less match when the active portion will start. */ if (position >= vbl_start && scanlines < position) position = min(crtc->vmax_vblank_start + scanlines, vtotal - 1); } else if (use_scanline_counter) { /* No obvious pixelcount register. Only query vertical * scanout position from Display scan line register. */ position = __intel_get_crtc_scanline(crtc); } else { /* * Have access to pixelcount since start of frame. * We can split this into vertical and horizontal * scanout position. */ position = (intel_de_read_fw(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; /* convert to pixel counts */ vbl_start *= htotal; vbl_end *= htotal; vtotal *= htotal; /* * In interlaced modes, the pixel counter counts all pixels, * so one field will have htotal more pixels. In order to avoid * the reported position from jumping backwards when the pixel * counter is beyond the length of the shorter field, just * clamp the position the length of the shorter field. This * matches how the scanline counter based position works since * the scanline counter doesn't count the two half lines. */ position = min(position, vtotal - 1); /* * Start of vblank interrupt is triggered at start of hsync, * just prior to the first active line of vblank. However we * consider lines to start at the leading edge of horizontal * active. So, should we get here before we've crossed into * the horizontal active of the first line in vblank, we would * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that, * always add htotal-hsync_start to the current pixel position. */ position = (position + htotal - hsync_start) % vtotal; } /* Get optional system timestamp after query. */ if (etime) *etime = ktime_get(); /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); /* * While in vblank, position will be negative * counting up towards 0 at vbl_end. And outside * vblank, position will be positive counting * up since vbl_end. */ if (position >= vbl_start) position -= vbl_end; else position += vtotal - vbl_end; if (use_scanline_counter) { *vpos = position; *hpos = 0; } else { *vpos = position / htotal; *hpos = position - (*vpos * htotal); } return true; } bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time, bool in_vblank_irq) { return drm_crtc_vblank_helper_get_vblank_timestamp_internal( crtc, max_error, vblank_time, in_vblank_irq, i915_get_crtc_scanoutpos); } int intel_get_crtc_scanline(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); unsigned long irqflags; int position; spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); position = __intel_get_crtc_scanline(crtc); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); return position; } static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv, enum pipe pipe) { i915_reg_t reg = PIPEDSL(pipe); u32 line1, line2; line1 = intel_de_read(dev_priv, reg) & PIPEDSL_LINE_MASK; drm_msleep(5); line2 = intel_de_read(dev_priv, reg) & PIPEDSL_LINE_MASK; return line1 != line2; } static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; /* Wait for the display line to settle/start moving */ if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100)) drm_err(&dev_priv->drm, "pipe %c scanline %s wait timed out\n", pipe_name(pipe), str_on_off(state)); } void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc) { wait_for_pipe_scanline_moving(crtc, false); } void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc) { wait_for_pipe_scanline_moving(crtc, true); } static int intel_crtc_scanline_offset(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; /* * The scanline counter increments at the leading edge of hsync. * * On most platforms it starts counting from vtotal-1 on the * first active line. That means the scanline counter value is * always one less than what we would expect. Ie. just after * start of vblank, which also occurs at start of hsync (on the * last active line), the scanline counter will read vblank_start-1. * * On gen2 the scanline counter starts counting from 1 instead * of vtotal-1, so we have to subtract one (or rather add vtotal-1 * to keep the value positive), instead of adding one. * * On HSW+ the behaviour of the scanline counter depends on the output * type. For DP ports it behaves like most other platforms, but on HDMI * there's an extra 1 line difference. So we need to add two instead of * one to the value. * * On VLV/CHV DSI the scanline counter would appear to increment * approx. 1/3 of a scanline before start of vblank. Unfortunately * that means we can't tell whether we're in vblank or not while * we're on that particular line. We must still set scanline_offset * to 1 so that the vblank timestamps come out correct when we query * the scanline counter from within the vblank interrupt handler. * However if queried just before the start of vblank we'll get an * answer that's slightly in the future. */ if (DISPLAY_VER(i915) == 2) { int vtotal; vtotal = adjusted_mode->crtc_vtotal; if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) vtotal /= 2; return vtotal - 1; } else if (HAS_DDI(i915) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { return 2; } else { return 1; } } void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state, bool vrr_enable) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *i915 = to_i915(crtc->base.dev); u8 mode_flags = crtc_state->mode_flags; struct drm_display_mode adjusted_mode; int vmax_vblank_start = 0; unsigned long irqflags; drm_mode_init(&adjusted_mode, &crtc_state->hw.adjusted_mode); if (vrr_enable) { drm_WARN_ON(&i915->drm, (mode_flags & I915_MODE_FLAG_VRR) == 0); adjusted_mode.crtc_vtotal = crtc_state->vrr.vmax; adjusted_mode.crtc_vblank_end = crtc_state->vrr.vmax; adjusted_mode.crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state); vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state); } else { mode_flags &= ~I915_MODE_FLAG_VRR; } /* * Belts and suspenders locking to guarantee everyone sees 100% * consistent state during fastset seamless refresh rate changes. * * vblank_time_lock takes care of all drm_vblank.c stuff, and * uncore.lock takes care of __intel_get_crtc_scanline() which * may get called elsewhere as well. * * TODO maybe just protect everything (including * __intel_get_crtc_scanline()) with vblank_time_lock? * Need to audit everything to make sure it's safe. */ spin_lock_irqsave(&i915->drm.vblank_time_lock, irqflags); spin_lock(&i915->uncore.lock); drm_calc_timestamping_constants(&crtc->base, &adjusted_mode); crtc->vmax_vblank_start = vmax_vblank_start; crtc->mode_flags = mode_flags; crtc->scanline_offset = intel_crtc_scanline_offset(crtc_state); spin_unlock(&i915->uncore.lock); spin_unlock_irqrestore(&i915->drm.vblank_time_lock, irqflags); }