/* * Copyright © 2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include "brw_context.h" #include "brw_defines.h" #include "intel_batchbuffer.h" #include "intel_fbo.h" /** * According to the latest documentation, any PIPE_CONTROL with the * "Command Streamer Stall" bit set must also have another bit set, * with five different options: * * - Render Target Cache Flush * - Depth Cache Flush * - Stall at Pixel Scoreboard * - Post-Sync Operation * - Depth Stall * - DC Flush Enable * * I chose "Stall at Pixel Scoreboard" since we've used it effectively * in the past, but the choice is fairly arbitrary. */ static void gen8_add_cs_stall_workaround_bits(uint32_t *flags) { uint32_t wa_bits = PIPE_CONTROL_RENDER_TARGET_FLUSH | PIPE_CONTROL_DEPTH_CACHE_FLUSH | PIPE_CONTROL_WRITE_IMMEDIATE | PIPE_CONTROL_WRITE_DEPTH_COUNT | PIPE_CONTROL_WRITE_TIMESTAMP | PIPE_CONTROL_STALL_AT_SCOREBOARD | PIPE_CONTROL_DEPTH_STALL | PIPE_CONTROL_DATA_CACHE_FLUSH; /* If we're doing a CS stall, and don't already have one of the * workaround bits set, add "Stall at Pixel Scoreboard." */ if ((*flags & PIPE_CONTROL_CS_STALL) != 0 && (*flags & wa_bits) == 0) *flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD; } /* Implement the WaCsStallAtEveryFourthPipecontrol workaround on IVB, BYT: * * "Every 4th PIPE_CONTROL command, not counting the PIPE_CONTROL with * only read-cache-invalidate bit(s) set, must have a CS_STALL bit set." * * Note that the kernel does CS stalls between batches, so we only need * to count them within a batch. */ static uint32_t gen7_cs_stall_every_four_pipe_controls(struct brw_context *brw, uint32_t flags) { if (brw->gen == 7 && !brw->is_haswell) { if (flags & PIPE_CONTROL_CS_STALL) { /* If we're doing a CS stall, reset the counter and carry on. */ brw->pipe_controls_since_last_cs_stall = 0; return 0; } /* If this is the fourth pipe control without a CS stall, do one now. */ if (++brw->pipe_controls_since_last_cs_stall == 4) { brw->pipe_controls_since_last_cs_stall = 0; return PIPE_CONTROL_CS_STALL; } } return 0; } /** * Emit a PIPE_CONTROL with various flushing flags. * * The caller is responsible for deciding what flags are appropriate for the * given generation. */ void brw_emit_pipe_control_flush(struct brw_context *brw, uint32_t flags) { if (brw->gen >= 6 && (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) && (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) { /* A pipe control command with flush and invalidate bits set * simultaneously is an inherently racy operation on Gen6+ if the * contents of the flushed caches were intended to become visible from * any of the invalidated caches. Split it in two PIPE_CONTROLs, the * first one should stall the pipeline to make sure that the flushed R/W * caches are coherent with memory once the specified R/O caches are * invalidated. On pre-Gen6 hardware the (implicit) R/O cache * invalidation seems to happen at the bottom of the pipeline together * with any write cache flush, so this shouldn't be a concern. */ brw_emit_pipe_control_flush(brw, (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) | PIPE_CONTROL_CS_STALL); flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL); } if (brw->gen >= 8) { if (brw->gen == 8) gen8_add_cs_stall_workaround_bits(&flags); if (brw->gen == 9 && (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE)) { /* Hardware workaround: SKL * * Emit Pipe Control with all bits set to zero before emitting * a Pipe Control with VF Cache Invalidate set. */ brw_emit_pipe_control_flush(brw, 0); } BEGIN_BATCH(6); OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2)); OUT_BATCH(flags); OUT_BATCH(0); OUT_BATCH(0); OUT_BATCH(0); OUT_BATCH(0); ADVANCE_BATCH(); } else if (brw->gen >= 6) { if (brw->gen == 6 && (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH)) { /* Hardware workaround: SNB B-Spec says: * * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush * Enable = 1, a PIPE_CONTROL with any non-zero post-sync-op is * required. */ brw_emit_post_sync_nonzero_flush(brw); } flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags); BEGIN_BATCH(5); OUT_BATCH(_3DSTATE_PIPE_CONTROL | (5 - 2)); OUT_BATCH(flags); OUT_BATCH(0); OUT_BATCH(0); OUT_BATCH(0); ADVANCE_BATCH(); } else { BEGIN_BATCH(4); OUT_BATCH(_3DSTATE_PIPE_CONTROL | flags | (4 - 2)); OUT_BATCH(0); OUT_BATCH(0); OUT_BATCH(0); ADVANCE_BATCH(); } } /** * Emit a PIPE_CONTROL that writes to a buffer object. * * \p flags should contain one of the following items: * - PIPE_CONTROL_WRITE_IMMEDIATE * - PIPE_CONTROL_WRITE_TIMESTAMP * - PIPE_CONTROL_WRITE_DEPTH_COUNT */ void brw_emit_pipe_control_write(struct brw_context *brw, uint32_t flags, drm_intel_bo *bo, uint32_t offset, uint32_t imm_lower, uint32_t imm_upper) { if (brw->gen >= 8) { if (brw->gen == 8) gen8_add_cs_stall_workaround_bits(&flags); BEGIN_BATCH(6); OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2)); OUT_BATCH(flags); OUT_RELOC64(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION, offset); OUT_BATCH(imm_lower); OUT_BATCH(imm_upper); ADVANCE_BATCH(); } else if (brw->gen >= 6) { flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags); /* PPGTT/GGTT is selected by DW2 bit 2 on Sandybridge, but DW1 bit 24 * on later platforms. We always use PPGTT on Gen7+. */ unsigned gen6_gtt = brw->gen == 6 ? PIPE_CONTROL_GLOBAL_GTT_WRITE : 0; BEGIN_BATCH(5); OUT_BATCH(_3DSTATE_PIPE_CONTROL | (5 - 2)); OUT_BATCH(flags); OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION, gen6_gtt | offset); OUT_BATCH(imm_lower); OUT_BATCH(imm_upper); ADVANCE_BATCH(); } else { BEGIN_BATCH(4); OUT_BATCH(_3DSTATE_PIPE_CONTROL | flags | (4 - 2)); OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION, PIPE_CONTROL_GLOBAL_GTT_WRITE | offset); OUT_BATCH(imm_lower); OUT_BATCH(imm_upper); ADVANCE_BATCH(); } } /** * Restriction [DevSNB, DevIVB]: * * Prior to changing Depth/Stencil Buffer state (i.e. any combination of * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER, * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set), * unless SW can otherwise guarantee that the pipeline from WM onwards is * already flushed (e.g., via a preceding MI_FLUSH). */ void brw_emit_depth_stall_flushes(struct brw_context *brw) { assert(brw->gen >= 6); /* Starting on BDW, these pipe controls are unnecessary. * * WM HW will internally manage the draining pipe and flushing of the caches * when this command is issued. The PIPE_CONTROL restrictions are removed. */ if (brw->gen >= 8) return; brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL); brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_CACHE_FLUSH); brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL); } /** * From the Ivybridge PRM, Volume 2 Part 1, Section 3.2 (VS Stage Input): * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth * stall needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS, * 3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS, * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one PIPE_CONTROL needs * to be sent before any combination of VS associated 3DSTATE." */ void gen7_emit_vs_workaround_flush(struct brw_context *brw) { assert(brw->gen == 7); brw_emit_pipe_control_write(brw, PIPE_CONTROL_WRITE_IMMEDIATE | PIPE_CONTROL_DEPTH_STALL, brw->workaround_bo, 0, 0, 0); } /** * Emit a PIPE_CONTROL command for gen7 with the CS Stall bit set. */ void gen7_emit_cs_stall_flush(struct brw_context *brw) { brw_emit_pipe_control_write(brw, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_WRITE_IMMEDIATE, brw->workaround_bo, 0, 0, 0); } /** * Emits a PIPE_CONTROL with a non-zero post-sync operation, for * implementing two workarounds on gen6. From section 1.4.7.1 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: * * [DevSNB-C+{W/A}] Before any depth stall flush (including those * produced by non-pipelined state commands), software needs to first * send a PIPE_CONTROL with no bits set except Post-Sync Operation != * 0. * * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. * * And the workaround for these two requires this workaround first: * * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent * BEFORE the pipe-control with a post-sync op and no write-cache * flushes. * * And this last workaround is tricky because of the requirements on * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM * volume 2 part 1: * * "1 of the following must also be set: * - Render Target Cache Flush Enable ([12] of DW1) * - Depth Cache Flush Enable ([0] of DW1) * - Stall at Pixel Scoreboard ([1] of DW1) * - Depth Stall ([13] of DW1) * - Post-Sync Operation ([13] of DW1) * - Notify Enable ([8] of DW1)" * * The cache flushes require the workaround flush that triggered this * one, so we can't use it. Depth stall would trigger the same. * Post-sync nonzero is what triggered this second workaround, so we * can't use that one either. Notify enable is IRQs, which aren't * really our business. That leaves only stall at scoreboard. */ void brw_emit_post_sync_nonzero_flush(struct brw_context *brw) { brw_emit_pipe_control_flush(brw, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD); brw_emit_pipe_control_write(brw, PIPE_CONTROL_WRITE_IMMEDIATE, brw->workaround_bo, 0, 0, 0); } /* Emit a pipelined flush to either flush render and texture cache for * reading from a FBO-drawn texture, or flush so that frontbuffer * render appears on the screen in DRI1. * * This is also used for the always_flush_cache driconf debug option. */ void brw_emit_mi_flush(struct brw_context *brw) { if (brw->batch.ring == BLT_RING && brw->gen >= 6) { BEGIN_BATCH_BLT(4); OUT_BATCH(MI_FLUSH_DW); OUT_BATCH(0); OUT_BATCH(0); OUT_BATCH(0); ADVANCE_BATCH(); } else { int flags = PIPE_CONTROL_NO_WRITE | PIPE_CONTROL_RENDER_TARGET_FLUSH; if (brw->gen >= 6) { flags |= PIPE_CONTROL_INSTRUCTION_INVALIDATE | PIPE_CONTROL_DEPTH_CACHE_FLUSH | PIPE_CONTROL_VF_CACHE_INVALIDATE | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | PIPE_CONTROL_CS_STALL; } brw_emit_pipe_control_flush(brw, flags); } } int brw_init_pipe_control(struct brw_context *brw, const struct gen_device_info *devinfo) { if (devinfo->gen < 6) return 0; /* We can't just use brw_state_batch to get a chunk of space for * the gen6 workaround because it involves actually writing to * the buffer, and the kernel doesn't let us write to the batch. */ brw->workaround_bo = drm_intel_bo_alloc(brw->bufmgr, "pipe_control workaround", 4096, 4096); if (brw->workaround_bo == NULL) return -ENOMEM; brw->pipe_controls_since_last_cs_stall = 0; return 0; } void brw_fini_pipe_control(struct brw_context *brw) { drm_intel_bo_unreference(brw->workaround_bo); }