diff options
Diffstat (limited to 'src/glsl/nir/nir_lower_io.c')
| -rw-r--r-- | src/glsl/nir/nir_lower_io.c | 129 |
1 files changed, 120 insertions, 9 deletions
diff --git a/src/glsl/nir/nir_lower_io.c b/src/glsl/nir/nir_lower_io.c index 80c5151..2c5fa16 100644 --- a/src/glsl/nir/nir_lower_io.c +++ b/src/glsl/nir/nir_lower_io.c @@ -160,12 +160,56 @@ load_op(struct lower_io_state *state, case nir_var_uniform: op = nir_intrinsic_load_uniform; break; + case nir_var_shared: + op = nir_intrinsic_load_shared; + break; default: unreachable("Unknown variable mode"); } return op; } +static nir_intrinsic_op +store_op(struct lower_io_state *state, + nir_variable_mode mode, bool per_vertex) +{ + nir_intrinsic_op op; + switch (mode) { + case nir_var_shader_in: + case nir_var_shader_out: + op = per_vertex ? nir_intrinsic_store_per_vertex_output : + nir_intrinsic_store_output; + break; + case nir_var_shared: + op = nir_intrinsic_store_shared; + break; + default: + unreachable("Unknown variable mode"); + } + return op; +} + +static nir_intrinsic_op +atomic_op(nir_intrinsic_op opcode) +{ + switch (opcode) { +#define OP(O) case nir_intrinsic_var_##O: return nir_intrinsic_shared_##O; + OP(atomic_exchange) + OP(atomic_comp_swap) + OP(atomic_add) + OP(atomic_imin) + OP(atomic_umin) + OP(atomic_imax) + OP(atomic_umax) + OP(atomic_and) + OP(atomic_or) + OP(atomic_xor) +#undef OP + default: + unreachable("Invalid atomic"); + } +} + static bool nir_lower_io_block(nir_block *block, void *void_state) { @@ -179,9 +223,25 @@ nir_lower_io_block(nir_block *block, void *void_state) nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); - if (intrin->intrinsic != nir_intrinsic_load_var && - intrin->intrinsic != nir_intrinsic_store_var) + switch (intrin->intrinsic) { + case nir_intrinsic_load_var: + case nir_intrinsic_store_var: + case nir_intrinsic_var_atomic_add: + case nir_intrinsic_var_atomic_imin: + case nir_intrinsic_var_atomic_umin: + case nir_intrinsic_var_atomic_imax: + case nir_intrinsic_var_atomic_umax: + case nir_intrinsic_var_atomic_and: + case nir_intrinsic_var_atomic_or: + case nir_intrinsic_var_atomic_xor: + case nir_intrinsic_var_atomic_exchange: + case nir_intrinsic_var_atomic_comp_swap: + /* We can lower the io for this nir instrinsic */ + break; + default: + /* We can't lower the io for this nir instrinsic, so skip it */ continue; + } nir_variable_mode mode = intrin->variables[0]->var->data.mode; @@ -190,6 +250,7 @@ nir_lower_io_block(nir_block *block, void *void_state) if (mode != nir_var_shader_in && mode != nir_var_shader_out && + mode != nir_var_shared && mode != nir_var_uniform) continue; @@ -216,6 +277,11 @@ nir_lower_io_block(nir_block *block, void *void_state) load->const_index[0] = intrin->variables[0]->var->data.driver_location; + if (load->intrinsic == nir_intrinsic_load_uniform) { + load->const_index[1] = + state->type_size(intrin->variables[0]->var->type); + } + if (per_vertex) load->src[0] = nir_src_for_ssa(vertex_index); @@ -236,7 +302,7 @@ nir_lower_io_block(nir_block *block, void *void_state) } case nir_intrinsic_store_var: { - assert(mode == nir_var_shader_out); + assert(mode == nir_var_shader_out || mode == nir_var_shared); nir_ssa_def *offset; nir_ssa_def *vertex_index; @@ -248,12 +314,9 @@ nir_lower_io_block(nir_block *block, void *void_state) per_vertex ? &vertex_index : NULL, state->type_size); - nir_intrinsic_op store_op = - per_vertex ? nir_intrinsic_store_per_vertex_output : - nir_intrinsic_store_output; - - nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->mem_ctx, - store_op); + nir_intrinsic_instr *store = + nir_intrinsic_instr_create(state->mem_ctx, + store_op(state, mode, per_vertex)); store->num_components = intrin->num_components; nir_src_copy(&store->src[0], &intrin->src[0], store); @@ -274,6 +337,51 @@ nir_lower_io_block(nir_block *block, void *void_state) break; } + case nir_intrinsic_var_atomic_add: + case nir_intrinsic_var_atomic_imin: + case nir_intrinsic_var_atomic_umin: + case nir_intrinsic_var_atomic_imax: + case nir_intrinsic_var_atomic_umax: + case nir_intrinsic_var_atomic_and: + case nir_intrinsic_var_atomic_or: + case nir_intrinsic_var_atomic_xor: + case nir_intrinsic_var_atomic_exchange: + case nir_intrinsic_var_atomic_comp_swap: { + assert(mode == nir_var_shared); + + nir_ssa_def *offset; + + offset = get_io_offset(b, intrin->variables[0], + NULL, state->type_size); + + nir_intrinsic_instr *atomic = + nir_intrinsic_instr_create(state->mem_ctx, + atomic_op(intrin->intrinsic)); + + atomic->src[0] = nir_src_for_ssa(offset); + + atomic->const_index[0] = + intrin->variables[0]->var->data.driver_location; + + nir_src_copy(&atomic->src[1], &intrin->src[0], atomic); + + if (intrin->intrinsic == nir_intrinsic_var_atomic_comp_swap) + nir_src_copy(&atomic->src[2], &intrin->src[1], atomic); + + if (intrin->dest.is_ssa) { + nir_ssa_dest_init(&atomic->instr, &atomic->dest, + intrin->dest.ssa.num_components, NULL); + nir_ssa_def_rewrite_uses(&intrin->dest.ssa, + nir_src_for_ssa(&atomic->dest.ssa)); + } else { + nir_dest_copy(&atomic->dest, &intrin->dest, state->mem_ctx); + } + + nir_instr_insert_before(&intrin->instr, &atomic->instr); + nir_instr_remove(&intrin->instr); + break; + } + default: break; } @@ -321,10 +429,13 @@ nir_get_io_offset_src(nir_intrinsic_instr *instr) case nir_intrinsic_load_output: case nir_intrinsic_load_uniform: return &instr->src[0]; + case nir_intrinsic_load_ubo: + case nir_intrinsic_load_ssbo: case nir_intrinsic_load_per_vertex_input: case nir_intrinsic_load_per_vertex_output: case nir_intrinsic_store_output: return &instr->src[1]; + case nir_intrinsic_store_ssbo: case nir_intrinsic_store_per_vertex_output: return &instr->src[2]; default: |