/************************************************************************** * * Copyright 2013 VMware, Inc. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * **************************************************************************/ /** * @file * Format conversion code for "special" float formats. * * @author Roland Scheidegger */ #include "util/u_debug.h" #include "lp_bld_type.h" #include "lp_bld_const.h" #include "lp_bld_arit.h" #include "lp_bld_bitarit.h" #include "lp_bld_logic.h" #include "lp_bld_format.h" /** * Convert float32 to a float-like value with less exponent and mantissa * bits. The mantissa is still biased, and the mantissa still has an implied 1, * and there may be a sign bit. * * @param src (vector) float value to convert * @param mantissa_bits the number of mantissa bits * @param exponent_bits the number of exponent bits * @param mantissa_start the start position of the small float in result value * @param has_sign if the small float has a sign bit * * This implements round-towards-zero (trunc) hence too large numbers get * converted to largest representable number, not infinity. * Small numbers may get converted to denorms, depending on normal * float denorm handling of the cpu. * Note that compared to the references, below, we skip any rounding bias * since we do rounding towards zero - OpenGL allows rounding towards zero * (though not preferred) and DX10 even seems to require it. * Note that this will pack mantissa, exponent and sign bit (if any) together, * and shift the result to mantissa_start. * * ref http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ * ref https://gist.github.com/rygorous/2156668 */ LLVMValueRef lp_build_float_to_smallfloat(struct gallivm_state *gallivm, struct lp_type i32_type, LLVMValueRef src, unsigned mantissa_bits, unsigned exponent_bits, unsigned mantissa_start, boolean has_sign) { LLVMBuilderRef builder = gallivm->builder; LLVMValueRef i32_floatexpmask, i32_smallexpmask, magic, normal; LLVMValueRef rescale_src, i32_roundmask, small_max; LLVMValueRef i32_qnanbit, shift, res; LLVMValueRef is_nan_or_inf, nan_or_inf, mask, i32_src; struct lp_type f32_type = lp_type_float_vec(32, 32 * i32_type.length); struct lp_build_context f32_bld, i32_bld; LLVMValueRef zero = lp_build_const_vec(gallivm, f32_type, 0.0f); unsigned exponent_start = mantissa_start + mantissa_bits; boolean always_preserve_nans = true; boolean maybe_correct_denorm_rounding = true; lp_build_context_init(&f32_bld, gallivm, f32_type); lp_build_context_init(&i32_bld, gallivm, i32_type); i32_smallexpmask = lp_build_const_int_vec(gallivm, i32_type, ((1 << exponent_bits) - 1) << 23); i32_floatexpmask = lp_build_const_int_vec(gallivm, i32_type, 0xff << 23); i32_src = LLVMBuildBitCast(builder, src, i32_bld.vec_type, ""); if (has_sign) { rescale_src = src; } else { /* clamp to pos range (can still have sign bit if NaN or negative zero) */ rescale_src = lp_build_max(&f32_bld, zero, src); } rescale_src = LLVMBuildBitCast(builder, rescale_src, i32_bld.vec_type, ""); /* "ordinary" number */ /* * get rid of excess mantissa bits and sign bit * This is only really needed for correct rounding of denorms I think * but only if we use the preserve NaN path does using * src_abs instead save us any instruction. */ if (maybe_correct_denorm_rounding || !always_preserve_nans) { i32_roundmask = lp_build_const_int_vec(gallivm, i32_type, ~((1 << (23 - mantissa_bits)) - 1) & 0x7fffffff); rescale_src = LLVMBuildBitCast(builder, rescale_src, i32_bld.vec_type, ""); rescale_src = lp_build_and(&i32_bld, rescale_src, i32_roundmask); rescale_src = LLVMBuildBitCast(builder, rescale_src, f32_bld.vec_type, ""); } else { rescale_src = lp_build_abs(&f32_bld, src); } /* bias exponent (and denormalize if necessary) */ magic = lp_build_const_int_vec(gallivm, i32_type, ((1 << (exponent_bits - 1)) - 1) << 23); magic = LLVMBuildBitCast(builder, magic, f32_bld.vec_type, ""); normal = lp_build_mul(&f32_bld, rescale_src, magic); /* clamp to max value - largest non-infinity number */ small_max = lp_build_const_int_vec(gallivm, i32_type, (((1 << exponent_bits) - 2) << 23) | (((1 << mantissa_bits) - 1) << (23 - mantissa_bits))); small_max = LLVMBuildBitCast(builder, small_max, f32_bld.vec_type, ""); normal = lp_build_min(&f32_bld, normal, small_max); normal = LLVMBuildBitCast(builder, normal, i32_bld.vec_type, ""); /* * handle nan/inf cases * a little bit tricky since -Inf -> 0, +Inf -> +Inf, +-Nan -> +Nan * (for no sign) else ->Inf -> ->Inf too. * could use explicit "unordered" comparison checking for NaNs * which might save us from calculating src_abs too. * (Cannot actually save the comparison since we need to distinguish * Inf and NaN cases anyway, but it would be better for AVX.) */ if (always_preserve_nans) { LLVMValueRef infcheck_src, is_inf, is_nan; LLVMValueRef src_abs = lp_build_abs(&f32_bld, src); src_abs = LLVMBuildBitCast(builder, src_abs, i32_bld.vec_type, ""); if (has_sign) { infcheck_src = src_abs; } else { infcheck_src = i32_src; } is_nan = lp_build_compare(gallivm, i32_type, PIPE_FUNC_GREATER, src_abs, i32_floatexpmask); is_inf = lp_build_compare(gallivm, i32_type, PIPE_FUNC_EQUAL, infcheck_src, i32_floatexpmask); is_nan_or_inf = lp_build_or(&i32_bld, is_nan, is_inf); /* could also set more mantissa bits but need at least the highest mantissa bit */ i32_qnanbit = lp_build_const_vec(gallivm, i32_type, 1 << 22); /* combine maxexp with qnanbit */ nan_or_inf = lp_build_or(&i32_bld, i32_smallexpmask, lp_build_and(&i32_bld, is_nan, i32_qnanbit)); } else { /* * A couple simplifications, with mostly 2 drawbacks (so disabled): * - it will promote some SNaNs (those which only had bits set * in the mantissa part which got chopped off) to +-Infinity. * (Those bits get chopped off anyway later so can as well use * rescale_src instead of src_abs here saving the calculation of that.) * - for no sign case, it relies on the max() being used for rescale_src * to give back the NaN (which is NOT ieee754r behavior, but should work * with sse2 on a full moon (rather if I got the operand order right) - * we _don't_ have well-defined behavior specified with min/max wrt NaNs, * however, and if it gets converted to cmp/select it may not work (we * don't really have specified behavior for cmp wrt NaNs neither). */ rescale_src = LLVMBuildBitCast(builder, rescale_src, i32_bld.vec_type, ""); is_nan_or_inf = lp_build_compare(gallivm, i32_type, PIPE_FUNC_GEQUAL, rescale_src, i32_floatexpmask); /* note this will introduce excess exponent bits */ nan_or_inf = rescale_src; } res = lp_build_select(&i32_bld, is_nan_or_inf, nan_or_inf, normal); if (mantissa_start > 0 || !always_preserve_nans) { /* mask off excess bits */ unsigned maskbits = (1 << (mantissa_bits + exponent_bits)) - 1; mask = lp_build_const_int_vec(gallivm, i32_type, maskbits << (23 - mantissa_bits)); res = lp_build_and(&i32_bld, res, mask); } /* add back sign bit at right position */ if (has_sign) { LLVMValueRef sign; struct lp_type u32_type = lp_type_uint_vec(32, 32 * i32_type.length); struct lp_build_context u32_bld; lp_build_context_init(&u32_bld, gallivm, u32_type); mask = lp_build_const_int_vec(gallivm, i32_type, 0x80000000); shift = lp_build_const_int_vec(gallivm, i32_type, 8 - exponent_bits); sign = lp_build_and(&i32_bld, mask, i32_src); sign = lp_build_shr(&u32_bld, sign, shift); res = lp_build_or(&i32_bld, sign, res); } /* shift to final position */ if (exponent_start < 23) { shift = lp_build_const_int_vec(gallivm, i32_type, 23 - exponent_start); res = lp_build_shr(&i32_bld, res, shift); } else { shift = lp_build_const_int_vec(gallivm, i32_type, exponent_start - 23); res = lp_build_shl(&i32_bld, res, shift); } return res; } /** * Convert rgba float SoA values to packed r11g11b10 values. * * @param src SoA float (vector) values to convert. */ LLVMValueRef lp_build_float_to_r11g11b10(struct gallivm_state *gallivm, LLVMValueRef *src) { LLVMValueRef dst, rcomp, bcomp, gcomp; struct lp_build_context i32_bld; LLVMTypeRef src_type = LLVMTypeOf(*src); unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ? LLVMGetVectorSize(src_type) : 1; struct lp_type i32_type = lp_type_int_vec(32, 32 * src_length); lp_build_context_init(&i32_bld, gallivm, i32_type); /* "rescale" and put in right position */ rcomp = lp_build_float_to_smallfloat(gallivm, i32_type, src[0], 6, 5, 0, false); gcomp = lp_build_float_to_smallfloat(gallivm, i32_type, src[1], 6, 5, 11, false); bcomp = lp_build_float_to_smallfloat(gallivm, i32_type, src[2], 5, 5, 22, false); /* combine the values */ dst = lp_build_or(&i32_bld, rcomp, gcomp); return lp_build_or(&i32_bld, dst, bcomp); } /** * Convert a float-like value with less exponent and mantissa * bits than a normal float32 to a float32. The mantissa of * the source value is assumed to have an implied 1, and the exponent * is biased. There may be a sign bit. * The source value to extract must be in a 32bit int (bits not part of * the value to convert will be masked off). * This works for things like 11-bit floats or half-floats, * mantissa, exponent (and sign if present) must be packed * the same as they are in a ordinary float. * * @param src (vector) value to convert * @param mantissa_bits the number of mantissa bits * @param exponent_bits the number of exponent bits * @param mantissa_start the bit start position of the packed component * @param has_sign if the small float has a sign bit * * ref http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ * ref https://gist.github.com/rygorous/2156668 */ LLVMValueRef lp_build_smallfloat_to_float(struct gallivm_state *gallivm, struct lp_type f32_type, LLVMValueRef src, unsigned mantissa_bits, unsigned exponent_bits, unsigned mantissa_start, boolean has_sign) { LLVMBuilderRef builder = gallivm->builder; LLVMValueRef smallexpmask, i32_floatexpmask, magic; LLVMValueRef wasinfnan, tmp, res, shift, maskabs, srcabs, sign; unsigned exponent_start = mantissa_start + mantissa_bits; struct lp_type i32_type = lp_type_int_vec(32, 32 * f32_type.length); struct lp_build_context f32_bld, i32_bld; lp_build_context_init(&f32_bld, gallivm, f32_type); lp_build_context_init(&i32_bld, gallivm, i32_type); /* extract the component to "float position" */ if (exponent_start < 23) { shift = lp_build_const_int_vec(gallivm, i32_type, 23 - exponent_start); src = lp_build_shl(&i32_bld, src, shift); } else { shift = lp_build_const_int_vec(gallivm, i32_type, exponent_start - 23); src = lp_build_shr(&i32_bld, src, shift); } maskabs = lp_build_const_int_vec(gallivm, i32_type, ((1 << (mantissa_bits + exponent_bits)) - 1) << (23 - mantissa_bits)); srcabs = lp_build_and(&i32_bld, src, maskabs); /* now do the actual scaling */ smallexpmask = lp_build_const_int_vec(gallivm, i32_type, ((1 << exponent_bits) - 1) << 23); i32_floatexpmask = lp_build_const_int_vec(gallivm, i32_type, 0xff << 23); if (0) { /* * Note that this code path, while simpler, will convert small * float denorms to floats according to current cpu denorm mode, if * denorms are disabled it will flush them to zero! * If cpu denorms are enabled, it should be faster though as long as * there's no denorms in the inputs, but if there are actually denorms * it's likely to be an order of magnitude slower (on x86 cpus). */ srcabs = LLVMBuildBitCast(builder, srcabs, f32_bld.vec_type, ""); /* * magic number has exponent new exp bias + (new exp bias - old exp bias), * mantissa is 0. */ magic = lp_build_const_int_vec(gallivm, i32_type, (255 - (1 << (exponent_bits - 1))) << 23); magic = LLVMBuildBitCast(builder, magic, f32_bld.vec_type, ""); /* adjust exponent and fix denorms */ res = lp_build_mul(&f32_bld, srcabs, magic); /* * if exp was max (== NaN or Inf) set new exp to max (keep mantissa), * so a simple "or" will do (because exp adjust will leave mantissa intact) */ /* use float compare (better for AVX 8-wide / no AVX2 but else should use int) */ smallexpmask = LLVMBuildBitCast(builder, smallexpmask, f32_bld.vec_type, ""); wasinfnan = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GEQUAL, srcabs, smallexpmask); res = LLVMBuildBitCast(builder, res, i32_bld.vec_type, ""); tmp = lp_build_and(&i32_bld, i32_floatexpmask, wasinfnan); res = lp_build_or(&i32_bld, tmp, res); } else { LLVMValueRef exp_one, isdenorm, denorm, normal, exp_adj; /* denorm (or zero) if exponent is zero */ exp_one = lp_build_const_int_vec(gallivm, i32_type, 1 << 23); isdenorm = lp_build_cmp(&i32_bld, PIPE_FUNC_LESS, srcabs, exp_one); /* inf or nan if exponent is max */ wasinfnan = lp_build_cmp(&i32_bld, PIPE_FUNC_GEQUAL, srcabs, smallexpmask); /* for denormal (or zero), add (== or) magic exp to mantissa (== srcabs) (as int) * then subtract it (as float). * Another option would be to just do inttofp then do a rescale mul. */ magic = lp_build_const_int_vec(gallivm, i32_type, (127 - ((1 << (exponent_bits - 1)) - 2)) << 23); denorm = lp_build_or(&i32_bld, srcabs, magic); denorm = LLVMBuildBitCast(builder, denorm, f32_bld.vec_type, ""); denorm = lp_build_sub(&f32_bld, denorm, LLVMBuildBitCast(builder, magic, f32_bld.vec_type, "")); denorm = LLVMBuildBitCast(builder, denorm, i32_bld.vec_type, ""); /* for normals, Infs, Nans fix up exponent */ exp_adj = lp_build_const_int_vec(gallivm, i32_type, (127 - ((1 << (exponent_bits - 1)) - 1)) << 23); normal = lp_build_add(&i32_bld, srcabs, exp_adj); tmp = lp_build_and(&i32_bld, wasinfnan, i32_floatexpmask); normal = lp_build_or(&i32_bld, tmp, normal); res = lp_build_select(&i32_bld, isdenorm, denorm, normal); } if (has_sign) { LLVMValueRef signmask = lp_build_const_int_vec(gallivm, i32_type, 0x80000000); shift = lp_build_const_int_vec(gallivm, i32_type, 8 - exponent_bits); sign = lp_build_shl(&i32_bld, src, shift); sign = lp_build_and(&i32_bld, signmask, sign); res = lp_build_or(&i32_bld, res, sign); } return LLVMBuildBitCast(builder, res, f32_bld.vec_type, ""); } /** * Convert packed float format (r11g11b10) value(s) to rgba float SoA values. * * @param src packed AoS r11g11b10 values (as (vector) int32) * @param dst pointer to the SoA result values */ void lp_build_r11g11b10_to_float(struct gallivm_state *gallivm, LLVMValueRef src, LLVMValueRef *dst) { LLVMTypeRef src_type = LLVMTypeOf(src); unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ? LLVMGetVectorSize(src_type) : 1; struct lp_type f32_type = lp_type_float_vec(32, 32 * src_length); dst[0] = lp_build_smallfloat_to_float(gallivm, f32_type, src, 6, 5, 0, false); dst[1] = lp_build_smallfloat_to_float(gallivm, f32_type, src, 6, 5, 11, false); dst[2] = lp_build_smallfloat_to_float(gallivm, f32_type, src, 5, 5, 22, false); /* Just set alpha to one */ dst[3] = lp_build_one(gallivm, f32_type); } static LLVMValueRef lp_build_rgb9_to_float_helper(struct gallivm_state *gallivm, struct lp_type f32_type, LLVMValueRef src, LLVMValueRef scale, unsigned mantissa_start) { LLVMValueRef shift, mask; struct lp_type i32_type = lp_type_int_vec(32, 32 * f32_type.length); struct lp_build_context i32_bld, f32_bld; lp_build_context_init(&i32_bld, gallivm, i32_type); lp_build_context_init(&f32_bld, gallivm, f32_type); /* * This is much easier as other weirdo float formats, since * there's no sign, no Inf/NaN, and there's nothing special * required for normals/denormals neither (as without the implied one * for the mantissa for other formats, everything looks like a denormal). * So just do (float)comp_bits * scale */ shift = lp_build_const_int_vec(gallivm, i32_type, mantissa_start); mask = lp_build_const_int_vec(gallivm, i32_type, 0x1ff); src = lp_build_shr(&i32_bld, src, shift); src = lp_build_and(&i32_bld, src, mask); src = lp_build_int_to_float(&f32_bld, src); return lp_build_mul(&f32_bld, src, scale); } /** * Convert shared exponent format (rgb9e5) value(s) to rgba float SoA values. * * @param src packed AoS rgb9e5 values (as (vector) int32) * @param dst pointer to the SoA result values */ void lp_build_rgb9e5_to_float(struct gallivm_state *gallivm, LLVMValueRef src, LLVMValueRef *dst) { LLVMBuilderRef builder = gallivm->builder; LLVMTypeRef src_type = LLVMTypeOf(src); LLVMValueRef shift, scale, bias, exp; unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ? LLVMGetVectorSize(src_type) : 1; struct lp_type i32_type = lp_type_int_vec(32, 32 * src_length); struct lp_type u32_type = lp_type_uint_vec(32, 32 * src_length); struct lp_type f32_type = lp_type_float_vec(32, 32 * src_length); struct lp_build_context i32_bld, u32_bld, f32_bld; lp_build_context_init(&i32_bld, gallivm, i32_type); lp_build_context_init(&u32_bld, gallivm, u32_type); lp_build_context_init(&f32_bld, gallivm, f32_type); /* extract exponent */ shift = lp_build_const_int_vec(gallivm, i32_type, 27); /* this shift needs to be unsigned otherwise need mask */ exp = lp_build_shr(&u32_bld, src, shift); /* * scale factor is 2 ^ (exp - bias) * (and additionally corrected here for the mantissa bits) * not using shift because * a) don't have vector shift in a lot of cases * b) shift direction changes hence need 2 shifts + conditional * (or rotate instruction which is even more rare (for instance XOP)) * so use whacky float 2 ^ function instead manipulating exponent * (saves us the float conversion at the end too) */ bias = lp_build_const_int_vec(gallivm, i32_type, 127 - (15 + 9)); scale = lp_build_add(&i32_bld, exp, bias); shift = lp_build_const_int_vec(gallivm, i32_type, 23); scale = lp_build_shl(&i32_bld, scale, shift); scale = LLVMBuildBitCast(builder, scale, f32_bld.vec_type, ""); dst[0] = lp_build_rgb9_to_float_helper(gallivm, f32_type, src, scale, 0); dst[1] = lp_build_rgb9_to_float_helper(gallivm, f32_type, src, scale, 9); dst[2] = lp_build_rgb9_to_float_helper(gallivm, f32_type, src, scale, 18); /* Just set alpha to one */ dst[3] = f32_bld.one; }