| use rustc_target::spec::Arch; | |
| use crate::compiler_builtins::CMP_RESULT_TY; | |
| use crate::prelude::*; | |
| pub(crate) fn f16_to_f32(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { | |
| let (value, arg_ty) = | |
| if fx.tcx.sess.target.is_like_darwin && fx.tcx.sess.target.arch == Arch::X86_64 { | |
| ( | |
| fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value), | |
| lib_call_arg_param(fx.tcx, types::I16, false), | |
| ) | |
| } else { | |
| (value, AbiParam::new(types::F16)) | |
| }; | |
| fx.lib_call("__extendhfsf2", vec![arg_ty], vec![AbiParam::new(types::F32)], &[value])[0] | |
| } | |
| fn f16_to_f64(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { | |
| let ret = f16_to_f32(fx, value); | |
| fx.bcx.ins().fpromote(types::F64, ret) | |
| } | |
| pub(crate) fn f32_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { | |
| let ret_ty = if fx.tcx.sess.target.is_like_darwin && fx.tcx.sess.target.arch == Arch::X86_64 { | |
| types::I16 | |
| } else { | |
| types::F16 | |
| }; | |
| let ret = fx.lib_call( | |
| "__truncsfhf2", | |
| vec![AbiParam::new(types::F32)], | |
| vec![AbiParam::new(ret_ty)], | |
| &[value], | |
| )[0]; | |
| if ret_ty == types::I16 { | |
| fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), ret) | |
| } else { | |
| ret | |
| } | |
| } | |
| fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { | |
| let ret_ty = if fx.tcx.sess.target.is_like_darwin && fx.tcx.sess.target.arch == Arch::X86_64 { | |
| types::I16 | |
| } else { | |
| types::F16 | |
| }; | |
| let ret = fx.lib_call( | |
| "__truncdfhf2", | |
| vec![AbiParam::new(types::F64)], | |
| vec![AbiParam::new(ret_ty)], | |
| &[value], | |
| )[0]; | |
| if ret_ty == types::I16 { | |
| fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), ret) | |
| } else { | |
| ret | |
| } | |
| } | |
| // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have | |
| // been added to Cranelift. | |
| pub(crate) fn maybe_with_f16_to_f32( | |
| fx: &mut FunctionCx<'_, '_, '_>, | |
| val: Value, | |
| f: impl FnOnce(&mut FunctionCx<'_, '_, '_>, Value) -> Value, | |
| ) -> Value { | |
| if fx.bcx.func.dfg.value_type(val) == types::F16 { | |
| let val = f16_to_f32(fx, val); | |
| let res = f(fx, val); | |
| f32_to_f16(fx, res) | |
| } else { | |
| f(fx, val) | |
| } | |
| } | |
| // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have | |
| // been added to Cranelift. | |
| pub(crate) fn maybe_with_f16_to_f32_pair( | |
| fx: &mut FunctionCx<'_, '_, '_>, | |
| a: Value, | |
| b: Value, | |
| f: impl FnOnce(&mut FunctionCx<'_, '_, '_>, Value, Value) -> Value, | |
| ) -> Value { | |
| if fx.bcx.func.dfg.value_type(a) == types::F16 { | |
| let a = f16_to_f32(fx, a); | |
| let b = f16_to_f32(fx, b); | |
| let res = f(fx, a, b); | |
| f32_to_f16(fx, res) | |
| } else { | |
| f(fx, a, b) | |
| } | |
| } | |
| pub(crate) fn fcmp(fx: &mut FunctionCx<'_, '_, '_>, cc: FloatCC, lhs: Value, rhs: Value) -> Value { | |
| let ty = fx.bcx.func.dfg.value_type(lhs); | |
| match ty { | |
| types::F32 | types::F64 => fx.bcx.ins().fcmp(cc, lhs, rhs), | |
| types::F16 => { | |
| let lhs = f16_to_f32(fx, lhs); | |
| let rhs = f16_to_f32(fx, rhs); | |
| fx.bcx.ins().fcmp(cc, lhs, rhs) | |
| } | |
| types::F128 => { | |
| let (name, int_cc) = match cc { | |
| FloatCC::Equal => ("__eqtf2", IntCC::Equal), | |
| FloatCC::NotEqual => ("__netf2", IntCC::NotEqual), | |
| FloatCC::LessThan => ("__lttf2", IntCC::SignedLessThan), | |
| FloatCC::LessThanOrEqual => ("__letf2", IntCC::SignedLessThanOrEqual), | |
| FloatCC::GreaterThan => ("__gttf2", IntCC::SignedGreaterThan), | |
| FloatCC::GreaterThanOrEqual => ("__getf2", IntCC::SignedGreaterThanOrEqual), | |
| _ => unreachable!("not currently used in rustc_codegen_cranelift: {cc:?}"), | |
| }; | |
| let res = fx.lib_call( | |
| name, | |
| vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], | |
| vec![AbiParam::new(CMP_RESULT_TY)], | |
| &[lhs, rhs], | |
| )[0]; | |
| let zero = fx.bcx.ins().iconst(CMP_RESULT_TY, 0); | |
| fx.bcx.ins().icmp(int_cc, res, zero) | |
| } | |
| _ => unreachable!("{ty:?}"), | |
| } | |
| } | |
| pub(crate) fn codegen_f128_binop( | |
| fx: &mut FunctionCx<'_, '_, '_>, | |
| bin_op: BinOp, | |
| lhs: Value, | |
| rhs: Value, | |
| ) -> Value { | |
| let name = match bin_op { | |
| BinOp::Add => "__addtf3", | |
| BinOp::Sub => "__subtf3", | |
| BinOp::Mul => "__multf3", | |
| BinOp::Div => "__divtf3", | |
| _ => unreachable!("handled in `codegen_float_binop`"), | |
| }; | |
| fx.lib_call( | |
| name, | |
| vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], | |
| vec![AbiParam::new(types::F128)], | |
| &[lhs, rhs], | |
| )[0] | |
| } | |
| pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { | |
| let bits = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value); | |
| let bits = fx.bcx.ins().bxor_imm_u(bits, 0x8000); | |
| fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), bits) | |
| } | |
| pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { | |
| let bits = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), value); | |
| let (low, high) = fx.bcx.ins().isplit(bits); | |
| let high = fx.bcx.ins().bxor_imm_u(high, 0x8000_0000_0000_0000_u64 as i64); | |
| let bits = fx.bcx.ins().iconcat(low, high); | |
| fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits) | |
| } | |
| pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { | |
| let bits = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value); | |
| let bits = fx.bcx.ins().band_imm_u(bits, 0x7fff); | |
| fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), bits) | |
| } | |
| pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { | |
| let bits = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), value); | |
| let (low, high) = fx.bcx.ins().isplit(bits); | |
| let high = fx.bcx.ins().band_imm_u(high, 0x7fff_ffff_ffff_ffff_u64 as i64); | |
| let bits = fx.bcx.ins().iconcat(low, high); | |
| fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits) | |
| } | |
| pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { | |
| let lhs = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), lhs); | |
| let rhs = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), rhs); | |
| let res = fx.bcx.ins().band_imm_u(lhs, 0x7fff); | |
| let sign = fx.bcx.ins().band_imm_u(rhs, 0x8000); | |
| let res = fx.bcx.ins().bor(res, sign); | |
| fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), res) | |
| } | |
| pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { | |
| let lhs = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), lhs); | |
| let rhs = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), rhs); | |
| let (low, lhs_high) = fx.bcx.ins().isplit(lhs); | |
| let (_, rhs_high) = fx.bcx.ins().isplit(rhs); | |
| let high = fx.bcx.ins().band_imm_u(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64); | |
| let sign = fx.bcx.ins().band_imm_u(rhs_high, 0x8000_0000_0000_0000_u64 as i64); | |
| let high = fx.bcx.ins().bor(high, sign); | |
| let res = fx.bcx.ins().iconcat(low, high); | |
| fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), res) | |
| } | |
| pub(crate) fn codegen_cast( | |
| fx: &mut FunctionCx<'_, '_, '_>, | |
| from: Value, | |
| from_signed: bool, | |
| to_ty: Type, | |
| to_signed: bool, | |
| ) -> Value { | |
| let from_ty = fx.bcx.func.dfg.value_type(from); | |
| if from_ty.is_float() && to_ty.is_float() { | |
| let name = match (from_ty, to_ty) { | |
| (types::F16, types::F32) => return f16_to_f32(fx, from), | |
| (types::F16, types::F64) => return f16_to_f64(fx, from), | |
| (types::F16, types::F128) => "__extendhftf2", | |
| (types::F32, types::F128) => "__extendsftf2", | |
| (types::F64, types::F128) => "__extenddftf2", | |
| (types::F128, types::F64) => "__trunctfdf2", | |
| (types::F128, types::F32) => "__trunctfsf2", | |
| (types::F128, types::F16) => "__trunctfhf2", | |
| (types::F64, types::F16) => return f64_to_f16(fx, from), | |
| (types::F32, types::F16) => return f32_to_f16(fx, from), | |
| _ => unreachable!("{from_ty:?} -> {to_ty:?}"), | |
| }; | |
| fx.lib_call(name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])[0] | |
| } else if from_ty.is_int() && to_ty == types::F16 { | |
| let res = clif_int_or_float_cast(fx, from, from_signed, types::F32, false); | |
| f32_to_f16(fx, res) | |
| } else if from_ty == types::F16 && to_ty.is_int() { | |
| let from = f16_to_f32(fx, from); | |
| clif_int_or_float_cast(fx, from, false, to_ty, to_signed) | |
| } else if from_ty.is_int() && to_ty == types::F128 { | |
| let (from, from_ty) = if from_ty.bits() < 32 { | |
| (clif_int_or_float_cast(fx, from, from_signed, types::I32, from_signed), types::I32) | |
| } else { | |
| (from, from_ty) | |
| }; | |
| let name = format!( | |
| "__float{sign}{size}itf", | |
| sign = if from_signed { "" } else { "un" }, | |
| size = match from_ty { | |
| types::I32 => 's', | |
| types::I64 => 'd', | |
| types::I128 => 't', | |
| _ => unreachable!("{from_ty:?}"), | |
| }, | |
| ); | |
| fx.lib_call( | |
| &name, | |
| vec![lib_call_arg_param(fx.tcx, from_ty, from_signed)], | |
| vec![AbiParam::new(to_ty)], | |
| &[from], | |
| )[0] | |
| } else if from_ty == types::F128 && to_ty.is_int() { | |
| let ret_ty = if to_ty.bits() < 32 { types::I32 } else { to_ty }; | |
| let name = format!( | |
| "__fix{sign}tf{size}i", | |
| sign = if to_signed { "" } else { "uns" }, | |
| size = match ret_ty { | |
| types::I32 => 's', | |
| types::I64 => 'd', | |
| types::I128 => 't', | |
| _ => unreachable!("{from_ty:?}"), | |
| }, | |
| ); | |
| let ret = | |
| fx.lib_call(&name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from]) | |
| [0]; | |
| let val = if ret_ty == to_ty { | |
| ret | |
| } else { | |
| let (min, max) = match (to_ty, to_signed) { | |
| (types::I8, false) => (0, i64::from(u8::MAX)), | |
| (types::I16, false) => (0, i64::from(u16::MAX)), | |
| (types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)), | |
| (types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)), | |
| _ => unreachable!("{to_ty:?}"), | |
| }; | |
| let min_val = fx.bcx.ins().iconst(types::I32, min); | |
| let max_val = fx.bcx.ins().iconst(types::I32, max); | |
| let val = if to_signed { | |
| let has_underflow = fx.bcx.ins().icmp_imm_s(IntCC::SignedLessThan, ret, min); | |
| let has_overflow = fx.bcx.ins().icmp_imm_s(IntCC::SignedGreaterThan, ret, max); | |
| let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, ret); | |
| fx.bcx.ins().select(has_overflow, max_val, bottom_capped) | |
| } else { | |
| let has_overflow = fx.bcx.ins().icmp_imm_u(IntCC::UnsignedGreaterThan, ret, max); | |
| fx.bcx.ins().select(has_overflow, max_val, ret) | |
| }; | |
| fx.bcx.ins().ireduce(to_ty, val) | |
| }; | |
| if let Some(false) = fx.tcx.sess.opts.unstable_opts.saturating_float_casts { | |
| return val; | |
| } | |
| let is_not_nan = fcmp(fx, FloatCC::Equal, from, from); | |
| let zero = type_zero_value(&mut fx.bcx, to_ty); | |
| fx.bcx.ins().select(is_not_nan, val, zero) | |
| } else { | |
| unreachable!("{from_ty:?} -> {to_ty:?}"); | |
| } | |
| } | |
| pub(crate) fn fma_f16(fx: &mut FunctionCx<'_, '_, '_>, x: Value, y: Value, z: Value) -> Value { | |
| let x = f16_to_f64(fx, x); | |
| let y = f16_to_f64(fx, y); | |
| let z = f16_to_f64(fx, z); | |
| let res = fx.bcx.ins().fma(x, y, z); | |
| f64_to_f16(fx, res) | |
| } | |
| pub(crate) fn fmin_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { | |
| fx.lib_call( | |
| "fminimumf128", | |
| vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], | |
| vec![AbiParam::new(types::F128)], | |
| &[a, b], | |
| )[0] | |
| } | |
| pub(crate) fn fmax_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { | |
| fx.lib_call( | |
| "fmaximumf128", | |
| vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], | |
| vec![AbiParam::new(types::F128)], | |
| &[a, b], | |
| )[0] | |
| } |