| use crate::prelude::*; |
| |
| pub(crate) fn f16_to_f32(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { |
| let (value, arg_ty) = |
| if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { |
| ( |
| fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value), |
| lib_call_arg_param(fx.tcx, types::I16, false), |
| ) |
| } else { |
| (value, AbiParam::new(types::F16)) |
| }; |
| fx.lib_call("__extendhfsf2", vec![arg_ty], vec![AbiParam::new(types::F32)], &[value])[0] |
| } |
| |
| fn f16_to_f64(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { |
| let ret = f16_to_f32(fx, value); |
| fx.bcx.ins().fpromote(types::F64, ret) |
| } |
| |
| pub(crate) fn f32_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { |
| let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { |
| types::I16 |
| } else { |
| types::F16 |
| }; |
| let ret = fx.lib_call( |
| "__truncsfhf2", |
| vec![AbiParam::new(types::F32)], |
| vec![AbiParam::new(ret_ty)], |
| &[value], |
| )[0]; |
| if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret } |
| } |
| |
| fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { |
| let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { |
| types::I16 |
| } else { |
| types::F16 |
| }; |
| let ret = fx.lib_call( |
| "__truncdfhf2", |
| vec![AbiParam::new(types::F64)], |
| vec![AbiParam::new(ret_ty)], |
| &[value], |
| )[0]; |
| if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret } |
| } |
| |
| pub(crate) fn fcmp(fx: &mut FunctionCx<'_, '_, '_>, cc: FloatCC, lhs: Value, rhs: Value) -> Value { |
| let ty = fx.bcx.func.dfg.value_type(lhs); |
| match ty { |
| types::F32 | types::F64 => fx.bcx.ins().fcmp(cc, lhs, rhs), |
| types::F16 => { |
| let lhs = f16_to_f32(fx, lhs); |
| let rhs = f16_to_f32(fx, rhs); |
| fx.bcx.ins().fcmp(cc, lhs, rhs) |
| } |
| types::F128 => { |
| let (name, int_cc) = match cc { |
| FloatCC::Equal => ("__eqtf2", IntCC::Equal), |
| FloatCC::NotEqual => ("__netf2", IntCC::NotEqual), |
| FloatCC::LessThan => ("__lttf2", IntCC::SignedLessThan), |
| FloatCC::LessThanOrEqual => ("__letf2", IntCC::SignedLessThanOrEqual), |
| FloatCC::GreaterThan => ("__gttf2", IntCC::SignedGreaterThan), |
| FloatCC::GreaterThanOrEqual => ("__getf2", IntCC::SignedGreaterThanOrEqual), |
| _ => unreachable!("not currently used in rustc_codegen_cranelift: {cc:?}"), |
| }; |
| let res = fx.lib_call( |
| name, |
| vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], |
| // FIXME(rust-lang/compiler-builtins#919): This should be `I64` on non-AArch64 |
| // architectures, but switching it before compiler-builtins is fixed causes test |
| // failures. |
| vec![AbiParam::new(types::I32)], |
| &[lhs, rhs], |
| )[0]; |
| let zero = fx.bcx.ins().iconst(types::I32, 0); |
| let res = fx.bcx.ins().icmp(int_cc, res, zero); |
| res |
| } |
| _ => unreachable!("{ty:?}"), |
| } |
| } |
| |
| pub(crate) fn codegen_f128_binop( |
| fx: &mut FunctionCx<'_, '_, '_>, |
| bin_op: BinOp, |
| lhs: Value, |
| rhs: Value, |
| ) -> Value { |
| let name = match bin_op { |
| BinOp::Add => "__addtf3", |
| BinOp::Sub => "__subtf3", |
| BinOp::Mul => "__multf3", |
| BinOp::Div => "__divtf3", |
| _ => unreachable!("handled in `codegen_float_binop`"), |
| }; |
| fx.lib_call( |
| name, |
| vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], |
| vec![AbiParam::new(types::F128)], |
| &[lhs, rhs], |
| )[0] |
| } |
| |
| pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { |
| let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value); |
| let bits = fx.bcx.ins().bxor_imm(bits, 0x8000); |
| fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits) |
| } |
| |
| pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { |
| let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value); |
| let (low, high) = fx.bcx.ins().isplit(bits); |
| let high = fx.bcx.ins().bxor_imm(high, 0x8000_0000_0000_0000_u64 as i64); |
| let bits = fx.bcx.ins().iconcat(low, high); |
| fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits) |
| } |
| |
| pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { |
| let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value); |
| let bits = fx.bcx.ins().band_imm(bits, 0x7fff); |
| fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits) |
| } |
| |
| pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { |
| let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value); |
| let (low, high) = fx.bcx.ins().isplit(bits); |
| let high = fx.bcx.ins().band_imm(high, 0x7fff_ffff_ffff_ffff_u64 as i64); |
| let bits = fx.bcx.ins().iconcat(low, high); |
| fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits) |
| } |
| |
| pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { |
| let lhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), lhs); |
| let rhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), rhs); |
| let res = fx.bcx.ins().band_imm(lhs, 0x7fff); |
| let sign = fx.bcx.ins().band_imm(rhs, 0x8000); |
| let res = fx.bcx.ins().bor(res, sign); |
| fx.bcx.ins().bitcast(types::F16, MemFlags::new(), res) |
| } |
| |
| pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { |
| let lhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), lhs); |
| let rhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), rhs); |
| let (low, lhs_high) = fx.bcx.ins().isplit(lhs); |
| let (_, rhs_high) = fx.bcx.ins().isplit(rhs); |
| let high = fx.bcx.ins().band_imm(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64); |
| let sign = fx.bcx.ins().band_imm(rhs_high, 0x8000_0000_0000_0000_u64 as i64); |
| let high = fx.bcx.ins().bor(high, sign); |
| let res = fx.bcx.ins().iconcat(low, high); |
| fx.bcx.ins().bitcast(types::F128, MemFlags::new(), res) |
| } |
| |
| pub(crate) fn codegen_cast( |
| fx: &mut FunctionCx<'_, '_, '_>, |
| from: Value, |
| from_signed: bool, |
| to_ty: Type, |
| to_signed: bool, |
| ) -> Value { |
| let from_ty = fx.bcx.func.dfg.value_type(from); |
| if from_ty.is_float() && to_ty.is_float() { |
| let name = match (from_ty, to_ty) { |
| (types::F16, types::F32) => return f16_to_f32(fx, from), |
| (types::F16, types::F64) => return f16_to_f64(fx, from), |
| (types::F16, types::F128) => "__extendhftf2", |
| (types::F32, types::F128) => "__extendsftf2", |
| (types::F64, types::F128) => "__extenddftf2", |
| (types::F128, types::F64) => "__trunctfdf2", |
| (types::F128, types::F32) => "__trunctfsf2", |
| (types::F128, types::F16) => "__trunctfhf2", |
| (types::F64, types::F16) => return f64_to_f16(fx, from), |
| (types::F32, types::F16) => return f32_to_f16(fx, from), |
| _ => unreachable!("{from_ty:?} -> {to_ty:?}"), |
| }; |
| fx.lib_call(name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])[0] |
| } else if from_ty.is_int() && to_ty == types::F16 { |
| let res = clif_int_or_float_cast(fx, from, from_signed, types::F32, false); |
| f32_to_f16(fx, res) |
| } else if from_ty == types::F16 && to_ty.is_int() { |
| let from = f16_to_f32(fx, from); |
| clif_int_or_float_cast(fx, from, false, to_ty, to_signed) |
| } else if from_ty.is_int() && to_ty == types::F128 { |
| let (from, from_ty) = if from_ty.bits() < 32 { |
| (clif_int_or_float_cast(fx, from, from_signed, types::I32, from_signed), types::I32) |
| } else { |
| (from, from_ty) |
| }; |
| let name = format!( |
| "__float{sign}{size}itf", |
| sign = if from_signed { "" } else { "un" }, |
| size = match from_ty { |
| types::I32 => 's', |
| types::I64 => 'd', |
| types::I128 => 't', |
| _ => unreachable!("{from_ty:?}"), |
| }, |
| ); |
| fx.lib_call( |
| &name, |
| vec![lib_call_arg_param(fx.tcx, from_ty, from_signed)], |
| vec![AbiParam::new(to_ty)], |
| &[from], |
| )[0] |
| } else if from_ty == types::F128 && to_ty.is_int() { |
| let ret_ty = if to_ty.bits() < 32 { types::I32 } else { to_ty }; |
| let name = format!( |
| "__fix{sign}tf{size}i", |
| sign = if from_signed { "" } else { "un" }, |
| size = match ret_ty { |
| types::I32 => 's', |
| types::I64 => 'd', |
| types::I128 => 't', |
| _ => unreachable!("{from_ty:?}"), |
| }, |
| ); |
| let ret = |
| fx.lib_call(&name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from]) |
| [0]; |
| let val = if ret_ty == to_ty { |
| ret |
| } else { |
| let (min, max) = match (to_ty, to_signed) { |
| (types::I8, false) => (0, i64::from(u8::MAX)), |
| (types::I16, false) => (0, i64::from(u16::MAX)), |
| (types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)), |
| (types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)), |
| _ => unreachable!("{to_ty:?}"), |
| }; |
| let min_val = fx.bcx.ins().iconst(types::I32, min); |
| let max_val = fx.bcx.ins().iconst(types::I32, max); |
| |
| let val = if to_signed { |
| let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, ret, min); |
| let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, ret, max); |
| let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, ret); |
| fx.bcx.ins().select(has_overflow, max_val, bottom_capped) |
| } else { |
| let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, ret, max); |
| fx.bcx.ins().select(has_overflow, max_val, ret) |
| }; |
| fx.bcx.ins().ireduce(to_ty, val) |
| }; |
| |
| if let Some(false) = fx.tcx.sess.opts.unstable_opts.saturating_float_casts { |
| return val; |
| } |
| |
| let is_not_nan = fcmp(fx, FloatCC::Equal, from, from); |
| let zero = type_zero_value(&mut fx.bcx, to_ty); |
| fx.bcx.ins().select(is_not_nan, val, zero) |
| } else { |
| unreachable!("{from_ty:?} -> {to_ty:?}"); |
| } |
| } |
| |
| pub(crate) fn fma_f16(fx: &mut FunctionCx<'_, '_, '_>, x: Value, y: Value, z: Value) -> Value { |
| let x = f16_to_f64(fx, x); |
| let y = f16_to_f64(fx, y); |
| let z = f16_to_f64(fx, z); |
| let res = fx.bcx.ins().fma(x, y, z); |
| f64_to_f16(fx, res) |
| } |
| |
| pub(crate) fn fmin_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { |
| fx.lib_call( |
| "fminimumf128", |
| vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], |
| vec![AbiParam::new(types::F128)], |
| &[a, b], |
| )[0] |
| } |
| |
| pub(crate) fn fmax_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { |
| fx.lib_call( |
| "fmaximumf128", |
| vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], |
| vec![AbiParam::new(types::F128)], |
| &[a, b], |
| )[0] |
| } |
