| pub mod llvm; | |
| mod simd; | |
| #[cfg(feature = "master")] | |
| use std::iter; | |
| use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, Type, UnaryOp}; | |
| use rustc_abi::{Align, BackendRepr, HasDataLayout, WrappingRange}; | |
| use rustc_codegen_ssa::base::wants_msvc_seh; | |
| use rustc_codegen_ssa::common::IntPredicate; | |
| use rustc_codegen_ssa::diagnostics::InvalidMonomorphization; | |
| use rustc_codegen_ssa::mir::IntrinsicResult; | |
| use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; | |
| use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; | |
| #[cfg(feature = "master")] | |
| use rustc_codegen_ssa::traits::MiscCodegenMethods; | |
| use rustc_codegen_ssa::traits::{ | |
| ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, | |
| IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, | |
| }; | |
| use rustc_codegen_ssa::{MemFlags, RetagInfo}; | |
| use rustc_data_structures::fx::FxHashSet; | |
| #[cfg(feature = "master")] | |
| use rustc_middle::ty::layout::FnAbiOf; | |
| use rustc_middle::ty::layout::LayoutOf; | |
| use rustc_middle::ty::{self, Instance, Ty}; | |
| use rustc_middle::{bug, span_bug}; | |
| use rustc_span::{Span, Symbol, sym}; | |
| use rustc_target::callconv::{ArgAbi, PassMode}; | |
| #[cfg(feature = "master")] | |
| use crate::abi::FnAbiGccExt; | |
| use crate::abi::GccType; | |
| use crate::builder::Builder; | |
| use crate::common::{SignType, TypeReflection}; | |
| use crate::context::CodegenCx; | |
| use crate::intrinsic::simd::generic_simd_intrinsic; | |
| use crate::type_of::LayoutGccExt; | |
| fn float_intrinsic<'gcc, 'tcx>( | |
| cx: &CodegenCx<'gcc, 'tcx>, | |
| typ: Type<'gcc>, | |
| name: &str, | |
| ) -> Option<Function<'gcc>> { | |
| // GCC doesn't have the intrinsic we want so we use the compiler-builtins one | |
| Some(cx.context.new_function( | |
| None, | |
| FunctionType::Extern, | |
| typ, | |
| &[cx.context.new_parameter(None, typ, "a"), cx.context.new_parameter(None, typ, "b")], | |
| name, | |
| false, | |
| )) | |
| } | |
| fn get_simple_intrinsic<'gcc, 'tcx>( | |
| cx: &CodegenCx<'gcc, 'tcx>, | |
| name: Symbol, | |
| ) -> Option<Function<'gcc>> { | |
| let gcc_name = match name { | |
| sym::sqrtf32 => "sqrtf", | |
| sym::sqrtf64 => "sqrt", | |
| sym::powif32 => "__builtin_powif", | |
| sym::powif64 => "__builtin_powi", | |
| sym::sinf32 => "sinf", | |
| sym::sinf64 => "sin", | |
| sym::cosf32 => "cosf", | |
| sym::cosf64 => "cos", | |
| sym::powf32 => "powf", | |
| sym::powf64 => "pow", | |
| sym::expf32 => "expf", | |
| sym::expf64 => "exp", | |
| sym::exp2f32 => "exp2f", | |
| sym::exp2f64 => "exp2", | |
| sym::logf32 => "logf", | |
| sym::logf64 => "log", | |
| sym::log10f32 => "log10f", | |
| sym::log10f64 => "log10", | |
| sym::log2f32 => "log2f", | |
| sym::log2f64 => "log2", | |
| sym::fmaf32 => "fmaf", | |
| sym::fmaf64 => "fma", | |
| // FIXME: calling `fma` from libc without FMA target feature uses expensive software emulation | |
| sym::fmuladdf32 => "fmaf", // FIXME: use gcc intrinsic analogous to llvm.fmuladd.f32 | |
| sym::fmuladdf64 => "fma", // FIXME: use gcc intrinsic analogous to llvm.fmuladd.f64 | |
| sym::minimumf32 => return float_intrinsic(cx, cx.type_f32(), "fminimumf"), | |
| sym::minimumf64 => return float_intrinsic(cx, cx.type_f64(), "fminimum"), | |
| sym::minimumf128 => return float_intrinsic(cx, cx.type_f128(), "fminimumf128"), | |
| sym::maximumf32 => return float_intrinsic(cx, cx.type_f32(), "fmaximumf"), | |
| sym::maximumf64 => return float_intrinsic(cx, cx.type_f64(), "fmaximum"), | |
| sym::maximumf128 => return float_intrinsic(cx, cx.type_f128(), "fmaximumf128"), | |
| sym::copysignf32 => "copysignf", | |
| sym::copysignf64 => "copysign", | |
| sym::floorf32 => "floorf", | |
| sym::floorf64 => "floor", | |
| sym::ceilf32 => "ceilf", | |
| sym::ceilf64 => "ceil", | |
| sym::powf128 => return float_intrinsic(cx, cx.type_f128(), "powf128"), | |
| sym::truncf32 => "truncf", | |
| sym::truncf64 => "trunc", | |
| // We match the LLVM backend and lower this to `rint`. | |
| sym::round_ties_even_f32 => "rintf", | |
| sym::round_ties_even_f64 => "rint", | |
| sym::roundf32 => "roundf", | |
| sym::roundf64 => "round", | |
| sym::abort => "abort", | |
| _ => return None, | |
| }; | |
| Some(cx.context.get_builtin_function(gcc_name)) | |
| } | |
| fn get_simple_function_f128<'gcc, 'tcx>( | |
| span: Span, | |
| cx: &CodegenCx<'gcc, 'tcx>, | |
| name: Symbol, | |
| ) -> Function<'gcc> { | |
| let f128_type = cx.type_f128(); | |
| let func_name = match name { | |
| sym::ceilf128 => "ceilf128", | |
| sym::fabs => "fabsf128", | |
| sym::expf128 => "expf128", | |
| sym::exp2f128 => "exp2f128", | |
| sym::floorf128 => "floorf128", | |
| sym::logf128 => "logf128", | |
| sym::log2f128 => "log2f128", | |
| sym::log10f128 => "log10f128", | |
| sym::truncf128 => "truncf128", | |
| sym::roundf128 => "roundf128", | |
| sym::round_ties_even_f128 => "roundevenf128", | |
| sym::sqrtf128 => "sqrtf128", | |
| _ => span_bug!(span, "used get_simple_function_f128 for non-unary f128 intrinsic"), | |
| }; | |
| cx.context.new_function( | |
| None, | |
| FunctionType::Extern, | |
| f128_type, | |
| &[cx.context.new_parameter(None, f128_type, "a")], | |
| func_name, | |
| false, | |
| ) | |
| } | |
| fn generic_f16_builtin<'gcc, 'tcx>( | |
| cx: &CodegenCx<'gcc, 'tcx>, | |
| name: Symbol, | |
| args: &[OperandRef<'tcx, RValue<'gcc>>], | |
| ) -> RValue<'gcc> { | |
| let f32_type = cx.type_f32(); | |
| let builtin_name = match name { | |
| sym::fabs => "fabsf", | |
| _ => unreachable!(), | |
| }; | |
| let func = cx.context.get_builtin_function(builtin_name); | |
| let args: Vec<_> = | |
| args.iter().map(|arg| cx.context.new_cast(None, arg.immediate(), f32_type)).collect(); | |
| let result = cx.context.new_call(None, func, &args); | |
| cx.context.new_cast(None, result, cx.type_f16()) | |
| } | |
| fn f16_builtin<'gcc, 'tcx>( | |
| cx: &CodegenCx<'gcc, 'tcx>, | |
| name: Symbol, | |
| args: &[OperandRef<'tcx, RValue<'gcc>>], | |
| ) -> RValue<'gcc> { | |
| let f32_type = cx.type_f32(); | |
| let builtin_name = match name { | |
| sym::ceilf16 => "__builtin_ceilf", | |
| sym::copysignf16 => "__builtin_copysignf", | |
| sym::expf16 => "expf", | |
| sym::exp2f16 => "exp2f", | |
| sym::fabs => "fabsf", | |
| sym::floorf16 => "__builtin_floorf", | |
| sym::fmaf16 => "fmaf", | |
| sym::logf16 => "logf", | |
| sym::log2f16 => "log2f", | |
| sym::log10f16 => "log10f", | |
| sym::powf16 => "__builtin_powf", | |
| sym::roundf16 => "__builtin_roundf", | |
| sym::round_ties_even_f16 => "__builtin_rintf", | |
| sym::sqrtf16 => "__builtin_sqrtf", | |
| sym::truncf16 => "__builtin_truncf", | |
| _ => unreachable!(), | |
| }; | |
| let func = cx.context.get_builtin_function(builtin_name); | |
| let args: Vec<_> = | |
| args.iter().map(|arg| cx.context.new_cast(None, arg.immediate(), f32_type)).collect(); | |
| let result = cx.context.new_call(None, func, &args); | |
| cx.context.new_cast(None, result, cx.type_f16()) | |
| } | |
| impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { | |
| fn codegen_intrinsic_call( | |
| &mut self, | |
| instance: Instance<'tcx>, | |
| args: &[OperandRef<'tcx, RValue<'gcc>>], | |
| result_layout: ty::layout::TyAndLayout<'tcx>, | |
| result_place: Option<PlaceValue<RValue<'gcc>>>, | |
| span: Span, | |
| ) -> IntrinsicResult<'tcx, RValue<'gcc>> { | |
| let tcx = self.tcx; | |
| let result = PlaceRef { val: result_place.unwrap(), layout: result_layout }; | |
| let name = tcx.item_name(instance.def_id()); | |
| let name_str = name.as_str(); | |
| let fn_args = instance.args; | |
| let simple = get_simple_intrinsic(self, name); | |
| let value = match name { | |
| _ if simple.is_some() => { | |
| let func = simple.expect("simple intrinsic function"); | |
| self.cx.context.new_call( | |
| self.location, | |
| func, | |
| &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
| ) | |
| } | |
| // FIXME(antoyo): We can probably remove these and use the fallback intrinsic implementation. | |
| sym::minimumf32 | sym::minimumf64 | sym::maximumf32 | sym::maximumf64 => { | |
| let (ty, func_name) = match name { | |
| sym::minimumf32 => (self.cx.float_type, "fminimumf"), | |
| sym::maximumf32 => (self.cx.float_type, "fmaximumf"), | |
| sym::minimumf64 => (self.cx.double_type, "fminimum"), | |
| sym::maximumf64 => (self.cx.double_type, "fmaximum"), | |
| _ => unreachable!(), | |
| }; | |
| let func = self.cx.context.new_function( | |
| None, | |
| FunctionType::Extern, | |
| ty, | |
| &[ | |
| self.cx.context.new_parameter(None, ty, "a"), | |
| self.cx.context.new_parameter(None, ty, "b"), | |
| ], | |
| func_name, | |
| false, | |
| ); | |
| self.cx.context.new_call( | |
| self.location, | |
| func, | |
| &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
| ) | |
| } | |
| sym::ceilf16 | |
| | sym::copysignf16 | |
| | sym::expf16 | |
| | sym::exp2f16 | |
| | sym::floorf16 | |
| | sym::fmaf16 | |
| | sym::logf16 | |
| | sym::log2f16 | |
| | sym::log10f16 | |
| | sym::powf16 | |
| | sym::roundf16 | |
| | sym::round_ties_even_f16 | |
| | sym::sqrtf16 | |
| | sym::truncf16 => f16_builtin(self, name, args), | |
| sym::ceilf128 | |
| | sym::floorf128 | |
| | sym::truncf128 | |
| | sym::roundf128 | |
| | sym::round_ties_even_f128 | |
| | sym::sqrtf128 | |
| | sym::expf128 | |
| | sym::exp2f128 | |
| | sym::logf128 | |
| | sym::log2f128 | |
| | sym::log10f128 | |
| if self.cx.supports_f128_type => | |
| { | |
| let func = get_simple_function_f128(span, self, name); | |
| self.cx.context.new_call( | |
| self.location, | |
| func, | |
| &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
| ) | |
| } | |
| sym::copysignf128 if self.cx.supports_f128_type => { | |
| let f128_type = self.cx.type_f128(); | |
| let func = self.cx.context.new_function( | |
| None, | |
| FunctionType::Extern, | |
| f128_type, | |
| &[ | |
| self.cx.context.new_parameter(None, f128_type, "a"), | |
| self.cx.context.new_parameter(None, f128_type, "b"), | |
| ], | |
| "copysignf128", | |
| false, | |
| ); | |
| self.cx.context.new_call( | |
| self.location, | |
| func, | |
| &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
| ) | |
| } | |
| sym::fmaf128 => { | |
| let f128_type = self.cx.type_f128(); | |
| let func = self.cx.context.new_function( | |
| None, | |
| FunctionType::Extern, | |
| f128_type, | |
| &[ | |
| self.cx.context.new_parameter(None, f128_type, "a"), | |
| self.cx.context.new_parameter(None, f128_type, "b"), | |
| self.cx.context.new_parameter(None, f128_type, "c"), | |
| ], | |
| "fmaf128", | |
| false, | |
| ); | |
| self.cx.context.new_call( | |
| self.location, | |
| func, | |
| &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
| ) | |
| } | |
| sym::powif16 => { | |
| let func = self.cx.context.get_builtin_function("__builtin_powif"); | |
| let arg0 = self.cx.context.new_cast(None, args[0].immediate(), self.cx.type_f32()); | |
| let args = [arg0, args[1].immediate()]; | |
| let result = self.cx.context.new_call(None, func, &args); | |
| self.cx.context.new_cast(None, result, self.cx.type_f16()) | |
| } | |
| sym::powif128 => { | |
| let f128_type = self.cx.type_f128(); | |
| let func = self.cx.context.new_function( | |
| None, | |
| FunctionType::Extern, | |
| f128_type, | |
| &[ | |
| self.cx.context.new_parameter(None, f128_type, "a"), | |
| self.cx.context.new_parameter(None, self.int_type, "b"), | |
| ], | |
| "__powitf2", | |
| false, | |
| ); | |
| self.cx.context.new_call( | |
| self.location, | |
| func, | |
| &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
| ) | |
| } | |
| sym::is_val_statically_known => { | |
| let a = args[0].immediate(); | |
| let builtin = self.context.get_builtin_function("__builtin_constant_p"); | |
| let res = self.context.new_call(None, builtin, &[a]); | |
| self.icmp(IntPredicate::IntEQ, res, self.const_i32(0)) | |
| } | |
| sym::catch_unwind => { | |
| try_intrinsic( | |
| self, | |
| args[0].immediate(), | |
| args[1].immediate(), | |
| args[2].immediate(), | |
| result, | |
| ); | |
| return IntrinsicResult::WroteIntoPlace; | |
| } | |
| sym::breakpoint => { | |
| unimplemented!(); | |
| } | |
| sym::va_arg => { | |
| unimplemented!(); | |
| } | |
| sym::volatile_load | sym::unaligned_volatile_load => { | |
| let ptr = args[0].immediate(); | |
| let abi_align = result_layout.align.abi; | |
| let ptr_align = if name == sym::volatile_load { abi_align } else { Align::ONE }; | |
| let load = self.volatile_load(result.layout.gcc_type(self), ptr, ptr_align); | |
| if let BackendRepr::Scalar(scalar) = result.layout.backend_repr { | |
| self.to_immediate_scalar(load, scalar) | |
| } else { | |
| load | |
| } | |
| } | |
| sym::prefetch_read_data | |
| | sym::prefetch_write_data | |
| | sym::prefetch_read_instruction | |
| | sym::prefetch_write_instruction => { | |
| unimplemented!(); | |
| } | |
| sym::ctlz | |
| | sym::ctlz_nonzero | |
| | sym::cttz | |
| | sym::cttz_nonzero | |
| | sym::ctpop | |
| | sym::bswap | |
| | sym::bitreverse | |
| | sym::rotate_left | |
| | sym::rotate_right | |
| | sym::saturating_add | |
| | sym::saturating_sub => { | |
| match int_type_width_signed(args[0].layout.ty, self) { | |
| Some((width, signed)) => match name { | |
| sym::ctlz => self.count_leading_zeroes(width, args[0].immediate()), | |
| sym::ctlz_nonzero => { | |
| self.count_leading_zeroes_nonzero(width, args[0].immediate()) | |
| } | |
| sym::cttz => self.count_trailing_zeroes(width, args[0].immediate()), | |
| sym::cttz_nonzero => { | |
| self.count_trailing_zeroes_nonzero(width, args[0].immediate()) | |
| } | |
| sym::ctpop => self.pop_count(args[0].immediate()), | |
| sym::bswap => { | |
| if width == 8 { | |
| args[0].immediate() // byte swap a u8/i8 is just a no-op | |
| } else { | |
| self.gcc_bswap(args[0].immediate(), width) | |
| } | |
| } | |
| sym::bitreverse => self.bit_reverse(width, args[0].immediate()), | |
| sym::rotate_left | sym::rotate_right => { | |
| // Using optimized branchless algorithm from: | |
| // https://blog.regehr.org/archives/1063 | |
| // This implementation uses the pattern (x<<n) | (x>>(-n&(width-1))) | |
| // which generates efficient code for other platforms. | |
| let is_left = name == sym::rotate_left; | |
| let val = args[0].immediate(); | |
| let raw_shift = args[1].immediate(); | |
| if is_left { | |
| self.rotate_left(val, raw_shift, width) | |
| } else { | |
| self.rotate_right(val, raw_shift, width) | |
| } | |
| } | |
| sym::saturating_add => self.saturating_add( | |
| args[0].immediate(), | |
| args[1].immediate(), | |
| signed, | |
| width, | |
| ), | |
| sym::saturating_sub => self.saturating_sub( | |
| args[0].immediate(), | |
| args[1].immediate(), | |
| signed, | |
| width, | |
| ), | |
| _ => bug!(), | |
| }, | |
| None => { | |
| let err = tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { | |
| span, | |
| name, | |
| ty: args[0].layout.ty, | |
| }); | |
| return IntrinsicResult::Err(err); | |
| } | |
| } | |
| } | |
| sym::fabs => 'fabs: { | |
| let ty = args[0].layout.ty; | |
| let ty::Float(float_ty) = *ty.kind() else { | |
| span_bug!(span, "expected float type for fabs intrinsic: {:?}", ty); | |
| }; | |
| let func = match float_ty { | |
| ty::FloatTy::F16 => break 'fabs generic_f16_builtin(self, name, args), | |
| ty::FloatTy::F32 => self.context.get_builtin_function("fabsf"), | |
| ty::FloatTy::F64 => self.context.get_builtin_function("fabs"), | |
| ty::FloatTy::F128 => get_simple_function_f128(span, self, name), | |
| }; | |
| self.cx.context.new_call( | |
| self.location, | |
| func, | |
| &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
| ) | |
| } | |
| sym::raw_eq => { | |
| use rustc_abi::BackendRepr::*; | |
| let tp_ty = fn_args.type_at(0); | |
| let layout = self.layout_of(tp_ty).layout; | |
| let _use_integer_compare = match layout.backend_repr() { | |
| Scalar(_) | ScalarPair { a: _, b: _, b_offset: _ } => true, | |
| SimdVector { .. } | SimdScalableVector { .. } => false, | |
| Memory { .. } => { | |
| // For rusty ABIs, small aggregates are actually passed | |
| // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), | |
| // so we re-use that same threshold here. | |
| layout.size() <= self.data_layout().pointer_size() * 2 | |
| } | |
| }; | |
| let a = args[0].immediate(); | |
| let b = args[1].immediate(); | |
| if layout.size().bytes() == 0 { | |
| self.const_bool(true) | |
| } | |
| /*else if use_integer_compare { | |
| let integer_ty = self.type_ix(layout.size.bits()); // FIXME(antoyo): LLVM creates an integer of 96 bits for [i32; 3], but gcc doesn't support this, so it creates an integer of 128 bits. | |
| let ptr_ty = self.type_ptr_to(integer_ty); | |
| let a_ptr = self.bitcast(a, ptr_ty); | |
| let a_val = self.load(integer_ty, a_ptr, layout.align.abi); | |
| let b_ptr = self.bitcast(b, ptr_ty); | |
| let b_val = self.load(integer_ty, b_ptr, layout.align.abi); | |
| self.icmp(IntPredicate::IntEQ, a_val, b_val) | |
| }*/ | |
| else { | |
| let void_ptr_type = self.context.new_type::<*const ()>(); | |
| let a_ptr = self.bitcast(a, void_ptr_type); | |
| let b_ptr = self.bitcast(b, void_ptr_type); | |
| let n = self.context.new_cast( | |
| None, | |
| self.const_usize(layout.size().bytes()), | |
| self.sizet_type, | |
| ); | |
| let builtin = self.context.get_builtin_function("memcmp"); | |
| let cmp = self.context.new_call(None, builtin, &[a_ptr, b_ptr, n]); | |
| self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)) | |
| } | |
| } | |
| sym::compare_bytes => { | |
| let a = args[0].immediate(); | |
| let b = args[1].immediate(); | |
| let n = args[2].immediate(); | |
| let void_ptr_type = self.context.new_type::<*const ()>(); | |
| let a_ptr = self.bitcast(a, void_ptr_type); | |
| let b_ptr = self.bitcast(b, void_ptr_type); | |
| // Here we assume that the `memcmp` provided by the target is a NOP for size 0. | |
| let builtin = self.context.get_builtin_function("memcmp"); | |
| let cmp = self.context.new_call(None, builtin, &[a_ptr, b_ptr, n]); | |
| self.sext(cmp, self.type_ix(32)) | |
| } | |
| sym::black_box => { | |
| args[0].val.store(self, result); | |
| let block = self.llbb(); | |
| let extended_asm = block.add_extended_asm(None, ""); | |
| extended_asm.add_input_operand(None, "r", result.val.llval); | |
| extended_asm.add_clobber("memory"); | |
| extended_asm.set_volatile_flag(true); | |
| // We have copied the value to `result` already. | |
| return IntrinsicResult::WroteIntoPlace; | |
| } | |
| sym::ptr_mask => { | |
| let usize_type = self.context.new_type::<usize>(); | |
| let void_ptr_type = self.context.new_type::<*const ()>(); | |
| let ptr = args[0].immediate(); | |
| let mask = args[1].immediate(); | |
| let addr = self.bitcast(ptr, usize_type); | |
| let masked = self.and(addr, mask); | |
| self.bitcast(masked, void_ptr_type) | |
| } | |
| _ if name_str.starts_with("simd_") => { | |
| match generic_simd_intrinsic( | |
| self, | |
| name, | |
| args, | |
| result.layout.ty, | |
| result.layout.gcc_type(self), | |
| span, | |
| ) { | |
| Ok(value) => value, | |
| Err(err) => return IntrinsicResult::Err(err), | |
| } | |
| } | |
| // Fall back to default body | |
| _ => { | |
| let fallback = Instance::new_raw(instance.def_id(), instance.args); | |
| return IntrinsicResult::Fallback(fallback); | |
| } | |
| }; | |
| if result.layout.ty.is_bool() { | |
| let val = self.from_immediate(value); | |
| self.store_to_place(val, result.val); | |
| } else if !result.layout.ty.is_unit() { | |
| self.store_to_place(value, result.val); | |
| } | |
| IntrinsicResult::WroteIntoPlace | |
| } | |
| fn codegen_llvm_intrinsic_call( | |
| &mut self, | |
| instance: ty::Instance<'tcx>, | |
| args: &[OperandRef<'tcx, Self::Value>], | |
| is_cleanup: bool, | |
| ) -> Self::Value { | |
| let func = if let Some(&func) = self.intrinsic_instances.borrow().get(&instance) { | |
| func | |
| } else { | |
| let sym = self.tcx.symbol_name(instance).name; | |
| let func = if let Some(func) = self.intrinsics.borrow().get(sym) { | |
| *func | |
| } else { | |
| self.linkage.set(FunctionType::Extern); | |
| let func = match sym { | |
| "llvm.fma.f16" => { | |
| // fma is not a target builtin, but a normal builtin, so we handle it differently | |
| // here. | |
| self.context.get_builtin_function("fma") | |
| } | |
| _ => llvm::intrinsic(sym, self), | |
| }; | |
| self.intrinsics.borrow_mut().insert(sym.to_string(), func); | |
| self.on_stack_function_params.borrow_mut().insert(func, FxHashSet::default()); | |
| crate::attributes::from_fn_attrs(self, func, instance); | |
| func | |
| }; | |
| self.intrinsic_instances.borrow_mut().insert(instance, func); | |
| func | |
| }; | |
| let fn_ptr = func.get_address(None); | |
| let fn_ty = fn_ptr.get_type(); | |
| let mut call_args = vec![]; | |
| for arg in args { | |
| match arg.val { | |
| OperandValue::ZeroSized => {} | |
| OperandValue::Immediate(_) => call_args.push(arg.immediate()), | |
| OperandValue::Pair(a, b) => { | |
| call_args.push(a); | |
| call_args.push(b); | |
| } | |
| OperandValue::Ref(op_place_val) => { | |
| let mut llval = op_place_val.llval; | |
| // We can't use `PlaceRef::load` here because the argument | |
| // may have a type we don't treat as immediate, but the ABI | |
| // used for this call is passing it by-value. In that case, | |
| // the load would just produce `OperandValue::Ref` instead | |
| // of the `OperandValue::Immediate` we need for the call. | |
| llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); | |
| if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { | |
| if scalar.is_bool() { | |
| self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); | |
| } | |
| // We store bools as `i8` so we need to truncate to `i1`. | |
| llval = self.to_immediate_scalar(llval, scalar); | |
| } | |
| call_args.push(llval); | |
| } | |
| } | |
| } | |
| // FIXME directly use the llvm intrinsic adjustment functions here | |
| let llret = self.call(fn_ty, None, None, fn_ptr, &call_args, None, None); | |
| if is_cleanup { | |
| self.apply_attrs_to_cleanup_callsite(llret); | |
| } | |
| llret | |
| } | |
| fn abort(&mut self) { | |
| let func = self.context.get_builtin_function("abort"); | |
| let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; | |
| self.call(self.type_void(), None, None, func, &[], None, None); | |
| } | |
| fn assume(&mut self, value: Self::Value) { | |
| // FIXME(antoyo): switch to assume when it exists. | |
| // Or use something like this: | |
| // #define __assume(cond) do { if (!(cond)) __builtin_unreachable(); } while (0) | |
| self.expect(value, true); | |
| } | |
| fn expect(&mut self, cond: Self::Value, _expected: bool) -> Self::Value { | |
| // FIXME(antoyo) | |
| cond | |
| } | |
| fn type_checked_load( | |
| &mut self, | |
| _vtable: Self::Value, | |
| _vtable_byte_offset: u64, | |
| _typeid: &[u8], | |
| ) -> Self::Value { | |
| // Unsupported. | |
| self.context.new_rvalue_from_int(self.int_type, 0) | |
| } | |
| fn va_start(&mut self, _va_list: RValue<'gcc>) { | |
| unimplemented!(); | |
| } | |
| fn retag_reg(&mut self, _ptr: Self::Value, _info: &RetagInfo<Self::Value>) -> Self::Value { | |
| unimplemented!() | |
| } | |
| fn retag_mem(&mut self, _ptr: Self::Value, _info: &RetagInfo<Self::Value>) { | |
| unimplemented!() | |
| } | |
| } | |
| impl<'a, 'gcc, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { | |
| fn store_fn_arg( | |
| &mut self, | |
| arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, | |
| idx: &mut usize, | |
| dst: PlaceRef<'tcx, Self::Value>, | |
| ) { | |
| arg_abi.store_fn_arg(self, idx, dst) | |
| } | |
| fn store_arg( | |
| &mut self, | |
| arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, | |
| val: RValue<'gcc>, | |
| dst: PlaceRef<'tcx, RValue<'gcc>>, | |
| ) { | |
| arg_abi.store(self, val, dst) | |
| } | |
| } | |
| pub trait ArgAbiExt<'gcc, 'tcx> { | |
| fn store( | |
| &self, | |
| bx: &mut Builder<'_, 'gcc, 'tcx>, | |
| val: RValue<'gcc>, | |
| dst: PlaceRef<'tcx, RValue<'gcc>>, | |
| ); | |
| fn store_fn_arg( | |
| &self, | |
| bx: &mut Builder<'_, 'gcc, 'tcx>, | |
| idx: &mut usize, | |
| dst: PlaceRef<'tcx, RValue<'gcc>>, | |
| ); | |
| } | |
| impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { | |
| /// Stores a direct/indirect value described by this ArgAbi into a | |
| /// place for the original Rust type of this argument/return. | |
| /// Can be used for both storing formal arguments into Rust variables | |
| /// or results of call/invoke instructions into their destinations. | |
| fn store( | |
| &self, | |
| bx: &mut Builder<'_, 'gcc, 'tcx>, | |
| val: RValue<'gcc>, | |
| dst: PlaceRef<'tcx, RValue<'gcc>>, | |
| ) { | |
| if self.is_ignore() { | |
| return; | |
| } | |
| if self.is_sized_indirect() { | |
| OperandValue::Ref(PlaceValue::new_sized(val, self.layout.align.abi)).store(bx, dst) | |
| } else if self.is_unsized_indirect() { | |
| bug!("unsized `ArgAbi` cannot be stored"); | |
| } else if let PassMode::Cast { ref cast, .. } = self.mode { | |
| // FIXME(eddyb): Figure out when the simpler Store is safe, clang | |
| // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. | |
| let can_store_through_cast_ptr = false; | |
| if can_store_through_cast_ptr { | |
| let cast_ptr_llty = bx.type_ptr_to(cast.gcc_type(bx)); | |
| let cast_dst = bx.pointercast(dst.val.llval, cast_ptr_llty); | |
| bx.store(val, cast_dst, self.layout.align.abi); | |
| } else { | |
| // The actual return type is a struct, but the ABI | |
| // adaptation code has cast it into some scalar type. The | |
| // code that follows is the only reliable way I have | |
| // found to do a transform like i64 -> {i32,i32}. | |
| // Basically we dump the data onto the stack then memcpy it. | |
| // | |
| // Other approaches I tried: | |
| // - Casting rust ret pointer to the foreign type and using Store | |
| // is (a) unsafe if size of foreign type > size of rust type and | |
| // (b) runs afoul of strict aliasing rules, yielding invalid | |
| // assembly under -O (specifically, the store gets removed). | |
| // - Truncating foreign type to correct integral type and then | |
| // bitcasting to the struct type yields invalid cast errors. | |
| // We instead thus allocate some scratch space... | |
| let scratch_size = cast.size(bx); | |
| let scratch_align = cast.align(bx); | |
| let scratch = bx.alloca(scratch_size, scratch_align); | |
| bx.lifetime_start(scratch, scratch_size); | |
| // ... where we first store the value... | |
| rustc_codegen_ssa::mir::store_cast(bx, cast, val, scratch, scratch_align); | |
| // ... and then memcpy it to the intended destination. | |
| bx.memcpy( | |
| dst.val.llval, | |
| self.layout.align.abi, | |
| scratch, | |
| scratch_align, | |
| bx.const_usize(self.layout.size.bytes()), | |
| MemFlags::empty(), | |
| None, | |
| ); | |
| bx.lifetime_end(scratch, scratch_size); | |
| } | |
| } else { | |
| OperandValue::Immediate(val).store(bx, dst); | |
| } | |
| } | |
| fn store_fn_arg<'a>( | |
| &self, | |
| bx: &mut Builder<'a, 'gcc, 'tcx>, | |
| idx: &mut usize, | |
| dst: PlaceRef<'tcx, RValue<'gcc>>, | |
| ) { | |
| let mut next = || { | |
| let val = bx.current_func().get_param(*idx as i32); | |
| *idx += 1; | |
| val.to_rvalue() | |
| }; | |
| match self.mode { | |
| PassMode::Ignore => {} | |
| PassMode::Pair(..) => { | |
| OperandValue::Pair(next(), next()).store(bx, dst); | |
| } | |
| PassMode::Indirect { meta_attrs: Some(_), .. } => { | |
| bug!("unsized `ArgAbi` cannot be stored"); | |
| } | |
| PassMode::Direct(_) | |
| | PassMode::Indirect { meta_attrs: None, .. } | |
| | PassMode::Cast { .. } => { | |
| let next_arg = next(); | |
| self.store(bx, next_arg, dst); | |
| } | |
| } | |
| } | |
| } | |
| fn int_type_width_signed<'gcc, 'tcx>( | |
| ty: Ty<'tcx>, | |
| cx: &CodegenCx<'gcc, 'tcx>, | |
| ) -> Option<(u64, bool)> { | |
| match *ty.kind() { | |
| ty::Int(t) => Some(( | |
| match t { | |
| rustc_middle::ty::IntTy::Isize => u64::from(cx.tcx.sess.target.pointer_width), | |
| rustc_middle::ty::IntTy::I8 => 8, | |
| rustc_middle::ty::IntTy::I16 => 16, | |
| rustc_middle::ty::IntTy::I32 => 32, | |
| rustc_middle::ty::IntTy::I64 => 64, | |
| rustc_middle::ty::IntTy::I128 => 128, | |
| }, | |
| true, | |
| )), | |
| ty::Uint(t) => Some(( | |
| match t { | |
| rustc_middle::ty::UintTy::Usize => u64::from(cx.tcx.sess.target.pointer_width), | |
| rustc_middle::ty::UintTy::U8 => 8, | |
| rustc_middle::ty::UintTy::U16 => 16, | |
| rustc_middle::ty::UintTy::U32 => 32, | |
| rustc_middle::ty::UintTy::U64 => 64, | |
| rustc_middle::ty::UintTy::U128 => 128, | |
| }, | |
| false, | |
| )), | |
| _ => None, | |
| } | |
| } | |
| impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { | |
| fn bit_reverse(&mut self, width: u64, value: RValue<'gcc>) -> RValue<'gcc> { | |
| let result_type = value.get_type(); | |
| let typ = result_type.to_unsigned(self.cx); | |
| let value = | |
| if result_type.is_signed(self.cx) { self.gcc_int_cast(value, typ) } else { value }; | |
| let context = &self.cx.context; | |
| let result = match width { | |
| 8 | 16 | 32 | 64 => { | |
| let mask = ((1u128 << width) - 1) as u64; | |
| let (m0, m1, m2) = if width > 16 { | |
| ( | |
| context.new_rvalue_from_long(typ, (0x5555555555555555u64 & mask) as i64), | |
| context.new_rvalue_from_long(typ, (0x3333333333333333u64 & mask) as i64), | |
| context.new_rvalue_from_long(typ, (0x0f0f0f0f0f0f0f0fu64 & mask) as i64), | |
| ) | |
| } else { | |
| ( | |
| context.new_rvalue_from_int(typ, (0x5555u64 & mask) as i32), | |
| context.new_rvalue_from_int(typ, (0x3333u64 & mask) as i32), | |
| context.new_rvalue_from_int(typ, (0x0f0fu64 & mask) as i32), | |
| ) | |
| }; | |
| let one = context.new_rvalue_from_int(typ, 1); | |
| let two = context.new_rvalue_from_int(typ, 2); | |
| let four = context.new_rvalue_from_int(typ, 4); | |
| // First step. | |
| let left = self.lshr(value, one); | |
| let left = self.and(left, m0); | |
| let right = self.and(value, m0); | |
| let right = self.shl(right, one); | |
| let step1 = self.or(left, right); | |
| // Second step. | |
| let left = self.lshr(step1, two); | |
| let left = self.and(left, m1); | |
| let right = self.and(step1, m1); | |
| let right = self.shl(right, two); | |
| let step2 = self.or(left, right); | |
| // Third step. | |
| let left = self.lshr(step2, four); | |
| let left = self.and(left, m2); | |
| let right = self.and(step2, m2); | |
| let right = self.shl(right, four); | |
| let step3 = self.or(left, right); | |
| // Fourth step. | |
| if width == 8 { step3 } else { self.gcc_bswap(step3, width) } | |
| } | |
| 128 => { | |
| // FIXME(antoyo): find a more efficient implementation? | |
| let sixty_four = self.gcc_int(typ, 64); | |
| let right_shift = self.gcc_lshr(value, sixty_four); | |
| let high = self.gcc_int_cast(right_shift, self.u64_type); | |
| let low = self.gcc_int_cast(value, self.u64_type); | |
| let reversed_high = self.bit_reverse(64, high); | |
| let reversed_low = self.bit_reverse(64, low); | |
| let new_low = self.gcc_int_cast(reversed_high, typ); | |
| let new_high = self.shl(self.gcc_int_cast(reversed_low, typ), sixty_four); | |
| self.gcc_or(new_low, new_high, self.location) | |
| } | |
| _ => { | |
| panic!("cannot bit reverse with width = {}", width); | |
| } | |
| }; | |
| self.gcc_int_cast(result, result_type) | |
| } | |
| fn count_zeroes(&mut self, width: u64, arg: RValue<'gcc>, count_leading: bool) -> RValue<'gcc> { | |
| // if arg is 0, early return 0, else call count_leading_zeroes_nonzero or count_trailing_zeroes_nonzero | |
| let func = self.current_func(); | |
| let then_block = func.new_block("then"); | |
| let else_block = func.new_block("else"); | |
| let after_block = func.new_block("after"); | |
| let result = func.new_local(None, self.u32_type, "zeros"); | |
| let zero = self.cx.gcc_zero(arg.get_type()); | |
| let cond = self.gcc_icmp(IntPredicate::IntEQ, arg, zero); | |
| self.llbb().end_with_conditional(None, cond, then_block, else_block); | |
| let zero_result = self.cx.gcc_uint(self.u32_type, width); | |
| then_block.add_assignment(None, result, zero_result); | |
| then_block.end_with_jump(None, after_block); | |
| // NOTE: since jumps were added in a place count_xxxxing_zeroes_nonzero() does not expect, | |
| // the current block in the state need to be updated. | |
| self.switch_to_block(else_block); | |
| let zeros = if count_leading { | |
| self.count_leading_zeroes_nonzero(width, arg) | |
| } else { | |
| self.count_trailing_zeroes_nonzero(width, arg) | |
| }; | |
| self.llbb().add_assignment(None, result, zeros); | |
| self.llbb().end_with_jump(None, after_block); | |
| // NOTE: since jumps were added in a place rustc does not | |
| // expect, the current block in the state need to be updated. | |
| self.switch_to_block(after_block); | |
| result.to_rvalue() | |
| } | |
| fn count_zeroes_nonzero( | |
| &mut self, | |
| width: u64, | |
| arg: RValue<'gcc>, | |
| count_leading: bool, | |
| ) -> RValue<'gcc> { | |
| // Pre-condition: arg is guaranteed to not be 0 by caller | |
| fn use_builtin_function<'a, 'gcc, 'tcx>( | |
| builder: &mut Builder<'a, 'gcc, 'tcx>, | |
| builtin: &str, | |
| arg: RValue<'gcc>, | |
| arg_type: gccjit::Type<'gcc>, | |
| expected_type: gccjit::Type<'gcc>, | |
| ) -> RValue<'gcc> { | |
| let arg = if arg_type != expected_type { | |
| builder.context.new_cast(builder.location, arg, expected_type) | |
| } else { | |
| arg | |
| }; | |
| let builtin = builder.context.get_builtin_function(builtin); | |
| let res = builder.context.new_call(builder.location, builtin, &[arg]); | |
| builder.context.new_cast(builder.location, res, builder.u32_type) | |
| } | |
| // FIXME(antoyo): use width? | |
| let result_type = self.u32_type; | |
| let mut arg_type = arg.get_type(); | |
| let arg = if arg_type.is_signed(self.cx) { | |
| arg_type = arg_type.to_unsigned(self.cx); | |
| self.gcc_int_cast(arg, arg_type) | |
| } else { | |
| arg | |
| }; | |
| // FIXME(antoyo): write a new function Type::is_compatible_with(&Type) and use it here | |
| // instead of using is_uint(). | |
| if arg_type.is_uchar(self.cx) || arg_type.is_ushort(self.cx) || arg_type.is_uint(self.cx) { | |
| let builtin = if count_leading { "__builtin_clz" } else { "__builtin_ctz" }; | |
| use_builtin_function(self, builtin, arg, arg_type, self.cx.uint_type) | |
| } else if arg_type.is_ulong(self.cx) { | |
| let builtin = if count_leading { "__builtin_clzl" } else { "__builtin_ctzl" }; | |
| use_builtin_function(self, builtin, arg, arg_type, self.cx.uint_type) | |
| } else if arg_type.is_ulonglong(self.cx) { | |
| let builtin = if count_leading { "__builtin_clzll" } else { "__builtin_ctzll" }; | |
| use_builtin_function(self, builtin, arg, arg_type, self.cx.uint_type) | |
| } else if width == 128 { | |
| // arg is guaranteed to not be 0, so either its 64 high or 64 low bits are not 0 | |
| // __buildin_clzll is UB when called with 0, so call it on the 64 high bits if they are not 0, | |
| // else call it on the 64 low bits and add 64. In the else case, 64 low bits can't be 0 | |
| // because arg is not 0. | |
| // __buildin_ctzll is UB when called with 0, so call it on the 64 low bits if they are not 0, | |
| // else call it on the 64 high bits and add 64. In the else case, 64 high bits can't be 0 | |
| // because arg is not 0. | |
| let result = self.current_func().new_local(None, result_type, "count_zeroes_results"); | |
| let cz_then_block = self.current_func().new_block("cz_then"); | |
| let cz_else_block = self.current_func().new_block("cz_else"); | |
| let cz_after_block = self.current_func().new_block("cz_after"); | |
| let low = self.gcc_int_cast(arg, self.u64_type); | |
| let sixty_four = self.const_uint(arg_type, 64); | |
| let shift = self.lshr(arg, sixty_four); | |
| let high = self.gcc_int_cast(shift, self.u64_type); | |
| let (first, second, builtin) = if count_leading { | |
| (low, high, self.context.get_builtin_function("__builtin_clzll")) | |
| } else { | |
| (high, low, self.context.get_builtin_function("__builtin_ctzll")) | |
| }; | |
| let zero_64 = self.const_uint(self.u64_type, 0); | |
| let cond = self.gcc_icmp(IntPredicate::IntNE, second, zero_64); | |
| self.llbb().end_with_conditional(self.location, cond, cz_then_block, cz_else_block); | |
| self.switch_to_block(cz_then_block); | |
| let result_128 = | |
| self.gcc_int_cast(self.context.new_call(None, builtin, &[second]), result_type); | |
| cz_then_block.add_assignment(self.location, result, result_128); | |
| cz_then_block.end_with_jump(self.location, cz_after_block); | |
| self.switch_to_block(cz_else_block); | |
| let count_more_zeroes = | |
| self.gcc_int_cast(self.context.new_call(None, builtin, &[first]), result_type); | |
| let sixty_four_result_type = self.const_uint(result_type, 64); | |
| let count_result_type = self.add(count_more_zeroes, sixty_four_result_type); | |
| cz_else_block.add_assignment(self.location, result, count_result_type); | |
| cz_else_block.end_with_jump(self.location, cz_after_block); | |
| self.switch_to_block(cz_after_block); | |
| result.to_rvalue() | |
| } else { | |
| let byte_diff = self.ulonglong_type.get_size() as i64 - arg_type.get_size() as i64; | |
| let diff = self.context.new_rvalue_from_long(self.int_type, byte_diff * 8); | |
| let ull_arg = self.context.new_cast(self.location, arg, self.ulonglong_type); | |
| let res = if count_leading { | |
| let count_leading_zeroes = self.context.get_builtin_function("__builtin_clzll"); | |
| self.context.new_call(self.location, count_leading_zeroes, &[ull_arg]) - diff | |
| } else { | |
| let count_trailing_zeroes = self.context.get_builtin_function("__builtin_ctzll"); | |
| let mask = self.context.new_rvalue_from_long(arg_type, -1); // To get the value with all bits set. | |
| let masked = mask | |
| & self.context.new_unary_op( | |
| self.location, | |
| UnaryOp::BitwiseNegate, | |
| arg_type, | |
| arg, | |
| ); | |
| let cond = | |
| self.context.new_comparison(self.location, ComparisonOp::Equals, masked, mask); | |
| let diff = diff * self.context.new_cast(self.location, cond, self.int_type); | |
| self.context.new_call(self.location, count_trailing_zeroes, &[ull_arg]) - diff | |
| }; | |
| self.context.new_cast(self.location, res, result_type) | |
| } | |
| } | |
| fn count_leading_zeroes(&mut self, width: u64, arg: RValue<'gcc>) -> RValue<'gcc> { | |
| self.count_zeroes(width, arg, true) | |
| } | |
| fn count_leading_zeroes_nonzero(&mut self, width: u64, arg: RValue<'gcc>) -> RValue<'gcc> { | |
| // Pre-condition: arg is guaranteed to not be 0 by caller, else count_leading_zeros should be used | |
| self.count_zeroes_nonzero(width, arg, true) | |
| } | |
| fn count_trailing_zeroes(&mut self, width: u64, arg: RValue<'gcc>) -> RValue<'gcc> { | |
| self.count_zeroes(width, arg, false) | |
| } | |
| fn count_trailing_zeroes_nonzero(&mut self, width: u64, arg: RValue<'gcc>) -> RValue<'gcc> { | |
| // Pre-condition: arg is guaranteed to not be 0 by caller, else count_trailing_zeros should be used | |
| self.count_zeroes_nonzero(width, arg, false) | |
| } | |
| fn pop_count(&mut self, value: RValue<'gcc>) -> RValue<'gcc> { | |
| // FIXME(antoyo): use the optimized version with fewer operations. | |
| let result_type = self.u32_type; | |
| let arg_type = value.get_type(); | |
| let value_type = arg_type.to_unsigned(self.cx); | |
| let value = | |
| if arg_type.is_signed(self.cx) { self.gcc_int_cast(value, value_type) } else { value }; | |
| // only break apart 128-bit ints if they're not natively supported | |
| // FIXME(antoyo): remove this if/when native 128-bit integers land in libgccjit | |
| if value_type.is_u128(self.cx) && !self.cx.supports_128bit_integers { | |
| let sixty_four = self.gcc_int(value_type, 64); | |
| let right_shift = self.gcc_lshr(value, sixty_four); | |
| let high = self.gcc_int_cast(right_shift, self.cx.ulonglong_type); | |
| let high = self.pop_count(high); | |
| let low = self.gcc_int_cast(value, self.cx.ulonglong_type); | |
| let low = self.pop_count(low); | |
| let res = high + low; | |
| return self.gcc_int_cast(res, result_type); | |
| } | |
| // Use Wenger's algorithm for population count, gcc's seems to play better with it | |
| // for (int counter = 0; value != 0; counter++) { | |
| // value &= value - 1; | |
| // } | |
| let func = self.current_func(); | |
| let loop_head = func.new_block("head"); | |
| let loop_body = func.new_block("body"); | |
| let loop_tail = func.new_block("tail"); | |
| let counter_type = self.int_type; | |
| let counter = self.current_func().new_local(None, counter_type, "popcount_counter"); | |
| let val = self.current_func().new_local(None, value_type, "popcount_value"); | |
| let zero = self.gcc_zero(counter_type); | |
| self.llbb().add_assignment(self.location, counter, zero); | |
| self.llbb().add_assignment(self.location, val, value); | |
| self.br(loop_head); | |
| // check if value isn't zero | |
| self.switch_to_block(loop_head); | |
| let zero = self.gcc_zero(value_type); | |
| let cond = self.gcc_icmp(IntPredicate::IntNE, val.to_rvalue(), zero); | |
| self.cond_br(cond, loop_body, loop_tail); | |
| // val &= val - 1; | |
| self.switch_to_block(loop_body); | |
| let one = self.gcc_int(value_type, 1); | |
| let sub = self.gcc_sub(val.to_rvalue(), one); | |
| let op = self.gcc_and(val.to_rvalue(), sub); | |
| loop_body.add_assignment(self.location, val, op); | |
| // counter += 1 | |
| let one = self.gcc_int(counter_type, 1); | |
| let op = self.gcc_add(counter.to_rvalue(), one); | |
| loop_body.add_assignment(self.location, counter, op); | |
| self.br(loop_head); | |
| // end of loop | |
| self.switch_to_block(loop_tail); | |
| self.gcc_int_cast(counter.to_rvalue(), result_type) | |
| } | |
| // Algorithm from: https://blog.regehr.org/archives/1063 | |
| fn rotate_left( | |
| &mut self, | |
| value: RValue<'gcc>, | |
| shift: RValue<'gcc>, | |
| width: u64, | |
| ) -> RValue<'gcc> { | |
| let max = self.const_uint(shift.get_type(), width); | |
| let shift = self.urem(shift, max); | |
| let lhs = self.shl(value, shift); | |
| let result_neg = self.neg(shift); | |
| let result_and = self.and(result_neg, self.const_uint(shift.get_type(), width - 1)); | |
| let rhs = self.lshr(value, result_and); | |
| self.or(lhs, rhs) | |
| } | |
| // Algorithm from: https://blog.regehr.org/archives/1063 | |
| fn rotate_right( | |
| &mut self, | |
| value: RValue<'gcc>, | |
| shift: RValue<'gcc>, | |
| width: u64, | |
| ) -> RValue<'gcc> { | |
| let max = self.const_uint(shift.get_type(), width); | |
| let shift = self.urem(shift, max); | |
| let lhs = self.lshr(value, shift); | |
| let result_neg = self.neg(shift); | |
| let result_and = self.and(result_neg, self.const_uint(shift.get_type(), width - 1)); | |
| let rhs = self.shl(value, result_and); | |
| self.or(lhs, rhs) | |
| } | |
| fn saturating_add( | |
| &mut self, | |
| lhs: RValue<'gcc>, | |
| rhs: RValue<'gcc>, | |
| signed: bool, | |
| width: u64, | |
| ) -> RValue<'gcc> { | |
| let result_type = lhs.get_type(); | |
| if signed { | |
| // Based on algorithm from: https://stackoverflow.com/a/56531252/389119 | |
| let func = self.current_func(); | |
| let res = func.new_local(self.location, result_type, "saturating_sum"); | |
| let supports_native_type = self.is_native_int_type(result_type); | |
| let overflow = if supports_native_type { | |
| let func_name = "__builtin_add_overflow"; | |
| let overflow_func = self.context.get_builtin_function(func_name); | |
| self.overflow_call(overflow_func, &[lhs, rhs, res.get_address(self.location)], None) | |
| } else { | |
| let func_name = match width { | |
| 128 => "__rust_i128_addo", | |
| _ => unreachable!(), | |
| }; | |
| let (int_result, overflow) = | |
| self.operation_with_overflow(func_name, lhs, rhs, width); | |
| self.llbb().add_assignment(self.location, res, int_result); | |
| overflow | |
| }; | |
| let then_block = func.new_block("then"); | |
| let after_block = func.new_block("after"); | |
| // Return `result_type`'s maximum or minimum value on overflow | |
| // NOTE: convert the type to unsigned to have an unsigned shift. | |
| let unsigned_type = result_type.to_unsigned(self.cx); | |
| let shifted = self.gcc_lshr( | |
| self.gcc_int_cast(lhs, unsigned_type), | |
| self.gcc_int(unsigned_type, width as i64 - 1), | |
| ); | |
| let uint_max = self.gcc_not(self.gcc_int(unsigned_type, 0)); | |
| let int_max = self.gcc_lshr(uint_max, self.gcc_int(unsigned_type, 1)); | |
| then_block.add_assignment( | |
| self.location, | |
| res, | |
| self.gcc_int_cast(self.gcc_add(shifted, int_max), result_type), | |
| ); | |
| then_block.end_with_jump(self.location, after_block); | |
| self.llbb().end_with_conditional(self.location, overflow, then_block, after_block); | |
| // NOTE: since jumps were added in a place rustc does not | |
| // expect, the current block in the state need to be updated. | |
| self.switch_to_block(after_block); | |
| res.to_rvalue() | |
| } else { | |
| // Algorithm from: http://locklessinc.com/articles/sat_arithmetic/ | |
| let res = self.gcc_add(lhs, rhs); | |
| let cond = self.gcc_icmp(IntPredicate::IntULT, res, lhs); | |
| let value = self.gcc_neg(self.gcc_int_cast(cond, result_type)); | |
| self.gcc_or(res, value, self.location) | |
| } | |
| } | |
| // Algorithm from: https://locklessinc.com/articles/sat_arithmetic/ | |
| fn saturating_sub( | |
| &mut self, | |
| lhs: RValue<'gcc>, | |
| rhs: RValue<'gcc>, | |
| signed: bool, | |
| width: u64, | |
| ) -> RValue<'gcc> { | |
| let result_type = lhs.get_type(); | |
| if signed { | |
| // Based on algorithm from: https://stackoverflow.com/a/56531252/389119 | |
| let func = self.current_func(); | |
| let res = func.new_local(self.location, result_type, "saturating_diff"); | |
| let supports_native_type = self.is_native_int_type(result_type); | |
| let overflow = if supports_native_type { | |
| let func_name = "__builtin_sub_overflow"; | |
| let overflow_func = self.context.get_builtin_function(func_name); | |
| self.overflow_call(overflow_func, &[lhs, rhs, res.get_address(self.location)], None) | |
| } else { | |
| let func_name = match width { | |
| 128 => "__rust_i128_subo", | |
| _ => unreachable!(), | |
| }; | |
| let (int_result, overflow) = | |
| self.operation_with_overflow(func_name, lhs, rhs, width); | |
| self.llbb().add_assignment(self.location, res, int_result); | |
| overflow | |
| }; | |
| let then_block = func.new_block("then"); | |
| let after_block = func.new_block("after"); | |
| // Return `result_type`'s maximum or minimum value on overflow | |
| // NOTE: convert the type to unsigned to have an unsigned shift. | |
| let unsigned_type = result_type.to_unsigned(self.cx); | |
| let shifted = self.gcc_lshr( | |
| self.gcc_int_cast(lhs, unsigned_type), | |
| self.gcc_int(unsigned_type, width as i64 - 1), | |
| ); | |
| let uint_max = self.gcc_not(self.gcc_int(unsigned_type, 0)); | |
| let int_max = self.gcc_lshr(uint_max, self.gcc_int(unsigned_type, 1)); | |
| then_block.add_assignment( | |
| self.location, | |
| res, | |
| self.gcc_int_cast(self.gcc_add(shifted, int_max), result_type), | |
| ); | |
| then_block.end_with_jump(self.location, after_block); | |
| self.llbb().end_with_conditional(self.location, overflow, then_block, after_block); | |
| // NOTE: since jumps were added in a place rustc does not | |
| // expect, the current block in the state need to be updated. | |
| self.switch_to_block(after_block); | |
| res.to_rvalue() | |
| } else { | |
| let res = self.gcc_sub(lhs, rhs); | |
| let comparison = self.gcc_icmp(IntPredicate::IntULE, res, lhs); | |
| let value = self.gcc_neg(self.gcc_int_cast(comparison, result_type)); | |
| self.gcc_and(res, value) | |
| } | |
| } | |
| } | |
| fn try_intrinsic<'a, 'b, 'gcc, 'tcx>( | |
| bx: &'b mut Builder<'a, 'gcc, 'tcx>, | |
| try_func: RValue<'gcc>, | |
| data: RValue<'gcc>, | |
| _catch_func: RValue<'gcc>, | |
| dest: PlaceRef<'tcx, RValue<'gcc>>, | |
| ) { | |
| if !bx.sess().panic_strategy().unwinds() { | |
| let param_type = bx.u8_type.make_pointer(); | |
| let fn_type = | |
| bx.context.new_function_pointer_type(None, bx.type_void(), &[param_type], false); | |
| bx.call(fn_type, None, None, try_func, &[data], None, None); | |
| // Return 0 unconditionally from the intrinsic call; | |
| // we can never unwind. | |
| OperandValue::Immediate(bx.const_bool(false)).store(bx, dest); | |
| } else { | |
| if wants_msvc_seh(bx.sess()) { | |
| unimplemented!(); | |
| } | |
| #[cfg(feature = "master")] | |
| codegen_gnu_try(bx, try_func, data, _catch_func, dest); | |
| #[cfg(not(feature = "master"))] | |
| unimplemented!(); | |
| } | |
| } | |
| // Definition of the standard `try` function for Rust using the GNU-like model | |
| // of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke` | |
| // instructions). | |
| // | |
| // This codegen is a little surprising because we always call a shim | |
| // function instead of inlining the call to `invoke` manually here. This is done | |
| // because in LLVM we're only allowed to have one personality per function | |
| // definition. The call to the `try` intrinsic is being inlined into the | |
| // function calling it, and that function may already have other personality | |
| // functions in play. By calling a shim we're guaranteed that our shim will have | |
| // the right personality function. | |
| #[cfg(feature = "master")] | |
| fn codegen_gnu_try<'gcc, 'tcx>( | |
| bx: &mut Builder<'_, 'gcc, 'tcx>, | |
| try_func: RValue<'gcc>, | |
| data: RValue<'gcc>, | |
| catch_func: RValue<'gcc>, | |
| dest: PlaceRef<'tcx, RValue<'gcc>>, | |
| ) { | |
| let cx: &CodegenCx<'gcc, '_> = bx.cx; | |
| let (llty, func) = get_rust_try_fn(cx, &mut |mut bx| { | |
| // Codegens the shims described above: | |
| // | |
| // bx: | |
| // invoke %try_func(%data) normal %normal unwind %catch | |
| // | |
| // normal: | |
| // ret 0 | |
| // | |
| // catch: | |
| // (%ptr, _) = landingpad | |
| // call %catch_func(%data, %ptr) | |
| // ret 1 | |
| let then = bx.append_sibling_block("then"); | |
| let catch = bx.append_sibling_block("catch"); | |
| let func = bx.current_func(); | |
| let try_func = func.get_param(0).to_rvalue(); | |
| let data = func.get_param(1).to_rvalue(); | |
| let catch_func = func.get_param(2).to_rvalue(); | |
| let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); | |
| let current_block = bx.block; | |
| bx.switch_to_block(then); | |
| bx.ret(bx.const_bool(false)); | |
| // Type indicator for the exception being thrown. | |
| // | |
| // The value is a pointer to the exception object | |
| // being thrown. | |
| bx.switch_to_block(catch); | |
| bx.set_personality_fn(bx.eh_personality()); | |
| let eh_pointer_builtin = bx.cx.context.get_target_builtin_function("__builtin_eh_pointer"); | |
| let zero = bx.cx.context.new_rvalue_zero(bx.int_type); | |
| let ptr = bx.cx.context.new_call(None, eh_pointer_builtin, &[zero]); | |
| let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); | |
| bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None); | |
| bx.ret(bx.const_bool(true)); | |
| // NOTE: the blocks must be filled before adding the try/catch, otherwise gcc will not | |
| // generate a try/catch. | |
| // FIXME(antoyo): add a check in the libgccjit API to prevent this. | |
| bx.switch_to_block(current_block); | |
| bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None); | |
| }); | |
| let func = unsafe { std::mem::transmute::<Function<'gcc>, RValue<'gcc>>(func) }; | |
| // Note that no invoke is used here because by definition this function | |
| // can't panic (that's what it's catching). | |
| let ret = bx.call(llty, None, None, func, &[try_func, data, catch_func], None, None); | |
| OperandValue::Immediate(ret).store(bx, dest); | |
| } | |
| // Helper function used to get a handle to the `__rust_try` function used to | |
| // catch exceptions. | |
| // | |
| // This function is only generated once and is then cached. | |
| #[cfg(feature = "master")] | |
| fn get_rust_try_fn<'a, 'gcc, 'tcx>( | |
| cx: &'a CodegenCx<'gcc, 'tcx>, | |
| codegen: &mut dyn FnMut(Builder<'a, 'gcc, 'tcx>), | |
| ) -> (Type<'gcc>, Function<'gcc>) { | |
| if let Some(llfn) = cx.rust_try_fn.get() { | |
| return llfn; | |
| } | |
| // Define the type up front for the signature of the rust_try function. | |
| let tcx = cx.tcx; | |
| let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8); | |
| // `unsafe fn(*mut Data) -> ()` | |
| let try_fn_ty = Ty::new_fn_ptr( | |
| tcx, | |
| ty::Binder::dummy(tcx.mk_fn_sig_rust_abi( | |
| iter::once(i8p), | |
| tcx.types.unit, | |
| rustc_hir::Safety::Unsafe, | |
| )), | |
| ); | |
| // `unsafe fn(*mut Data, *mut i8) -> ()` | |
| let catch_fn_ty = Ty::new_fn_ptr( | |
| tcx, | |
| ty::Binder::dummy(tcx.mk_fn_sig_rust_abi( | |
| [i8p, i8p].iter().cloned(), | |
| tcx.types.unit, | |
| rustc_hir::Safety::Unsafe, | |
| )), | |
| ); | |
| // `unsafe fn(unsafe fn(*mut Data) -> (), *mut Data, unsafe fn(*mut Data, *mut i8) -> ()) -> bool` | |
| let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig_rust_abi( | |
| [try_fn_ty, i8p, catch_fn_ty], | |
| tcx.types.bool, | |
| rustc_hir::Safety::Unsafe, | |
| )); | |
| let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen); | |
| cx.rust_try_fn.set(Some(rust_try)); | |
| rust_try | |
| } | |
| // Helper function to give a Block to a closure to codegen a shim function. | |
| // This is currently primarily used for the `try` intrinsic functions above. | |
| #[cfg(feature = "master")] | |
| fn gen_fn<'a, 'gcc, 'tcx>( | |
| cx: &'a CodegenCx<'gcc, 'tcx>, | |
| name: &str, | |
| rust_fn_sig: ty::PolyFnSig<'tcx>, | |
| codegen: &mut dyn FnMut(Builder<'a, 'gcc, 'tcx>), | |
| ) -> (Type<'gcc>, Function<'gcc>) { | |
| let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); | |
| let return_type = fn_abi.gcc_type(cx).return_type; | |
| // FIXME(eddyb) find a nicer way to do this. | |
| cx.linkage.set(FunctionType::Internal); | |
| let func = cx.declare_fn(name, fn_abi); | |
| cx.set_frame_pointer_type(func); | |
| cx.apply_target_cpu_attr(func); | |
| let block = Builder::append_block(cx, func, "entry-block"); | |
| let bx = Builder::build(cx, block); | |
| codegen(bx); | |
| (return_type, func) | |
| } |