| use std::ops::Deref as _; | |
| use rustc_abi::{ | |
| Align, BackendRepr, FieldIdx, FieldsShape, Size, TagEncoding, VariantIdx, Variants, | |
| }; | |
| use rustc_middle::mir::PlaceTy; | |
| use rustc_middle::mir::interpret::Scalar; | |
| use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout}; | |
| use rustc_middle::ty::{self, Ty}; | |
| use rustc_middle::{bug, mir}; | |
| use tracing::{debug, instrument}; | |
| use super::operand::OperandValue; | |
| use super::{FunctionCx, LocalRef}; | |
| use crate::common::IntPredicate; | |
| use crate::size_of_val; | |
| use crate::traits::*; | |
| /// The location and extra runtime properties of the place. | |
| /// | |
| /// Typically found in a [`PlaceRef`] or an [`OperandValue::Ref`]. | |
| /// | |
| /// As a location in memory, this has no specific type. If you want to | |
| /// load or store it using a typed operation, use [`Self::with_type`]. | |
| #[derive(Copy, Clone, Debug)] | |
| pub struct PlaceValue<V> { | |
| /// A pointer to the contents of the place. | |
| pub llval: V, | |
| /// This place's extra data if it is unsized, or `None` if null. | |
| pub llextra: Option<V>, | |
| /// The alignment we know for this place. | |
| pub align: Align, | |
| } | |
| impl<V: CodegenObject> PlaceValue<V> { | |
| /// Constructor for the ordinary case of `Sized` types. | |
| /// | |
| /// Sets `llextra` to `None`. | |
| pub fn new_sized(llval: V, align: Align) -> PlaceValue<V> { | |
| PlaceValue { llval, llextra: None, align } | |
| } | |
| /// Allocates a stack slot in the function for a value | |
| /// of the specified size and alignment. | |
| /// | |
| /// The allocation itself is untyped. | |
| pub fn alloca<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| bx: &mut Bx, | |
| size: Size, | |
| align: Align, | |
| ) -> PlaceValue<V> { | |
| let llval = bx.alloca(size, align); | |
| PlaceValue::new_sized(llval, align) | |
| } | |
| /// Creates a `PlaceRef` to this location with the given type. | |
| pub fn with_type<'tcx>(self, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> { | |
| assert!( | |
| layout.is_unsized() || layout.is_uninhabited() || self.llextra.is_none(), | |
| "Had pointer metadata {:?} for sized type {layout:?}", | |
| self.llextra, | |
| ); | |
| PlaceRef { val: self, layout } | |
| } | |
| /// Gets the pointer to this place as an [`OperandValue::Immediate`] | |
| /// or, for those needing metadata, an [`OperandValue::Pair`]. | |
| /// | |
| /// This is the inverse of [`OperandValue::deref`]. | |
| pub fn address(self) -> OperandValue<V> { | |
| if let Some(llextra) = self.llextra { | |
| OperandValue::Pair(self.llval, llextra) | |
| } else { | |
| OperandValue::Immediate(self.llval) | |
| } | |
| } | |
| } | |
| #[derive(Copy, Clone, Debug)] | |
| pub struct PlaceRef<'tcx, V> { | |
| /// The location and extra runtime properties of the place. | |
| pub val: PlaceValue<V>, | |
| /// The monomorphized type of this place, including variant information. | |
| /// | |
| /// You probably shouldn't use the alignment from this layout; | |
| /// rather you should use the `.val.align` of the actual place, | |
| /// which might be different from the type's normal alignment. | |
| pub layout: TyAndLayout<'tcx>, | |
| } | |
| impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { | |
| pub fn new_sized(llval: V, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> { | |
| PlaceRef::new_sized_aligned(llval, layout, layout.align.abi) | |
| } | |
| pub fn new_sized_aligned( | |
| llval: V, | |
| layout: TyAndLayout<'tcx>, | |
| align: Align, | |
| ) -> PlaceRef<'tcx, V> { | |
| assert!(layout.is_sized()); | |
| PlaceValue::new_sized(llval, align).with_type(layout) | |
| } | |
| // FIXME(eddyb) pass something else for the name so no work is done | |
| // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`). | |
| pub fn alloca<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| bx: &mut Bx, | |
| layout: TyAndLayout<'tcx>, | |
| ) -> Self { | |
| if layout.deref().is_scalable_vector() { | |
| Self::alloca_scalable(bx, layout) | |
| } else { | |
| Self::alloca_size(bx, layout.size, layout) | |
| } | |
| } | |
| pub fn alloca_size<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| bx: &mut Bx, | |
| size: Size, | |
| layout: TyAndLayout<'tcx>, | |
| ) -> Self { | |
| assert!(layout.is_sized(), "tried to statically allocate unsized place"); | |
| PlaceValue::alloca(bx, size, layout.align.abi).with_type(layout) | |
| } | |
| /// Returns a place for an indirect reference to an unsized place. | |
| // FIXME(eddyb) pass something else for the name so no work is done | |
| // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`). | |
| pub fn alloca_unsized_indirect<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| bx: &mut Bx, | |
| layout: TyAndLayout<'tcx>, | |
| ) -> Self { | |
| assert!(layout.is_unsized(), "tried to allocate indirect place for sized values"); | |
| let ptr_ty = Ty::new_mut_ptr(bx.cx().tcx(), layout.ty); | |
| let ptr_layout = bx.cx().layout_of(ptr_ty); | |
| Self::alloca(bx, ptr_layout) | |
| } | |
| pub fn len<Cx: ConstCodegenMethods<Value = V>>(&self, cx: &Cx) -> V { | |
| if let FieldsShape::Array { count, .. } = self.layout.fields { | |
| if self.layout.is_unsized() { | |
| assert_eq!(count, 0); | |
| self.val.llextra.unwrap() | |
| } else { | |
| cx.const_usize(count) | |
| } | |
| } else { | |
| bug!("unexpected layout `{:#?}` in PlaceRef::len", self.layout) | |
| } | |
| } | |
| fn alloca_scalable<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| bx: &mut Bx, | |
| layout: TyAndLayout<'tcx>, | |
| ) -> Self { | |
| PlaceValue::new_sized(bx.alloca_with_ty(layout), layout.align.abi).with_type(layout) | |
| } | |
| } | |
| impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { | |
| /// Access a field, at a point when the value's case is known. | |
| pub fn project_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| self, | |
| bx: &mut Bx, | |
| ix: usize, | |
| ) -> Self { | |
| let field = self.layout.field(bx.cx(), ix); | |
| let offset = self.layout.fields.offset(ix); | |
| let effective_field_align = self.val.align.restrict_for_offset(offset); | |
| // `simple` is called when we don't need to adjust the offset to | |
| // the dynamic alignment of the field. | |
| let mut simple = || { | |
| let llval = if offset.bytes() == 0 { | |
| self.val.llval | |
| } else { | |
| bx.inbounds_ptradd(self.val.llval, bx.const_usize(offset.bytes())) | |
| }; | |
| let val = PlaceValue { | |
| llval, | |
| llextra: if bx.cx().tcx().type_has_metadata(field.ty, bx.cx().typing_env()) { | |
| self.val.llextra | |
| } else { | |
| None | |
| }, | |
| align: effective_field_align, | |
| }; | |
| val.with_type(field) | |
| }; | |
| // Simple cases, which don't need DST adjustment: | |
| // * known alignment - sized types, `[T]`, `str` | |
| // * offset 0 -- rounding up to alignment cannot change the offset | |
| // Note that looking at `field.align` is incorrect since that is not necessarily equal | |
| // to the dynamic alignment of the type. | |
| match field.ty.kind() { | |
| _ if field.is_sized() => return simple(), | |
| ty::Slice(..) | ty::Str => return simple(), | |
| _ if offset.bytes() == 0 => return simple(), | |
| _ => {} | |
| } | |
| // We need to get the pointer manually now. | |
| // We do this by casting to a `*i8`, then offsetting it by the appropriate amount. | |
| // We do this instead of, say, simply adjusting the pointer from the result of a GEP | |
| // because the field may have an arbitrary alignment in the LLVM representation. | |
| // | |
| // To demonstrate: | |
| // | |
| // struct Foo<T: ?Sized> { | |
| // x: u16, | |
| // y: T | |
| // } | |
| // | |
| // The type `Foo<Foo<Trait>>` is represented in LLVM as `{ u16, { u16, u8 }}`, meaning that | |
| // the `y` field has 16-bit alignment. | |
| let meta = self.val.llextra; | |
| let unaligned_offset = bx.cx().const_usize(offset.bytes()); | |
| // Get the alignment of the field | |
| let (_, mut unsized_align) = size_of_val::size_and_align_of_dst(bx, field.ty, meta); | |
| // For packed types, we need to cap alignment. | |
| if let ty::Adt(def, _) = self.layout.ty.kind() | |
| && let Some(packed) = def.repr().pack | |
| { | |
| let packed = bx.const_usize(packed.bytes()); | |
| let cmp = bx.icmp(IntPredicate::IntULT, unsized_align, packed); | |
| unsized_align = bx.select(cmp, unsized_align, packed) | |
| } | |
| // Bump the unaligned offset up to the appropriate alignment | |
| let offset = round_up_const_value_to_alignment(bx, unaligned_offset, unsized_align); | |
| debug!("struct_field_ptr: DST field offset: {:?}", offset); | |
| // Adjust pointer. | |
| let ptr = bx.inbounds_ptradd(self.val.llval, offset); | |
| let val = | |
| PlaceValue { llval: ptr, llextra: self.val.llextra, align: effective_field_align }; | |
| val.with_type(field) | |
| } | |
| /// Sets the discriminant for a new value of the given case of the given | |
| /// representation. | |
| pub fn codegen_set_discr<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| &self, | |
| bx: &mut Bx, | |
| variant_index: VariantIdx, | |
| ) { | |
| match codegen_tag_value(bx.cx(), variant_index, self.layout) { | |
| Err(UninhabitedVariantError) => { | |
| // We play it safe by using a well-defined `abort`, but we could go for immediate UB | |
| // if that turns out to be helpful. | |
| bx.abort(); | |
| } | |
| Ok(Some((tag_field, imm))) => { | |
| let tag_place = self.project_field(bx, tag_field.as_usize()); | |
| OperandValue::Immediate(imm).store(bx, tag_place); | |
| } | |
| Ok(None) => {} | |
| } | |
| } | |
| pub fn project_index<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| &self, | |
| bx: &mut Bx, | |
| llindex: V, | |
| ) -> Self { | |
| // Statically compute the offset if we can, otherwise just use the element size, | |
| // as this will yield the lowest alignment. | |
| let layout = self.layout.field(bx, 0); | |
| let offset = if let Some(llindex) = bx.const_to_opt_uint(llindex) { | |
| layout.size.checked_mul(llindex, bx).unwrap_or(layout.size) | |
| } else { | |
| layout.size | |
| }; | |
| let llval = bx.inbounds_nuw_gep(bx.cx().backend_type(layout), self.val.llval, &[llindex]); | |
| let align = self.val.align.restrict_for_offset(offset); | |
| PlaceValue::new_sized(llval, align).with_type(layout) | |
| } | |
| pub fn project_downcast<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| &self, | |
| bx: &mut Bx, | |
| variant_index: VariantIdx, | |
| ) -> Self { | |
| let mut downcast = *self; | |
| downcast.layout = self.layout.for_variant(bx.cx(), variant_index); | |
| downcast | |
| } | |
| pub fn project_type<Bx: BuilderMethods<'a, 'tcx, Value = V>>( | |
| &self, | |
| bx: &mut Bx, | |
| ty: Ty<'tcx>, | |
| ) -> Self { | |
| let mut downcast = *self; | |
| downcast.layout = bx.cx().layout_of(ty); | |
| downcast | |
| } | |
| pub fn storage_live<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) { | |
| bx.lifetime_start(self.val.llval, self.layout.size); | |
| } | |
| pub fn storage_dead<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) { | |
| bx.lifetime_end(self.val.llval, self.layout.size); | |
| } | |
| } | |
| impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | |
| #[instrument(level = "trace", skip(self, bx))] | |
| pub fn codegen_place( | |
| &mut self, | |
| bx: &mut Bx, | |
| place_ref: mir::PlaceRef<'tcx>, | |
| ) -> PlaceRef<'tcx, Bx::Value> { | |
| let cx = self.cx; | |
| let tcx = self.cx.tcx(); | |
| let mut base = 0; | |
| let mut cg_base = match self.locals[place_ref.local] { | |
| LocalRef::Place(place) => place, | |
| LocalRef::UnsizedPlace(place) => bx.load_operand(place).deref(cx), | |
| LocalRef::Operand(..) => { | |
| if place_ref.is_indirect_first_projection() { | |
| base = 1; | |
| let cg_base = self.codegen_consume( | |
| bx, | |
| mir::PlaceRef { projection: &place_ref.projection[..0], ..place_ref }, | |
| ); | |
| cg_base.deref(bx.cx()) | |
| } else { | |
| bug!("using operand local {:?} as place", place_ref); | |
| } | |
| } | |
| LocalRef::PendingOperand => { | |
| bug!("using still-pending operand local {:?} as place", place_ref); | |
| } | |
| }; | |
| for elem in place_ref.projection[base..].iter() { | |
| cg_base = match *elem { | |
| mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()), | |
| mir::ProjectionElem::Field(ref field, _) => { | |
| assert!( | |
| !cg_base.layout.ty.is_any_ptr(), | |
| "Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \ | |
| but tried to access field {field:?} of pointer {cg_base:?}", | |
| ); | |
| cg_base.project_field(bx, field.index()) | |
| } | |
| mir::ProjectionElem::OpaqueCast(ty) => { | |
| bug!("encountered OpaqueCast({ty}) in codegen") | |
| } | |
| mir::ProjectionElem::UnwrapUnsafeBinder(ty) => { | |
| cg_base.project_type(bx, self.monomorphize(ty)) | |
| } | |
| mir::ProjectionElem::Index(index) => { | |
| let index = &mir::Operand::Copy(mir::Place::from(index)); | |
| let index = self.codegen_operand(bx, index); | |
| let llindex = index.immediate(); | |
| cg_base.project_index(bx, llindex) | |
| } | |
| mir::ProjectionElem::ConstantIndex { offset, from_end: false, min_length: _ } => { | |
| let lloffset = bx.cx().const_usize(offset); | |
| cg_base.project_index(bx, lloffset) | |
| } | |
| mir::ProjectionElem::ConstantIndex { offset, from_end: true, min_length: _ } => { | |
| let lloffset = bx.cx().const_usize(offset); | |
| let lllen = cg_base.len(bx.cx()); | |
| let llindex = bx.sub(lllen, lloffset); | |
| cg_base.project_index(bx, llindex) | |
| } | |
| mir::ProjectionElem::Subslice { from, to, from_end } => { | |
| let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from)); | |
| let projected_ty = | |
| PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, *elem).ty; | |
| subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty)); | |
| if subslice.layout.is_unsized() { | |
| assert!(from_end, "slice subslices should be `from_end`"); | |
| subslice.val.llextra = Some( | |
| bx.sub(cg_base.val.llextra.unwrap(), bx.cx().const_usize(from + to)), | |
| ); | |
| } | |
| subslice | |
| } | |
| mir::ProjectionElem::Downcast(_, v) => cg_base.project_downcast(bx, v), | |
| }; | |
| } | |
| debug!("codegen_place(place={:?}) => {:?}", place_ref, cg_base); | |
| cg_base | |
| } | |
| pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> { | |
| let tcx = self.cx.tcx(); | |
| let place_ty = place_ref.ty(self.mir, tcx); | |
| self.monomorphize(place_ty.ty) | |
| } | |
| } | |
| fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( | |
| bx: &mut Bx, | |
| value: Bx::Value, | |
| align: Bx::Value, | |
| ) -> Bx::Value { | |
| // In pseudo code: | |
| // | |
| // if value & (align - 1) == 0 { | |
| // value | |
| // } else { | |
| // (value & !(align - 1)) + align | |
| // } | |
| // | |
| // Usually this is written without branches as | |
| // | |
| // (value + align - 1) & !(align - 1) | |
| // | |
| // But this formula cannot take advantage of constant `value`. E.g. if `value` is known | |
| // at compile time to be `1`, this expression should be optimized to `align`. However, | |
| // optimization only holds if `align` is a power of two. Since the optimizer doesn't know | |
| // that `align` is a power of two, it cannot perform this optimization. | |
| // | |
| // Instead we use | |
| // | |
| // value + (-value & (align - 1)) | |
| // | |
| // Since `align` is used only once, the expression can be optimized. For `value = 0` | |
| // its optimized to `0` even in debug mode. | |
| // | |
| // NB: The previous version of this code used | |
| // | |
| // (value + align - 1) & -align | |
| // | |
| // Even though `-align == !(align - 1)`, LLVM failed to optimize this even for | |
| // `value = 0`. Bug report: https://bugs.llvm.org/show_bug.cgi?id=48559 | |
| let one = bx.const_usize(1); | |
| let align_minus_1 = bx.sub(align, one); | |
| let neg_value = bx.neg(value); | |
| let offset = bx.and(neg_value, align_minus_1); | |
| bx.add(value, offset) | |
| } | |
| /// Calculates the value that needs to be stored to mark the discriminant. | |
| /// | |
| /// This might be `None` for a `struct` or a niched variant (like `Some(&3)`). | |
| /// | |
| /// If it's `Some`, it returns the value to store and the field in which to | |
| /// store it. Note that this value is *not* the same as the discriminant, in | |
| /// general, as it might be a niche value or have a different size. | |
| /// | |
| /// It might also be an `Err` because the variant is uninhabited. | |
| pub(super) fn codegen_tag_value<'tcx, V>( | |
| cx: &impl CodegenMethods<'tcx, Value = V>, | |
| variant_index: VariantIdx, | |
| layout: TyAndLayout<'tcx>, | |
| ) -> Result<Option<(FieldIdx, V)>, UninhabitedVariantError> { | |
| // By checking uninhabited-ness first we don't need to worry about types | |
| // like `(u32, !)` which are single-variant but weird. | |
| if layout.for_variant(cx, variant_index).is_uninhabited() { | |
| return Err(UninhabitedVariantError); | |
| } | |
| Ok(match layout.variants { | |
| Variants::Empty => unreachable!("we already handled uninhabited types"), | |
| Variants::Single { index } => { | |
| assert_eq!(index, variant_index); | |
| None | |
| } | |
| Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => { | |
| let discr = layout.ty.discriminant_for_variant(cx.tcx(), variant_index); | |
| let to = discr.unwrap().val; | |
| let tag_layout = layout.field(cx, tag_field.as_usize()); | |
| let tag_llty = cx.immediate_backend_type(tag_layout); | |
| let imm = cx.const_uint_big(tag_llty, to); | |
| Some((tag_field, imm)) | |
| } | |
| Variants::Multiple { | |
| tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, | |
| tag_field, | |
| .. | |
| } => { | |
| if variant_index != untagged_variant { | |
| let niche_layout = layout.field(cx, tag_field.as_usize()); | |
| let niche_llty = cx.immediate_backend_type(niche_layout); | |
| let BackendRepr::Scalar(scalar) = niche_layout.backend_repr else { | |
| bug!("expected a scalar placeref for the niche"); | |
| }; | |
| // We are supposed to compute `niche_value.wrapping_add(niche_start)` wrapping | |
| // around the `niche`'s type. | |
| // The easiest way to do that is to do wrapping arithmetic on `u128` and then | |
| // masking off any extra bits that occur because we did the arithmetic with too many bits. | |
| let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); | |
| let niche_value = (niche_value as u128).wrapping_add(niche_start); | |
| let niche_value = niche_value & niche_layout.size.unsigned_int_max(); | |
| let niche_llval = cx.scalar_to_backend( | |
| Scalar::from_uint(niche_value, niche_layout.size), | |
| scalar, | |
| niche_llty, | |
| ); | |
| Some((tag_field, niche_llval)) | |
| } else { | |
| None | |
| } | |
| } | |
| }) | |
| } | |
| #[derive(Debug)] | |
| pub(super) struct UninhabitedVariantError; |