| //! Lock-free atomics for the sizes GCC lowers to libatomic calls even where LLVM inlines them, | |
| //! through the `__sync_*` builtins: GCC still inlines those, but they are always SeqCst. | |
| use gccjit::{BinaryOp, ComparisonOp, RValue, ToRValue, Type, UnaryOp}; | |
| use rustc_abi::Size; | |
| use rustc_codegen_ssa::common::AtomicRmwBinOp; | |
| use rustc_codegen_ssa::traits::BuilderMethods; | |
| use rustc_middle::ty::layout::HasTyCtxt; | |
| use rustc_span::Symbol; | |
| use rustc_target::spec::Arch; | |
| use crate::builder::Builder; | |
| use crate::common::SignType; | |
| impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { | |
| /// GCC turns every 16-byte `__atomic_*` builtin into a libatomic call, even where LLVM inlines a | |
| /// lock-free sequence. The always-SeqCst `__sync_*` builtins are still inlined there. | |
| pub fn use_sync_atomics(&self, size: u64) -> bool { | |
| let sess = self.tcx().sess; | |
| size == 16 | |
| && self.supports_128bit_integers | |
| && match sess.target.arch { | |
| Arch::AArch64 => true, | |
| Arch::X86_64 => { | |
| sess.internal_target_features.contains(&Symbol::intern("cmpxchg16b")) | |
| } | |
| _ => false, | |
| } | |
| } | |
| pub fn sync_atomic_load( | |
| &mut self, | |
| typ: Type<'gcc>, | |
| ptr: RValue<'gcc>, | |
| size: Size, | |
| ) -> RValue<'gcc> { | |
| let zero = self.context.new_rvalue_zero(self.type_ix(size.bits())); | |
| let value = self.sync_compare_and_swap(ptr, zero, zero, size.bytes()); | |
| self.context.new_cast(self.location, value, typ) | |
| } | |
| pub fn sync_atomic_store(&mut self, value: RValue<'gcc>, ptr: RValue<'gcc>, size: Size) { | |
| self.sync_compare_and_swap_loop(ptr, value.get_type(), size, |builder, current| { | |
| builder.context.new_cast(builder.location, value, current.get_type()) | |
| }); | |
| } | |
| /// Returns the previous value and whether the swap happened. | |
| pub fn sync_atomic_cmpxchg( | |
| &mut self, | |
| dst: RValue<'gcc>, | |
| cmp: RValue<'gcc>, | |
| src: RValue<'gcc>, | |
| size: u64, | |
| ) -> (RValue<'gcc>, RValue<'gcc>) { | |
| let func = self.current_func(); | |
| let expected = self.new_temp(func, self.location, cmp.get_type()); | |
| self.llbb().add_assignment(self.location, expected, cmp); | |
| let previous = self.sync_compare_and_swap(dst, expected.to_rvalue(), src, size); | |
| let previous = self.context.new_cast(self.location, previous, cmp.get_type()); | |
| let success = self.new_temp(func, self.location, self.bool_type); | |
| let comparison = self.context.new_comparison( | |
| self.location, | |
| ComparisonOp::Equals, | |
| previous, | |
| expected.to_rvalue(), | |
| ); | |
| self.llbb().add_assignment(self.location, success, comparison); | |
| (previous, success.to_rvalue()) | |
| } | |
| pub fn sync_atomic_rmw( | |
| &mut self, | |
| op: AtomicRmwBinOp, | |
| dst: RValue<'gcc>, | |
| src: RValue<'gcc>, | |
| size: Size, | |
| ) -> RValue<'gcc> { | |
| let src_type = src.get_type(); | |
| let name = match op { | |
| AtomicRmwBinOp::AtomicAdd => "add", | |
| AtomicRmwBinOp::AtomicSub => "sub", | |
| AtomicRmwBinOp::AtomicAnd => "and", | |
| AtomicRmwBinOp::AtomicOr => "or", | |
| AtomicRmwBinOp::AtomicXor => "xor", | |
| AtomicRmwBinOp::AtomicXchg => { | |
| return self.sync_compare_and_swap_loop(dst, src_type, size, |builder, current| { | |
| builder.context.new_cast(builder.location, src, current.get_type()) | |
| }); | |
| } | |
| // `__sync_fetch_and_nand` emits a note that crashes libgccjit's diagnostic printer. | |
| AtomicRmwBinOp::AtomicNand => { | |
| return self.sync_compare_and_swap_loop(dst, src_type, size, |builder, current| { | |
| let int_type = current.get_type(); | |
| let src = builder.context.new_cast(builder.location, src, int_type); | |
| let and = builder.context.new_binary_op( | |
| builder.location, | |
| BinaryOp::BitwiseAnd, | |
| int_type, | |
| current, | |
| src, | |
| ); | |
| builder.context.new_unary_op( | |
| builder.location, | |
| UnaryOp::BitwiseNegate, | |
| int_type, | |
| and, | |
| ) | |
| }); | |
| } | |
| AtomicRmwBinOp::AtomicMax => { | |
| return self.sync_atomic_extremum( | |
| dst, | |
| src, | |
| size, | |
| src_type.to_signed(self), | |
| ComparisonOp::GreaterThanEquals, | |
| ); | |
| } | |
| AtomicRmwBinOp::AtomicMin => { | |
| return self.sync_atomic_extremum( | |
| dst, | |
| src, | |
| size, | |
| src_type.to_signed(self), | |
| ComparisonOp::LessThanEquals, | |
| ); | |
| } | |
| AtomicRmwBinOp::AtomicUMax => { | |
| return self.sync_atomic_extremum( | |
| dst, | |
| src, | |
| size, | |
| src_type.to_unsigned(self), | |
| ComparisonOp::GreaterThanEquals, | |
| ); | |
| } | |
| AtomicRmwBinOp::AtomicUMin => { | |
| return self.sync_atomic_extremum( | |
| dst, | |
| src, | |
| size, | |
| src_type.to_unsigned(self), | |
| ComparisonOp::LessThanEquals, | |
| ); | |
| } | |
| }; | |
| let fetch_and_op = self.context.get_builtin_function(format!( | |
| "__sync_fetch_and_{}_{}", | |
| name, | |
| size.bytes() | |
| )); | |
| let pointer_type = fetch_and_op.get_param(0).to_rvalue().get_type(); | |
| let dst = self.context.new_cast(self.location, dst, pointer_type); | |
| let int_type = fetch_and_op.get_param(1).to_rvalue().get_type(); | |
| let src = self.context.new_cast(self.location, src, int_type); | |
| let result = self.context.new_call(self.location, fetch_and_op, &[dst, src]); | |
| self.context.new_cast(self.location, result, src_type) | |
| } | |
| /// Min/max: keeps the current value if `current keep_current_operator src` holds, with both | |
| /// compared as `comparison_type`; stores `src` otherwise. | |
| fn sync_atomic_extremum( | |
| &mut self, | |
| dst: RValue<'gcc>, | |
| src: RValue<'gcc>, | |
| size: Size, | |
| comparison_type: Type<'gcc>, | |
| keep_current_operator: ComparisonOp, | |
| ) -> RValue<'gcc> { | |
| self.sync_compare_and_swap_loop(dst, src.get_type(), size, |builder, current| { | |
| let keep_current = builder.context.new_comparison( | |
| builder.location, | |
| keep_current_operator, | |
| builder.context.new_cast(builder.location, current, comparison_type), | |
| builder.context.new_cast(builder.location, src, comparison_type), | |
| ); | |
| let src = builder.context.new_cast(builder.location, src, current.get_type()); | |
| builder.select(keep_current, current, src) | |
| }) | |
| } | |
| /// Returns the value `dst` held before the operation. | |
| fn sync_compare_and_swap( | |
| &mut self, | |
| dst: RValue<'gcc>, | |
| expected: RValue<'gcc>, | |
| desired: RValue<'gcc>, | |
| size: u64, | |
| ) -> RValue<'gcc> { | |
| let compare_and_swap = | |
| self.context.get_builtin_function(format!("__sync_val_compare_and_swap_{}", size)); | |
| let pointer_type = compare_and_swap.get_param(0).to_rvalue().get_type(); | |
| let dst = self.context.new_cast(self.location, dst, pointer_type); | |
| let int_type = compare_and_swap.get_param(1).to_rvalue().get_type(); | |
| let expected = self.context.new_cast(self.location, expected, int_type); | |
| let desired = self.context.new_cast(self.location, desired, int_type); | |
| // A temporary keeps the call from being evaluated again wherever the result is used. | |
| let previous = self.new_temp(self.current_func(), self.location, int_type); | |
| let call = | |
| self.context.new_call(self.location, compare_and_swap, &[dst, expected, desired]); | |
| self.llbb().add_assignment(self.location, previous, call); | |
| previous.to_rvalue() | |
| } | |
| /// Stores `new_value(current)` with a compare-and-swap loop and returns the replaced value. | |
| fn sync_compare_and_swap_loop( | |
| &mut self, | |
| dst: RValue<'gcc>, | |
| typ: Type<'gcc>, | |
| size: Size, | |
| new_value: impl FnOnce(&mut Self, RValue<'gcc>) -> RValue<'gcc>, | |
| ) -> RValue<'gcc> { | |
| let func = self.current_func(); | |
| let int_type = self.type_ix(size.bits()); | |
| let current = self.new_temp(func, self.location, int_type); | |
| // Start from 0 instead of loading first: if `dst` holds something else, the first swap | |
| // fails and returns the actual value, which the next iteration uses. | |
| self.llbb().add_assignment(self.location, current, self.context.new_rvalue_zero(int_type)); | |
| let loop_block = func.new_block("sync_compare_and_swap_loop"); | |
| let after_block = func.new_block("after_sync_compare_and_swap_loop"); | |
| self.llbb().end_with_jump(self.location, loop_block); | |
| self.switch_to_block(loop_block); | |
| let desired = new_value(self, current.to_rvalue()); | |
| let previous = self.sync_compare_and_swap(dst, current.to_rvalue(), desired, size.bytes()); | |
| let previous = self.context.new_cast(self.location, previous, int_type); | |
| let swapped = self.new_temp(func, self.location, self.bool_type); | |
| let comparison = self.context.new_comparison( | |
| self.location, | |
| ComparisonOp::Equals, | |
| previous, | |
| current.to_rvalue(), | |
| ); | |
| self.llbb().add_assignment(self.location, swapped, comparison); | |
| self.llbb().add_assignment(self.location, current, previous); | |
| self.llbb().end_with_conditional( | |
| self.location, | |
| swapped.to_rvalue(), | |
| after_block, | |
| loop_block, | |
| ); | |
| self.switch_to_block(after_block); | |
| self.context.new_cast(self.location, current.to_rvalue(), typ) | |
| } | |
| } |