| use rustc_hir::attrs::InlineAttr; | |
| use rustc_hir::def::DefKind; | |
| use rustc_hir::def_id::LocalDefId; | |
| use rustc_hir::{self as hir, find_attr}; | |
| use rustc_middle::bug; | |
| use rustc_middle::mir::visit::Visitor; | |
| use rustc_middle::mir::*; | |
| use rustc_middle::query::Providers; | |
| use rustc_middle::ty::TyCtxt; | |
| use rustc_session::config::{InliningThreshold, OptLevel}; | |
| use crate::{inline, pass_manager as pm}; | |
| pub(super) fn provide(providers: &mut Providers) { | |
| providers.cross_crate_inlinable = cross_crate_inlinable; | |
| } | |
| fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { | |
| let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); | |
| // If this has an extern indicator, then this function is globally shared and thus will not | |
| // generate cgu-internal copies which would make it cross-crate inlinable. | |
| if codegen_fn_attrs.contains_extern_indicator() { | |
| return false; | |
| } | |
| // This just reproduces the logic from Instance::requires_inline. | |
| match tcx.def_kind(def_id) { | |
| DefKind::Ctor(..) | DefKind::Closure | DefKind::SyntheticCoroutineBody => return true, | |
| DefKind::Fn | DefKind::AssocFn => {} | |
| _ => return false, | |
| } | |
| // From this point on, it is valid to return true or false. | |
| if tcx.sess.opts.unstable_opts.cross_crate_inline_threshold == InliningThreshold::Always { | |
| return true; | |
| } | |
| if find_attr!(tcx, def_id, RustcIntrinsic) { | |
| // Intrinsic fallback bodies are always cross-crate inlineable. | |
| // To ensure that the MIR inliner doesn't cluelessly try to inline fallback | |
| // bodies even when the backend would implement something better, we stop | |
| // the MIR inliner from ever inlining an intrinsic. | |
| return true; | |
| } | |
| if let hir::Constness::Const { always: true } = tcx.constness(def_id) { | |
| // Comptime functions only exist during const eval and can never be passed | |
| // to codegen. The const eval MIR pipeline also doesn't inline anything at all. | |
| return false; | |
| } | |
| // Obey source annotations first; this is important because it means we can use | |
| // #[inline(never)] to force code generation. | |
| match codegen_fn_attrs.inline { | |
| InlineAttr::Never => return false, | |
| InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true, | |
| _ => {} | |
| } | |
| // If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until | |
| // the function is referenced, in order to skip codegen for unused functions. This is | |
| // intentionally after the check for `inline(never)`, so that `inline(never)` wins. | |
| if tcx.sess.opts.unstable_opts.hint_mostly_unused { | |
| return true; | |
| } | |
| let sig = tcx.fn_sig(def_id).instantiate_identity().skip_norm_wip(); | |
| for ty in sig.inputs().skip_binder().iter().chain(std::iter::once(&sig.output().skip_binder())) | |
| { | |
| // FIXME(f16_f128): in order to avoid crashes building `core`, always inline to skip | |
| // codegen if the function is not used. | |
| if ty == &tcx.types.f16 || ty == &tcx.types.f128 { | |
| return true; | |
| } | |
| } | |
| // Don't do any inference when incremental compilation is enabled; the additional inlining that | |
| // inference permits also creates more work for small edits. | |
| if tcx.sess.opts.incremental.is_some() { | |
| return false; | |
| } | |
| // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not | |
| // enabled. This ensures that we do inference even if someone only passes -Zinline-mir, | |
| // which is less confusing than having to also enable -Copt-level=1. | |
| let inliner_will_run = | |
| pm::should_run_pass(&inline::Inline, &pm::PassCtx::for_body(tcx, def_id.to_def_id())) | |
| || inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id()); | |
| if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run { | |
| return false; | |
| } | |
| if !tcx.is_mir_available(def_id) { | |
| return false; | |
| } | |
| let threshold = match tcx.sess.opts.unstable_opts.cross_crate_inline_threshold { | |
| InliningThreshold::Always => return true, | |
| InliningThreshold::Sometimes(threshold) => threshold, | |
| InliningThreshold::Never => return false, | |
| }; | |
| let mir = tcx.optimized_mir(def_id); | |
| let mut checker = | |
| CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 }; | |
| checker.visit_body(mir); | |
| checker.calls == 0 | |
| && checker.resumes == 0 | |
| && checker.landing_pads == 0 | |
| && checker.statements <= threshold | |
| } | |
| // The threshold that CostChecker computes is balancing the desire to make more things | |
| // inlinable cross crates against the growth in incremental CGU size that happens when too many | |
| // things in the sysroot are made inlinable. | |
| // Permitting calls causes the size of some incremental CGUs to grow, because more functions are | |
| // made inlinable out of the sysroot or dependencies. | |
| // Assert terminators are similar to calls, but do not have the same impact on compile time, so | |
| // those are just treated as statements. | |
| // A threshold exists at all because we don't want to blindly mark a huge function as inlinable. | |
| struct CostChecker<'b, 'tcx> { | |
| tcx: TyCtxt<'tcx>, | |
| callee_body: &'b Body<'tcx>, | |
| calls: usize, | |
| statements: usize, | |
| landing_pads: usize, | |
| resumes: usize, | |
| } | |
| impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { | |
| fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) { | |
| // Don't count StorageLive/StorageDead in the inlining cost. | |
| match statement.kind { | |
| StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Nop => {} | |
| _ => self.statements += 1, | |
| } | |
| } | |
| fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) { | |
| self.statements += 1; | |
| let tcx = self.tcx; | |
| match &terminator.kind { | |
| TerminatorKind::Drop { place, unwind, .. } => { | |
| let ty = place.ty(self.callee_body, tcx).ty; | |
| if !ty.is_trivially_pure_clone_copy() { | |
| self.calls += 1; | |
| if let UnwindAction::Cleanup(_) = unwind { | |
| self.landing_pads += 1; | |
| } | |
| } | |
| } | |
| TerminatorKind::Call { func, unwind, .. } => { | |
| // We track calls because they make our function not a leaf (and in theory, the | |
| // number of calls indicates how likely this function is to perturb other CGUs). | |
| // But there are a handful of intrinsics such as raw_eq that should not block | |
| // cross-crate-inlining. Adding a broad exception for all intrinsics benchmarks well | |
| // and seems more sustainable than an ever-growing list of intrinsics to ignore. | |
| if let Some((fn_def_id, _)) = func.const_fn_def() | |
| && find_attr!(tcx, fn_def_id, RustcIntrinsic) | |
| { | |
| return; | |
| } | |
| self.calls += 1; | |
| if let UnwindAction::Cleanup(_) = unwind { | |
| self.landing_pads += 1; | |
| } | |
| } | |
| TerminatorKind::TailCall { .. } => { | |
| self.calls += 1; | |
| } | |
| TerminatorKind::Assert { unwind, .. } => { | |
| if let UnwindAction::Cleanup(_) = unwind { | |
| self.landing_pads += 1; | |
| } | |
| } | |
| TerminatorKind::UnwindResume => self.resumes += 1, | |
| TerminatorKind::InlineAsm { unwind, .. } => { | |
| if let UnwindAction::Cleanup(_) = unwind { | |
| self.landing_pads += 1; | |
| } | |
| } | |
| TerminatorKind::Return | |
| | TerminatorKind::Goto { .. } | |
| | TerminatorKind::SwitchInt { .. } | |
| | TerminatorKind::Unreachable | |
| | TerminatorKind::UnwindTerminate(_) => {} | |
| kind @ (TerminatorKind::FalseUnwind { .. } | |
| | TerminatorKind::FalseEdge { .. } | |
| | TerminatorKind::Yield { .. } | |
| | TerminatorKind::CoroutineDrop) => { | |
| bug!("{kind:?} should not be in runtime MIR"); | |
| } | |
| } | |
| } | |
| } |