compiler/rustc_mir_transform/src/pass_manager.rs - rust - Git at Google

 use std::cell::RefCell;
 use std::collections::hash_map::Entry;
 use std::sync::atomic::Ordering;

 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_middle::mir::{Body, MirDumper, MirPhase, RuntimePhase};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Session;
 use tracing::trace;

 use crate::lint::lint_body;
 use crate::{diagnostics, validate};

 thread_local! {
     /// Maps MIR pass names to a snake case form to match profiling naming style
     static PASS_TO_PROFILER_NAMES: RefCell<FxHashMap<&'static str, &'static str>> = {
         RefCell::new(FxHashMap::default())
     };
 }

 /// Converts a MIR pass name into a snake case form to match the profiling naming style.
 fn to_profiler_name(type_name: &'static str) -> &'static str {
     PASS_TO_PROFILER_NAMES.with(|names| match names.borrow_mut().entry(type_name) {
         Entry::Occupied(e) => *e.get(),
         Entry::Vacant(e) => {
             let snake_case: String = type_name
                 .chars()
                 .flat_map(|c| {
                     if c.is_ascii_uppercase() {
                         vec!['_', c.to_ascii_lowercase()]
                     } else if c == '-' {
                         vec!['_']
                     } else {
                         vec![c]
                     }
                 })
                 .collect();
             let result = &*String::leak(format!("mir_pass{}", snake_case));
             e.insert(result);
             result
         }
     })
 }

 // A function that simplifies a pass's type_name. E.g. `Baz`, `Baz<'_>`,
 // `foo::bar::Baz`, and `foo::bar::Baz<'a, 'b>` all become `Baz`.
 //
 // It's `const` for perf reasons: it's called a lot, and doing the string
 // operations at runtime causes a non-trivial slowdown. If
 // `split_once`/`rsplit_once` become `const` its body could be simplified to
 // this:
 // ```ignore (fragment)
 // let name = if let Some((_, tail)) = name.rsplit_once(':') { tail } else { name };
 // let name = if let Some((head, _)) = name.split_once('<') { head } else { name };
 // name
 // ```
 const fn simplify_pass_type_name(name: &'static str) -> &'static str {
     // FIXME(const-hack) Simplify the implementation once more `str` methods get const-stable.

     // Work backwards from the end. If a ':' is hit, strip it and everything before it.
     let bytes = name.as_bytes();
     let mut i = bytes.len();
     while i > 0 && bytes[i - 1] != b':' {
         i -= 1;
     }
     let (_, bytes) = bytes.split_at(i);

     // Work forwards from the start of what's left. If a '<' is hit, strip it and everything after
     // it.
     let mut i = 0;
     while i < bytes.len() && bytes[i] != b'<' {
         i += 1;
     }
     let (bytes, _) = bytes.split_at(i);

     match std::str::from_utf8(bytes) {
         Ok(name) => name,
         Err(_) => panic!(),
     }
 }

 /// Rules outlining when this pass may be overridden or suppressed.
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub(crate) enum PassPolicy {
     /// This pass implements a mandatory lowering step, either to implement parts of the MIR semantics
     /// or to bring MIR into a shape that is easier to deal with for later passes/codegen.
     /// Passes using this cannot be disabled via any means. They must not remove any UB, as they will
     /// run in Miri. They must also come with a comment justifying why they must always run.
     Required,
     /// An optional pass that may be configured by `-Zmir-enable-passes`.
     Optional {
         /// Whether this pass should be enabled by default in this session in the absence of
         /// an explicit `-Zmir-enable-passes` or `#[optimize(none)]`.
         generally_enabled: bool,
         /// Whether this is an optimization pass. `#[optimize(none)]` only disables optimization
         /// passes.
         /// A pass may be optional without being an optimization pass,
         /// e.g. if it just adds extra debug checks that one can turn off.
         optimization: bool,
     },
 }

 impl PassPolicy {
     fn and_enabled(self, enabled: bool) -> Self {
         match self {
             PassPolicy::Required => PassPolicy::Required,
             PassPolicy::Optional { generally_enabled: enabled_by_default, optimization } => {
                 PassPolicy::Optional {
                     generally_enabled: enabled_by_default && enabled,
                     optimization,
                 }
             }
         }
     }

     /// Create a [`PassPolicy::Optional`] that is not an optimization,
     /// enabled by default under the given condition.
     pub(crate) fn optional_non_optimization(condition: bool) -> Self {
         Self::Optional { generally_enabled: condition, optimization: false }
     }

     /// Create a [`PassPolicy::Optional`] optimization, enabled by default under the given condition.
     pub(crate) fn optimization(condition: bool) -> Self {
         Self::Optional { generally_enabled: condition, optimization: true }
     }
 }

 /// A streamlined trait that you can implement to create a pass; the
 /// pass will be named after the type, and it will consist of a main
 /// loop that goes over each available MIR and applies `run_pass`.
 pub(super) trait MirPass<'tcx> {
     fn name(&self) -> &'static str {
         const { simplify_pass_type_name(std::any::type_name::<Self>()) }
     }

     fn profiler_name(&self) -> &'static str {
         to_profiler_name(self.name())
     }

     /// Describes how this pass is enabled and which mechanisms may disable it.
     fn policy(&self, sess: &Session) -> PassPolicy;

     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>);

     fn is_mir_dump_enabled(&self) -> bool {
         true
     }
 }

 /// Just like `MirPass`, except it cannot mutate `Body`, and MIR dumping is
 /// disabled (via the `Lint` adapter).
 pub(super) trait MirLint<'tcx> {
     fn name(&self) -> &'static str {
         const { simplify_pass_type_name(std::any::type_name::<Self>()) }
     }

     fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>);
 }

 /// An adapter for `MirLint`s that implements `MirPass`.
 #[derive(Debug, Clone)]
 pub(super) struct Lint<T>(pub T);

 impl<'tcx, T> MirPass<'tcx> for Lint<T>
 where
     T: MirLint<'tcx>,
 {
     fn name(&self) -> &'static str {
         self.0.name()
     }

     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         self.0.run_lint(tcx, body)
     }

     fn is_mir_dump_enabled(&self) -> bool {
         false
     }

     fn policy(&self, _sess: &Session) -> PassPolicy {
         PassPolicy::optional_non_optimization(true)
     }
 }

 pub(super) struct WithMinOptLevel<T>(pub u32, pub T);

 impl<'tcx, T> MirPass<'tcx> for WithMinOptLevel<T>
 where
     T: MirPass<'tcx>,
 {
     fn name(&self) -> &'static str {
         self.1.name()
     }

     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         self.1.run_pass(tcx, body)
     }

     fn policy(&self, sess: &Session) -> PassPolicy {
         self.1.policy(sess).and_enabled(sess.mir_opt_level() >= self.0 as usize)
     }
 }

 /// Whether to allow [optimization passes].
 ///
 /// [optimization passes]: PassPolicy::Optional::optimization
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub(crate) enum Optimizations {
     /// The current function has `#[optimize(none)]`.
     Suppressed,
     /// Normal optimizations may run.
     Allowed,
 }

 /// Run the sequence of passes without validating the MIR after each pass. The MIR is still
 /// validated at the end.
 pub(super) fn run_passes_no_validate<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
     passes: &[&dyn MirPass<'tcx>],
     phase_change: Option<MirPhase>,
 ) {
     run_passes_inner(tcx, body, passes, phase_change, false);
 }

 /// The optional `phase_change` is applied after executing all the passes, if present
 pub(super) fn run_passes<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
     passes: &[&dyn MirPass<'tcx>],
     phase_change: Option<MirPhase>,
 ) {
     run_passes_inner(tcx, body, passes, phase_change, true);
 }

 pub(super) fn should_run_pass<'tcx, P>(
     tcx: TyCtxt<'tcx>,
     pass: &P,
     optimizations: Optimizations,
 ) -> bool
 where
     P: MirPass<'tcx> + ?Sized,
 {
     let name = pass.name();
     let pass_override = || {
         tcx.sess
             .opts
             .unstable_opts
             .mir_enable_passes
             .iter()
             .rev()
             .find_map(|(name_, polarity)| if name == name_ { Some(*polarity) } else { None })
     };

     match pass.policy(tcx.sess) {
         PassPolicy::Required => true,
         PassPolicy::Optional { generally_enabled: enabled_by_default, optimization } => {
             if let Some(o) = pass_override() {
                 trace!(
                     pass = %name,
                     "{} as requested by flag",
                     if o { "Running" } else { "Not running" }
                 );
                 o
             } else if optimization && optimizations == Optimizations::Suppressed {
                 trace!(pass = %name, "Not running as requested by `#[optimize(none)]`");
                 false
             } else {
                 enabled_by_default
             }
         }
     }
 }

 fn run_passes_inner<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
     passes: &[&dyn MirPass<'tcx>],
     phase_change: Option<MirPhase>,
     validate_each: bool,
 ) {
     let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes;
     trace!(?overridden_passes);

     let named_passes: FxIndexSet<_> =
         overridden_passes.iter().map(|(name, _)| name.as_str()).collect();

     let mut unknown_found = false;
     for &name in named_passes.difference(&*crate::PASS_NAMES) {
         tcx.dcx().emit_warn(diagnostics::UnknownPassName { name });
         unknown_found = true;
     }

     if unknown_found {
         let mut valid_pass_names = crate::PASS_NAMES.iter().copied().collect::<Vec<_>>();
         valid_pass_names.sort();
         tcx.dcx().emit_note(diagnostics::ValidPassNames { valid_passes: valid_pass_names.into() });
     }

     // Verify that no passes are missing from the `declare_passes` invocation
     #[cfg(debug_assertions)]
     {
         let used_passes: FxIndexSet<_> = passes.iter().map(|p| p.name()).collect();

         let undeclared = used_passes.difference(&*crate::PASS_NAMES).collect::<Vec<_>>();
         if let Some((name, rest)) = undeclared.split_first() {
             let mut err =
                 tcx.dcx().struct_bug(format!("pass `{name}` is not declared in `PASS_NAMES`"));
             for name in rest {
                 err.note(format!("pass `{name}` is also not declared in `PASS_NAMES`"));
             }
             err.emit();
         }
     }

     let prof_arg = tcx.sess.prof.enabled().then(|| format!("{:?}", body.source.def_id()));

     if !body.should_skip() {
         let validate = validate_each & tcx.sess.opts.unstable_opts.validate_mir;
         let lint = tcx.sess.opts.unstable_opts.lint_mir;

         let def_id = body.source.def_id();
         let optimizations = if tcx.def_kind(def_id).has_codegen_attrs()
             && tcx.codegen_fn_attrs(def_id).optimize.do_not_optimize()
         {
             Optimizations::Suppressed
         } else {
             Optimizations::Allowed
         };

         for pass in passes {
             let pass_name = pass.name();

             if !should_run_pass(tcx, *pass, optimizations) {
                 continue;
             };

             if is_optimization_stage(body, phase_change, optimizations)
                 && let Some(limit) = &tcx.sess.opts.unstable_opts.mir_opt_bisect_limit
                 && limited_by_opt_bisect(
                     tcx,
                     tcx.def_path_debug_str(body.source.def_id()),
                     *limit,
                     *pass,
                 )
             {
                 continue;
             }

             let dumper = if pass.is_mir_dump_enabled()
                 && let Some(dumper) = MirDumper::new(tcx, pass_name, body)
             {
                 Some(dumper.set_show_pass_num().set_disambiguator(&"before"))
             } else {
                 None
             };

             if let Some(dumper) = dumper.as_ref() {
                 dumper.dump_mir(body);
             }

             if let Some(prof_arg) = &prof_arg {
                 tcx.sess
                     .prof
                     .generic_activity_with_arg(pass.profiler_name(), &**prof_arg)
                     .run(|| pass.run_pass(tcx, body));
             } else {
                 pass.run_pass(tcx, body);
             }

             if let Some(dumper) = dumper {
                 dumper.set_disambiguator(&"after").dump_mir(body);
             }

             if validate {
                 validate_body(tcx, body, format!("after pass {pass_name}"));
             }
             if lint {
                 lint_body(tcx, body, format!("after pass {pass_name}"));
             }

             body.pass_count += 1;
         }
     }

     if let Some(new_phase) = phase_change {
         if body.phase >= new_phase {
             panic!("Invalid MIR phase transition from {:?} to {:?}", body.phase, new_phase);
         }

         body.phase = new_phase;
         body.pass_count = 0;

         dump_mir_for_phase_change(tcx, body);

         let validate =
             (validate_each & tcx.sess.opts.unstable_opts.validate_mir & !body.should_skip())
                 || new_phase == MirPhase::Runtime(RuntimePhase::Optimized);
         let lint = tcx.sess.opts.unstable_opts.lint_mir & !body.should_skip();
         if validate {
             validate_body(tcx, body, format!("after phase change to {}", new_phase.name()));
         }
         if lint {
             lint_body(tcx, body, format!("after phase change to {}", new_phase.name()));
         }

         body.pass_count = 1;
     }
 }

 pub(super) fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) {
     validate::Validator { when }.run_pass(tcx, body);
 }

 pub(super) fn dump_mir_for_phase_change<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
     assert_eq!(body.pass_count, 0);
     if let Some(dumper) = MirDumper::new(tcx, body.phase.name(), body) {
         dumper.set_show_pass_num().set_disambiguator(&"after").dump_mir(body)
     }
 }

 fn is_optimization_stage(
     body: &Body<'_>,
     phase_change: Option<MirPhase>,
     optimizations: Optimizations,
 ) -> bool {
     optimizations == Optimizations::Allowed
         && body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup)
         && phase_change == Some(MirPhase::Runtime(RuntimePhase::Optimized))
 }

 fn limited_by_opt_bisect<'tcx, P>(
     tcx: TyCtxt<'tcx>,
     def_path: String,
     limit: usize,
     pass: &P,
 ) -> bool
 where
     P: MirPass<'tcx> + ?Sized,
 {
     let current_opt_bisect_count =
         tcx.sess.mir_opt_bisect_eval_count.fetch_add(1, Ordering::Relaxed);

     let can_run = current_opt_bisect_count < limit;

     if can_run {
         eprintln!(
             "BISECT: running pass ({}) {} on {}",
             current_opt_bisect_count + 1,
             pass.name(),
             def_path
         );
     } else {
         eprintln!(
             "BISECT: NOT running pass ({}) {} on {}",
             current_opt_bisect_count + 1,
             pass.name(),
             def_path
         );
     }

     !can_run
 }
	use std::cell::RefCell;
	use std::collections::hash_map::Entry;
	use std::sync::atomic::Ordering;

	use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
	use rustc_middle::mir::{Body, MirDumper, MirPhase, RuntimePhase};
	use rustc_middle::ty::TyCtxt;
	use rustc_session::Session;
	use tracing::trace;

	use crate::lint::lint_body;
	use crate::{diagnostics, validate};

	thread_local! {
	/// Maps MIR pass names to a snake case form to match profiling naming style
	static PASS_TO_PROFILER_NAMES: RefCell<FxHashMap<&'static str, &'static str>> = {
	RefCell::new(FxHashMap::default())
	};
	}

	/// Converts a MIR pass name into a snake case form to match the profiling naming style.
	fn to_profiler_name(type_name: &'static str) -> &'static str {
	PASS_TO_PROFILER_NAMES.with(\|names\| match names.borrow_mut().entry(type_name) {
	Entry::Occupied(e) => *e.get(),
	Entry::Vacant(e) => {
	let snake_case: String = type_name
	.chars()
	.flat_map(\|c\| {
	if c.is_ascii_uppercase() {
	vec!['_', c.to_ascii_lowercase()]
	} else if c == '-' {
	vec!['_']
	} else {
	vec![c]
	}
	})
	.collect();
	let result = &*String::leak(format!("mir_pass{}", snake_case));
	e.insert(result);
	result
	}
	})
	}

	// A function that simplifies a pass's type_name. E.g. `Baz`, `Baz<'_>`,
	// `foo::bar::Baz`, and `foo::bar::Baz<'a, 'b>` all become `Baz`.
	//
	// It's `const` for perf reasons: it's called a lot, and doing the string
	// operations at runtime causes a non-trivial slowdown. If
	// `split_once`/`rsplit_once` become `const` its body could be simplified to
	// this:
	// ```ignore (fragment)
	// let name = if let Some((_, tail)) = name.rsplit_once(':') { tail } else { name };
	// let name = if let Some((head, _)) = name.split_once('<') { head } else { name };
	// name
	// ```
	const fn simplify_pass_type_name(name: &'static str) -> &'static str {
	// FIXME(const-hack) Simplify the implementation once more `str` methods get const-stable.

	// Work backwards from the end. If a ':' is hit, strip it and everything before it.
	let bytes = name.as_bytes();
	let mut i = bytes.len();
	while i > 0 && bytes[i - 1] != b':' {
	i -= 1;
	}
	let (_, bytes) = bytes.split_at(i);

	// Work forwards from the start of what's left. If a '<' is hit, strip it and everything after
	// it.
	let mut i = 0;
	while i < bytes.len() && bytes[i] != b'<' {
	i += 1;
	}
	let (bytes, _) = bytes.split_at(i);

	match std::str::from_utf8(bytes) {
	Ok(name) => name,
	Err(_) => panic!(),
	}
	}

	/// Rules outlining when this pass may be overridden or suppressed.
	#[derive(Copy, Clone, Debug, PartialEq, Eq)]
	pub(crate) enum PassPolicy {
	/// This pass implements a mandatory lowering step, either to implement parts of the MIR semantics
	/// or to bring MIR into a shape that is easier to deal with for later passes/codegen.
	/// Passes using this cannot be disabled via any means. They must not remove any UB, as they will
	/// run in Miri. They must also come with a comment justifying why they must always run.
	Required,
	/// An optional pass that may be configured by `-Zmir-enable-passes`.
	Optional {
	/// Whether this pass should be enabled by default in this session in the absence of
	/// an explicit `-Zmir-enable-passes` or `#[optimize(none)]`.
	generally_enabled: bool,
	/// Whether this is an optimization pass. `#[optimize(none)]` only disables optimization
	/// passes.
	/// A pass may be optional without being an optimization pass,
	/// e.g. if it just adds extra debug checks that one can turn off.
	optimization: bool,
	},
	}

	impl PassPolicy {
	fn and_enabled(self, enabled: bool) -> Self {
	match self {
	PassPolicy::Required => PassPolicy::Required,
	PassPolicy::Optional { generally_enabled: enabled_by_default, optimization } => {
	PassPolicy::Optional {
	generally_enabled: enabled_by_default && enabled,
	optimization,
	}
	}
	}
	}

	/// Create a [`PassPolicy::Optional`] that is not an optimization,
	/// enabled by default under the given condition.
	pub(crate) fn optional_non_optimization(condition: bool) -> Self {
	Self::Optional { generally_enabled: condition, optimization: false }
	}

	/// Create a [`PassPolicy::Optional`] optimization, enabled by default under the given condition.
	pub(crate) fn optimization(condition: bool) -> Self {
	Self::Optional { generally_enabled: condition, optimization: true }
	}
	}

	/// A streamlined trait that you can implement to create a pass; the
	/// pass will be named after the type, and it will consist of a main
	/// loop that goes over each available MIR and applies `run_pass`.
	pub(super) trait MirPass<'tcx> {
	fn name(&self) -> &'static str {
	const { simplify_pass_type_name(std::any::type_name::<Self>()) }
	}

	fn profiler_name(&self) -> &'static str {
	to_profiler_name(self.name())
	}

	/// Describes how this pass is enabled and which mechanisms may disable it.
	fn policy(&self, sess: &Session) -> PassPolicy;

	fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>);

	fn is_mir_dump_enabled(&self) -> bool {
	true
	}
	}

	/// Just like `MirPass`, except it cannot mutate `Body`, and MIR dumping is
	/// disabled (via the `Lint` adapter).
	pub(super) trait MirLint<'tcx> {
	fn name(&self) -> &'static str {
	const { simplify_pass_type_name(std::any::type_name::<Self>()) }
	}

	fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>);
	}

	/// An adapter for `MirLint`s that implements `MirPass`.
	#[derive(Debug, Clone)]
	pub(super) struct Lint<T>(pub T);

	impl<'tcx, T> MirPass<'tcx> for Lint<T>
	where
	T: MirLint<'tcx>,
	{
	fn name(&self) -> &'static str {
	self.0.name()
	}

	fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
	self.0.run_lint(tcx, body)
	}

	fn is_mir_dump_enabled(&self) -> bool {
	false
	}

	fn policy(&self, _sess: &Session) -> PassPolicy {
	PassPolicy::optional_non_optimization(true)
	}
	}

	pub(super) struct WithMinOptLevel<T>(pub u32, pub T);

	impl<'tcx, T> MirPass<'tcx> for WithMinOptLevel<T>
	where
	T: MirPass<'tcx>,
	{
	fn name(&self) -> &'static str {
	self.1.name()
	}

	fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
	self.1.run_pass(tcx, body)
	}

	fn policy(&self, sess: &Session) -> PassPolicy {
	self.1.policy(sess).and_enabled(sess.mir_opt_level() >= self.0 as usize)
	}
	}

	/// Whether to allow [optimization passes].
	///
	/// [optimization passes]: PassPolicy::Optional::optimization
	#[derive(Copy, Clone, Debug, PartialEq, Eq)]
	pub(crate) enum Optimizations {
	/// The current function has `#[optimize(none)]`.
	Suppressed,
	/// Normal optimizations may run.
	Allowed,
	}

	/// Run the sequence of passes without validating the MIR after each pass. The MIR is still
	/// validated at the end.
	pub(super) fn run_passes_no_validate<'tcx>(
	tcx: TyCtxt<'tcx>,
	body: &mut Body<'tcx>,
	passes: &[&dyn MirPass<'tcx>],
	phase_change: Option<MirPhase>,
	) {
	run_passes_inner(tcx, body, passes, phase_change, false);
	}

	/// The optional `phase_change` is applied after executing all the passes, if present
	pub(super) fn run_passes<'tcx>(
	tcx: TyCtxt<'tcx>,
	body: &mut Body<'tcx>,
	passes: &[&dyn MirPass<'tcx>],
	phase_change: Option<MirPhase>,
	) {
	run_passes_inner(tcx, body, passes, phase_change, true);
	}

	pub(super) fn should_run_pass<'tcx, P>(
	tcx: TyCtxt<'tcx>,
	pass: &P,
	optimizations: Optimizations,
	) -> bool
	where
	P: MirPass<'tcx> + ?Sized,
	{
	let name = pass.name();
	let pass_override = \|\| {
	tcx.sess
	.opts
	.unstable_opts
	.mir_enable_passes
	.iter()
	.rev()
	.find_map(\|(name_, polarity)\| if name == name_ { Some(*polarity) } else { None })
	};

	match pass.policy(tcx.sess) {
	PassPolicy::Required => true,
	PassPolicy::Optional { generally_enabled: enabled_by_default, optimization } => {
	if let Some(o) = pass_override() {
	trace!(
	pass = %name,
	"{} as requested by flag",
	if o { "Running" } else { "Not running" }
	);
	o
	} else if optimization && optimizations == Optimizations::Suppressed {
	trace!(pass = %name, "Not running as requested by `#[optimize(none)]`");
	false
	} else {
	enabled_by_default
	}
	}
	}
	}

	fn run_passes_inner<'tcx>(
	tcx: TyCtxt<'tcx>,
	body: &mut Body<'tcx>,
	passes: &[&dyn MirPass<'tcx>],
	phase_change: Option<MirPhase>,
	validate_each: bool,
	) {
	let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes;
	trace!(?overridden_passes);

	let named_passes: FxIndexSet<_> =
	overridden_passes.iter().map(\|(name, _)\| name.as_str()).collect();

	let mut unknown_found = false;
	for &name in named_passes.difference(&*crate::PASS_NAMES) {
	tcx.dcx().emit_warn(diagnostics::UnknownPassName { name });
	unknown_found = true;
	}

	if unknown_found {
	let mut valid_pass_names = crate::PASS_NAMES.iter().copied().collect::<Vec<_>>();
	valid_pass_names.sort();
	tcx.dcx().emit_note(diagnostics::ValidPassNames { valid_passes: valid_pass_names.into() });
	}

	// Verify that no passes are missing from the `declare_passes` invocation
	#[cfg(debug_assertions)]
	{
	let used_passes: FxIndexSet<_> = passes.iter().map(\|p\| p.name()).collect();

	let undeclared = used_passes.difference(&*crate::PASS_NAMES).collect::<Vec<_>>();
	if let Some((name, rest)) = undeclared.split_first() {
	let mut err =
	tcx.dcx().struct_bug(format!("pass `{name}` is not declared in `PASS_NAMES`"));
	for name in rest {
	err.note(format!("pass `{name}` is also not declared in `PASS_NAMES`"));
	}
	err.emit();
	}
	}

	let prof_arg = tcx.sess.prof.enabled().then(\|\| format!("{:?}", body.source.def_id()));

	if !body.should_skip() {
	let validate = validate_each & tcx.sess.opts.unstable_opts.validate_mir;
	let lint = tcx.sess.opts.unstable_opts.lint_mir;

	let def_id = body.source.def_id();
	let optimizations = if tcx.def_kind(def_id).has_codegen_attrs()
	&& tcx.codegen_fn_attrs(def_id).optimize.do_not_optimize()
	{
	Optimizations::Suppressed
	} else {
	Optimizations::Allowed
	};

	for pass in passes {
	let pass_name = pass.name();

	if !should_run_pass(tcx, *pass, optimizations) {
	continue;
	};

	if is_optimization_stage(body, phase_change, optimizations)
	&& let Some(limit) = &tcx.sess.opts.unstable_opts.mir_opt_bisect_limit
	&& limited_by_opt_bisect(
	tcx,
	tcx.def_path_debug_str(body.source.def_id()),
	*limit,
	*pass,
	)
	{
	continue;
	}

	let dumper = if pass.is_mir_dump_enabled()
	&& let Some(dumper) = MirDumper::new(tcx, pass_name, body)
	{
	Some(dumper.set_show_pass_num().set_disambiguator(&"before"))
	} else {
	None
	};

	if let Some(dumper) = dumper.as_ref() {
	dumper.dump_mir(body);
	}

	if let Some(prof_arg) = &prof_arg {
	tcx.sess
	.prof
	.generic_activity_with_arg(pass.profiler_name(), &**prof_arg)
	.run(\|\| pass.run_pass(tcx, body));
	} else {
	pass.run_pass(tcx, body);
	}

	if let Some(dumper) = dumper {
	dumper.set_disambiguator(&"after").dump_mir(body);
	}

	if validate {
	validate_body(tcx, body, format!("after pass {pass_name}"));
	}
	if lint {
	lint_body(tcx, body, format!("after pass {pass_name}"));
	}

	body.pass_count += 1;
	}
	}

	if let Some(new_phase) = phase_change {
	if body.phase >= new_phase {
	panic!("Invalid MIR phase transition from {:?} to {:?}", body.phase, new_phase);
	}

	body.phase = new_phase;
	body.pass_count = 0;

	dump_mir_for_phase_change(tcx, body);

	let validate =
	(validate_each & tcx.sess.opts.unstable_opts.validate_mir & !body.should_skip())
	\|\| new_phase == MirPhase::Runtime(RuntimePhase::Optimized);
	let lint = tcx.sess.opts.unstable_opts.lint_mir & !body.should_skip();
	if validate {
	validate_body(tcx, body, format!("after phase change to {}", new_phase.name()));
	}
	if lint {
	lint_body(tcx, body, format!("after phase change to {}", new_phase.name()));
	}

	body.pass_count = 1;
	}
	}

	pub(super) fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) {
	validate::Validator { when }.run_pass(tcx, body);
	}

	pub(super) fn dump_mir_for_phase_change<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
	assert_eq!(body.pass_count, 0);
	if let Some(dumper) = MirDumper::new(tcx, body.phase.name(), body) {
	dumper.set_show_pass_num().set_disambiguator(&"after").dump_mir(body)
	}
	}

	fn is_optimization_stage(
	body: &Body<'_>,
	phase_change: Option<MirPhase>,
	optimizations: Optimizations,
	) -> bool {
	optimizations == Optimizations::Allowed
	&& body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup)
	&& phase_change == Some(MirPhase::Runtime(RuntimePhase::Optimized))
	}

	fn limited_by_opt_bisect<'tcx, P>(
	tcx: TyCtxt<'tcx>,
	def_path: String,
	limit: usize,
	pass: &P,
	) -> bool
	where
	P: MirPass<'tcx> + ?Sized,
	{
	let current_opt_bisect_count =
	tcx.sess.mir_opt_bisect_eval_count.fetch_add(1, Ordering::Relaxed);

	let can_run = current_opt_bisect_count < limit;

	if can_run {
	eprintln!(
	"BISECT: running pass ({}) {} on {}",
	current_opt_bisect_count + 1,
	pass.name(),
	def_path
	);
	} else {
	eprintln!(
	"BISECT: NOT running pass ({}) {} on {}",
	current_opt_bisect_count + 1,
	pass.name(),
	def_path
	);
	}

	!can_run
	}