profile-guided-optimization.html - rust-lang/rustc-dev-guide - Git at Google

 <!DOCTYPE HTML>
 <html lang="en" class="light sidebar-visible" dir="ltr">
     <head>
         <!-- Book generated using mdBook -->
         <meta charset="UTF-8">
         <title>Profile-guided optimization - Rust Compiler Development Guide</title>


         <!-- Custom HTML head -->

         <meta name="description" content="A guide to developing the Rust compiler (rustc)">
         <meta name="viewport" content="width=device-width, initial-scale=1">
         <meta name="theme-color" content="#ffffff">

         <link rel="icon" href="favicon.svg">
         <link rel="shortcut icon" href="favicon.png">
         <link rel="stylesheet" href="css/variables.css">
         <link rel="stylesheet" href="css/general.css">
         <link rel="stylesheet" href="css/chrome.css">
         <link rel="stylesheet" href="css/print.css" media="print">

         <!-- Fonts -->
         <link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
         <link rel="stylesheet" href="fonts/fonts.css">

         <!-- Highlight.js Stylesheets -->
         <link rel="stylesheet" id="highlight-css" href="highlight.css">
         <link rel="stylesheet" id="tomorrow-night-css" href="tomorrow-night.css">
         <link rel="stylesheet" id="ayu-highlight-css" href="ayu-highlight.css">

         <!-- Custom theme stylesheets -->


         <!-- Provide site root and default themes to javascript -->
         <script>
             const path_to_root = "";
             const default_light_theme = "light";
             const default_dark_theme = "navy";
         </script>
         <!-- Start loading toc.js asap -->
         <script src="toc.js"></script>
     </head>
     <body>
     <div id="body-container">
         <!-- Work around some values being stored in localStorage wrapped in quotes -->
         <script>
             try {
                 let theme = localStorage.getItem('mdbook-theme');
                 let sidebar = localStorage.getItem('mdbook-sidebar');

                 if (theme.startsWith('"') && theme.endsWith('"')) {
                     localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
                 }

                 if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
                     localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
                 }
             } catch (e) { }
         </script>

         <!-- Set the theme before any content is loaded, prevents flash -->
         <script>
             const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
             let theme;
             try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
             if (theme === null || theme === undefined) { theme = default_theme; }
             const html = document.documentElement;
             html.classList.remove('light')
             html.classList.add(theme);
             html.classList.add("js");
         </script>

         <input type="checkbox" id="sidebar-toggle-anchor" class="hidden">

         <!-- Hide / unhide sidebar before it is displayed -->
         <script>
             let sidebar = null;
             const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
             if (document.body.clientWidth >= 1080) {
                 try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
                 sidebar = sidebar || 'visible';
             } else {
                 sidebar = 'hidden';
             }
             sidebar_toggle.checked = sidebar === 'visible';
             html.classList.remove('sidebar-visible');
             html.classList.add("sidebar-" + sidebar);
         </script>

         <nav id="sidebar" class="sidebar" aria-label="Table of contents">
             <!-- populated by js -->
             <mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
             <noscript>
                 <iframe class="sidebar-iframe-outer" src="toc.html"></iframe>
             </noscript>
             <div id="sidebar-resize-handle" class="sidebar-resize-handle">
                 <div class="sidebar-resize-indicator"></div>
             </div>
         </nav>

         <div id="page-wrapper" class="page-wrapper">

             <div class="page">
                 <div id="menu-bar-hover-placeholder"></div>
                 <div id="menu-bar" class="menu-bar sticky">
                     <div class="left-buttons">
                         <label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
                             <i class="fa fa-bars"></i>
                         </label>
                         <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
                             <i class="fa fa-paint-brush"></i>
                         </button>
                         <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
                             <li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
                         </ul>
                         <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
                             <i class="fa fa-search"></i>
                         </button>
                     </div>

                     <h1 class="menu-title">Rust Compiler Development Guide</h1>

                     <div class="right-buttons">
                         <a href="print.html" title="Print this book" aria-label="Print this book">
                             <i id="print-button" class="fa fa-print"></i>
                         </a>
                         <a href="https://github.com/rust-lang/rustc-dev-guide" title="Git repository" aria-label="Git repository">
                             <i id="git-repository-button" class="fa fa-github"></i>
                         </a>
                         <a href="https://github.com/rust-lang/rustc-dev-guide/edit/master/src/profile-guided-optimization.md" title="Suggest an edit" aria-label="Suggest an edit">
                             <i id="git-edit-button" class="fa fa-edit"></i>
                         </a>

                     </div>
                 </div>

                 <div id="search-wrapper" class="hidden">
                     <form id="searchbar-outer" class="searchbar-outer">
                         <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
                     </form>
                     <div id="searchresults-outer" class="searchresults-outer hidden">
                         <div id="searchresults-header" class="searchresults-header"></div>
                         <ul id="searchresults">
                         </ul>
                     </div>
                 </div>

                 <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
                 <script>
                     document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
                     document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
                     Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
                         link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
                     });
                 </script>

                 <div id="content" class="content">
                     <main>
                         <h1 id="profile-guided-optimization"><a class="header" href="#profile-guided-optimization">Profile-guided optimization</a></h1>
 <ul>
 <li><a href="#what-is-profiled-guided-optimization">What is profiled-guided optimization?</a></li>
 <li><a href="#how-is-pgo-implemented-in-rustc">How is PGO implemented in <code>rustc</code>?</a>
 <ul>
 <li><a href="#overall-workflow">Overall workflow</a></li>
 <li><a href="#compile-time-aspects">Compile-time aspects</a>
 <ul>
 <li><a href="#create-binaries-with-instrumentation">Create binaries with instrumentation</a></li>
 <li><a href="#compile-binaries-where-optimizations-make-use-of-profiling-data">Compile binaries where optimizations make use of profiling data</a></li>
 </ul>
 </li>
 <li><a href="#runtime-aspects">Runtime aspects</a></li>
 </ul>
 </li>
 <li><a href="#testing-pgo">Testing PGO</a></li>
 <li><a href="#additional-information">Additional information</a></li>
 </ul>
 <p><code>rustc</code> supports doing profile-guided optimization (PGO).
 This chapter describes what PGO is and how the support for it is
 implemented in <code>rustc</code>.</p>
 <h2 id="what-is-profiled-guided-optimization"><a class="header" href="#what-is-profiled-guided-optimization">What is profiled-guided optimization?</a></h2>
 <p>The basic concept of PGO is to collect data about the typical execution of
 a program (e.g. which branches it is likely to take) and then use this data
 to inform optimizations such as inlining, machine-code layout,
 register allocation, etc.</p>
 <p>There are different ways of collecting data about a program's execution.
 One is to run the program inside a profiler (such as <code>perf</code>) and another
 is to create an instrumented binary, that is, a binary that has data
 collection built into it, and run that.
 The latter usually provides more accurate data.</p>
 <h2 id="how-is-pgo-implemented-in-rustc"><a class="header" href="#how-is-pgo-implemented-in-rustc">How is PGO implemented in <code>rustc</code>?</a></h2>
 <p><code>rustc</code> current PGO implementation relies entirely on LLVM.
 LLVM actually <a href="https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization">supports multiple forms</a> of PGO:</p>
 <ul>
 <li>Sampling-based PGO where an external profiling tool like <code>perf</code> is used
 to collect data about a program's execution.</li>
 <li>GCOV-based profiling, where code coverage infrastructure is used to collect
 profiling information.</li>
 <li>Front-end based instrumentation, where the compiler front-end (e.g. Clang)
 inserts instrumentation intrinsics into the LLVM IR it generates (but see the
 <sup class="footnote-reference" id="fr-note-instrument-coverage-1"><a href="#footnote-note-instrument-coverage">1</a></sup>"Note").</li>
 <li>IR-level instrumentation, where LLVM inserts the instrumentation intrinsics
 itself during optimization passes.</li>
 </ul>
 <p><code>rustc</code> supports only the last approach, IR-level instrumentation, mainly
 because it is almost exclusively implemented in LLVM and needs little
 maintenance on the Rust side. Fortunately, it is also the most modern approach,
 yielding the best results.</p>
 <p>So, we are dealing with an instrumentation-based approach, i.e. profiling data
 is generated by a specially instrumented version of the program that's being
 optimized. Instrumentation-based PGO has two components: a compile-time
 component and run-time component, and one needs to understand the overall
 workflow to see how they interact.</p>
 <h3 id="overall-workflow"><a class="header" href="#overall-workflow">Overall workflow</a></h3>
 <p>Generating a PGO-optimized program involves the following four steps:</p>
 <ol>
 <li>Compile the program with instrumentation enabled (e.g. <code>rustc -C profile-generate main.rs</code>)</li>
 <li>Run the instrumented program (e.g. <code>./main</code>) which generates a <code>default-&lt;id&gt;.profraw</code> file</li>
 <li>Convert the <code>.profraw</code> file into a <code>.profdata</code> file using LLVM's <code>llvm-profdata</code> tool.</li>
 <li>Compile the program again, this time making use of the profiling data
 (e.g. <code>rustc -C profile-use=merged.profdata main.rs</code>)</li>
 </ol>
 <h3 id="compile-time-aspects"><a class="header" href="#compile-time-aspects">Compile-time aspects</a></h3>
 <p>Depending on which step in the above workflow we are in, two different things
 can happen at compile time:</p>
 <h4 id="create-binaries-with-instrumentation"><a class="header" href="#create-binaries-with-instrumentation">Create binaries with instrumentation</a></h4>
 <p>As mentioned above, the profiling instrumentation is added by LLVM.
 <code>rustc</code> instructs LLVM to do so <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/rustllvm/PassWrapper.cpp#L412-L416">by setting the appropriate</a>
 flags when creating LLVM <code>PassManager</code>s:</p>
 <pre><code class="language-C">	// `PMBR` is an `LLVMPassManagerBuilderRef`
     unwrap(PMBR)-&gt;EnablePGOInstrGen = true;
     // Instrumented binaries have a default output path for the `.profraw` file
     // hard-coded into them:
     unwrap(PMBR)-&gt;PGOInstrGen = PGOGenPath;
 </code></pre>
 <p><code>rustc</code> also has to make sure that some of the symbols from LLVM's profiling
 runtime are not removed <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/librustc_codegen_ssa/back/symbol_export.rs#L212-L225">by marking the with the right export level</a>.</p>
 <h4 id="compile-binaries-where-optimizations-make-use-of-profiling-data"><a class="header" href="#compile-binaries-where-optimizations-make-use-of-profiling-data">Compile binaries where optimizations make use of profiling data</a></h4>
 <p>In the final step of the workflow described above, the program is compiled
 again, with the compiler using the gathered profiling data in order to drive
 optimization decisions. <code>rustc</code> again leaves most of the work to LLVM here,
 basically <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/rustllvm/PassWrapper.cpp#L417-L420">just telling</a> the LLVM <code>PassManagerBuilder</code>
 where the profiling data can be found:</p>
 <pre><code class="language-C">	unwrap(PMBR)-&gt;PGOInstrUse = PGOUsePath;
 </code></pre>
 <p>LLVM does the rest (e.g. setting branch weights, marking functions with
 <code>cold</code> or <code>inlinehint</code>, etc).</p>
 <h3 id="runtime-aspects"><a class="header" href="#runtime-aspects">Runtime aspects</a></h3>
 <p>Instrumentation-based approaches always also have a runtime component, i.e.
 once we have an instrumented program, that program needs to be run in order
 to generate profiling data, and collecting and persisting this profiling
 data needs some infrastructure in place.</p>
 <p>In the case of LLVM, these runtime components are implemented in
 <a href="https://github.com/llvm/llvm-project/tree/main/compiler-rt/lib/profile">compiler-rt</a> and statically linked into any instrumented
 binaries.
 The <code>rustc</code> version of this can be found in <code>library/profiler_builtins</code> which
 basically packs the C code from <code>compiler-rt</code> into a Rust crate.</p>
 <p>In order for <code>profiler_builtins</code> to be built, <code>profiler = true</code> must be set
 in <code>rustc</code>'s <code>bootstrap.toml</code>.</p>
 <h2 id="testing-pgo"><a class="header" href="#testing-pgo">Testing PGO</a></h2>
 <p>Since the PGO workflow spans multiple compiler invocations most testing happens
 in <a href="https://github.com/rust-lang/rust/tree/master/tests/run-make">run-make tests</a> (the relevant tests have <code>pgo</code> in their name).
 There is also a <a href="https://github.com/rust-lang/rust/blob/master/tests/codegen/pgo-instrumentation.rs">codegen test</a> that checks that some expected
 instrumentation artifacts show up in LLVM IR.</p>
 <h2 id="additional-information"><a class="header" href="#additional-information">Additional information</a></h2>
 <p>Clang's documentation contains a good overview on <a href="https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization">PGO in LLVM</a>.</p>
 <hr>
 <ol class="footnote-definition"><li id="footnote-note-instrument-coverage">
 <p>Note: <code>rustc</code> now supports front-end-based coverage
 instrumentation, via the experimental option
 <a href="./llvm-coverage-instrumentation.html"><code>-C instrument-coverage</code></a>, but using these
 coverage results for PGO has not been attempted at this time. <a href="#fr-note-instrument-coverage-1">↩</a></p>
 </li>
 </ol>
                     </main>

                     <nav class="nav-wrapper" aria-label="Page navigation">
                         <!-- Mobile navigation buttons -->
                             <a rel="prev" href="backend/libs-and-metadata.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
                                 <i class="fa fa-angle-left"></i>
                             </a>

                             <a rel="next prefetch" href="llvm-coverage-instrumentation.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
                                 <i class="fa fa-angle-right"></i>
                             </a>

                         <div style="clear: both"></div>
                     </nav>
                 </div>
             </div>

             <nav class="nav-wide-wrapper" aria-label="Page navigation">
                     <a rel="prev" href="backend/libs-and-metadata.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
                         <i class="fa fa-angle-left"></i>
                     </a>

                     <a rel="next prefetch" href="llvm-coverage-instrumentation.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
                         <i class="fa fa-angle-right"></i>
                     </a>
             </nav>

         </div>


         <script>
             window.playground_copyable = true;
         </script>


         <script src="elasticlunr.min.js"></script>
         <script src="mark.min.js"></script>
         <script src="searcher.js"></script>

         <script src="clipboard.min.js"></script>
         <script src="highlight.js"></script>
         <script src="book.js"></script>

         <!-- Custom JS scripts -->
         <script src="mermaid.min.js"></script>
         <script src="mermaid-init.js"></script>


     </div>
     </body>
 </html>
	<!DOCTYPE HTML>
	<html lang="en" class="light sidebar-visible" dir="ltr">
	<head>
	<!-- Book generated using mdBook -->
	<meta charset="UTF-8">
	<title>Profile-guided optimization - Rust Compiler Development Guide</title>


	<!-- Custom HTML head -->

	<meta name="description" content="A guide to developing the Rust compiler (rustc)">
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<meta name="theme-color" content="#ffffff">

	<link rel="icon" href="favicon.svg">
	<link rel="shortcut icon" href="favicon.png">
	<link rel="stylesheet" href="css/variables.css">
	<link rel="stylesheet" href="css/general.css">
	<link rel="stylesheet" href="css/chrome.css">
	<link rel="stylesheet" href="css/print.css" media="print">

	<!-- Fonts -->
	<link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
	<link rel="stylesheet" href="fonts/fonts.css">

	<!-- Highlight.js Stylesheets -->
	<link rel="stylesheet" id="highlight-css" href="highlight.css">
	<link rel="stylesheet" id="tomorrow-night-css" href="tomorrow-night.css">
	<link rel="stylesheet" id="ayu-highlight-css" href="ayu-highlight.css">

	<!-- Custom theme stylesheets -->


	<!-- Provide site root and default themes to javascript -->
	<script>
	const path_to_root = "";
	const default_light_theme = "light";
	const default_dark_theme = "navy";
	</script>
	<!-- Start loading toc.js asap -->
	<script src="toc.js"></script>
	</head>
	<body>
	<div id="body-container">
	<!-- Work around some values being stored in localStorage wrapped in quotes -->
	<script>
	try {
	let theme = localStorage.getItem('mdbook-theme');
	let sidebar = localStorage.getItem('mdbook-sidebar');

	if (theme.startsWith('"') && theme.endsWith('"')) {
	localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
	}

	if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
	localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
	}
	} catch (e) { }
	</script>

	<!-- Set the theme before any content is loaded, prevents flash -->
	<script>
	const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
	let theme;
	try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
	if (theme === null \|\| theme === undefined) { theme = default_theme; }
	const html = document.documentElement;
	html.classList.remove('light')
	html.classList.add(theme);
	html.classList.add("js");
	</script>

	<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">

	<!-- Hide / unhide sidebar before it is displayed -->
	<script>
	let sidebar = null;
	const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
	if (document.body.clientWidth >= 1080) {
	try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
	sidebar = sidebar \|\| 'visible';
	} else {
	sidebar = 'hidden';
	}
	sidebar_toggle.checked = sidebar === 'visible';
	html.classList.remove('sidebar-visible');
	html.classList.add("sidebar-" + sidebar);
	</script>

	<nav id="sidebar" class="sidebar" aria-label="Table of contents">
	<!-- populated by js -->
	<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
	<noscript>
	<iframe class="sidebar-iframe-outer" src="toc.html"></iframe>
	</noscript>
	<div id="sidebar-resize-handle" class="sidebar-resize-handle">
	<div class="sidebar-resize-indicator"></div>
	</div>
	</nav>

	<div id="page-wrapper" class="page-wrapper">

	<div class="page">
	<div id="menu-bar-hover-placeholder"></div>
	<div id="menu-bar" class="menu-bar sticky">
	<div class="left-buttons">
	<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
	<i class="fa fa-bars"></i>
	</label>
	<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
	<i class="fa fa-paint-brush"></i>
	</button>
	<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
	<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
	<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
	<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
	<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
	<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
	<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
	</ul>
	<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
	<i class="fa fa-search"></i>
	</button>
	</div>

	<h1 class="menu-title">Rust Compiler Development Guide</h1>

	<div class="right-buttons">
	<a href="print.html" title="Print this book" aria-label="Print this book">
	<i id="print-button" class="fa fa-print"></i>
	</a>
	<a href="https://github.com/rust-lang/rustc-dev-guide" title="Git repository" aria-label="Git repository">
	<i id="git-repository-button" class="fa fa-github"></i>
	</a>
	<a href="https://github.com/rust-lang/rustc-dev-guide/edit/master/src/profile-guided-optimization.md" title="Suggest an edit" aria-label="Suggest an edit">
	<i id="git-edit-button" class="fa fa-edit"></i>
	</a>

	</div>
	</div>

	<div id="search-wrapper" class="hidden">
	<form id="searchbar-outer" class="searchbar-outer">
	<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
	</form>
	<div id="searchresults-outer" class="searchresults-outer hidden">
	<div id="searchresults-header" class="searchresults-header"></div>
	<ul id="searchresults">
	</ul>
	</div>
	</div>

	<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
	<script>
	document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
	document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
	Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
	link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
	});
	</script>

	<div id="content" class="content">
	<main>
	<h1 id="profile-guided-optimization"><a class="header" href="#profile-guided-optimization">Profile-guided optimization</a></h1>
	<ul>
	<li><a href="#what-is-profiled-guided-optimization">What is profiled-guided optimization?</a></li>
	<li><a href="#how-is-pgo-implemented-in-rustc">How is PGO implemented in <code>rustc</code>?</a>
	<ul>
	<li><a href="#overall-workflow">Overall workflow</a></li>
	<li><a href="#compile-time-aspects">Compile-time aspects</a>
	<ul>
	<li><a href="#create-binaries-with-instrumentation">Create binaries with instrumentation</a></li>
	<li><a href="#compile-binaries-where-optimizations-make-use-of-profiling-data">Compile binaries where optimizations make use of profiling data</a></li>
	</ul>
	</li>
	<li><a href="#runtime-aspects">Runtime aspects</a></li>
	</ul>
	</li>
	<li><a href="#testing-pgo">Testing PGO</a></li>
	<li><a href="#additional-information">Additional information</a></li>
	</ul>
	<p><code>rustc</code> supports doing profile-guided optimization (PGO).
	This chapter describes what PGO is and how the support for it is
	implemented in <code>rustc</code>.</p>
	<h2 id="what-is-profiled-guided-optimization"><a class="header" href="#what-is-profiled-guided-optimization">What is profiled-guided optimization?</a></h2>
	<p>The basic concept of PGO is to collect data about the typical execution of
	a program (e.g. which branches it is likely to take) and then use this data
	to inform optimizations such as inlining, machine-code layout,
	register allocation, etc.</p>
	<p>There are different ways of collecting data about a program's execution.
	One is to run the program inside a profiler (such as <code>perf</code>) and another
	is to create an instrumented binary, that is, a binary that has data
	collection built into it, and run that.
	The latter usually provides more accurate data.</p>
	<h2 id="how-is-pgo-implemented-in-rustc"><a class="header" href="#how-is-pgo-implemented-in-rustc">How is PGO implemented in <code>rustc</code>?</a></h2>
	<p><code>rustc</code> current PGO implementation relies entirely on LLVM.
	LLVM actually <a href="https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization">supports multiple forms</a> of PGO:</p>
	<ul>
	<li>Sampling-based PGO where an external profiling tool like <code>perf</code> is used
	to collect data about a program's execution.</li>
	<li>GCOV-based profiling, where code coverage infrastructure is used to collect
	profiling information.</li>
	<li>Front-end based instrumentation, where the compiler front-end (e.g. Clang)
	inserts instrumentation intrinsics into the LLVM IR it generates (but see the
	<sup class="footnote-reference" id="fr-note-instrument-coverage-1"><a href="#footnote-note-instrument-coverage">1</a></sup>"Note").</li>
	<li>IR-level instrumentation, where LLVM inserts the instrumentation intrinsics
	itself during optimization passes.</li>
	</ul>
	<p><code>rustc</code> supports only the last approach, IR-level instrumentation, mainly
	because it is almost exclusively implemented in LLVM and needs little
	maintenance on the Rust side. Fortunately, it is also the most modern approach,
	yielding the best results.</p>
	<p>So, we are dealing with an instrumentation-based approach, i.e. profiling data
	is generated by a specially instrumented version of the program that's being
	optimized. Instrumentation-based PGO has two components: a compile-time
	component and run-time component, and one needs to understand the overall
	workflow to see how they interact.</p>
	<h3 id="overall-workflow"><a class="header" href="#overall-workflow">Overall workflow</a></h3>
	<p>Generating a PGO-optimized program involves the following four steps:</p>
	<ol>
	<li>Compile the program with instrumentation enabled (e.g. <code>rustc -C profile-generate main.rs</code>)</li>
	<li>Run the instrumented program (e.g. <code>./main</code>) which generates a <code>default-<id>.profraw</code> file</li>
	<li>Convert the <code>.profraw</code> file into a <code>.profdata</code> file using LLVM's <code>llvm-profdata</code> tool.</li>
	<li>Compile the program again, this time making use of the profiling data
	(e.g. <code>rustc -C profile-use=merged.profdata main.rs</code>)</li>
	</ol>
	<h3 id="compile-time-aspects"><a class="header" href="#compile-time-aspects">Compile-time aspects</a></h3>
	<p>Depending on which step in the above workflow we are in, two different things
	can happen at compile time:</p>
	<h4 id="create-binaries-with-instrumentation"><a class="header" href="#create-binaries-with-instrumentation">Create binaries with instrumentation</a></h4>
	<p>As mentioned above, the profiling instrumentation is added by LLVM.
	<code>rustc</code> instructs LLVM to do so <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/rustllvm/PassWrapper.cpp#L412-L416">by setting the appropriate</a>
	flags when creating LLVM <code>PassManager</code>s:</p>
	<pre><code class="language-C"> // `PMBR` is an `LLVMPassManagerBuilderRef`
	unwrap(PMBR)->EnablePGOInstrGen = true;
	// Instrumented binaries have a default output path for the `.profraw` file
	// hard-coded into them:
	unwrap(PMBR)->PGOInstrGen = PGOGenPath;
	</code></pre>
	<p><code>rustc</code> also has to make sure that some of the symbols from LLVM's profiling
	runtime are not removed <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/librustc_codegen_ssa/back/symbol_export.rs#L212-L225">by marking the with the right export level</a>.</p>
	<h4 id="compile-binaries-where-optimizations-make-use-of-profiling-data"><a class="header" href="#compile-binaries-where-optimizations-make-use-of-profiling-data">Compile binaries where optimizations make use of profiling data</a></h4>
	<p>In the final step of the workflow described above, the program is compiled
	again, with the compiler using the gathered profiling data in order to drive
	optimization decisions. <code>rustc</code> again leaves most of the work to LLVM here,
	basically <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/rustllvm/PassWrapper.cpp#L417-L420">just telling</a> the LLVM <code>PassManagerBuilder</code>
	where the profiling data can be found:</p>
	<pre><code class="language-C"> unwrap(PMBR)->PGOInstrUse = PGOUsePath;
	</code></pre>
	<p>LLVM does the rest (e.g. setting branch weights, marking functions with
	<code>cold</code> or <code>inlinehint</code>, etc).</p>
	<h3 id="runtime-aspects"><a class="header" href="#runtime-aspects">Runtime aspects</a></h3>
	<p>Instrumentation-based approaches always also have a runtime component, i.e.
	once we have an instrumented program, that program needs to be run in order
	to generate profiling data, and collecting and persisting this profiling
	data needs some infrastructure in place.</p>
	<p>In the case of LLVM, these runtime components are implemented in
	<a href="https://github.com/llvm/llvm-project/tree/main/compiler-rt/lib/profile">compiler-rt</a> and statically linked into any instrumented
	binaries.
	The <code>rustc</code> version of this can be found in <code>library/profiler_builtins</code> which
	basically packs the C code from <code>compiler-rt</code> into a Rust crate.</p>
	<p>In order for <code>profiler_builtins</code> to be built, <code>profiler = true</code> must be set
	in <code>rustc</code>'s <code>bootstrap.toml</code>.</p>
	<h2 id="testing-pgo"><a class="header" href="#testing-pgo">Testing PGO</a></h2>
	<p>Since the PGO workflow spans multiple compiler invocations most testing happens
	in <a href="https://github.com/rust-lang/rust/tree/master/tests/run-make">run-make tests</a> (the relevant tests have <code>pgo</code> in their name).
	There is also a <a href="https://github.com/rust-lang/rust/blob/master/tests/codegen/pgo-instrumentation.rs">codegen test</a> that checks that some expected
	instrumentation artifacts show up in LLVM IR.</p>
	<h2 id="additional-information"><a class="header" href="#additional-information">Additional information</a></h2>
	<p>Clang's documentation contains a good overview on <a href="https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization">PGO in LLVM</a>.</p>
	<hr>
	<ol class="footnote-definition"><li id="footnote-note-instrument-coverage">
	<p>Note: <code>rustc</code> now supports front-end-based coverage
	instrumentation, via the experimental option
	<a href="./llvm-coverage-instrumentation.html"><code>-C instrument-coverage</code></a>, but using these
	coverage results for PGO has not been attempted at this time. <a href="#fr-note-instrument-coverage-1">↩</a></p>
	</li>
	</ol>
	</main>

	<nav class="nav-wrapper" aria-label="Page navigation">
	<!-- Mobile navigation buttons -->
	<a rel="prev" href="backend/libs-and-metadata.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
	<i class="fa fa-angle-left"></i>
	</a>

	<a rel="next prefetch" href="llvm-coverage-instrumentation.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
	<i class="fa fa-angle-right"></i>
	</a>

	<div style="clear: both"></div>
	</nav>
	</div>
	</div>

	<nav class="nav-wide-wrapper" aria-label="Page navigation">
	<a rel="prev" href="backend/libs-and-metadata.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
	<i class="fa fa-angle-left"></i>
	</a>

	<a rel="next prefetch" href="llvm-coverage-instrumentation.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
	<i class="fa fa-angle-right"></i>
	</a>
	</nav>

	</div>




	<script>
	window.playground_copyable = true;
	</script>


	<script src="elasticlunr.min.js"></script>
	<script src="mark.min.js"></script>
	<script src="searcher.js"></script>

	<script src="clipboard.min.js"></script>
	<script src="highlight.js"></script>
	<script src="book.js"></script>

	<!-- Custom JS scripts -->
	<script src="mermaid.min.js"></script>
	<script src="mermaid-init.js"></script>


	</div>
	</body>
	</html>